summaryrefslogtreecommitdiffstats
path: root/cache
diff options
context:
space:
mode:
Diffstat (limited to 'cache')
-rw-r--r--cache/dynacache/dynacache.go550
-rw-r--r--cache/dynacache/dynacache_test.go175
-rw-r--r--cache/filecache/filecache.go8
-rw-r--r--cache/filecache/filecache_test.go12
-rw-r--r--cache/filecache/integration_test.go8
-rw-r--r--cache/namedmemcache/named_cache.go78
-rw-r--r--cache/namedmemcache/named_cache_test.go80
7 files changed, 734 insertions, 177 deletions
diff --git a/cache/dynacache/dynacache.go b/cache/dynacache/dynacache.go
new file mode 100644
index 000000000..bb3f7b098
--- /dev/null
+++ b/cache/dynacache/dynacache.go
@@ -0,0 +1,550 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dynacache
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "path"
+ "regexp"
+ "runtime"
+ "sync"
+ "time"
+
+ "github.com/bep/lazycache"
+ "github.com/bep/logg"
+ "github.com/gohugoio/hugo/common/herrors"
+ "github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/common/paths"
+ "github.com/gohugoio/hugo/common/rungroup"
+ "github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/identity"
+ "github.com/gohugoio/hugo/resources/resource"
+)
+
+const minMaxSize = 10
+
+// New creates a new cache.
+func New(opts Options) *Cache {
+ if opts.CheckInterval == 0 {
+ opts.CheckInterval = time.Second * 2
+ }
+
+ if opts.MaxSize == 0 {
+ opts.MaxSize = 100000
+ }
+ if opts.Log == nil {
+ panic("nil Log")
+ }
+
+ if opts.MinMaxSize == 0 {
+ opts.MinMaxSize = 30
+ }
+
+ stats := &stats{
+ opts: opts,
+ adjustmentFactor: 1.0,
+ currentMaxSize: opts.MaxSize,
+ availableMemory: config.GetMemoryLimit(),
+ }
+
+ infol := opts.Log.InfoCommand("dynacache")
+
+ c := &Cache{
+ partitions: make(map[string]PartitionManager),
+ opts: opts,
+ stats: stats,
+ infol: infol,
+ }
+
+ c.stop = c.start()
+
+ return c
+}
+
+// Options for the cache.
+type Options struct {
+ Log loggers.Logger
+ CheckInterval time.Duration
+ MaxSize int
+ MinMaxSize int
+ Running bool
+}
+
+// Options for a partition.
+type OptionsPartition struct {
+ // When to clear the this partition.
+ ClearWhen ClearWhen
+
+ // Weight is a number between 1 and 100 that indicates how, in general, how big this partition may get.
+ Weight int
+}
+
+func (o OptionsPartition) WeightFraction() float64 {
+ return float64(o.Weight) / 100
+}
+
+func (o OptionsPartition) CalculateMaxSize(maxSizePerPartition int) int {
+ return int(math.Floor(float64(maxSizePerPartition) * o.WeightFraction()))
+}
+
+// A dynamic partitioned cache.
+type Cache struct {
+ mu sync.RWMutex
+
+ partitions map[string]PartitionManager
+ opts Options
+ infol logg.LevelLogger
+
+ stats *stats
+ stopOnce sync.Once
+ stop func()
+}
+
+// ClearMatching clears all partition for which the predicate returns true.
+func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
+ g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
+ NumWorkers: len(c.partitions),
+ Handle: func(ctx context.Context, partition PartitionManager) error {
+ partition.clearMatching(predicate)
+ return nil
+ },
+ })
+
+ for _, p := range c.partitions {
+ g.Enqueue(p)
+ }
+
+ g.Wait()
+}
+
+// ClearOnRebuild prepares the cache for a new rebuild taking the given changeset into account.
+func (c *Cache) ClearOnRebuild(changeset ...identity.Identity) {
+ g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
+ NumWorkers: len(c.partitions),
+ Handle: func(ctx context.Context, partition PartitionManager) error {
+ partition.clearOnRebuild(changeset...)
+ return nil
+ },
+ })
+
+ for _, p := range c.partitions {
+ g.Enqueue(p)
+ }
+
+ g.Wait()
+
+ // Clear any entries marked as stale above.
+ g = rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
+ NumWorkers: len(c.partitions),
+ Handle: func(ctx context.Context, partition PartitionManager) error {
+ partition.clearStale()
+ return nil
+ },
+ })
+
+ for _, p := range c.partitions {
+ g.Enqueue(p)
+ }
+
+ g.Wait()
+}
+
+type keysProvider interface {
+ Keys() []string
+}
+
+// Keys returns a list of keys in all partitions.
+func (c *Cache) Keys(predicate func(s string) bool) []string {
+ if predicate == nil {
+ predicate = func(s string) bool { return true }
+ }
+ var keys []string
+ for pn, g := range c.partitions {
+ pkeys := g.(keysProvider).Keys()
+ for _, k := range pkeys {
+ p := path.Join(pn, k)
+ if predicate(p) {
+ keys = append(keys, p)
+ }
+ }
+
+ }
+ return keys
+}
+
+func calculateMaxSizePerPartition(maxItemsTotal, totalWeightQuantity, numPartitions int) int {
+ if numPartitions == 0 {
+ panic("numPartitions must be > 0")
+ }
+ if totalWeightQuantity == 0 {
+ panic("totalWeightQuantity must be > 0")
+ }
+
+ avgWeight := float64(totalWeightQuantity) / float64(numPartitions)
+ return int(math.Floor(float64(maxItemsTotal) / float64(numPartitions) * (100.0 / avgWeight)))
+}
+
+// Stop stops the cache.
+func (c *Cache) Stop() {
+ c.stopOnce.Do(func() {
+ c.stop()
+ })
+}
+
+func (c *Cache) adjustCurrentMaxSize() {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+
+ if len(c.partitions) == 0 {
+ return
+ }
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ s := c.stats
+ s.memstatsCurrent = m
+ // fmt.Printf("\n\nAvailable = %v\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMaxSize = %d\nAdjustmentFactor=%f\n\n", helpers.FormatByteCount(s.availableMemory), helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, c.stats.currentMaxSize, s.adjustmentFactor)
+
+ if s.availableMemory >= s.memstatsCurrent.Alloc {
+ if s.adjustmentFactor <= 1.0 {
+ s.adjustmentFactor += 0.2
+ }
+ } else {
+ // We're low on memory.
+ s.adjustmentFactor -= 0.4
+ }
+
+ if s.adjustmentFactor <= 0 {
+ s.adjustmentFactor = 0.05
+ }
+
+ if !s.adjustCurrentMaxSize() {
+ return
+ }
+
+ totalWeight := 0
+ for _, pm := range c.partitions {
+ totalWeight += pm.getOptions().Weight
+ }
+
+ maxSizePerPartition := calculateMaxSizePerPartition(c.stats.currentMaxSize, totalWeight, len(c.partitions))
+
+ evicted := 0
+ for _, p := range c.partitions {
+ evicted += p.adjustMaxSize(p.getOptions().CalculateMaxSize(maxSizePerPartition))
+ }
+
+ if evicted > 0 {
+ c.infol.
+ WithFields(
+ logg.Fields{
+ {Name: "evicted", Value: evicted},
+ {Name: "numGC", Value: m.NumGC},
+ {Name: "limit", Value: helpers.FormatByteCount(c.stats.availableMemory)},
+ {Name: "alloc", Value: helpers.FormatByteCount(m.Alloc)},
+ {Name: "totalAlloc", Value: helpers.FormatByteCount(m.TotalAlloc)},
+ },
+ ).Logf("adjusted partitions' max size")
+ }
+}
+
+func (c *Cache) start() func() {
+ ticker := time.NewTicker(c.opts.CheckInterval)
+ quit := make(chan struct{})
+
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ c.adjustCurrentMaxSize()
+ case <-quit:
+ ticker.Stop()
+ return
+ }
+ }
+ }()
+
+ return func() {
+ close(quit)
+ }
+}
+
+var partitionNameRe = regexp.MustCompile(`^\/[a-zA-Z0-9]{4}(\/[a-zA-Z0-9]+)?(\/[a-zA-Z0-9]+)?`)
+
+// GetOrCreatePartition gets or creates a partition with the given name.
+func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts OptionsPartition) *Partition[K, V] {
+ if c == nil {
+ panic("nil Cache")
+ }
+ if opts.Weight < 1 || opts.Weight > 100 {
+ panic("invalid Weight, must be between 1 and 100")
+ }
+
+ if partitionNameRe.FindString(name) != name {
+ panic(fmt.Sprintf("invalid partition name %q", name))
+ }
+
+ c.mu.RLock()
+ p, found := c.partitions[name]
+ c.mu.RUnlock()
+ if found {
+ return p.(*Partition[K, V])
+ }
+
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ // Double check.
+ p, found = c.partitions[name]
+ if found {
+ return p.(*Partition[K, V])
+ }
+
+ // At this point, we don't know the the number of partitions or their configuration, but
+ // this will be re-adjusted later.
+ const numberOfPartitionsEstimate = 10
+ maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate)
+
+ // Create a new partition and cache it.
+ partition := &Partition[K, V]{
+ c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}),
+ maxSize: maxSize,
+ trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name),
+ opts: opts,
+ }
+ c.partitions[name] = partition
+
+ return partition
+}
+
+// Partition is a partition in the cache.
+type Partition[K comparable, V any] struct {
+ c *lazycache.Cache[K, V]
+
+ zero V
+
+ trace logg.LevelLogger
+ opts OptionsPartition
+
+ maxSize int
+}
+
+// GetOrCreate gets or creates a value for the given key.
+func (p *Partition[K, V]) GetOrCreate(key K, create func(key K) (V, error)) (V, error) {
+ v, _, err := p.c.GetOrCreate(key, create)
+ return v, err
+}
+
+// GetOrCreateWitTimeout gets or creates a value for the given key and times out if the create function
+// takes too long.
+func (p *Partition[K, V]) GetOrCreateWitTimeout(key K, duration time.Duration, create func(key K) (V, error)) (V, error) {
+ resultch := make(chan V, 1)
+ errch := make(chan error, 1)
+
+ go func() {
+ v, _, err := p.c.GetOrCreate(key, create)
+ if err != nil {
+ errch <- err
+ return
+ }
+ resultch <- v
+ }()
+
+ select {
+ case v := <-resultch:
+ return v, nil
+ case err := <-errch:
+ return p.zero, err
+ case <-time.After(duration):
+ return p.zero, &herrors.TimeoutError{
+ Duration: duration,
+ }
+ }
+}
+
+func (p *Partition[K, V]) clearMatching(predicate func(k, v any) bool) {
+ p.c.DeleteFunc(func(key K, v V) bool {
+ if predicate(key, v) {
+ p.trace.Log(
+ logg.StringFunc(
+ func() string {
+ return fmt.Sprintf("clearing cache key %v", key)
+ },
+ ),
+ )
+ return true
+ }
+ return false
+ })
+}
+
+func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
+ opts := p.getOptions()
+ if opts.ClearWhen == ClearNever {
+ return
+ }
+
+ if opts.ClearWhen == ClearOnRebuild {
+ // Clear all.
+ p.Clear()
+ return
+ }
+
+ depsFinder := identity.NewFinder(identity.FinderConfig{})
+
+ shouldDelete := func(key K, v V) bool {
+ // We always clear elements marked as stale.
+ if resource.IsStaleAny(v) {
+ return true
+ }
+
+ // Now check if this entry has changed based on the changeset
+ // based on filesystem events.
+ if len(changeset) == 0 {
+ // Nothing changed.
+ return false
+ }
+
+ var probablyDependent bool
+ identity.WalkIdentitiesShallow(v, func(level int, id2 identity.Identity) bool {
+ for _, id := range changeset {
+ if r := depsFinder.Contains(id, id2, -1); r > 0 {
+ // It's probably dependent, evict from cache.
+ probablyDependent = true
+ return true
+ }
+ }
+ return false
+ })
+
+ return probablyDependent
+ }
+
+ // First pass.
+ // Second pass needs to be done in a separate loop to catch any
+ // elements marked as stale in the other partitions.
+ p.c.DeleteFunc(func(key K, v V) bool {
+ if shouldDelete(key, v) {
+ p.trace.Log(
+ logg.StringFunc(
+ func() string {
+ return fmt.Sprintf("first pass: clearing cache key %v", key)
+ },
+ ),
+ )
+ resource.MarkStale(v)
+ return true
+ }
+ return false
+ })
+}
+
+func (p *Partition[K, V]) Keys() []K {
+ var keys []K
+ p.c.DeleteFunc(func(key K, v V) bool {
+ keys = append(keys, key)
+ return false
+ })
+ return keys
+}
+
+func (p *Partition[K, V]) clearStale() {
+ p.c.DeleteFunc(func(key K, v V) bool {
+ isStale := resource.IsStaleAny(v)
+ if isStale {
+ p.trace.Log(
+ logg.StringFunc(
+ func() string {
+ return fmt.Sprintf("second pass: clearing cache key %v", key)
+ },
+ ),
+ )
+ }
+
+ return isStale
+ })
+}
+
+// adjustMaxSize adjusts the max size of the and returns the number of items evicted.
+func (p *Partition[K, V]) adjustMaxSize(newMaxSize int) int {
+ if newMaxSize < minMaxSize {
+ newMaxSize = minMaxSize
+ }
+ p.maxSize = newMaxSize
+ // fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize)
+ return p.c.Resize(newMaxSize)
+}
+
+func (p *Partition[K, V]) getMaxSize() int {
+ return p.maxSize
+}
+
+func (p *Partition[K, V]) getOptions() OptionsPartition {
+ return p.opts
+}
+
+func (p *Partition[K, V]) Clear() {
+ p.c.DeleteFunc(func(key K, v V) bool {
+ return true
+ })
+}
+
+func (p *Partition[K, V]) Get(ctx context.Context, key K) (V, bool) {
+ return p.c.Get(key)
+}
+
+type PartitionManager interface {
+ adjustMaxSize(addend int) int
+ getMaxSize() int
+ getOptions() OptionsPartition
+ clearOnRebuild(changeset ...identity.Identity)
+ clearMatching(predicate func(k, v any) bool)
+ clearStale()
+}
+
+const (
+ ClearOnRebuild ClearWhen = iota + 1
+ ClearOnChange
+ ClearNever
+)
+
+type ClearWhen int
+
+type stats struct {
+ opts Options
+ memstatsCurrent runtime.MemStats
+ currentMaxSize int
+ availableMemory uint64
+
+ adjustmentFactor float64
+}
+
+func (s *stats) adjustCurrentMaxSize() bool {
+ newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor))
+
+ if newCurrentMaxSize < s.opts.MaxSize {
+ newCurrentMaxSize = int(s.opts.MinMaxSize)
+ }
+ changed := newCurrentMaxSize != s.currentMaxSize
+ s.currentMaxSize = newCurrentMaxSize
+ return changed
+}
+
+// CleanKey turns s into a format suitable for a cache key for this package.
+// The key will be a Unix-styled path with a leading slash but no trailing slash.
+func CleanKey(s string) string {
+ return path.Clean(paths.ToSlashPreserveLeading(s))
+}
diff --git a/cache/dynacache/dynacache_test.go b/cache/dynacache/dynacache_test.go
new file mode 100644
index 000000000..53de2385e
--- /dev/null
+++ b/cache/dynacache/dynacache_test.go
@@ -0,0 +1,175 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dynacache
+
+import (
+ "path/filepath"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/identity"
+ "github.com/gohugoio/hugo/resources/resource"
+)
+
+var (
+ _ resource.StaleInfo = (*testItem)(nil)
+ _ identity.Identity = (*testItem)(nil)
+)
+
+type testItem struct {
+ name string
+ isStale bool
+}
+
+func (t testItem) IsStale() bool {
+ return t.isStale
+}
+
+func (t testItem) IdentifierBase() string {
+ return t.name
+}
+
+func TestCache(t *testing.T) {
+ t.Parallel()
+ c := qt.New(t)
+
+ cache := New(Options{
+ Log: loggers.NewDefault(),
+ })
+
+ c.Cleanup(func() {
+ cache.Stop()
+ })
+
+ opts := OptionsPartition{Weight: 30}
+
+ c.Assert(cache, qt.Not(qt.IsNil))
+
+ p1 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", opts)
+ c.Assert(p1, qt.Not(qt.IsNil))
+
+ p2 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", opts)
+
+ c.Assert(func() { GetOrCreatePartition[string, testItem](cache, "foo bar", opts) }, qt.PanicMatches, ".*invalid partition name.*")
+ c.Assert(func() { GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", OptionsPartition{Weight: 1234}) }, qt.PanicMatches, ".*invalid Weight.*")
+
+ c.Assert(p2, qt.Equals, p1)
+
+ p3 := GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", opts)
+ c.Assert(p3, qt.Not(qt.IsNil))
+ c.Assert(p3, qt.Not(qt.Equals), p1)
+
+ c.Assert(func() { New(Options{}) }, qt.PanicMatches, ".*nil Log.*")
+}
+
+func TestCalculateMaxSizePerPartition(t *testing.T) {
+ t.Parallel()
+ c := qt.New(t)
+
+ c.Assert(calculateMaxSizePerPartition(1000, 500, 5), qt.Equals, 200)
+ c.Assert(calculateMaxSizePerPartition(1000, 250, 5), qt.Equals, 400)
+ c.Assert(func() { calculateMaxSizePerPartition(1000, 250, 0) }, qt.PanicMatches, ".*must be > 0.*")
+ c.Assert(func() { calculateMaxSizePerPartition(1000, 0, 1) }, qt.PanicMatches, ".*must be > 0.*")
+}
+
+func TestCleanKey(t *testing.T) {
+ c := qt.New(t)
+
+ c.Assert(CleanKey("a/b/c"), qt.Equals, "/a/b/c")
+ c.Assert(CleanKey("/a/b/c"), qt.Equals, "/a/b/c")
+ c.Assert(CleanKey("a/b/c/"), qt.Equals, "/a/b/c")
+ c.Assert(CleanKey(filepath.FromSlash("/a/b/c/")), qt.Equals, "/a/b/c")
+}
+
+func newTestCache(t *testing.T) *Cache {
+ cache := New(
+ Options{
+ Log: loggers.NewDefault(),
+ },
+ )
+
+ p1 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", OptionsPartition{Weight: 30, ClearWhen: ClearOnRebuild})
+ p2 := GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", OptionsPartition{Weight: 30, ClearWhen: ClearOnChange})
+
+ p1.GetOrCreate("clearOnRebuild", func(string) (testItem, error) {
+ return testItem{}, nil
+ })
+
+ p2.GetOrCreate("clearBecauseStale", func(string) (testItem, error) {
+ return testItem{
+ isStale: true,
+ }, nil
+ })
+
+ p2.GetOrCreate("clearBecauseIdentityChanged", func(string) (testItem, error) {
+ return testItem{
+ name: "changed",
+ }, nil
+ })
+
+ p2.GetOrCreate("clearNever", func(string) (testItem, error) {
+ return testItem{
+ isStale: false,
+ }, nil
+ })
+
+ t.Cleanup(func() {
+ cache.Stop()
+ })
+
+ return cache
+}
+
+func TestClear(t *testing.T) {
+ t.Parallel()
+ c := qt.New(t)
+
+ predicateAll := func(string) bool {
+ return true
+ }
+
+ cache := newTestCache(t)
+
+ c.Assert(cache.Keys(predicateAll), qt.HasLen, 4)
+
+ cache.ClearOnRebuild()
+
+ // Stale items are always cleared.
+ c.Assert(cache.Keys(predicateAll), qt.HasLen, 2)
+
+ cache = newTestCache(t)
+ cache.ClearOnRebuild(identity.StringIdentity("changed"))
+
+ c.Assert(cache.Keys(nil), qt.HasLen, 1)
+
+ cache = newTestCache(t)
+
+ cache.ClearMatching(func(k, v any) bool {
+ return k.(string) == "clearOnRebuild"
+ })
+
+ c.Assert(cache.Keys(predicateAll), qt.HasLen, 3)
+
+ cache.adjustCurrentMaxSize()
+}
+
+func TestAdjustCurrentMaxSize(t *testing.T) {
+ t.Parallel()
+ c := qt.New(t)
+ cache := newTestCache(t)
+ alloc := cache.stats.memstatsCurrent.Alloc
+ cache.adjustCurrentMaxSize()
+ c.Assert(cache.stats.memstatsCurrent.Alloc, qt.Not(qt.Equals), alloc)
+}
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
index 414478ee2..093d2941c 100644
--- a/cache/filecache/filecache.go
+++ b/cache/filecache/filecache.go
@@ -24,6 +24,7 @@ import (
"time"
"github.com/gohugoio/hugo/common/hugio"
+ "github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/helpers"
@@ -109,7 +110,7 @@ func (l *lockedFile) Close() error {
func (c *Cache) init() error {
c.initOnce.Do(func() {
// Create the base dir if it does not exist.
- if err := c.Fs.MkdirAll("", 0777); err != nil && !os.IsExist(err) {
+ if err := c.Fs.MkdirAll("", 0o777); err != nil && !os.IsExist(err) {
c.initErr = err
}
})
@@ -146,7 +147,8 @@ func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
// it when done.
func (c *Cache) ReadOrCreate(id string,
read func(info ItemInfo, r io.ReadSeeker) error,
- create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
+ create func(info ItemInfo, w io.WriteCloser) error,
+) (info ItemInfo, err error) {
if err := c.init(); err != nil {
return ItemInfo{}, err
}
@@ -380,7 +382,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
baseDir := v.DirCompiled
- bfs := afero.NewBasePathFs(cfs, baseDir)
+ bfs := hugofs.NewBasePathFs(cfs, baseDir)
var pruneAllRootDir string
if k == CacheKeyModules {
diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
index 61f9eda64..59fb09276 100644
--- a/cache/filecache/filecache_test.go
+++ b/cache/filecache/filecache_test.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@ import (
"errors"
"fmt"
"io"
- "path/filepath"
"strings"
"sync"
"testing"
@@ -86,17 +85,8 @@ dir = ":cacheDir/c"
cache := caches.Get("GetJSON")
c.Assert(cache, qt.Not(qt.IsNil))
- bfs, ok := cache.Fs.(*afero.BasePathFs)
- c.Assert(ok, qt.Equals, true)
- filename, err := bfs.RealPath("key")
- c.Assert(err, qt.IsNil)
-
cache = caches.Get("Images")
c.Assert(cache, qt.Not(qt.IsNil))
- bfs, ok = cache.Fs.(*afero.BasePathFs)
- c.Assert(ok, qt.Equals, true)
- filename, _ = bfs.RealPath("key")
- c.Assert(filename, qt.Equals, filepath.FromSlash("_gen/images/key"))
rf := func(s string) func() (io.ReadCloser, error) {
return func() (io.ReadCloser, error) {
diff --git a/cache/filecache/integration_test.go b/cache/filecache/integration_test.go
index a8a45988e..1e920c29f 100644
--- a/cache/filecache/integration_test.go
+++ b/cache/filecache/integration_test.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@ package filecache_test
import (
"path/filepath"
-
"testing"
"time"
@@ -47,7 +46,6 @@ title: "Home"
_, err := b.H.BaseFs.ResourcesCache.Stat(filepath.Join("_gen", "images"))
b.Assert(err, qt.IsNil)
-
}
func TestPruneImages(t *testing.T) {
@@ -55,6 +53,7 @@ func TestPruneImages(t *testing.T) {
// TODO(bep)
t.Skip("skip flaky test on CI server")
}
+ t.Skip("skip flaky test")
files := `
-- hugo.toml --
baseURL = "https://example.com"
@@ -92,7 +91,7 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
// TODO(bep) we need a way to test full rebuilds.
// For now, just sleep a little so the cache elements expires.
- time.Sleep(300 * time.Millisecond)
+ time.Sleep(500 * time.Millisecond)
b.RenameFile("assets/a/pixel.png", "assets/b/pixel2.png").Build()
@@ -104,5 +103,4 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
b.Assert(err, qt.Not(qt.IsNil))
_, err = b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir)
b.Assert(err, qt.IsNil)
-
}
diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go
deleted file mode 100644
index 7fb4fe8ed..000000000
--- a/cache/namedmemcache/named_cache.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package namedmemcache provides a memory cache with a named lock. This is suitable
-// for situations where creating the cached resource can be time consuming or otherwise
-// resource hungry, or in situations where a "once only per key" is a requirement.
-package namedmemcache
-
-import (
- "sync"
-
- "github.com/BurntSushi/locker"
-)
-
-// Cache holds the cached values.
-type Cache struct {
- nlocker *locker.Locker
- cache map[string]cacheEntry
- mu sync.RWMutex
-}
-
-type cacheEntry struct {
- value any
- err error
-}
-
-// New creates a new cache.
-func New() *Cache {
- return &Cache{
- nlocker: locker.NewLocker(),
- cache: make(map[string]cacheEntry),
- }
-}
-
-// Clear clears the cache state.
-func (c *Cache) Clear() {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- c.cache = make(map[string]cacheEntry)
- c.nlocker = locker.NewLocker()
-}
-
-// GetOrCreate tries to get the value with the given cache key, if not found
-// create will be called and cached.
-// This method is thread safe. It also guarantees that the create func for a given
-// key is invoked only once for this cache.
-func (c *Cache) GetOrCreate(key string, create func() (any, error)) (any, error) {
- c.mu.RLock()
- entry, found := c.cache[key]
- c.mu.RUnlock()
-
- if found {
- return entry.value, entry.err
- }
-
- c.nlocker.Lock(key)
- defer c.nlocker.Unlock(key)
-
- // Create it.
- value, err := create()
-
- c.mu.Lock()
- c.cache[key] = cacheEntry{value: value, err: err}
- c.mu.Unlock()
-
- return value, err
-}
diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go
deleted file mode 100644
index 2db923d76..000000000
--- a/cache/namedmemcache/named_cache_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package namedmemcache
-
-import (
- "fmt"
- "sync"
- "testing"
-
- qt "github.com/frankban/quicktest"
-)
-
-func TestNamedCache(t *testing.T) {
- t.Parallel()
- c := qt.New(t)
-
- cache := New()
-
- counter := 0
- create := func() (any, error) {
- counter++
- return counter, nil
- }
-
- for i := 0; i < 5; i++ {
- v1, err := cache.GetOrCreate("a1", create)
- c.Assert(err, qt.IsNil)
- c.Assert(v1, qt.Equals, 1)
- v2, err := cache.GetOrCreate("a2", create)
- c.Assert(err, qt.IsNil)
- c.Assert(v2, qt.Equals, 2)
- }
-
- cache.Clear()
-
- v3, err := cache.GetOrCreate("a2", create)
- c.Assert(err, qt.IsNil)
- c.Assert(v3, qt.Equals, 3)
-}
-
-func TestNamedCacheConcurrent(t *testing.T) {
- t.Parallel()
-
- c := qt.New(t)
-
- var wg sync.WaitGroup
-
- cache := New()
-
- create := func(i int) func() (any, error) {
- return func() (any, error) {
- return i, nil
- }
- }
-
- for i := 0; i < 10; i++ {
- wg.Add(1)
- go func() {
- defer wg.Done()
- for j := 0; j < 100; j++ {
- id := fmt.Sprintf("id%d", j)
- v, err := cache.GetOrCreate(id, create(j))
- c.Assert(err, qt.IsNil)
- c.Assert(v, qt.Equals, j)
- }
- }()
- }
- wg.Wait()
-}
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-06 22:19:22 +0000