From c4bbc2805c7092deafabae01e71ff2dda37bd877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 23 May 2026 12:12:51 +0200 Subject: [PATCH] all: Replace RWMutex struct caches with ConcurrentMap --- cache/filecache/filecache.go | 17 ++------ cache/filecache/filecache_pruner.go | 5 +-- common/hstrings/strings.go | 43 +++---------------- go.mod | 2 +- go.sum | 4 +- hugolib/hugo_sites_build.go | 5 ++- hugolib/shortcode.go | 15 +++---- markup/highlight/chromalexers/chromalexers.go | 30 +++---------- resources/jsconfig/jsconfig.go | 34 +++++---------- resources/post_publish.go | 28 +++--------- resources/resource_spec.go | 6 +-- tpl/images/images.go | 42 ++++++------------ tpl/images/images_test.go | 3 +- 13 files changed, 66 insertions(+), 168 deletions(-) diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 1d5ba7650..f5d5b09bf 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -31,6 +31,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/BurntSushi/locker" + "github.com/bep/helpers/maphelpers" "github.com/spf13/afero" ) @@ -56,8 +57,7 @@ type Cache struct { } type lockTracker struct { - seenMu sync.RWMutex - seen map[string]struct{} + seen *maphelpers.ConcurrentSet[string] *locker.Locker } @@ -65,16 +65,7 @@ type lockTracker struct { // Lock tracks the ids in use. We use this information to do garbage collection // after a Hugo build. func (l *lockTracker) Lock(id string) { - l.seenMu.RLock() - if _, seen := l.seen[id]; !seen { - l.seenMu.RUnlock() - l.seenMu.Lock() - l.seen[id] = struct{}{} - l.seenMu.Unlock() - } else { - l.seenMu.RUnlock() - } - + l.seen.AddIfAbsent(id) l.Locker.Lock(id) } @@ -92,7 +83,7 @@ func NewCache(fs afero.Fs, cfg FileCacheConfig) *Cache { return &Cache{ Fs: fs, - entryLocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})}, + entryLocker: &lockTracker{Locker: locker.NewLocker(), seen: maphelpers.NewConcurrentSet[string]()}, cfg: cfg, } } diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go index 890aee965..d6322cb70 100644 --- a/cache/filecache/filecache_pruner.go +++ b/cache/filecache/filecache_pruner.go @@ -93,10 +93,9 @@ func (c *Cache) Prune(force bool) (int, error) { shouldRemove := force || c.isExpired(info.ModTime()) - if !shouldRemove && len(c.entryLocker.seen) > 0 { + if !shouldRemove && c.entryLocker.seen.Len() > 0 { // Remove it if it's not been touched/used in the last build. - _, seen := c.entryLocker.seen[name] - shouldRemove = !seen + shouldRemove = !c.entryLocker.seen.Has(name) } if shouldRemove { diff --git a/common/hstrings/strings.go b/common/hstrings/strings.go index ee7a62a3a..49adb61a8 100644 --- a/common/hstrings/strings.go +++ b/common/hstrings/strings.go @@ -19,8 +19,8 @@ import ( "slices" "sort" "strings" - "sync" + "github.com/bep/helpers/maphelpers" "github.com/gohugoio/hugo/compare" ) @@ -55,46 +55,17 @@ func EqualAny(a string, b ...string) bool { return slices.Contains(b, a) } -// regexpCache represents a cache of regexp objects protected by a mutex. -type regexpCache struct { - mu sync.RWMutex - re map[string]*regexp.Regexp -} - -func (rc *regexpCache) getOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) { - var ok bool - - if re, ok = rc.get(pattern); !ok { - re, err = regexp.Compile(pattern) - if err != nil { - return nil, err - } - rc.set(pattern, re) - } - - return re, nil -} - -func (rc *regexpCache) get(key string) (re *regexp.Regexp, ok bool) { - rc.mu.RLock() - re, ok = rc.re[key] - rc.mu.RUnlock() - return -} - -func (rc *regexpCache) set(key string, re *regexp.Regexp) { - rc.mu.Lock() - rc.re[key] = re - rc.mu.Unlock() -} - -var reCache = regexpCache{re: make(map[string]*regexp.Regexp)} +var reCache = *maphelpers.NewConcurrentMap[string, *regexp.Regexp]() // GetOrCompileRegexp retrieves a regexp object from the cache based upon the pattern. // If the pattern is not found in the cache, the pattern is compiled and added to // the cache. func GetOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) { - return reCache.getOrCompileRegexp(pattern) + return reCache.GetOrCreate(pattern, + func() (*regexp.Regexp, error) { + return regexp.Compile(pattern) + }, + ) } // HasAnyPrefix checks if the string s has any of the prefixes given. diff --git a/go.mod b/go.mod index 89dee3907..bc935dc5e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/bep/golibsass v1.2.0 github.com/bep/golocales v0.1.0 github.com/bep/goportabletext v0.2.0 - github.com/bep/helpers v0.8.0 + github.com/bep/helpers v0.12.0 github.com/bep/imagemeta v0.17.2 github.com/bep/lazycache v0.8.1 github.com/bep/logg v0.4.0 diff --git a/go.sum b/go.sum index 88d2dc748..7c34b13fd 100644 --- a/go.sum +++ b/go.sum @@ -162,8 +162,8 @@ github.com/bep/golocales v0.1.0 h1:rjWf1S4basIje+G+je5WMW8G+yzaoz4gEDFolrFVdvA= github.com/bep/golocales v0.1.0/go.mod h1:Hl78nje8mNL3LzLeJvYN9NsIZgyFJGrGfvgO9r1+mwE= github.com/bep/goportabletext v0.2.0 h1:CZ9f8jADBWqHwBymQiJJPCTSV/tHSA+PYzlUf86Yze0= github.com/bep/goportabletext v0.2.0/go.mod h1:xDeA5+qcgKzJq6Q6XjAiBKtxLD3Yn7f6XP4joD3J3qU= -github.com/bep/helpers v0.8.0 h1:plg2BFgA9AgIHF2XemyZdZLqixjzQk3uyyArV48FngQ= -github.com/bep/helpers v0.8.0/go.mod h1:PfE7MGdA8sSQ19nyDh4tYbs5rAlStlJaDI21f/fnNps= +github.com/bep/helpers v0.12.0 h1:tD6V2DQW0B+FUynF2etR/106S/TO9akm+vA/Hk24GxY= +github.com/bep/helpers v0.12.0/go.mod h1:PfE7MGdA8sSQ19nyDh4tYbs5rAlStlJaDI21f/fnNps= github.com/bep/imagemeta v0.17.2 h1:fDyXM1eAqCfBeqGLqS6UsN4OfuLM0cdu70KuLCehjOg= github.com/bep/imagemeta v0.17.2/go.mod h1:+Hlp195TfZpzsqCxtDKTG6eWdyz2+F2V/oCYfr3CZKA= github.com/bep/lazycache v0.8.1 h1:ko6ASLjkPxyV5DMWoNNZ8B2M0weyjqXX8IZkjBoBtvg= diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 0b674207c..4fef1bc77 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bep/debounce" + "github.com/bep/helpers/maphelpers" "github.com/bep/logg" "github.com/gohugoio/go-radix" "github.com/gohugoio/hugo/bufferpool" @@ -692,7 +693,7 @@ func (h *HugoSites) postProcess(l logg.LevelLogger) error { } var toPostProcess []postpub.PostPublishedResource - for _, r := range h.ResourceSpec.PostProcessResources { + for _, r := range h.ResourceSpec.PostProcessResources.All() { toPostProcess = append(toPostProcess, r) } @@ -758,7 +759,7 @@ func (h *HugoSites) postProcess(l logg.LevelLogger) error { // Prepare for a new build. for _, s := range h.Sites { - s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource) + s.ResourceSpec.PostProcessResources = maphelpers.NewConcurrentMap[string, postpub.PostPublishedResource]() } return g.Wait() diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index aae033051..d283a901b 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -27,6 +27,7 @@ import ( "strings" "sync" + "github.com/bep/helpers/maphelpers" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hstore" "github.com/gohugoio/hugo/common/types" @@ -286,24 +287,18 @@ type shortcodeParseInfo struct { shortcodes []*shortcode // All the shortcode names in this set. - nameSetMu sync.RWMutex - nameSet map[string]bool + nameSet *maphelpers.ConcurrentSet[string] // Configuration enableInlineShortcodes bool } func (s *shortcodeParseInfo) addName(name string) { - s.nameSetMu.Lock() - defer s.nameSetMu.Unlock() - s.nameSet[name] = true + s.nameSet.Add(name) } func (s *shortcodeParseInfo) hasName(name string) bool { - s.nameSetMu.RLock() - defer s.nameSetMu.RUnlock() - _, ok := s.nameSet[name] - return ok + return s.nameSet.Has(name) } func newShortcodeHandler(filename string, d *deps.Deps) *shortcodeParseInfo { @@ -312,7 +307,7 @@ func newShortcodeHandler(filename string, d *deps.Deps) *shortcodeParseInfo { firstTemplateStore: d.TemplateStore, enableInlineShortcodes: d.ExecHelper.Sec().EnableInlineShortcodes, shortcodes: make([]*shortcode, 0, 4), - nameSet: make(map[string]bool), + nameSet: maphelpers.NewConcurrentSet[string](), } return sh diff --git a/markup/highlight/chromalexers/chromalexers.go b/markup/highlight/chromalexers/chromalexers.go index 6ab4a7bbe..f0a18a148 100644 --- a/markup/highlight/chromalexers/chromalexers.go +++ b/markup/highlight/chromalexers/chromalexers.go @@ -14,37 +14,21 @@ package chromalexers import ( - "sync" - "github.com/alecthomas/chroma/v2" "github.com/alecthomas/chroma/v2/lexers" + "github.com/bep/helpers/maphelpers" ) -type lexersMap struct { - lexers map[string]chroma.Lexer - mu sync.RWMutex -} - -var lexerCache = &lexersMap{lexers: make(map[string]chroma.Lexer)} +var lexerCache = *maphelpers.NewConcurrentMap[string, chroma.Lexer]() // Get returns a lexer for the given language name, nil if not found. // This is just a wrapper around chromalexers.Get that caches the result. // Reasoning for this is that chromalexers.Get is slow in the case where the lexer is not found, // which is a common case in Hugo. func Get(name string) chroma.Lexer { - lexerCache.mu.RLock() - lexer, found := lexerCache.lexers[name] - lexerCache.mu.RUnlock() - - if found { - return lexer - } - - lexer = lexers.Get(name) - - lexerCache.mu.Lock() - lexerCache.lexers[name] = lexer - lexerCache.mu.Unlock() - - return lexer + l, _ := lexerCache.GetOrCreate(name, func() (chroma.Lexer, error) { + l := lexers.Get(name) + return l, nil + }) + return l } diff --git a/resources/jsconfig/jsconfig.go b/resources/jsconfig/jsconfig.go index b6e867995..dfc2f235d 100644 --- a/resources/jsconfig/jsconfig.go +++ b/resources/jsconfig/jsconfig.go @@ -16,59 +16,47 @@ package jsconfig import ( "path/filepath" "sort" - "sync" + + "github.com/bep/helpers/maphelpers" ) // Builder builds a jsconfig.json file that, currently, is used only to assist // IntelliSense in editors. type Builder struct { - sourceRootsMu sync.RWMutex - sourceRoots map[string]bool + sourceRoots *maphelpers.ConcurrentSet[string] } // NewBuilder creates a new Builder. func NewBuilder() *Builder { - return &Builder{sourceRoots: make(map[string]bool)} + return &Builder{sourceRoots: maphelpers.NewConcurrentSet[string]()} } // Build builds a new Config with paths relative to dir. // This method is thread safe. func (b *Builder) Build(dir string) *Config { - b.sourceRootsMu.RLock() - defer b.sourceRootsMu.RUnlock() - - if len(b.sourceRoots) == 0 { + if b.sourceRoots.Len() == 0 { return nil } - conf := newJSConfig() - var roots []string - for root := range b.sourceRoots { + for root := range b.sourceRoots.All() { rel, err := filepath.Rel(dir, filepath.Join(root, "*")) if err == nil { roots = append(roots, rel) } } + if len(roots) == 0 { + return nil + } sort.Strings(roots) + conf := newJSConfig() conf.CompilerOptions.Paths["*"] = roots - return conf } // AddSourceRoot adds a new source root. // This method is thread safe. func (b *Builder) AddSourceRoot(root string) { - b.sourceRootsMu.RLock() - found := b.sourceRoots[root] - b.sourceRootsMu.RUnlock() - - if found { - return - } - - b.sourceRootsMu.Lock() - b.sourceRoots[root] = true - b.sourceRootsMu.Unlock() + b.sourceRoots.AddIfAbsent(root) } // CompilerOptions holds compilerOptions for jsonconfig.json. diff --git a/resources/post_publish.go b/resources/post_publish.go index b2adfa5ce..848755782 100644 --- a/resources/post_publish.go +++ b/resources/post_publish.go @@ -25,27 +25,11 @@ type transformationKeyer interface { // PostProcess wraps the given Resource for later processing. func (spec *Spec) PostProcess(r resource.Resource) (postpub.PostPublishedResource, error) { key := r.(transformationKeyer).TransformationKey() - spec.postProcessMu.RLock() - result, found := spec.PostProcessResources[key] - spec.postProcessMu.RUnlock() - if found { + return spec.PostProcessResources.GetOrCreate(key, func() (postpub.PostPublishedResource, error) { + result := postpub.NewPostPublishResource(spec.incr.Incr(), r) + if result == nil { + panic("got nil result") + } return result, nil - } - - spec.postProcessMu.Lock() - defer spec.postProcessMu.Unlock() - - // Double check - result, found = spec.PostProcessResources[key] - if found { - return result, nil - } - - result = postpub.NewPostPublishResource(spec.incr.Incr(), r) - if result == nil { - panic("got nil result") - } - spec.PostProcessResources[key] = result - - return result, nil + }) } diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 2ffa4f9f2..861cdc435 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -18,6 +18,7 @@ import ( "path" "sync" + "github.com/bep/helpers/maphelpers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/internal/warpc" @@ -88,7 +89,7 @@ func NewSpec( incr: incr, FileCaches: fileCaches, PostBuildAssets: &PostBuildAssets{ - PostProcessResources: make(map[string]postpub.PostPublishedResource), + PostProcessResources: maphelpers.NewConcurrentMap[string, postpub.PostPublishedResource](), JSConfigBuilder: jsconfig.NewBuilder(), }, } @@ -155,8 +156,7 @@ type SpecCommon struct { } type PostBuildAssets struct { - postProcessMu sync.RWMutex - PostProcessResources map[string]postpub.PostPublishedResource + PostProcessResources *maphelpers.ConcurrentMap[string, postpub.PostPublishedResource] JSConfigBuilder *jsconfig.Builder } diff --git a/tpl/images/images.go b/tpl/images/images.go index 8185bf186..2dcad8662 100644 --- a/tpl/images/images.go +++ b/tpl/images/images.go @@ -20,8 +20,8 @@ import ( "image" "path" "path/filepath" - "sync" + "github.com/bep/helpers/maphelpers" "github.com/bep/overlayfs" "github.com/gohugoio/hugo/common/hashing" "github.com/gohugoio/hugo/common/hugio" @@ -52,7 +52,7 @@ func New(d *deps.Deps) *Namespace { return &Namespace{ readFileFs: readFileFs, Filters: &images.Filters{}, - cache: map[string]image.Config{}, + cache: maphelpers.NewConcurrentMap[string, image.Config](), deps: d, createClient: create.New(d.ResourceSpec), } @@ -62,8 +62,7 @@ func New(d *deps.Deps) *Namespace { type Namespace struct { *images.Filters readFileFs afero.Fs - cacheMu sync.RWMutex - cache map[string]image.Config + cache *maphelpers.ConcurrentMap[string, image.Config] deps *deps.Deps createClient *create.Client } @@ -80,34 +79,19 @@ func (ns *Namespace) Config(path any) (image.Config, error) { return image.Config{}, errors.New("config needs a filename") } - // Check cache for image config. - ns.cacheMu.RLock() - config, ok := ns.cache[filename] - ns.cacheMu.RUnlock() + return ns.cache.GetOrCreate(filename, func() (image.Config, error) { + f, err := ns.readFileFs.Open(filename) + if err != nil { + return image.Config{}, err + } + defer f.Close() - if ok { - return config, nil - } + ext := filepath.Ext(filename) + format, _ := images.ImageFormatFromExt(ext) - f, err := ns.readFileFs.Open(filename) - if err != nil { - return image.Config{}, err - } - defer f.Close() - - ext := filepath.Ext(filename) - format, _ := images.ImageFormatFromExt(ext) - - config, _, err = ns.deps.ResourceSpec.Imaging.Codec.DecodeConfig(format, f) - if err != nil { + config, _, err := ns.deps.ResourceSpec.Imaging.Codec.DecodeConfig(format, f) return config, err - } - - ns.cacheMu.Lock() - ns.cache[filename] = config - ns.cacheMu.Unlock() - - return config, nil + }) } // Filter applies the given filters to the image given as the last element in args. diff --git a/tpl/images/images_test.go b/tpl/images/images_test.go index be5db69fe..95cc8a48e 100644 --- a/tpl/images/images_test.go +++ b/tpl/images/images_test.go @@ -113,7 +113,8 @@ func TestNSConfig(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(result, qt.Equals, test.expect) - c.Assert(len(ns.cache), qt.Not(qt.Equals), 0) + _, cached := ns.cache.Lookup(sp) + c.Assert(cached, qt.IsTrue) } }