cache/filecache: Don't prune used cache entries with mixed-case dir names

On case-insensitive filesystems, entries created before we started lowercasing
the content paths in v0.123 are the same file as today's lowercased cache key,
and were removed on every hugo --gc.

Fixes #15101

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Bjørn Erik Pedersen
2026-07-26 11:40:27 +02:00
committed by GitHub
parent f228c87d41
commit 861ede6d10
5 changed files with 217 additions and 13 deletions
+53 -12
View File
@@ -17,6 +17,7 @@ import (
"fmt"
"io"
"os"
"strings"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/hugofs"
@@ -59,6 +60,29 @@ func (c *Cache) Prune(force bool) (int, error) {
counter := 0
seen := c.entryLocker.seen
seenByLower := make(map[string]string, seen.Len())
for id := range seen.All() {
seenByLower[strings.ToLower(id)] = id
}
// Names on disk matching a used cache key except for the case, and the used keys
// actually walked. See the note about case-insensitive filesystems below.
var candidates map[string]string
visited := make(map[string]bool, seen.Len())
remove := func(name string) error {
err := c.Fs.Remove(name)
if err == nil {
counter++
return nil
}
if !herrors.IsNotExist(err) {
return err
}
return nil
}
err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {
if info == nil {
return nil
@@ -93,29 +117,46 @@ func (c *Cache) Prune(force bool) (int, error) {
shouldRemove := force || c.isExpired(info.ModTime())
if !shouldRemove && c.entryLocker.seen.Len() > 0 {
if seen.Has(name) {
visited[name] = true
} else if !shouldRemove && seen.Len() > 0 {
if id, found := seenByLower[strings.ToLower(name)]; found {
// On case-insensitive filesystems this is the same file as id; e.g. an
// entry created before Hugo started lowercasing the content paths in
// v0.123 (content/MyBundle => _gen/images/MyBundle). Decided once the
// walk is done: if id is walked too, they are distinct files and this
// one is stale. See issue 15101.
if candidates == nil {
candidates = make(map[string]string)
}
candidates[name] = id
} else {
// Remove it if it's not been touched/used in the last build.
shouldRemove = !c.entryLocker.seen.Has(name)
shouldRemove = true
}
}
if shouldRemove {
err := c.Fs.Remove(name)
if err == nil {
counter++
}
if err != nil && !herrors.IsNotExist(err) {
return err
}
return remove(name)
}
return nil
})
if err != nil {
return counter, err
}
for name, id := range candidates {
if visited[id] {
if err := remove(name); err != nil {
return counter, err
}
}
}
return counter, nil
}
func (c *Cache) pruneRootDirs(force bool) (int, error) {
dirs, err := afero.ReadDir(c.Fs, "")
if err != nil {
+61
View File
@@ -20,11 +20,72 @@ import (
"time"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/htesting"
"github.com/spf13/afero"
qt "github.com/frankban/quicktest"
)
// A cache entry created before Hugo started lowercasing content paths in v0.123
// (e.g. _gen/images/MyBundle) is on a case-insensitive filesystem the same file as
// the lowercased cache key used today, and must not be pruned.
// See issue 15101.
func TestPruneCacheEntryWithOtherCase(t *testing.T) {
t.Parallel()
c := qt.New(t)
dir := t.TempDir()
if isCaseInsensitive, err := htesting.IsCaseInsensitiveFs(dir); err != nil {
t.Fatal(err)
} else if !isCaseInsensitive {
t.Skip("skip test on case-sensitive filesystem")
}
fs := afero.NewBasePathFs(afero.NewOsFs(), dir)
newCache := func() *filecache.Cache {
return filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
}
c.Assert(newCache().SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil)
cache := newCache()
_, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) {
return []byte("def"), nil
})
c.Assert(err, qt.IsNil)
c.Assert(string(b), qt.Equals, "abc")
count, err := cache.Prune(false)
c.Assert(err, qt.IsNil)
c.Assert(count, qt.Equals, 0)
c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "abc")
}
// On a case-sensitive filesystem the entries above are distinct files,
// and the one not used in this build should be pruned.
func TestPruneCacheEntryWithOtherCaseCaseSensitiveFs(t *testing.T) {
t.Parallel()
c := qt.New(t)
fs := afero.NewMemMapFs()
cache := filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
c.Assert(cache.SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil)
cache = filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
_, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) {
return []byte("def"), nil
})
c.Assert(err, qt.IsNil)
c.Assert(string(b), qt.Equals, "def")
count, err := cache.Prune(false)
c.Assert(err, qt.IsNil)
c.Assert(count, qt.Equals, 1)
c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "")
c.Assert(cache.GetString("mybundle/i1"), qt.Equals, "def")
}
func TestPrune(t *testing.T) {
t.Parallel()
+22
View File
@@ -16,6 +16,7 @@ package htesting
import (
"math/rand"
"os"
"path/filepath"
"regexp"
"runtime"
"strconv"
@@ -58,6 +59,27 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) {
return tempDir, func() { fs.RemoveAll(tempDir) }, nil
}
// IsCaseInsensitiveFs reports whether dir lives on a case-insensitive filesystem
// (e.g. the default on macOS and Windows).
func IsCaseInsensitiveFs(dir string) (bool, error) {
f, err := os.CreateTemp(dir, "case-*.tmp")
if err != nil {
return false, err
}
f.Close()
defer os.Remove(f.Name())
_, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(f.Name()))))
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// BailOut panics with a stack trace after the given duration. Useful for
// hanging tests.
func BailOut(after time.Duration) {
+42
View File
@@ -15,6 +15,7 @@ package main
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -27,6 +28,7 @@ import (
"runtime"
"strconv"
"strings"
"sync"
"testing"
"time"
@@ -78,10 +80,22 @@ func TestMain(m *testing.M) {
})
}
var isCaseInsensitiveFs = sync.OnceValues(func() (bool, error) {
return htesting.IsCaseInsensitiveFs(os.TempDir())
})
var commonTestScriptsParam = testscript.Params{
Setup: func(env *testscript.Env) error {
return testSetupFunc()(env)
},
Condition: func(cond string) (bool, error) {
switch cond {
case "caseinsensitivefs":
return isCaseInsensitiveFs()
default:
return false, fmt.Errorf("unknown condition %q", cond)
}
},
Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
// log prints to stderr.
"log": func(ts *testscript.TestScript, neg bool, args []string) {
@@ -252,6 +266,34 @@ var commonTestScriptsParam = testscript.Params{
ts.Fatalf("failed to create symlink: %v", err)
}
},
// base64decode decodes a base64-encoded file into a binary file.
"base64decode": func(ts *testscript.TestScript, neg bool, args []string) {
if len(args) != 2 {
ts.Fatalf("usage: base64decode src_base64_file dest_binary_file")
}
// Resolve paths relative to the test sandbox's current directory
src := ts.MkAbs(args[0])
dest := ts.MkAbs(args[1])
// Read the base64 text
base64Data, err := os.ReadFile(src)
if err != nil {
ts.Fatalf("failed to read base64 file: %v", err)
}
// Decode base64 back to raw bytes.
binaryBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(base64Data)))
if err != nil {
ts.Fatalf("failed to decode base64 data: %v", err)
}
// Write the binary file.
err = os.WriteFile(dest, binaryBytes, 0o644)
if err != nil {
ts.Fatalf("failed to write binary file: %v", err)
}
},
// httpget checks that a HTTP resource's body matches (if it compiles as a regexp) or contains all of the strings given as arguments.
"httpget": func(ts *testscript.TestScript, neg bool, args []string) {
@@ -0,0 +1,38 @@
# In Hugo v0.123 we started lowercasing the content paths, but on case-insensitive
# filesystems existing caches keep their mixed-case dir names (e.g. MyBundle).
# The pruner would then consider those entries unused and remove them on every
# hugo --gc. See issue 15101.
[!caseinsensitivefs] skip
base64decode pixel.png.txt content/MyBundle/pix.png
hugo
stdout 'Processed images │ 1 '
tree resources/_gen
stdout 'pix_hu_a354833fd576551d.png'
# Simulate a cache created by an older Hugo version.
mv resources/_gen/images/mybundle resources/_gen/images/MyBundle
hugo --gc
stdout 'Cleaned │ 0 '
tree resources/_gen
stdout 'pix_hu_a354833fd576551d.png'
-- hugo.toml --
disableKinds = ["term", "taxonomy", "home"]
-- layouts/all.html --
All.
-- layouts/page.html --
{{ $pix := .Resources.Get "pix.png" }}
{{ $resized := $pix.Resize "10x10" }}
Resized: {{ $resized.RelPermalink }}
-- content/MyBundle/index.md --
---
title: MyBundle
---
-- pixel.png.txt --
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==