diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go index d6322cb70..cd95766be 100644 --- a/cache/filecache/filecache_pruner.go +++ b/cache/filecache/filecache_pruner.go @@ -17,6 +17,7 @@ import ( "fmt" "io" "os" + "strings" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/hugofs" @@ -59,6 +60,29 @@ func (c *Cache) Prune(force bool) (int, error) { counter := 0 + seen := c.entryLocker.seen + seenByLower := make(map[string]string, seen.Len()) + for id := range seen.All() { + seenByLower[strings.ToLower(id)] = id + } + + // Names on disk matching a used cache key except for the case, and the used keys + // actually walked. See the note about case-insensitive filesystems below. + var candidates map[string]string + visited := make(map[string]bool, seen.Len()) + + remove := func(name string) error { + err := c.Fs.Remove(name) + if err == nil { + counter++ + return nil + } + if !herrors.IsNotExist(err) { + return err + } + return nil + } + err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error { if info == nil { return nil @@ -93,27 +117,44 @@ func (c *Cache) Prune(force bool) (int, error) { shouldRemove := force || c.isExpired(info.ModTime()) - if !shouldRemove && c.entryLocker.seen.Len() > 0 { - // Remove it if it's not been touched/used in the last build. - shouldRemove = !c.entryLocker.seen.Has(name) + if seen.Has(name) { + visited[name] = true + } else if !shouldRemove && seen.Len() > 0 { + if id, found := seenByLower[strings.ToLower(name)]; found { + // On case-insensitive filesystems this is the same file as id; e.g. an + // entry created before Hugo started lowercasing the content paths in + // v0.123 (content/MyBundle => _gen/images/MyBundle). Decided once the + // walk is done: if id is walked too, they are distinct files and this + // one is stale. See issue 15101. + if candidates == nil { + candidates = make(map[string]string) + } + candidates[name] = id + } else { + // Remove it if it's not been touched/used in the last build. + shouldRemove = true + } } if shouldRemove { - err := c.Fs.Remove(name) - if err == nil { - counter++ - } - - if err != nil && !herrors.IsNotExist(err) { - return err - } - + return remove(name) } return nil }) + if err != nil { + return counter, err + } - return counter, err + for name, id := range candidates { + if visited[id] { + if err := remove(name); err != nil { + return counter, err + } + } + } + + return counter, nil } func (c *Cache) pruneRootDirs(force bool) (int, error) { diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index 117ff3430..dc01e417a 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -20,11 +20,72 @@ import ( "time" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" qt "github.com/frankban/quicktest" ) +// A cache entry created before Hugo started lowercasing content paths in v0.123 +// (e.g. _gen/images/MyBundle) is on a case-insensitive filesystem the same file as +// the lowercased cache key used today, and must not be pruned. +// See issue 15101. +func TestPruneCacheEntryWithOtherCase(t *testing.T) { + t.Parallel() + c := qt.New(t) + + dir := t.TempDir() + if isCaseInsensitive, err := htesting.IsCaseInsensitiveFs(dir); err != nil { + t.Fatal(err) + } else if !isCaseInsensitive { + t.Skip("skip test on case-sensitive filesystem") + } + + fs := afero.NewBasePathFs(afero.NewOsFs(), dir) + newCache := func() *filecache.Cache { + return filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1}) + } + + c.Assert(newCache().SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil) + + cache := newCache() + _, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) { + return []byte("def"), nil + }) + c.Assert(err, qt.IsNil) + c.Assert(string(b), qt.Equals, "abc") + + count, err := cache.Prune(false) + c.Assert(err, qt.IsNil) + c.Assert(count, qt.Equals, 0) + c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "abc") +} + +// On a case-sensitive filesystem the entries above are distinct files, +// and the one not used in this build should be pruned. +func TestPruneCacheEntryWithOtherCaseCaseSensitiveFs(t *testing.T) { + t.Parallel() + c := qt.New(t) + + fs := afero.NewMemMapFs() + cache := filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1}) + + c.Assert(cache.SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil) + + cache = filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1}) + _, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) { + return []byte("def"), nil + }) + c.Assert(err, qt.IsNil) + c.Assert(string(b), qt.Equals, "def") + + count, err := cache.Prune(false) + c.Assert(err, qt.IsNil) + c.Assert(count, qt.Equals, 1) + c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "") + c.Assert(cache.GetString("mybundle/i1"), qt.Equals, "def") +} + func TestPrune(t *testing.T) { t.Parallel() diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index 384695df4..46f6bd9ec 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -16,6 +16,7 @@ package htesting import ( "math/rand" "os" + "path/filepath" "regexp" "runtime" "strconv" @@ -58,6 +59,27 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) { return tempDir, func() { fs.RemoveAll(tempDir) }, nil } +// IsCaseInsensitiveFs reports whether dir lives on a case-insensitive filesystem +// (e.g. the default on macOS and Windows). +func IsCaseInsensitiveFs(dir string) (bool, error) { + f, err := os.CreateTemp(dir, "case-*.tmp") + if err != nil { + return false, err + } + f.Close() + defer os.Remove(f.Name()) + + _, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(f.Name())))) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + // BailOut panics with a stack trace after the given duration. Useful for // hanging tests. func BailOut(after time.Duration) { diff --git a/main_test.go b/main_test.go index 12e708dc0..8cea78818 100644 --- a/main_test.go +++ b/main_test.go @@ -15,6 +15,7 @@ package main import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "io" @@ -27,6 +28,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -78,10 +80,22 @@ func TestMain(m *testing.M) { }) } +var isCaseInsensitiveFs = sync.OnceValues(func() (bool, error) { + return htesting.IsCaseInsensitiveFs(os.TempDir()) +}) + var commonTestScriptsParam = testscript.Params{ Setup: func(env *testscript.Env) error { return testSetupFunc()(env) }, + Condition: func(cond string) (bool, error) { + switch cond { + case "caseinsensitivefs": + return isCaseInsensitiveFs() + default: + return false, fmt.Errorf("unknown condition %q", cond) + } + }, Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){ // log prints to stderr. "log": func(ts *testscript.TestScript, neg bool, args []string) { @@ -252,6 +266,34 @@ var commonTestScriptsParam = testscript.Params{ ts.Fatalf("failed to create symlink: %v", err) } }, + // base64decode decodes a base64-encoded file into a binary file. + "base64decode": func(ts *testscript.TestScript, neg bool, args []string) { + if len(args) != 2 { + ts.Fatalf("usage: base64decode src_base64_file dest_binary_file") + } + + // Resolve paths relative to the test sandbox's current directory + src := ts.MkAbs(args[0]) + dest := ts.MkAbs(args[1]) + + // Read the base64 text + base64Data, err := os.ReadFile(src) + if err != nil { + ts.Fatalf("failed to read base64 file: %v", err) + } + + // Decode base64 back to raw bytes. + binaryBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(base64Data))) + if err != nil { + ts.Fatalf("failed to decode base64 data: %v", err) + } + + // Write the binary file. + err = os.WriteFile(dest, binaryBytes, 0o644) + if err != nil { + ts.Fatalf("failed to write binary file: %v", err) + } + }, // httpget checks that a HTTP resource's body matches (if it compiles as a regexp) or contains all of the strings given as arguments. "httpget": func(ts *testscript.TestScript, neg bool, args []string) { diff --git a/testscripts/commands/hugo_gc_mixed_case_cache_dir.txt b/testscripts/commands/hugo_gc_mixed_case_cache_dir.txt new file mode 100644 index 000000000..7c7af1373 --- /dev/null +++ b/testscripts/commands/hugo_gc_mixed_case_cache_dir.txt @@ -0,0 +1,38 @@ +# In Hugo v0.123 we started lowercasing the content paths, but on case-insensitive +# filesystems existing caches keep their mixed-case dir names (e.g. MyBundle). +# The pruner would then consider those entries unused and remove them on every +# hugo --gc. See issue 15101. + +[!caseinsensitivefs] skip + +base64decode pixel.png.txt content/MyBundle/pix.png + +hugo + +stdout 'Processed images │ 1 ' +tree resources/_gen +stdout 'pix_hu_a354833fd576551d.png' + +# Simulate a cache created by an older Hugo version. +mv resources/_gen/images/mybundle resources/_gen/images/MyBundle + +hugo --gc +stdout 'Cleaned │ 0 ' + +tree resources/_gen +stdout 'pix_hu_a354833fd576551d.png' + +-- hugo.toml -- +disableKinds = ["term", "taxonomy", "home"] +-- layouts/all.html -- +All. +-- layouts/page.html -- +{{ $pix := .Resources.Get "pix.png" }} +{{ $resized := $pix.Resize "10x10" }} +Resized: {{ $resized.RelPermalink }} +-- content/MyBundle/index.md -- +--- +title: MyBundle +--- +-- pixel.png.txt -- +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==