mirror of
https://github.com/gohugoio/hugo.git
synced 2026-08-24 07:18:54 +00:00
cache/filecache: Don't prune used cache entries with mixed-case dir names
On case-insensitive filesystems, entries created before we started lowercasing the content paths in v0.123 are the same file as today's lowercased cache key, and were removed on every hugo --gc. Fixes #15101 Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
f228c87d41
commit
861ede6d10
Vendored
+53
-12
@@ -17,6 +17,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/common/herrors"
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
@@ -59,6 +60,29 @@ func (c *Cache) Prune(force bool) (int, error) {
|
||||
|
||||
counter := 0
|
||||
|
||||
seen := c.entryLocker.seen
|
||||
seenByLower := make(map[string]string, seen.Len())
|
||||
for id := range seen.All() {
|
||||
seenByLower[strings.ToLower(id)] = id
|
||||
}
|
||||
|
||||
// Names on disk matching a used cache key except for the case, and the used keys
|
||||
// actually walked. See the note about case-insensitive filesystems below.
|
||||
var candidates map[string]string
|
||||
visited := make(map[string]bool, seen.Len())
|
||||
|
||||
remove := func(name string) error {
|
||||
err := c.Fs.Remove(name)
|
||||
if err == nil {
|
||||
counter++
|
||||
return nil
|
||||
}
|
||||
if !herrors.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {
|
||||
if info == nil {
|
||||
return nil
|
||||
@@ -93,27 +117,44 @@ func (c *Cache) Prune(force bool) (int, error) {
|
||||
|
||||
shouldRemove := force || c.isExpired(info.ModTime())
|
||||
|
||||
if !shouldRemove && c.entryLocker.seen.Len() > 0 {
|
||||
if seen.Has(name) {
|
||||
visited[name] = true
|
||||
} else if !shouldRemove && seen.Len() > 0 {
|
||||
if id, found := seenByLower[strings.ToLower(name)]; found {
|
||||
// On case-insensitive filesystems this is the same file as id; e.g. an
|
||||
// entry created before Hugo started lowercasing the content paths in
|
||||
// v0.123 (content/MyBundle => _gen/images/MyBundle). Decided once the
|
||||
// walk is done: if id is walked too, they are distinct files and this
|
||||
// one is stale. See issue 15101.
|
||||
if candidates == nil {
|
||||
candidates = make(map[string]string)
|
||||
}
|
||||
candidates[name] = id
|
||||
} else {
|
||||
// Remove it if it's not been touched/used in the last build.
|
||||
shouldRemove = !c.entryLocker.seen.Has(name)
|
||||
shouldRemove = true
|
||||
}
|
||||
}
|
||||
|
||||
if shouldRemove {
|
||||
err := c.Fs.Remove(name)
|
||||
if err == nil {
|
||||
counter++
|
||||
}
|
||||
|
||||
if err != nil && !herrors.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return remove(name)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return counter, err
|
||||
}
|
||||
|
||||
for name, id := range candidates {
|
||||
if visited[id] {
|
||||
if err := remove(name); err != nil {
|
||||
return counter, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return counter, nil
|
||||
}
|
||||
|
||||
func (c *Cache) pruneRootDirs(force bool) (int, error) {
|
||||
|
||||
+61
@@ -20,11 +20,72 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gohugoio/hugo/cache/filecache"
|
||||
"github.com/gohugoio/hugo/htesting"
|
||||
"github.com/spf13/afero"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
||||
// A cache entry created before Hugo started lowercasing content paths in v0.123
|
||||
// (e.g. _gen/images/MyBundle) is on a case-insensitive filesystem the same file as
|
||||
// the lowercased cache key used today, and must not be pruned.
|
||||
// See issue 15101.
|
||||
func TestPruneCacheEntryWithOtherCase(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := qt.New(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
if isCaseInsensitive, err := htesting.IsCaseInsensitiveFs(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !isCaseInsensitive {
|
||||
t.Skip("skip test on case-sensitive filesystem")
|
||||
}
|
||||
|
||||
fs := afero.NewBasePathFs(afero.NewOsFs(), dir)
|
||||
newCache := func() *filecache.Cache {
|
||||
return filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
|
||||
}
|
||||
|
||||
c.Assert(newCache().SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil)
|
||||
|
||||
cache := newCache()
|
||||
_, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) {
|
||||
return []byte("def"), nil
|
||||
})
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(string(b), qt.Equals, "abc")
|
||||
|
||||
count, err := cache.Prune(false)
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(count, qt.Equals, 0)
|
||||
c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "abc")
|
||||
}
|
||||
|
||||
// On a case-sensitive filesystem the entries above are distinct files,
|
||||
// and the one not used in this build should be pruned.
|
||||
func TestPruneCacheEntryWithOtherCaseCaseSensitiveFs(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := qt.New(t)
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
cache := filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
|
||||
|
||||
c.Assert(cache.SetBytes("MyBundle/i1", []byte("abc")), qt.IsNil)
|
||||
|
||||
cache = filecache.NewCache(fs, filecache.FileCacheConfig{Dir: "cache", MaxAge: -1})
|
||||
_, b, err := cache.GetOrCreateBytes("mybundle/i1", func() ([]byte, error) {
|
||||
return []byte("def"), nil
|
||||
})
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(string(b), qt.Equals, "def")
|
||||
|
||||
count, err := cache.Prune(false)
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(count, qt.Equals, 1)
|
||||
c.Assert(cache.GetString("MyBundle/i1"), qt.Equals, "")
|
||||
c.Assert(cache.GetString("mybundle/i1"), qt.Equals, "def")
|
||||
}
|
||||
|
||||
func TestPrune(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ package htesting
|
||||
import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -58,6 +59,27 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) {
|
||||
return tempDir, func() { fs.RemoveAll(tempDir) }, nil
|
||||
}
|
||||
|
||||
// IsCaseInsensitiveFs reports whether dir lives on a case-insensitive filesystem
|
||||
// (e.g. the default on macOS and Windows).
|
||||
func IsCaseInsensitiveFs(dir string) (bool, error) {
|
||||
f, err := os.CreateTemp(dir, "case-*.tmp")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(f.Name()))))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// BailOut panics with a stack trace after the given duration. Useful for
|
||||
// hanging tests.
|
||||
func BailOut(after time.Duration) {
|
||||
|
||||
@@ -15,6 +15,7 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -78,10 +80,22 @@ func TestMain(m *testing.M) {
|
||||
})
|
||||
}
|
||||
|
||||
var isCaseInsensitiveFs = sync.OnceValues(func() (bool, error) {
|
||||
return htesting.IsCaseInsensitiveFs(os.TempDir())
|
||||
})
|
||||
|
||||
var commonTestScriptsParam = testscript.Params{
|
||||
Setup: func(env *testscript.Env) error {
|
||||
return testSetupFunc()(env)
|
||||
},
|
||||
Condition: func(cond string) (bool, error) {
|
||||
switch cond {
|
||||
case "caseinsensitivefs":
|
||||
return isCaseInsensitiveFs()
|
||||
default:
|
||||
return false, fmt.Errorf("unknown condition %q", cond)
|
||||
}
|
||||
},
|
||||
Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
|
||||
// log prints to stderr.
|
||||
"log": func(ts *testscript.TestScript, neg bool, args []string) {
|
||||
@@ -252,6 +266,34 @@ var commonTestScriptsParam = testscript.Params{
|
||||
ts.Fatalf("failed to create symlink: %v", err)
|
||||
}
|
||||
},
|
||||
// base64decode decodes a base64-encoded file into a binary file.
|
||||
"base64decode": func(ts *testscript.TestScript, neg bool, args []string) {
|
||||
if len(args) != 2 {
|
||||
ts.Fatalf("usage: base64decode src_base64_file dest_binary_file")
|
||||
}
|
||||
|
||||
// Resolve paths relative to the test sandbox's current directory
|
||||
src := ts.MkAbs(args[0])
|
||||
dest := ts.MkAbs(args[1])
|
||||
|
||||
// Read the base64 text
|
||||
base64Data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
ts.Fatalf("failed to read base64 file: %v", err)
|
||||
}
|
||||
|
||||
// Decode base64 back to raw bytes.
|
||||
binaryBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(base64Data)))
|
||||
if err != nil {
|
||||
ts.Fatalf("failed to decode base64 data: %v", err)
|
||||
}
|
||||
|
||||
// Write the binary file.
|
||||
err = os.WriteFile(dest, binaryBytes, 0o644)
|
||||
if err != nil {
|
||||
ts.Fatalf("failed to write binary file: %v", err)
|
||||
}
|
||||
},
|
||||
|
||||
// httpget checks that a HTTP resource's body matches (if it compiles as a regexp) or contains all of the strings given as arguments.
|
||||
"httpget": func(ts *testscript.TestScript, neg bool, args []string) {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# In Hugo v0.123 we started lowercasing the content paths, but on case-insensitive
|
||||
# filesystems existing caches keep their mixed-case dir names (e.g. MyBundle).
|
||||
# The pruner would then consider those entries unused and remove them on every
|
||||
# hugo --gc. See issue 15101.
|
||||
|
||||
[!caseinsensitivefs] skip
|
||||
|
||||
base64decode pixel.png.txt content/MyBundle/pix.png
|
||||
|
||||
hugo
|
||||
|
||||
stdout 'Processed images │ 1 '
|
||||
tree resources/_gen
|
||||
stdout 'pix_hu_a354833fd576551d.png'
|
||||
|
||||
# Simulate a cache created by an older Hugo version.
|
||||
mv resources/_gen/images/mybundle resources/_gen/images/MyBundle
|
||||
|
||||
hugo --gc
|
||||
stdout 'Cleaned │ 0 '
|
||||
|
||||
tree resources/_gen
|
||||
stdout 'pix_hu_a354833fd576551d.png'
|
||||
|
||||
-- hugo.toml --
|
||||
disableKinds = ["term", "taxonomy", "home"]
|
||||
-- layouts/all.html --
|
||||
All.
|
||||
-- layouts/page.html --
|
||||
{{ $pix := .Resources.Get "pix.png" }}
|
||||
{{ $resized := $pix.Resize "10x10" }}
|
||||
Resized: {{ $resized.RelPermalink }}
|
||||
-- content/MyBundle/index.md --
|
||||
---
|
||||
title: MyBundle
|
||||
---
|
||||
-- pixel.png.txt --
|
||||
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==
|
||||
Reference in New Issue
Block a user