Compare commits

...

7 Commits

Author SHA1 Message Date
hugoreleaser ee48d9692a releaser: Bump versions for release of 0.143.0
[ci skip]
2025-02-01 15:56:04 +00:00
Bjørn Erik Pedersen db28695ff5 Fix some server/watch rebuild issues
Two issues:

1. Fixe potential edit-loop in server/watch mode (see below)
2. Drain the cache eviction stack before we start calculating the change set. This should allow more fine grained rebuilds for bigger sites and/or in low memory situations.

The fix in 6c68142cc1 wasn't really fixing the complete problem.

In Hugo we have some steps that takes more time than others, one example being CSS building with TailwindCSS.

The symptom here is that sometimes when you:

1. Edit content or templates that does not trigger a CSS rebuild => Snappy rebuild.
2. Edit stylesheet or add a CSS class to template that triggers a CSS rebuild => relatively slow rebuild (expected)
3. Then back to content editing or template edits that should not trigger a CSS rebuild => relatively slow rebuild (not expected)

This commit fixes this by pulling the dynacache GC step up and merge it with the cache buster step.

Fixes #13316
2025-02-01 16:29:14 +01:00
Bjørn Erik Pedersen 778f0d9002 Don't re-render aliases on server rebuilds
This can lead to stale aliases when rebuilding, but that's a trade-off we need to take for snappier rebuilds on bigger sites.

Note that it should be possible to detect alias changes, but I'm not sure it's worth it.
2025-02-01 15:23:36 +01:00
Alex Shpak 13b208e2f7 tpl/tplimpl: Remove leading whitespaces produced by Youtube shortcode 2025-01-31 09:29:10 +01:00
Bjørn Erik Pedersen 329b2342f0 Fix "concurrent map iteration and map write" in pages from data
Fixes #13254
2025-01-30 19:57:09 +01:00
Joe Mooring 33b46d8a41 resources: Remove debug statement
Closes #13320
2025-01-30 11:32:57 +01:00
Bjørn Erik Pedersen 6c68142cc1 Fix TailwindCSS related server rebuild issue
Fixes #13316
2025-01-29 18:46:12 +01:00
15 changed files with 117 additions and 51 deletions
+11 -5
View File
@@ -176,11 +176,12 @@ func (c *Cache) ClearMatching(predicatePartition func(k string, p PartitionManag
}
// ClearOnRebuild prepares the cache for a new rebuild taking the given changeset into account.
func (c *Cache) ClearOnRebuild(changeset ...identity.Identity) {
// predicate is optional and will clear any entry for which it returns true.
func (c *Cache) ClearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) {
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
NumWorkers: len(c.partitions),
Handle: func(ctx context.Context, partition PartitionManager) error {
partition.clearOnRebuild(changeset...)
partition.clearOnRebuild(predicate, changeset...)
return nil
},
})
@@ -479,7 +480,12 @@ func (p *Partition[K, V]) clearMatching(predicate func(k, v any) bool) {
})
}
func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
func (p *Partition[K, V]) clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity) {
if predicate == nil {
predicate = func(k, v any) bool {
return false
}
}
opts := p.getOptions()
if opts.ClearWhen == ClearNever {
return
@@ -525,7 +531,7 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
// Second pass needs to be done in a separate loop to catch any
// elements marked as stale in the other partitions.
p.c.DeleteFunc(func(key K, v V) bool {
if shouldDelete(key, v) {
if predicate(key, v) || shouldDelete(key, v) {
p.trace.Log(
logg.StringFunc(
func() string {
@@ -601,7 +607,7 @@ type PartitionManager interface {
adjustMaxSize(addend int) int
getMaxSize() int
getOptions() OptionsPartition
clearOnRebuild(changeset ...identity.Identity)
clearOnRebuild(predicate func(k, v any) bool, changeset ...identity.Identity)
clearMatching(predicate func(k, v any) bool)
clearStale()
}
+2 -2
View File
@@ -147,13 +147,13 @@ func TestClear(t *testing.T) {
c.Assert(cache.Keys(predicateAll), qt.HasLen, 4)
cache.ClearOnRebuild()
cache.ClearOnRebuild(nil)
// Stale items are always cleared.
c.Assert(cache.Keys(predicateAll), qt.HasLen, 2)
cache = newTestCache(t)
cache.ClearOnRebuild(identity.StringIdentity("changed"))
cache.ClearOnRebuild(nil, identity.StringIdentity("changed"))
c.Assert(cache.Keys(nil), qt.HasLen, 1)
+1 -1
View File
@@ -19,5 +19,5 @@ var CurrentVersion = Version{
Major: 0,
Minor: 143,
PatchLevel: 0,
Suffix: "-DEV",
Suffix: "",
}
+4
View File
@@ -73,10 +73,14 @@ func TestPrepareParams(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprint(i), func(t *testing.T) {
// PrepareParams modifies input.
prepareClone := PrepareParamsClone(test.input)
PrepareParams(test.input)
if !reflect.DeepEqual(test.expected, test.input) {
t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
}
if !reflect.DeepEqual(test.expected, prepareClone) {
t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, prepareClone)
}
})
}
}
+40 -1
View File
@@ -303,7 +303,7 @@ func toMergeStrategy(v any) ParamsMergeStrategy {
}
// PrepareParams
// * makes all the keys in the given map lower cased and will do so
// * makes all the keys in the given map lower cased and will do so recursively.
// * This will modify the map given.
// * Any nested map[interface{}]interface{}, map[string]interface{},map[string]string will be converted to Params.
// * Any _merge value will be converted to proper type and value.
@@ -343,3 +343,42 @@ func PrepareParams(m Params) {
}
}
}
// PrepareParamsClone is like PrepareParams, but it does not modify the input.
func PrepareParamsClone(m Params) Params {
m2 := make(Params)
for k, v := range m {
var retyped bool
lKey := strings.ToLower(k)
if lKey == MergeStrategyKey {
v = toMergeStrategy(v)
retyped = true
} else {
switch vv := v.(type) {
case map[any]any:
var p Params = cast.ToStringMap(v)
v = PrepareParamsClone(p)
retyped = true
case map[string]any:
var p Params = v.(map[string]any)
v = PrepareParamsClone(p)
retyped = true
case map[string]string:
p := make(Params)
for k, v := range vv {
p[k] = v
}
v = p
PrepareParams(p)
retyped = true
}
}
if retyped || k != lKey {
m2[lKey] = v
} else {
m2[k] = v
}
}
return m2
}
+27 -26
View File
@@ -1123,6 +1123,9 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
l logg.LevelLogger,
cachebuster func(s string) bool, changes []identity.Identity,
) error {
// Drain the cache eviction stack to start fresh.
h.Deps.MemCache.DrainEvictedIdentities()
h.Log.Debug().Log(logg.StringFunc(
func() string {
var sb strings.Builder
@@ -1163,17 +1166,32 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
}
// The order matters here:
// 1. Handle the cache busters first, as those may produce identities for the page reset step.
// 1. Then GC the cache, which may produce changes.
// 2. Then reset the page outputs, which may mark some resources as stale.
// 3. Then GC the cache.
if cachebuster != nil {
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache cachebuster")
h.dynacacheGCCacheBuster(cachebuster)
return ll, nil
}); err != nil {
return err
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache")
predicate := func(k any, v any) bool {
if cachebuster != nil {
if s, ok := k.(string); ok {
return cachebuster(s)
}
}
return false
}
h.MemCache.ClearOnRebuild(predicate, changes...)
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("dynacache keys:\n")
for _, key := range h.MemCache.Keys(nil) {
sb.WriteString(fmt.Sprintf(" %s\n", key))
}
return sb.String()
}))
return ll, nil
}); err != nil {
return err
}
// Drain the cache eviction stack.
@@ -1238,23 +1256,6 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
return err
}
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache")
h.MemCache.ClearOnRebuild(changes...)
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("dynacache keys:\n")
for _, key := range h.MemCache.Keys(nil) {
sb.WriteString(fmt.Sprintf(" %s\n", key))
}
return sb.String()
}))
return ll, nil
}); err != nil {
return err
}
return nil
}
+12 -5
View File
@@ -827,6 +827,11 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo
addedContentPaths []*paths.Path
)
var (
addedOrChangedContent []pathChange
changes []identity.Identity
)
for _, ev := range eventInfos {
cpss := h.BaseFs.ResolvePaths(ev.Name)
pss := make([]*paths.Path, len(cpss))
@@ -853,6 +858,13 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo
if err == nil && g != nil {
cacheBusters = append(cacheBusters, g)
}
if ev.added {
changes = append(changes, identity.StructuralChangeAdd)
}
if ev.removed {
changes = append(changes, identity.StructuralChangeRemove)
}
}
if ev.removed {
@@ -864,11 +876,6 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo
}
}
var (
addedOrChangedContent []pathChange
changes []identity.Identity
)
// Find the most specific identity possible.
handleChange := func(pathInfo *paths.Path, delete, isDir bool) {
switch pathInfo.Component() {
+1 -1
View File
@@ -1542,7 +1542,7 @@ func (s *Site) render(ctx *siteRenderContext) (err error) {
return err
}
if ctx.outIdx == 0 {
if ctx.outIdx == 0 && s.h.buildCounter.Load() == 0 {
// Note that even if disableAliases is set, the aliases themselves are
// preserved on page. The motivation with this is to be able to generate
// 301 redirects in a .htaccess file and similar using a custom output format.
+3
View File
@@ -33,6 +33,9 @@ const (
// GenghisKhan is an Identity everyone relates to.
GenghisKhan = StringIdentity("__genghiskhan")
StructuralChangeAdd = StringIdentity("__structural_change_add")
StructuralChangeRemove = StringIdentity("__structural_change_remove")
)
var NopManager = new(nopManager)
+4 -1
View File
@@ -158,8 +158,11 @@ func (p *PageConfig) Compile(basePath string, pagesFromData bool, ext string, lo
if p.Params == nil {
p.Params = make(maps.Params)
} else if pagesFromData {
p.Params = maps.PrepareParamsClone(p.Params)
} else {
maps.PrepareParams(p.Params)
}
maps.PrepareParams(p.Params)
if p.Content.Markup == "" && p.Content.MediaType == "" {
if ext == "" {
@@ -95,6 +95,10 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
}
idm := c.rs.Cfg.NewIdentityManager("concat")
// Re-create on structural changes.
idm.AddIdentity(identity.StructuralChangeAdd, identity.StructuralChangeRemove)
// Add the concatenated resources as dependencies to the composite resource
// so that we can track changes to the individual resources.
idm.AddIdentityForEach(identity.ForEeachIdentityProviderFunc(
@@ -36,8 +36,8 @@ func TestTailwindV4Basic(t *testing.T) {
"url": "https://github.com/bep/hugo-starter-tailwind-basic.git"
},
"devDependencies": {
"@tailwindcss/cli": "^4.0.0-alpha.26",
"tailwindcss": "^4.0.0-alpha.26"
"@tailwindcss/cli": "^4.0.1",
"tailwindcss": "^4.0.1"
},
"name": "hugo-starter-tailwind-basic",
"version": "0.1.0"
@@ -68,5 +68,5 @@ CSS: {{ $css.Content | safeCSS }}|
LogLevel: logg.LevelInfo,
}).Build()
b.AssertFileContent("public/index.html", "/*! tailwindcss v4.0.0")
b.AssertFileContent("public/index.html", "/*! tailwindcss v4.")
}
-1
View File
@@ -388,7 +388,6 @@ func (r *resourceAdapter) getImageOps() images.ImageResourceOps {
if r.MediaType().SubType == "svg" {
panic("this method is only available for raster images. To determine if an image is SVG, you can do {{ if eq .MediaType.SubType \"svg\" }}{{ end }}")
}
fmt.Println(r.MediaType().SubType)
panic("this method is only available for image resources")
}
r.init(false, false)
@@ -102,7 +102,7 @@ Renders an embedded YouTube video.
{{- $allow := "accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" }}
{{- $referrerpolicy := "strict-origin-when-cross-origin" }}
{{- /* Render. */}}
{{- /* Render. */ -}}
<div
{{- with $class }} class="{{ . }}" {{- end }}
{{- with $divStyle }} style="{{ . | safeCSS }}" {{- end -}}
+4 -4
View File
@@ -675,12 +675,12 @@ title: p2
b := hugolib.Test(t, files)
b.AssertFileContent("public/p1/index.html", "515600e76b272f51")
b.AssertFileContent("public/p2/index.html", "b5ceeace7dfa797a")
b.AssertFileContent("public/p1/index.html", "a0a6f5ade9cc3a9f")
b.AssertFileContent("public/p2/index.html", "289c655e727e596c")
files = strings.ReplaceAll(files, "privacy.youtube.privacyEnhanced = false", "privacy.youtube.privacyEnhanced = true")
b = hugolib.Test(t, files)
b.AssertFileContent("public/p1/index.html", "e92c7f4b768d7e23")
b.AssertFileContent("public/p2/index.html", "c384e83e035b71d9")
b.AssertFileContent("public/p1/index.html", "b76d790c20d2bd04")
b.AssertFileContent("public/p2/index.html", "a6db910a9cf54bc1")
}