Misc remote HTTP/content adapter enhancements

* Recover from server errors
* Improve go adapter rebuilds when adding new content

See #12502
Fixes #12570

.github/workflows/test-dart-sass-v1.yml

@@ -1,86 +0,0 @@
-on:
-  push:
-    branches: [master]
-  pull_request:
-name: TestDartSassV1
-env:
-  GOPROXY: https://proxy.golang.org
-  GO111MODULE: on
-  DART_SASS_VERSION: 1.62.1
-  DART_SASS_SHA_LINUX: 3574da75a7322a539034648b8ff84ff2cca162eb924d72b663d718cd3936f075
-permissions:
-  contents: read
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.22.x]
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - if: matrix.os == 'ubuntu-latest'
-        name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          docker-images: true
-          swap-storage: true
-      - name: Checkout code
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - name: Install Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491
-        with:
-          go-version: ${{ matrix.go-version }}
-          check-latest: true
-          cache: true
-          cache-dependency-path: |
-            **/go.sum
-            **/go.mod
-      - name: Install Ruby
-        uses: ruby/setup-ruby@5f19ec79cedfadb78ab837f95b87734d0003c899
-        with:
-          ruby-version: "2.7"
-          bundler-cache: true #
-      - name: Install Python
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
-        with:
-          python-version: "3.x"
-      - name: Install Mage
-        run: go install github.com/magefile/mage@v1.15.0
-      - name: Install asciidoctor
-        uses: reitzig/actions-asciidoctor@03fcc74cd74880b697950c4930c9ec8a67c69ecc
-      - name: Install docutils
-        run: |
-          pip install docutils
-          rst2html --version
-      - if: matrix.os == 'ubuntu-latest'
-        name: Install pandoc on Linux
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y pandoc
-      - if: matrix.os == 'macos-latest'
-        run: |
-          brew install pandoc
-      - if: matrix.os == 'windows-latest'
-        run: |
-          Choco-Install -PackageName pandoc
-      - run: pandoc -v
-      - name: Install dart-sass-embedded Linux
-        run: |
-          echo "Install Dart Sass version ${DART_SASS_VERSION} ..."
-          curl -LJO "https://github.com/sass/dart-sass-embedded/releases/download/${DART_SASS_VERSION}/sass_embedded-${DART_SASS_VERSION}-linux-x64.tar.gz";
-          echo "${DART_SASS_SHA_LINUX}  sass_embedded-${DART_SASS_VERSION}-linux-x64.tar.gz" | sha256sum -c;
-          tar -xvf "sass_embedded-${DART_SASS_VERSION}-linux-x64.tar.gz";
-          echo "$GITHUB_WORKSPACE/sass_embedded/" >> $GITHUB_PATH
-      - name: Check
-        run: |
-          dart-sass-embedded --version
-          mage -v check;
-        env:
-          HUGO_BUILD_TAGS: extended

cache/filecache/filecache.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/gohugoio/httpcache"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/hugofs"
 
@@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string,
 	return
 }
 
+// NamedLock locks the given id. The lock is released when the returned function is called.
+func (c *Cache) NamedLock(id string) func() {
+	id = cleanID(id)
+	c.nlocker.Lock(id)
+	return func() {
+		c.nlocker.Unlock(id)
+	}
+}
+
 // GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
 // be invoked and the result cached.
 // This method is protected by a named lock using the given id as identifier.
@@ -218,7 +228,23 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
 	var buff bytes.Buffer
 	return info,
 		hugio.ToReadCloser(&buff),
-		afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
+		c.writeReader(id, io.TeeReader(r, &buff))
+}
+
+func (c *Cache) writeReader(id string, r io.Reader) error {
+	dir := filepath.Dir(id)
+	if dir != "" {
+		_ = c.Fs.MkdirAll(dir, 0o777)
+	}
+	f, err := c.Fs.Create(id)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, _ = io.Copy(f, r)
+
+	return nil
 }
 
 // GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
@@ -253,9 +279,10 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
 		return info, b, nil
 	}
 
-	if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
+	if err := c.writeReader(id, bytes.NewReader(b)); err != nil {
 		return info, nil, err
 	}
+
 	return info, b, nil
 }
 
@@ -305,16 +332,8 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
 		return nil
 	}
 
-	if c.maxAge > 0 {
-		fi, err := c.Fs.Stat(id)
-		if err != nil {
-			return nil
-		}
-
-		if c.isExpired(fi.ModTime()) {
-			c.Fs.Remove(id)
-			return nil
-		}
+	if removed, err := c.removeIfExpired(id); err != nil || removed {
+		return nil
 	}
 
 	f, err := c.Fs.Open(id)
@@ -325,6 +344,49 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
 	return f
 }
 
+func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
+	if c.maxAge == 0 {
+		// No caching.
+		return nil, false
+	}
+
+	f, err := c.Fs.Open(id)
+	if err != nil {
+		return nil, false
+	}
+	defer f.Close()
+
+	b, err := io.ReadAll(f)
+	if err != nil {
+		return nil, false
+	}
+
+	removed, err := c.removeIfExpired(id)
+	if err != nil {
+		return nil, false
+	}
+
+	return b, removed
+}
+
+func (c *Cache) removeIfExpired(id string) (bool, error) {
+	if c.maxAge <= 0 {
+		return false, nil
+	}
+
+	fi, err := c.Fs.Stat(id)
+	if err != nil {
+		return false, err
+	}
+
+	if c.isExpired(fi.ModTime()) {
+		c.Fs.Remove(id)
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func (c *Cache) isExpired(modTime time.Time) bool {
 	if c.maxAge < 0 {
 		return false
@@ -398,3 +460,37 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 func cleanID(name string) string {
 	return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
 }
+
+// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
+// Note that none of the methods are protected by named locks, so you need to make sure
+// to do that in your own code.
+func (c *Cache) AsHTTPCache() httpcache.Cache {
+	return &httpCache{c: c}
+}
+
+type httpCache struct {
+	c *Cache
+}
+
+func (h *httpCache) Get(id string) (resp []byte, ok bool) {
+	id = cleanID(id)
+	b, removed := h.c.getBytesAndRemoveIfExpired(id)
+
+	return b, !removed
+}
+
+func (h *httpCache) Set(id string, resp []byte) {
+	if h.c.maxAge == 0 {
+		return
+	}
+
+	id = cleanID(id)
+
+	if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil {
+		panic(err)
+	}
+}
+
+func (h *httpCache) Delete(key string) {
+	h.c.Fs.Remove(key)
+}

cache/httpcache/httpcache.go

@@ -0,0 +1,208 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/gobwas/glob"
+	"github.com/gohugoio/hugo/common/predicate"
+	"github.com/gohugoio/hugo/config"
+	"github.com/mitchellh/mapstructure"
+)
+
+// DefaultConfig holds the default configuration for the HTTP cache.
+var DefaultConfig = Config{
+	Cache: Cache{
+		For: GlobMatcher{
+			Excludes: []string{"**"},
+		},
+	},
+	Polls: []PollConfig{
+		{
+			For: GlobMatcher{
+				Includes: []string{"**"},
+			},
+			Disable: true,
+		},
+	},
+}
+
+// Config holds the configuration for the HTTP cache.
+type Config struct {
+	// Configures the HTTP cache behaviour (RFC 9111).
+	// When this is not enabled for a resource, Hugo will go straight to the file cache.
+	Cache Cache
+
+	// Polls holds a list of configurations for polling remote resources to detect changes in watch mode.
+	// This can be disabled for some resources, typically if they are known to not change.
+	Polls []PollConfig
+}
+
+type Cache struct {
+	// Enable HTTP cache behaviour (RFC 9111) for these rsources.
+	For GlobMatcher
+}
+
+func (c *Config) Compile() (ConfigCompiled, error) {
+	var cc ConfigCompiled
+
+	p, err := c.Cache.For.CompilePredicate()
+	if err != nil {
+		return cc, err
+	}
+
+	cc.For = p
+
+	for _, pc := range c.Polls {
+
+		p, err := pc.For.CompilePredicate()
+		if err != nil {
+			return cc, err
+		}
+
+		cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{
+			For:    p,
+			Config: pc,
+		})
+	}
+
+	return cc, nil
+}
+
+// PollConfig holds the configuration for polling remote resources to detect changes in watch mode.
+// TODO1 make sure this enabled only in watch mode.
+type PollConfig struct {
+	// What remote resources to apply this configuration to.
+	For GlobMatcher
+
+	// Disable polling for this configuration.
+	Disable bool
+
+	// Low is the lower bound for the polling interval.
+	// This is the starting point when the resource has recently changed,
+	// if that resource stops changing, the polling interval will gradually increase towards High.
+	Low time.Duration
+
+	// High is the upper bound for the polling interval.
+	// This is the interval used when the resource is stable.
+	High time.Duration
+}
+
+func (c PollConfig) MarshalJSON() (b []byte, err error) {
+	// Marshal the durations as strings.
+	type Alias PollConfig
+	return json.Marshal(&struct {
+		Low  string
+		High string
+		Alias
+	}{
+		Low:   c.Low.String(),
+		High:  c.High.String(),
+		Alias: (Alias)(c),
+	})
+}
+
+type GlobMatcher struct {
+	// Excludes holds a list of glob patterns that will be excluded.
+	Excludes []string
+
+	// Includes holds a list of glob patterns that will be included.
+	Includes []string
+}
+
+type ConfigCompiled struct {
+	For         predicate.P[string]
+	PollConfigs []PollConfigCompiled
+}
+
+func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled {
+	for _, pc := range c.PollConfigs {
+		if pc.For(s) {
+			return pc
+		}
+	}
+	return PollConfigCompiled{}
+}
+
+func (c *ConfigCompiled) IsPollingDisabled() bool {
+	for _, pc := range c.PollConfigs {
+		if !pc.Config.Disable {
+			return false
+		}
+	}
+	return true
+}
+
+type PollConfigCompiled struct {
+	For    predicate.P[string]
+	Config PollConfig
+}
+
+func (p PollConfigCompiled) IsZero() bool {
+	return p.For == nil
+}
+
+func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) {
+	var p predicate.P[string]
+	for _, include := range gm.Includes {
+		g, err := glob.Compile(include, '/')
+		if err != nil {
+			return nil, err
+		}
+		fn := func(s string) bool {
+			return g.Match(s)
+		}
+		p = p.Or(fn)
+	}
+
+	for _, exclude := range gm.Excludes {
+		g, err := glob.Compile(exclude, '/')
+		if err != nil {
+			return nil, err
+		}
+		fn := func(s string) bool {
+			return !g.Match(s)
+		}
+		p = p.And(fn)
+	}
+
+	return p, nil
+}
+
+func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) {
+	if len(m) == 0 {
+		return DefaultConfig, nil
+	}
+
+	var c Config
+
+	dc := &mapstructure.DecoderConfig{
+		Result:           &c,
+		DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+		WeaklyTypedInput: true,
+	}
+
+	decoder, err := mapstructure.NewDecoder(dc)
+	if err != nil {
+		return c, err
+	}
+
+	if err := decoder.Decode(m); err != nil {
+		return c, err
+	}
+
+	return c, nil
+}

cache/httpcache/httpcache_integration_test.go

@@ -0,0 +1,64 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache_test
+
+import (
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+func TestConfigCustom(t *testing.T) {
+	files := `
+-- hugo.toml --
+[httpcache]
+[httpcache.cache.for]
+includes = ["**gohugo.io**"]
+[[httpcache.polls]]
+low = "5s"
+high = "32s"
+[httpcache.polls.for]
+includes = ["**gohugo.io**"]
+		
+	
+`
+
+	b := hugolib.Test(t, files)
+
+	httpcacheConf := b.H.Configs.Base.HTTPCache
+	compiled := b.H.Configs.Base.C.HTTPCache
+
+	b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"})
+	b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil)
+
+	pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg")
+	b.Assert(pc.Config.Low, qt.Equals, 5*time.Second)
+	b.Assert(pc.Config.High, qt.Equals, 32*time.Second)
+	b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue)
+}
+
+func TestConfigDefault(t *testing.T) {
+	files := `
+-- hugo.toml --
+`
+	b := hugolib.Test(t, files)
+
+	compiled := b.H.Configs.Base.C.HTTPCache
+
+	b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse)
+	b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse)
+	b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue)
+}

cache/httpcache/httpcache_test.go

@@ -0,0 +1,42 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestGlobMatcher(t *testing.T) {
+	c := qt.New(t)
+
+	g := GlobMatcher{
+		Includes: []string{"**/*.jpg", "**.png", "**/bar/**"},
+		Excludes: []string{"**/foo.jpg", "**.css"},
+	}
+
+	p, err := g.CompilePredicate()
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(p("foo.jpg"), qt.IsFalse)
+	c.Assert(p("foo.png"), qt.IsTrue)
+	c.Assert(p("foo/bar.jpg"), qt.IsTrue)
+	c.Assert(p("foo/bar.png"), qt.IsTrue)
+	c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse)
+	c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+	c.Assert(p("foo.css"), qt.IsFalse)
+	c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+	c.Assert(p("foo/bar/foo.xml"), qt.IsTrue)
+}

commands/commandeer.go

@@ -48,6 +48,7 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/resources/kinds"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -103,6 +104,9 @@ type rootCommand struct {
 	commonConfigs *lazycache.Cache[int32, *commonConfig]
 	hugoSites     *lazycache.Cache[int32, *hugolib.HugoSites]
 
+	// changesFromBuild received from Hugo in watch mode.
+	changesFromBuild chan []identity.Identity
+
 	commands []simplecobra.Commander
 
 	// Flags
@@ -304,7 +308,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
 
 func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
 	h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
-		depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
+		depsCfg := r.newDepsConfig(conf)
 		return hugolib.NewHugoSites(depsCfg)
 	})
 	return h, err
@@ -316,12 +320,16 @@ func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) {
 		if err != nil {
 			return nil, err
 		}
-		depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
+		depsCfg := r.newDepsConfig(conf)
 		return hugolib.NewHugoSites(depsCfg)
 	})
 	return h, err
 }
 
+func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg {
+	return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild}
+}
+
 func (r *rootCommand) Name() string {
 	return "hugo"
 }
@@ -408,6 +416,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
 		return err
 	}
 
+	r.changesFromBuild = make(chan []identity.Identity, 10)
+
 	r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5})
 	// We don't want to keep stale HugoSites in memory longer than needed.
 	r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{
@@ -506,7 +516,7 @@ Complete documentation is available at https://gohugo.io/.`
 	cmd.PersistentFlags().StringVar(&r.cfgDir, "configDir", "config", "config dir")
 	_ = cmd.MarkFlagDirname("configDir")
 	cmd.PersistentFlags().BoolVar(&r.quiet, "quiet", false, "build in quiet mode")
-	cmd.PersistentFlags().BoolVar(&r.renderToMemory, "renderToMemory", false, "render to memory (mostly useful when running the server)")
+	cmd.PersistentFlags().BoolVarP(&r.renderToMemory, "renderToMemory", "M", false, "render to memory (mostly useful when running the server)")
 
 	cmd.PersistentFlags().BoolVarP(&r.verbose, "verbose", "v", false, "verbose output")
 	cmd.PersistentFlags().BoolVarP(&r.debug, "debug", "", false, "debug output")

commands/hugobuilder.go

@@ -43,6 +43,7 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib"
 	"github.com/gohugoio/hugo/hugolib/filesystems"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/livereload"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/watcher"
@@ -343,6 +344,27 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
 	go func() {
 		for {
 			select {
+			case changes := <-c.r.changesFromBuild:
+				c.errState.setBuildErr(nil)
+				unlock, err := h.LockBuild()
+				if err != nil {
+					c.r.logger.Errorln("Failed to acquire a build lock: %s", err)
+					return
+				}
+				c.changeDetector.PrepareNew()
+				err = c.rebuildSitesForChanges(changes)
+				if err != nil {
+					c.r.logger.Errorln("Error while watching:", err)
+				}
+				if c.s != nil && c.s.doLiveReload {
+					doReload := c.changeDetector == nil || len(c.changeDetector.changed()) > 0
+					doReload = doReload || c.showErrorInBrowser && c.errCount() > 0
+					if doReload {
+						livereload.ForceRefresh()
+					}
+				}
+				unlock()
+
 			case evs := <-watcher.Events:
 				unlock, err := h.LockBuild()
 				if err != nil {
@@ -1019,6 +1041,19 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error {
 	return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
 }
 
+func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error {
+	c.errState.setBuildErr(nil)
+	h, err := c.hugo()
+	if err != nil {
+		return err
+	}
+	whatChanged := &hugolib.WhatChanged{}
+	whatChanged.Add(ids...)
+	err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
+	c.errState.setBuildErr(err)
+	return err
+}
+
 func (c *hugoBuilder) reloadConfig() error {
 	c.r.Reset()
 	c.r.configVersionID.Add(1)

commands/list.go

@@ -41,6 +41,8 @@ func newListCommand() *listCommand {
 			p.PublishDate().Format(time.RFC3339),
 			strconv.FormatBool(p.Draft()),
 			p.Permalink(),
+			p.Kind(),
+			p.Section(),
 		}
 	}
 
@@ -67,6 +69,8 @@ func newListCommand() *listCommand {
 			"publishDate",
 			"draft",
 			"permalink",
+			"kind",
+			"section",
 		})
 
 		for _, p := range h.Pages() {
@@ -75,9 +79,6 @@ func newListCommand() *listCommand {
 				if err := writer.Write(record); err != nil {
 					return err
 				}
-				if err != nil {
-					return err
-				}
 			}
 		}
 
@@ -88,8 +89,8 @@ func newListCommand() *listCommand {
 		commands: []simplecobra.Commander{
 			&simpleCommand{
 				name:  "drafts",
-				short: "List all drafts",
-				long:  `List all of the drafts in your content directory.`,
+				short: "List draft content",
+				long:  `List draft content.`,
 				run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
 					shouldInclude := func(p page.Page) bool {
 						if !p.Draft() || p.File() == nil {
@@ -109,8 +110,8 @@ func newListCommand() *listCommand {
 			},
 			&simpleCommand{
 				name:  "future",
-				short: "List all posts dated in the future",
-				long:  `List all of the posts in your content directory which will be posted in the future.`,
+				short: "List future content",
+				long:  `List content with a future publication date.`,
 				run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
 					shouldInclude := func(p page.Page) bool {
 						if !resource.IsFuture(p) || p.File() == nil {
@@ -129,8 +130,8 @@ func newListCommand() *listCommand {
 			},
 			&simpleCommand{
 				name:  "expired",
-				short: "List all posts already expired",
-				long:  `List all of the posts in your content directory which has already expired.`,
+				short: "List expired content",
+				long:  `List content with a past expiration date.`,
 				run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
 					shouldInclude := func(p page.Page) bool {
 						if !resource.IsExpired(p) || p.File() == nil {
@@ -149,8 +150,8 @@ func newListCommand() *listCommand {
 			},
 			&simpleCommand{
 				name:  "all",
-				short: "List all posts",
-				long:  `List all of the posts in your content directory, include drafts, future and expired pages.`,
+				short: "List all content",
+				long:  `List all content including draft, future, and expired.`,
 				run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
 					shouldInclude := func(p page.Page) bool {
 						return p.File() != nil
@@ -161,6 +162,20 @@ func newListCommand() *listCommand {
 					cmd.ValidArgsFunction = cobra.NoFileCompletions
 				},
 			},
+			&simpleCommand{
+				name:  "published",
+				short: "List published content",
+				long:  `List content that is not draft, future, or expired.`,
+				run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
+					shouldInclude := func(p page.Page) bool {
+						return !p.Draft() && !resource.IsFuture(p) && !resource.IsExpired(p) && p.File() != nil
+					}
+					return list(cd, r, shouldInclude)
+				},
+				withc: func(cmd *cobra.Command, r *rootCommand) {
+					cmd.ValidArgsFunction = cobra.NoFileCompletions
+				},
+			},
 		},
 	}
 }

commands/server.go

@@ -538,7 +538,7 @@ of a second, you will be able to save and see your changes nearly instantly.`
 	cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
 	cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")
 	cmd.Flags().BoolVar(&c.disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild")
-	cmd.Flags().BoolVar(&c.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload")
+	cmd.Flags().BoolVarP(&c.navigateToChanged, "navigateToChanged", "N", false, "navigate to changed content file on live browser reload")
 	cmd.Flags().BoolVar(&c.renderStaticToDisk, "renderStaticToDisk", false, "serve static files from disk and dynamic files from memory")
 	cmd.Flags().BoolVar(&c.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
 	cmd.Flags().BoolVar(&c.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")

common/maps/maps.go

@@ -112,17 +112,17 @@ func ToSliceStringMap(in any) ([]map[string]any, error) {
 }
 
 // LookupEqualFold finds key in m with case insensitive equality checks.
-func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) {
+func LookupEqualFold[T any | string](m map[string]T, key string) (T, string, bool) {
 	if v, found := m[key]; found {
-		return v, true
+		return v, key, true
 	}
 	for k, v := range m {
 		if strings.EqualFold(k, key) {
-			return v, true
+			return v, k, true
 		}
 	}
 	var s T
-	return s, false
+	return s, "", false
 }
 
 // MergeShallow merges src into dst, but only if the key does not already exist in dst.

common/maps/maps_test.go

@@ -180,16 +180,18 @@ func TestLookupEqualFold(t *testing.T) {
 		"B": "bv",
 	}
 
-	v, found := LookupEqualFold(m1, "b")
+	v, k, found := LookupEqualFold(m1, "b")
 	c.Assert(found, qt.IsTrue)
 	c.Assert(v, qt.Equals, "bv")
+	c.Assert(k, qt.Equals, "B")
 
 	m2 := map[string]string{
 		"a": "av",
 		"B": "bv",
 	}
 
-	v, found = LookupEqualFold(m2, "b")
+	v, k, found = LookupEqualFold(m2, "b")
 	c.Assert(found, qt.IsTrue)
+	c.Assert(k, qt.Equals, "B")
 	c.Assert(v, qt.Equals, "bv")
 }

common/predicate/predicate.go

@@ -24,6 +24,9 @@ func (p P[T]) And(ps ...P[T]) P[T] {
 				return false
 			}
 		}
+		if p == nil {
+			return true
+		}
 		return p(v)
 	}
 }
@@ -36,6 +39,9 @@ func (p P[T]) Or(ps ...P[T]) P[T] {
 				return true
 			}
 		}
+		if p == nil {
+			return false
+		}
 		return p(v)
 	}
 }

common/tasks/tasks.go

@@ -0,0 +1,153 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+	"sync"
+	"time"
+)
+
+// RunEvery runs a function at intervals defined by the function itself.
+// Functions can be added and removed while running.
+type RunEvery struct {
+	// Any error returned from the function will be passed to this function.
+	HandleError func(string, error)
+
+	// If set, the function will be run immediately.
+	RunImmediately bool
+
+	// The named functions to run.
+	funcs map[string]*Func
+
+	mu      sync.Mutex
+	started bool
+	closed  bool
+	quit    chan struct{}
+}
+
+type Func struct {
+	// The shortest interval between each run.
+	IntervalLow time.Duration
+
+	// The longest interval between each run.
+	IntervalHigh time.Duration
+
+	// The function to run.
+	F func(interval time.Duration) (time.Duration, error)
+
+	interval time.Duration
+	last     time.Time
+}
+
+func (r *RunEvery) Start() error {
+	if r.started {
+		return nil
+	}
+
+	r.started = true
+	r.quit = make(chan struct{})
+
+	go func() {
+		if r.RunImmediately {
+			r.run()
+		}
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-r.quit:
+				return
+			case <-ticker.C:
+				r.run()
+			}
+		}
+	}()
+
+	return nil
+}
+
+// Close stops the RunEvery from running.
+func (r *RunEvery) Close() error {
+	if r.closed {
+		return nil
+	}
+	r.closed = true
+	if r.quit != nil {
+		close(r.quit)
+	}
+	return nil
+}
+
+// Add adds a function to the RunEvery.
+func (r *RunEvery) Add(name string, f Func) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.funcs == nil {
+		r.funcs = make(map[string]*Func)
+	}
+	if f.IntervalLow == 0 {
+		f.IntervalLow = 500 * time.Millisecond
+	}
+	if f.IntervalHigh <= f.IntervalLow {
+		f.IntervalHigh = 20 * time.Second
+	}
+
+	start := f.IntervalHigh / 3
+	if start < f.IntervalLow {
+		start = f.IntervalLow
+	}
+	f.interval = start
+	f.last = time.Now()
+
+	r.funcs[name] = &f
+}
+
+// Remove removes a function from the RunEvery.
+func (r *RunEvery) Remove(name string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	delete(r.funcs, name)
+}
+
+// Has returns whether the RunEvery has a function with the given name.
+func (r *RunEvery) Has(name string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	_, found := r.funcs[name]
+	return found
+}
+
+func (r *RunEvery) run() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for name, f := range r.funcs {
+		if time.Now().Before(f.last.Add(f.interval)) {
+			continue
+		}
+		f.last = time.Now()
+		interval, err := f.F(f.interval)
+		if err != nil && r.HandleError != nil {
+			r.HandleError(name, err)
+		}
+
+		if interval < f.IntervalLow {
+			interval = f.IntervalLow
+		}
+
+		if interval > f.IntervalHigh {
+			interval = f.IntervalHigh
+		}
+		f.interval = interval
+	}
+}

common/types/closer.go

@@ -0,0 +1,47 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import "sync"
+
+type Closer interface {
+	Close() error
+}
+
+type CloseAdder interface {
+	Add(Closer)
+}
+
+type Closers struct {
+	mu sync.Mutex
+	cs []Closer
+}
+
+func (cs *Closers) Add(c Closer) {
+	cs.mu.Lock()
+	defer cs.mu.Unlock()
+	cs.cs = append(cs.cs, c)
+}
+
+func (cs *Closers) Close() error {
+	cs.mu.Lock()
+	defer cs.mu.Unlock()
+	for _, c := range cs.cs {
+		c.Close()
+	}
+
+	cs.cs = cs.cs[:0]
+
+	return nil
+}

config/allconfig/allconfig.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/cache/filecache"
+	"github.com/gohugoio/hugo/cache/httpcache"
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
@@ -119,6 +120,10 @@ type Config struct {
 	// <docsmeta>{"identifiers": ["caches"] }</docsmeta>
 	Caches filecache.Configs `mapstructure:"-"`
 
+	// The httpcache configuration section contains HTTP-cache-related configuration options.
+	// <docsmeta>{"identifiers": ["httpcache"] }</docsmeta>
+	HTTPCache httpcache.Config `mapstructure:"-"`
+
 	// The markup configuration section contains markup-related configuration options.
 	// <docsmeta>{"identifiers": ["markup"] }</docsmeta>
 	Markup markup_config.Config `mapstructure:"-"`
@@ -359,6 +364,11 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
 		}
 	}
 
+	httpCache, err := c.HTTPCache.Compile()
+	if err != nil {
+		return err
+	}
+
 	c.C = &ConfigCompiled{
 		Timeout:           timeout,
 		BaseURL:           baseURL,
@@ -374,6 +384,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
 		SegmentFilter:     c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...),
 		MainSections:      c.MainSections,
 		Clock:             clock,
+		HTTPCache:         httpCache,
 		transientErr:      transientErr,
 	}
 
@@ -413,6 +424,7 @@ type ConfigCompiled struct {
 	SegmentFilter     segments.SegmentFilter
 	MainSections      []string
 	Clock             time.Time
+	HTTPCache         httpcache.ConfigCompiled
 
 	// This is set to the last transient error found during config compilation.
 	// With themes/modules we compute the configuration in multiple passes, and

config/allconfig/alldecoders.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 
 	"github.com/gohugoio/hugo/cache/filecache"
+
+	"github.com/gohugoio/hugo/cache/httpcache"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
@@ -96,6 +98,18 @@ var allDecoderSetups = map[string]decodeWeight{
 			return err
 		},
 	},
+	"httpcache": {
+		key: "httpcache",
+		decode: func(d decodeWeight, p decodeConfig) error {
+			var err error
+			p.c.HTTPCache, err = httpcache.DecodeConfig(p.bcfg, p.p.GetStringMap(d.key))
+			if p.c.IgnoreCache {
+				p.c.HTTPCache.Cache.For.Excludes = []string{"**"}
+				p.c.HTTPCache.Cache.For.Includes = []string{}
+			}
+			return err
+		},
+	},
 	"build": {
 		key: "build",
 		decode: func(d decodeWeight, p decodeConfig) error {

config/allconfig/configlanguage.go

@@ -173,6 +173,8 @@ func (c ConfigLanguage) GetConfigSection(s string) any {
 		return c.m.Modules
 	case "deployment":
 		return c.config.Deployment
+	case "httpCacheCompiled":
+		return c.config.C.HTTPCache
 	default:
 		panic("not implemented: " + s)
 	}

config/commonConfig.go

@@ -98,10 +98,6 @@ type BuildConfig struct {
 	// One of never, fallback, always. Default is fallback
 	UseResourceCacheWhen string
 
-	// When enabled, will duplicate bundled resource files across languages that
-	// doesn't have a translated version.
-	DuplicateResourceFiles bool
-
 	// When enabled, will collect and write a hugo_stats.json with some build
 	// related aggregated data (e.g. CSS class names).
 	// Note that this was a bool <= v0.115.0.

deps/deps.go

@@ -15,11 +15,13 @@ import (
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/config/allconfig"
 	"github.com/gohugoio/hugo/config/security"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/resources/postpub"
@@ -85,7 +87,7 @@ type Deps struct {
 	BuildEndListeners *Listeners
 
 	// Resources that gets closed when the build is done or the server shuts down.
-	BuildClosers *Closers
+	BuildClosers *types.Closers
 
 	// This is common/global for all sites.
 	BuildState *BuildState
@@ -143,7 +145,7 @@ func (d *Deps) Init() error {
 	}
 
 	if d.BuildClosers == nil {
-		d.BuildClosers = &Closers{}
+		d.BuildClosers = &types.Closers{}
 	}
 
 	if d.Metrics == nil && d.Conf.TemplateMetrics() {
@@ -208,7 +210,7 @@ func (d *Deps) Init() error {
 		return fmt.Errorf("failed to create file caches from configuration: %w", err)
 	}
 
-	resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper)
+	resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState)
 	if err != nil {
 		return fmt.Errorf("failed to create resource spec: %w", err)
 	}
@@ -353,6 +355,9 @@ type DepsCfg struct {
 
 	// i18n handling.
 	TranslationProvider ResourceProvider
+
+	// ChangesFromBuild for changes passed back to the server/watch process.
+	ChangesFromBuild chan []identity.Identity
 }
 
 // BuildState are state used during a build.
@@ -361,11 +366,19 @@ type BuildState struct {
 
 	mu sync.Mutex // protects state below.
 
+	OnSignalRebuild func(ids ...identity.Identity)
+
 	// A set of filenames in /public that
 	// contains a post-processing prefix.
 	filenamesWithPostPrefix map[string]bool
 }
 
+var _ identity.SignalRebuilder = (*BuildState)(nil)
+
+func (b *BuildState) SignalRebuild(ids ...identity.Identity) {
+	b.OnSignalRebuild(ids...)
+}
+
 func (b *BuildState) AddFilenameWithPostPrefix(filename string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -389,30 +402,3 @@ func (b *BuildState) GetFilenamesWithPostPrefix() []string {
 func (b *BuildState) Incr() int {
 	return int(atomic.AddUint64(&b.counter, uint64(1)))
 }
-
-type Closer interface {
-	Close() error
-}
-
-type Closers struct {
-	mu sync.Mutex
-	cs []Closer
-}
-
-func (cs *Closers) Add(c Closer) {
-	cs.mu.Lock()
-	defer cs.mu.Unlock()
-	cs.cs = append(cs.cs, c)
-}
-
-func (cs *Closers) Close() error {
-	cs.mu.Lock()
-	defer cs.mu.Unlock()
-	for _, c := range cs.cs {
-		c.Close()
-	}
-
-	cs.cs = cs.cs[:0]
-
-	return nil
-}

docs/data/docs.yaml

@@ -939,7 +939,6 @@ config:
     cacheBusters:
     - source: (postcss|tailwind)\.config\.js
       target: (css|styles|scss|sass)
-    duplicateResourceFiles: false
     noJSConfigInAssets: false
     useResourceCacheWhen: fallback
   buildDrafts: false

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/gobuffalo/flect v1.0.2
 	github.com/gobwas/glob v0.2.3
 	github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e
+	github.com/gohugoio/httpcache v0.7.0
 	github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0
 	github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0
 	github.com/gohugoio/locales v0.14.0

go.sum

@@ -53,6 +53,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
@@ -67,9 +68,11 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
 github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
 github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.50.7 h1:odKb+uneeGgF2jgAerKjFzpljiyZxleV4SHB7oBK+YA=
@@ -171,6 +174,7 @@ github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dU
 github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
 github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -195,8 +199,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
 github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
-github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
-github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -207,12 +209,19 @@ github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1Rf
 github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw=
 github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI=
 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
+github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA=
 github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY=
 github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ=
+github.com/gohugoio/httpcache v0.5.0 h1:9xi4VuXd+KT3h0jOs8DlZxTMu5CtjDr0BvQMAuL/O5I=
+github.com/gohugoio/httpcache v0.5.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
+github.com/gohugoio/httpcache v0.6.0 h1:5pYJM43Yoc4uvIJ+/e770PS48srTumvuQZpuBfGFZV0=
+github.com/gohugoio/httpcache v0.6.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
+github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ3Vs=
+github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
 github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 h1:YhxZNU8y2vxV6Ibr7QJzzUlpr8oHHWX/l+Q1R/a5Zao=
 github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0/go.mod h1:0cuvOnGKW7WeXA3i7qK6IS07FH1bgJ2XzOjQ7BMJYH4=
 github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
@@ -274,12 +283,15 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE=
+github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
 github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
+github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -315,6 +327,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -394,6 +407,7 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -439,7 +453,9 @@ github.com/tdewolff/parse/v2 v2.7.13 h1:iSiwOUkCYLNfapHoqdLcqZVgvQ0jrsao8YYKP/UJ
 github.com/tdewolff/parse/v2 v2.7.13/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
 github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
+github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -806,6 +822,7 @@ google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=

hugolib/content_map.go

@@ -164,6 +164,55 @@ func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) {
 	return
 }
 
+func (m *pageMap) insertPageWithLock(s string, p *pageState) (contentNodeI, contentNodeI, bool) {
+	u, n, replaced := m.treePages.InsertIntoValuesDimensionWithLock(s, p)
+
+	if replaced && !m.s.h.isRebuild() && m.s.conf.PrintPathWarnings {
+		var messageDetail string
+		if p1, ok := n.(*pageState); ok && p1.File() != nil {
+			messageDetail = fmt.Sprintf(" file: %q", p1.File().Filename())
+		}
+		if p2, ok := u.(*pageState); ok && p2.File() != nil {
+			messageDetail += fmt.Sprintf(" file: %q", p2.File().Filename())
+		}
+
+		m.s.Log.Warnf("Duplicate content path: %q%s", s, messageDetail)
+	}
+
+	return u, n, replaced
+}
+
+func (m *pageMap) insertResourceWithLock(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) {
+	u, n, replaced := m.treeResources.InsertIntoValuesDimensionWithLock(s, r)
+	if replaced {
+		m.handleDuplicateResourcePath(s, r, n)
+	}
+	return u, n, replaced
+}
+
+func (m *pageMap) insertResource(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) {
+	u, n, replaced := m.treeResources.InsertIntoValuesDimension(s, r)
+	if replaced {
+		m.handleDuplicateResourcePath(s, r, n)
+	}
+	return u, n, replaced
+}
+
+func (m *pageMap) handleDuplicateResourcePath(s string, updated, existing contentNodeI) {
+	if m.s.h.isRebuild() || !m.s.conf.PrintPathWarnings {
+		return
+	}
+	var messageDetail string
+	if r1, ok := existing.(*resourceSource); ok && r1.fi != nil {
+		messageDetail = fmt.Sprintf(" file: %q", r1.fi.Meta().Filename)
+	}
+	if r2, ok := updated.(*resourceSource); ok && r2.fi != nil {
+		messageDetail += fmt.Sprintf(" file: %q", r2.fi.Meta().Filename)
+	}
+
+	m.s.Log.Warnf("Duplicate resource path: %q%s", s, messageDetail)
+}
+
 func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) {
 	if fi.IsDir() {
 		return
@@ -207,7 +256,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCoun
 			rs = &resourceSource{path: pi, opener: r, fi: fim, langIndex: fim.Meta().LangIndex}
 		}
 
-		tree.InsertIntoValuesDimension(key, rs)
+		_, _, _ = m.insertResource(key, rs)
 
 		return nil
 	}
@@ -261,7 +310,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCoun
 			return
 		}
 
-		m.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), p)
+		m.insertPageWithLock(pi.Base(), p)
 
 	}
 	return
@@ -286,12 +335,6 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil
 	f := source.NewFileInfo(fi)
 	h := s.h
 
-	// Make sure the layouts are initialized.
-	if _, err := h.init.layouts.Do(context.Background()); err != nil {
-		addErr = err
-		return
-	}
-
 	contentAdapter := s.pageMap.treePagesFromTemplateAdapters.Get(pi.Base())
 	var rebuild bool
 	if contentAdapter != nil {
@@ -336,13 +379,28 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil
 						return nil
 					}
 
-					u, n, replaced := s.pageMap.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), ps)
+					u, n, replaced := s.pageMap.insertPageWithLock(pi.Base(), ps)
 
 					if h.isRebuild() {
 						if replaced {
 							pt.AddChange(n.GetIdentity())
 						} else {
 							pt.AddChange(u.GetIdentity())
+							// New content not in use anywhere.
+							// To make sure that these gets listed in any site.RegularPages ranges or similar
+							// we could invalidate everything, but first try to collect a sample set
+							// from the surrounding pages.
+							var surroundingIDs []identity.Identity
+							ids := h.pageTrees.collectIdentitiesSurrounding(pi.Base(), 10)
+							if len(ids) > 0 {
+								surroundingIDs = append(surroundingIDs, ids...)
+							} else {
+								// No surrounding pages found, so invalidate everything.
+								surroundingIDs = []identity.Identity{identity.GenghisKhan}
+							}
+							for _, id := range surroundingIDs {
+								pt.AddChange(id)
+							}
 						}
 					}
 
@@ -358,9 +416,9 @@ func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *Buil
 						return err
 					}
 
-					rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: nil, langIndex: s.languagei}
+					rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: pt.GoTmplFi, langIndex: s.languagei}
 
-					_, n, replaced := s.pageMap.treeResources.InsertIntoValuesDimensionWithLock(rc.PathInfo.Base(), rs)
+					_, n, replaced := s.pageMap.insertResourceWithLock(rc.PathInfo.Base(), rs)
 
 					if h.isRebuild() && replaced {
 						pt.AddChange(n.GetIdentity())

hugolib/content_map_page.go

@@ -932,8 +932,8 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *
 				LockType: doctree.LockTypeRead,
 				Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
 					p := n.(*pageState)
-					if p.File() != nil {
-						add(p.File().FileInfo().Meta().PathInfo.BaseNameNoIdentifier(), p)
+					if p.PathInfo() != nil {
+						add(p.PathInfo().BaseNameNoIdentifier(), p)
 					}
 					return false, nil
 				},
@@ -975,7 +975,7 @@ type contentTreeReverseIndexMap struct {
 
 type sitePagesAssembler struct {
 	*Site
-	assembleChanges *whatChanged
+	assembleChanges *WhatChanged
 	ctx             context.Context
 }
 
@@ -1881,11 +1881,19 @@ func (sa *sitePagesAssembler) addStandalonePages() error {
 	}
 
 	if sitemapEnabled {
-		addStandalone("/_sitemap", kinds.KindSitemap, output.SitemapFormat)
-		skipSitemapIndex := s.Conf.IsMultihost() || !(s.Conf.DefaultContentLanguageInSubdir() || s.Conf.IsMultilingual())
+		of := output.SitemapFormat
+		if s.conf.Sitemap.Filename != "" {
+			of.BaseName = paths.Filename(s.conf.Sitemap.Filename)
+		}
+		addStandalone("/_sitemap", kinds.KindSitemap, of)
 
+		skipSitemapIndex := s.Conf.IsMultihost() || !(s.Conf.DefaultContentLanguageInSubdir() || s.Conf.IsMultilingual())
 		if !skipSitemapIndex {
-			addStandalone("/_sitemapindex", kinds.KindSitemapIndex, output.SitemapIndexFormat)
+			of = output.SitemapIndexFormat
+			if s.conf.Sitemap.Filename != "" {
+				of.BaseName = paths.Filename(s.conf.Sitemap.Filename)
+			}
+			addStandalone("/_sitemapindex", kinds.KindSitemapIndex, of)
 		}
 	}
 

hugolib/content_map_test.go

@@ -16,6 +16,7 @@ package hugolib
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
@@ -27,7 +28,7 @@ func TestContentMapSite(t *testing.T) {
 	pageTempl := `
 ---
 title: "Page %d"
-date: "2019-06-0%d"	
+date: "2019-06-0%d"
 lastMod: "2019-06-0%d"
 categories: [%q]
 ---
@@ -54,7 +55,7 @@ draft: true
 title: "Hugo Home"
 cascade:
     description: "Common Description"
-    
+
 ---
 
 Home Content.
@@ -110,7 +111,7 @@ IsDescendant overlap2: false: {{ $overlap2.IsDescendant $overlap1 }}
 IsAncestor overlap1: false: {{ $overlap1.IsAncestor $overlap2 }}
 IsAncestor overlap2: false: {{ $overlap2.IsAncestor $overlap1 }}
 FirstSection: {{ $blogSub.FirstSection.RelPermalink }} {{ $blog.FirstSection.RelPermalink }} {{ $home.FirstSection.RelPermalink }} {{ $page.FirstSection.RelPermalink }}
-InSection: true: {{ $page.InSection $blog }} false: {{ $page.InSection $blogSub }} 
+InSection: true: {{ $page.InSection $blog }} false: {{ $page.InSection $blogSub }}
 Next: {{ $page2.Next.RelPermalink }}
 NextInSection: {{ $page2.NextInSection.RelPermalink }}
 Pages: {{ range $blog.Pages }}{{ .RelPermalink }}|{{ end }}
@@ -140,11 +141,11 @@ Draft5: {{ if (.Site.GetPage "blog/draftsection/sub/page") }}FOUND{{ end }}|
 	 Num Regular: 9
         Main Sections: [blog]
         Pag Num Pages: 9
-        
-      Home: Hugo Home|/|2019-06-08|Current Section: /|Resources: 
-        Blog Section: Blogs|/blog/|2019-06-08|Current Section: /blog|Resources: 
+
+      Home: Hugo Home|/|2019-06-08|Current Section: /|Resources:
+        Blog Section: Blogs|/blog/|2019-06-08|Current Section: /blog|Resources:
         Blog Sub Section: Page 3|/blog/subsection/|2019-06-03|Current Section: /blog/subsection|Resources: application: /blog/subsection/subdata.json|
-        Page: Page 1|/blog/page1/|2019-06-01|Current Section: /blog|Resources: 
+        Page: Page 1|/blog/page1/|2019-06-01|Current Section: /blog|Resources:
         Bundle: Page 12|/blog/bundle/|0001-01-01|Current Section: /blog|Resources: application: /blog/bundle/data.json|page: |
         IsDescendant: true: true true: true true: true true: true true: true false: false false: false
         IsAncestor: true: true true: true true: true true: true true: true true: true false: false false: false false: false  false: false
@@ -153,7 +154,7 @@ Draft5: {{ if (.Site.GetPage "blog/draftsection/sub/page") }}FOUND{{ end }}|
         IsAncestor overlap1: false: false
         IsAncestor overlap2: false: false
         FirstSection: /blog/ /blog/ / /blog/
-        InSection: true: true false: false 
+        InSection: true: true false: false
         Next: /blog/page3/
         NextInSection: /blog/page3/
         Pages: /blog/page3/|/blog/subsection/|/blog/page2/|/blog/page1/|/blog/bundle/|
@@ -165,13 +166,13 @@ Draft5: {{ if (.Site.GetPage "blog/draftsection/sub/page") }}FOUND{{ end }}|
         Pag Blog Num Pages: 4
         Blog Num RegularPages: 4
         Blog Num Pages: 5
-        
+
         Draft1: |
         Draft2: FOUND|
         Draft3: FOUND|
         Draft4: FOUND|
         Draft5: FOUND|
-           
+
 `)
 }
 
@@ -185,7 +186,7 @@ title = "Integration Test"
 disableKinds=["page", "section", "taxonomy", "term", "sitemap", "robotsTXT", "RSS"]
 -- layouts/index.html --
 Home: {{ .Title }}|
-		
+
 	`
 
 	b := NewIntegrationTestBuilder(
@@ -358,3 +359,41 @@ p1-foo.txt
 	b.AssertFileExists("public/s1/p1-foo.txt", true) // failing test
 	b.AssertFileExists("public/s1/p1/index.html", true)
 }
+
+func TestSitemapOverrideFilename(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+baseURL = 'https://example.org/'
+disableKinds = ['page','rss','section','taxonomy','term']
+defaultContentLanguage = 'de'
+defaultContentLanguageInSubdir = true
+[languages.de]
+[languages.en]
+[sitemap]
+filename = 'foo.xml'
+-- layouts/index.html --
+irrelevant
+`
+
+	b := Test(t, files)
+
+	b.AssertFileExists("public/de/foo.xml", true)
+	b.AssertFileExists("public/en/foo.xml", true)
+	b.AssertFileContent("public/foo.xml",
+		"<loc>https://example.org/de/foo.xml</loc>",
+		"<loc>https://example.org/en/foo.xml</loc>",
+	)
+
+	files = strings.ReplaceAll(files, "filename = 'foo.xml'", "")
+
+	b = Test(t, files)
+
+	b.AssertFileExists("public/de/sitemap.xml", true)
+	b.AssertFileExists("public/en/sitemap.xml", true)
+	b.AssertFileContent("public/sitemap.xml",
+		"<loc>https://example.org/de/sitemap.xml</loc>",
+		"<loc>https://example.org/en/sitemap.xml</loc>",
+	)
+}

hugolib/hugo_sites.go

@@ -405,8 +405,9 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
 type BuildCfg struct {
 	// Skip rendering. Useful for testing.
 	SkipRender bool
+
 	// Use this to indicate what changed (for rebuilds).
-	whatChanged *whatChanged
+	WhatChanged *WhatChanged
 
 	// This is a partial re-render of some selected pages.
 	PartialReRender bool

hugolib/hugo_sites_build.go

@@ -114,9 +114,9 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 
 	// Need a pointer as this may be modified.
 	conf := &config
-	if conf.whatChanged == nil {
+	if conf.WhatChanged == nil {
 		// Assume everything has changed
-		conf.whatChanged = &whatChanged{needsPagesAssembly: true}
+		conf.WhatChanged = &WhatChanged{needsPagesAssembly: true}
 	}
 
 	var prepareErr error
@@ -128,7 +128,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 					s.Deps.BuildStartListeners.Notify()
 				}
 
-				if len(events) > 0 {
+				if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 {
 					// Rebuild
 					if err := h.initRebuild(conf); err != nil {
 						return fmt.Errorf("initRebuild: %w", err)
@@ -224,7 +224,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error {
 	})
 
 	for _, s := range h.Sites {
-		s.resetBuildState(config.whatChanged.needsPagesAssembly)
+		s.resetBuildState(config.WhatChanged.needsPagesAssembly)
 	}
 
 	h.reset(config)
@@ -239,9 +239,19 @@ func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *Bui
 	l = l.WithField("step", "process")
 	defer loggers.TimeTrackf(l, time.Now(), nil, "")
 
+	if _, err := h.init.layouts.Do(ctx); err != nil {
+		return err
+	}
+
 	if len(events) > 0 {
-		// This is a rebuild
-		return h.processPartial(ctx, l, config, init, events)
+		// This is a rebuild triggered from file events.
+		return h.processPartialFileEvents(ctx, l, config, init, events)
+	} else if len(config.WhatChanged.Changes()) > 0 {
+		// Rebuild triggered from remote events.
+		if err := init(config); err != nil {
+			return err
+		}
+		return h.processPartialRebuildChanges(ctx, l, config)
 	}
 	return h.processFull(ctx, l, config)
 }
@@ -252,8 +262,8 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 	l = l.WithField("step", "assemble")
 	defer loggers.TimeTrackf(l, time.Now(), nil, "")
 
-	if !bcfg.whatChanged.needsPagesAssembly {
-		changes := bcfg.whatChanged.Drain()
+	if !bcfg.WhatChanged.needsPagesAssembly {
+		changes := bcfg.WhatChanged.Drain()
 		if len(changes) > 0 {
 			if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
 				return err
@@ -269,7 +279,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 	for i, s := range h.Sites {
 		assemblers[i] = &sitePagesAssembler{
 			Site:            s,
-			assembleChanges: bcfg.whatChanged,
+			assembleChanges: bcfg.WhatChanged,
 			ctx:             ctx,
 		}
 	}
@@ -285,7 +295,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
 		return err
 	}
 
-	changes := bcfg.whatChanged.Drain()
+	changes := bcfg.WhatChanged.Drain()
 
 	// Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
 	// of what needs to be re-built.
@@ -324,10 +334,6 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error {
 		loggers.TimeTrackf(l, start, h.buildCounters.loggFields(), "")
 	}()
 
-	if _, err := h.init.layouts.Do(context.Background()); err != nil {
-		return err
-	}
-
 	siteRenderContext := &siteRenderContext{cfg: config, multihost: h.Configs.IsMultihost}
 
 	i := 0
@@ -612,8 +618,19 @@ func (p pathChange) isStructuralChange() bool {
 	return p.delete || p.isDir
 }
 
-// processPartial prepares the Sites' sources for a partial rebuild.
-func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
+func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error {
+	if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil {
+		return err
+	}
+
+	if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
+		return err
+	}
+	return nil
+}
+
+// processPartialFileEvents prepares the Sites' sources for a partial rebuild.
+func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
 	h.Log.Trace(logg.StringFunc(func() string {
 		var sb strings.Builder
 		sb.WriteString("File events:\n")
@@ -887,13 +904,13 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
 
 	resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
 
-	changed := &whatChanged{
+	changed := &WhatChanged{
 		needsPagesAssembly: needsPagesAssemble,
 		identitySet:        make(identity.Identities),
 	}
 	changed.Add(changes...)
 
-	config.whatChanged = changed
+	config.WhatChanged = changed
 
 	if err := init(config); err != nil {
 		return err
@@ -918,7 +935,6 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
 	if tmplChanged || i18nChanged {
 		// TODO(bep) we should split this, but currently the loading of i18n and layout files are tied together. See #12048.
 		h.init.layouts.Reset()
-
 		if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
 			// TODO(bep) this could probably be optimized to somehow
 			// only load the changed templates and its dependencies, but that is non-trivial.
@@ -978,19 +994,23 @@ func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConf
 	}
 
 	if len(bi.ChangedIdentities) > 0 {
-		buildConfig.whatChanged.Add(bi.ChangedIdentities...)
-		buildConfig.whatChanged.needsPagesAssembly = true
+		buildConfig.WhatChanged.Add(bi.ChangedIdentities...)
+		buildConfig.WhatChanged.needsPagesAssembly = true
 	}
 
 	for _, p := range bi.DeletedPaths {
 		pp := path.Join(bi.Path.Base(), p)
 		if v, ok := s.pageMap.treePages.Delete(pp); ok {
-			buildConfig.whatChanged.Add(v.GetIdentity())
+			buildConfig.WhatChanged.Add(v.GetIdentity())
 		}
 	}
 }
 
 func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error {
+	// Make sure the layouts are initialized.
+	if _, err := h.init.layouts.Do(context.Background()); err != nil {
+		return err
+	}
 	g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{
 		NumWorkers: h.numWorkers,
 		Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error {

hugolib/page__content.go

@@ -57,14 +57,14 @@ type pageContentReplacement struct {
 
 func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo, error) {
 	var (
-		sourceKey  string
-		openSource hugio.OpenReadSeekCloser
-		hasContent = m.pageConfig.IsFromContentAdapter
+		sourceKey            string
+		openSource           hugio.OpenReadSeekCloser
+		isFromContentAdapter = m.pageConfig.IsFromContentAdapter
 	)
 
-	if m.f != nil && !hasContent {
+	if m.f != nil && !isFromContentAdapter {
 		sourceKey = filepath.ToSlash(m.f.Filename())
-		if !hasContent {
+		if !isFromContentAdapter {
 			meta := m.f.FileInfo().Meta()
 			openSource = func() (hugio.ReadSeekCloser, error) {
 				r, err := meta.Open()
@@ -74,7 +74,7 @@ func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo
 				return r, nil
 			}
 		}
-	} else if hasContent {
+	} else if isFromContentAdapter {
 		openSource = m.pageConfig.Content.ValueAsOpenReadSeekCloser()
 	}
 
@@ -96,7 +96,9 @@ func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo
 
 	items, err := pageparser.ParseBytes(
 		source,
-		pageparser.Config{},
+		pageparser.Config{
+			NoFrontMatter: isFromContentAdapter,
+		},
 	)
 	if err != nil {
 		return nil, err
@@ -104,7 +106,7 @@ func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo
 
 	pi.itemsStep1 = items
 
-	if hasContent {
+	if isFromContentAdapter {
 		// No front matter.
 		return pi, nil
 	}

hugolib/page__menus.go

@@ -62,8 +62,22 @@ func (p *pageMenus) init() {
 			p.p,
 		)
 
+		params := p.p.Params()
+
+		var menus any
+		var ok bool
+
+		if p.p.m.pageConfig.Menus != nil {
+			menus = p.p.m.pageConfig.Menus
+		} else {
+			menus, ok = params["menus"]
+			if !ok {
+				menus = params["menu"]
+			}
+		}
+
 		var err error
-		p.pm, err = navigation.PageMenusFromPage(p.p)
+		p.pm, err = navigation.PageMenusFromPage(menus, p.p)
 		if err != nil {
 			p.p.s.Log.Errorln(p.p.wrapError(err))
 		}

hugolib/page__new.go

@@ -62,7 +62,8 @@ func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) {
 
 	if pcfg.Path != "" {
 		s := m.pageConfig.Path
-		if !paths.HasExt(s) {
+		// Paths from content adapters should never have any extension.
+		if pcfg.IsFromContentAdapter || !paths.HasExt(s) {
 			var (
 				isBranch    bool
 				isBranchSet bool

hugolib/page__per_output.go

@@ -307,7 +307,10 @@ func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (te
 	if pageparser.HasShortcode(contentToRender) {
 		contentToRenderb := []byte(contentToRender)
 		// String contains a shortcode.
-		parseInfo.itemsStep1, err = pageparser.ParseBytesMain(contentToRenderb, pageparser.Config{})
+		parseInfo.itemsStep1, err = pageparser.ParseBytes(contentToRenderb, pageparser.Config{
+			NoFrontMatter:    true,
+			NoSummaryDivider: true,
+		})
 		if err != nil {
 			return "", err
 		}

hugolib/pagecollections_test.go

@@ -693,3 +693,36 @@ draft: true
 	b.AssertFileContent("public/s1-foo/index.html", "/s1-foo/: Pages: /s1-foo/p2/|/s1-foo/s2-foo/|/s1-foo/s2/|$")
 	b.AssertFileContent("public/s1-foo/s2/index.html", "/s1-foo/s2/: Pages: /s1-foo/s2/p3/|$")
 }
+
+func TestGetPageContentAdapterBaseIssue12561(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['rss','section','sitemap','taxonomy','term']
+-- layouts/index.html --
+Test A: {{ (site.GetPage "/s1/p1").Title }}
+Test B: {{ (site.GetPage "p1").Title }}
+Test C: {{ (site.GetPage "/s2/p2").Title }}
+Test D: {{ (site.GetPage "p2").Title }}
+-- layouts/_default/single.html --
+{{ .Title }}
+-- content/s1/p1.md --
+---
+title: p1
+---
+-- content/s2/_content.gotmpl --
+{{ .AddPage (dict "path" "p2" "title" "p2") }}
+`
+
+	b := Test(t, files)
+
+	b.AssertFileExists("public/s1/p1/index.html", true)
+	b.AssertFileExists("public/s2/p2/index.html", true)
+	b.AssertFileContent("public/index.html",
+		"Test A: p1",
+		"Test B: p1",
+		"Test C: p2",
+		"Test D: p2", // fails
+	)
+}

hugolib/pages_language_merge_test.go

@@ -68,7 +68,7 @@ func TestMergeLanguages(t *testing.T) {
 
 	firstNN := nnSite.RegularPages()[0]
 	c.Assert(len(firstNN.Sites()), qt.Equals, 4)
-	c.Assert(firstNN.Sites().First().Language().Lang, qt.Equals, "en")
+	c.Assert(firstNN.Sites().Default().Language().Lang, qt.Equals, "en")
 
 	nnBundle := nnSite.getPageOldVersion("page", "bundle")
 	enBundle := enSite.getPageOldVersion("page", "bundle")

hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go

@@ -38,7 +38,11 @@ draft: false
 -- layouts/partials/get-value.html --
 {{ $val := "p1" }}
 {{ return $val }}
+-- layouts/_default/baseof.html --
+Baseof:
+{{ block "main" . }}{{ end }}
 -- layouts/_default/single.html --
+{{ define "main" }}
 Single: {{ .Title }}|{{ .Content }}|Params: {{ .Params.param1 }}|Path: {{ .Path }}|
 Dates: Date: {{ .Date.Format "2006-01-02" }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}|PublishDate: {{ .PublishDate.Format "2006-01-02" }}|ExpiryDate: {{ .ExpiryDate.Format "2006-01-02" }}|
 Len Resources: {{ .Resources | len }}
@@ -49,6 +53,7 @@ Featured Image: {{ .RelPermalink }}|{{ .Name }}|
 Resized Featured Image: {{ .RelPermalink }}|{{ .Width }}|
 {{ end}}
 {{ end }}
+{{ end }}
 -- layouts/_default/list.html --
 List: {{ .Title }}|{{ .Content }}|
 RegularPagesRecursive: {{ range .RegularPagesRecursive }}{{ .Title }}:{{ .Path }}|{{ end }}$
@@ -75,6 +80,7 @@ Pfile Content
 {{ $.AddPage  (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }}
 {{ $.AddPage  (dict "kind" "page" "path" "p3" "title" "p3title" "dates" $dates "content" $contentMarkdownDefault "draft" false ) }}
 {{ $.AddPage  (dict "kind" "page" "path" "p4" "title" "p4title" "dates" $dates "content" $contentMarkdownDefault "draft" $data.draft ) }}
+ADD_MORE_PLACEHOLDER
 
 
 {{ $resourceContent := dict "value" $dataResource }}
@@ -274,6 +280,14 @@ func TestPagesFromGoTmplRemovePage(t *testing.T) {
 	b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$")
 }
 
+func TestPagesFromGoTmplAddPage(t *testing.T) {
+	t.Parallel()
+	b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
+	b.EditFileReplaceAll("content/docs/_content.gotmpl", "ADD_MORE_PLACEHOLDER", `{{ $.AddPage  (dict "kind" "page" "path" "page_added" "title" "page_added_title" "dates" $dates "content" $contentHTML ) }}`).Build()
+	b.AssertFileExists("public/docs/page_added/index.html", true)
+	b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|page_added_title:/docs/page_added|pfile:/docs/pfile|$")
+}
+
 func TestPagesFromGoTmplDraftPage(t *testing.T) {
 	t.Parallel()
 	b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic)
@@ -477,3 +491,190 @@ baseURL = "https://example.com"
 	b.AssertFileExists("public/docs/p1/index.html", true)
 	b.AssertFileExists("public/docs/p2/index.html", false)
 }
+
+func TestPagesFromGoPathsWithDotsIssue12493(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['home','section','rss','sitemap','taxonomy','term']
+-- content/_content.gotmpl --
+{{ .AddPage (dict "path" "s-1.2.3/p-4.5.6" "title" "p-4.5.6") }}
+-- layouts/_default/single.html --
+{{ .Title }}
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileExists("public/s-1.2.3/p-4.5.6/index.html", true)
+}
+
+func TestPagesFromGoParamsIssue12497(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['home','section','rss','sitemap','taxonomy','term']
+-- content/_content.gotmpl --
+{{ .AddPage (dict "path" "p1" "title" "p1" "params" (dict "paraM1" "param1v" )) }}
+{{ .AddResource (dict "path" "p1/data1.yaml" "content" (dict "value" "data1" ) "params" (dict "paraM1" "param1v" )) }}
+-- layouts/_default/single.html --
+{{ .Title }}|{{ .Params.paraM1 }}
+{{ range .Resources }}
+{{ .Name }}|{{ .Params.paraM1 }}
+{{ end }}
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/p1/index.html",
+		"p1|param1v",
+		"data1.yaml|param1v",
+	)
+}
+
+func TestPagesFromGoTmplPathWarningsPathPage(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableKinds = ['home','section','rss','sitemap','taxonomy','term']
+printPathWarnings = true
+-- content/_content.gotmpl --
+{{ .AddPage (dict "path" "p1" "title" "p1" ) }}
+{{ .AddPage (dict "path" "p2" "title" "p2" ) }}
+-- content/p1.md --
+---
+title: "p1"
+---
+-- layouts/_default/single.html --
+{{ .Title }}|
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptWarn())
+
+	b.AssertFileContent("public/p1/index.html", "p1|")
+
+	b.AssertLogContains("Duplicate content path")
+
+	files = strings.ReplaceAll(files, `"path" "p1"`, `"path" "p1new"`)
+
+	b = hugolib.Test(t, files, hugolib.TestOptWarn())
+
+	b.AssertLogNotContains("WARN")
+}
+
+func TestPagesFromGoTmplPathWarningsPathResource(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+disableKinds = ['home','section','rss','sitemap','taxonomy','term']
+printPathWarnings = true
+-- content/_content.gotmpl --
+{{ .AddResource (dict "path" "p1/data1.yaml" "content" (dict "value" "data1" ) ) }}
+{{ .AddResource (dict "path" "p1/data2.yaml" "content" (dict "value" "data2" ) ) }}
+
+-- content/p1/index.md --
+---
+title: "p1"
+---
+-- content/p1/data1.yaml --
+value: data1
+-- layouts/_default/single.html --
+{{ .Title }}|
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptWarn())
+
+	b.AssertFileContent("public/p1/index.html", "p1|")
+
+	b.AssertLogContains("Duplicate resource path")
+
+	files = strings.ReplaceAll(files, `"path" "p1/data1.yaml"`, `"path" "p1/data1new.yaml"`)
+
+	b = hugolib.Test(t, files, hugolib.TestOptWarn())
+
+	b.AssertLogNotContains("WARN")
+}
+
+func TestPagesFromGoTmplShortcodeNoPreceddingCharacterIssue12544(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['home','rss','section','sitemap','taxonomy','term']
+-- content/_content.gotmpl --
+{{ $content := dict "mediaType" "text/html" "value" "x{{< sc >}}" }}
+{{ .AddPage (dict "content" $content "path" "a") }}
+
+{{ $content := dict "mediaType" "text/html" "value" "{{< sc >}}" }}
+{{ .AddPage (dict "content" $content "path" "b") }}
+-- layouts/_default/single.html --
+|{{ .Content }}|
+-- layouts/shortcodes/sc.html --
+foo
+{{- /**/ -}}
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/a/index.html", "|xfoo|")
+	b.AssertFileContent("public/b/index.html", "|foo|") // fails
+}
+
+func TestPagesFromGoTmplMenus(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['rss','section','sitemap','taxonomy','term']
+
+[menus]
+[[menus.main]]
+name = "Main"
+[[menus.footer]]
+name = "Footer"
+-- content/_content.gotmpl --
+{{ .AddPage (dict "path" "p1" "title" "p1" "menus" "main" ) }}
+{{ .AddPage (dict "path" "p2" "title" "p2" "menus" (slice "main" "footer")) }}
+-- layouts/index.html --
+Main: {{ range index site.Menus.main }}{{ .Name }}|{{ end }}|
+Footer: {{ range index site.Menus.footer }}{{ .Name }}|{{ end }}|
+
+`
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/index.html",
+		"Main: Main|p1|p2||",
+		"Footer: Footer|p2||",
+	)
+}
+
+func TestPagesFromGoTmplMore(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['home','rss','section','sitemap','taxonomy','term']
+[markup.goldmark.renderer]
+unsafe = true
+-- content/s1/_content.gotmpl --
+{{ $page := dict
+	"content" (dict "mediaType" "text/markdown" "value" "aaa <!--more--> bbb")
+	"title" "p1"
+	"path" "p1"
+  }}
+  {{ .AddPage $page }}
+-- layouts/_default/single.html --
+summary: {{ .Summary }}|content: {{ .Content}}
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/s1/p1/index.html",
+		"<p>aaa</p>|content: <p>aaa</p>\n<p>bbb</p>",
+	)
+}

hugolib/site.go

@@ -371,14 +371,14 @@ func (s *Site) watching() bool {
 	return s.h != nil && s.h.Configs.Base.Internal.Watch
 }
 
-type whatChanged struct {
+type WhatChanged struct {
 	mu sync.Mutex
 
 	needsPagesAssembly bool
 	identitySet        identity.Identities
 }
 
-func (w *whatChanged) Add(ids ...identity.Identity) {
+func (w *WhatChanged) Add(ids ...identity.Identity) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
@@ -391,24 +391,24 @@ func (w *whatChanged) Add(ids ...identity.Identity) {
 	}
 }
 
-func (w *whatChanged) Clear() {
+func (w *WhatChanged) Clear() {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	w.clear()
 }
 
-func (w *whatChanged) clear() {
+func (w *WhatChanged) clear() {
 	w.identitySet = identity.Identities{}
 }
 
-func (w *whatChanged) Changes() []identity.Identity {
+func (w *WhatChanged) Changes() []identity.Identity {
 	if w == nil || w.identitySet == nil {
 		return nil
 	}
 	return w.identitySet.AsSlice()
 }
 
-func (w *whatChanged) Drain() []identity.Identity {
+func (w *WhatChanged) Drain() []identity.Identity {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	ids := w.identitySet.AsSlice()

hugolib/site_new.go

@@ -141,10 +141,23 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 
 	memCache := dynacache.New(dynacache.Options{Watching: conf.Watching(), Log: logger})
 
+	var h *HugoSites
+	onSignalRebuild := func(ids ...identity.Identity) {
+		// This channel is buffered, but make sure we do this in a non-blocking way.
+		if cfg.ChangesFromBuild != nil {
+			go func() {
+				cfg.ChangesFromBuild <- ids
+			}()
+		}
+	}
+
 	firstSiteDeps := &deps.Deps{
-		Fs:                  cfg.Fs,
-		Log:                 logger,
-		Conf:                conf,
+		Fs:   cfg.Fs,
+		Log:  logger,
+		Conf: conf,
+		BuildState: &deps.BuildState{
+			OnSignalRebuild: onSignalRebuild,
+		},
 		MemCache:            memCache,
 		TemplateProvider:    tplimpl.DefaultTemplateProvider,
 		TranslationProvider: i18n.NewTranslationProvider(),
@@ -261,7 +274,8 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		return li.Lang < lj.Lang
 	})
 
-	h, err := newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
+	var err error
+	h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
 	if err == nil && h == nil {
 		panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")
 	}

hugoreleaser.env

@@ -1,7 +1,7 @@
 # Release env. 
 # These will be replaced by script before release.
-HUGORELEASER_TAG=v0.126.0
-HUGORELEASER_COMMITISH=32c967551be308fbd14e5f0dfba0ff50a60e7f5e
+HUGORELEASER_TAG=v0.126.3
+HUGORELEASER_COMMITISH=44f1edcb0647cdd6c59ba07530cec156f6622a77
 
 
 

identity/identity.go

@@ -241,6 +241,11 @@ type IdentityProvider interface {
 	GetIdentity() Identity
 }
 
+// SignalRebuilder is an optional interface for types that can signal a rebuild.
+type SignalRebuilder interface {
+	SignalRebuild(ids ...Identity)
+}
+
 // IncrementByOne implements Incrementer adding 1 every time Incr is called.
 type IncrementByOne struct {
 	counter uint64

magefile.go

@@ -167,19 +167,19 @@ func testGoFlags() string {
 // Note that we don't run with the extended tag. Currently not supported in 32 bit.
 func Test386() error {
 	env := map[string]string{"GOARCH": "386", "GOFLAGS": testGoFlags()}
-	return runCmd(env, goexe, "test", "./...")
+	return runCmd(env, goexe, "test", "-p", "2", "./...")
 }
 
 // Run tests
 func Test() error {
 	env := map[string]string{"GOFLAGS": testGoFlags()}
-	return runCmd(env, goexe, "test", "./...", "-tags", buildTags())
+	return runCmd(env, goexe, "test", "-p", "2", "./...", "-tags", buildTags())
 }
 
 // Run tests with race detector
 func TestRace() error {
 	env := map[string]string{"GOFLAGS": testGoFlags()}
-	return runCmd(env, goexe, "test", "-race", "./...", "-tags", buildTags())
+	return runCmd(env, goexe, "test", "-p", "2", "-race", "./...", "-tags", buildTags())
 }
 
 // Run gofmt linter

media/config.go

@@ -194,7 +194,7 @@ func DecodeTypes(in map[string]any) (*config.ConfigNamespace[map[string]MediaTyp
 				return nil, nil, err
 			}
 			mm := maps.ToStringMap(v)
-			suffixes, found := maps.LookupEqualFold(mm, "suffixes")
+			suffixes, _, found := maps.LookupEqualFold(mm, "suffixes")
 			if found {
 				mediaType.SuffixesCSV = strings.TrimSpace(strings.ToLower(strings.Join(cast.ToStringSlice(suffixes), ",")))
 			}

metrics/metrics.go

@@ -168,22 +168,21 @@ func (s *Store) WriteMetrics(w io.Writer) {
 	s.mu.Unlock()
 
 	if s.calculateHints {
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "cumulative", "average", "maximum", "cache", "percent", "cached", "total", "")
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "duration", "duration", "duration", "potential", "cached", "count", "count", "template")
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "----------", "--------", "--------", "---------", "-------", "------", "-----", "--------")
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "cumulative", "average", "maximum", "cache", "percent", "cached", "total", "")
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "duration", "duration", "duration", "potential", "cached", "count", "count", "template")
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %9s  %7s  %6s  %5s  %s\n", "----------", "--------", "--------", "---------", "-------", "------", "-----", "--------")
 	} else {
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %5s  %s\n", "cumulative", "average", "maximum", "", "")
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %5s  %s\n", "duration", "duration", "duration", "count", "template")
-		fmt.Fprintf(w, "  %13s  %12s  %12s  %5s  %s\n", "----------", "--------", "--------", "-----", "--------")
-
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %5s  %s\n", "cumulative", "average", "maximum", "", "")
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %5s  %s\n", "duration", "duration", "duration", "count", "template")
+		fmt.Fprintf(w, "  %15s  %12s  %12s  %5s  %s\n", "----------", "--------", "--------", "-----", "--------")
 	}
 
 	sort.Sort(bySum(results))
 	for _, v := range results {
 		if s.calculateHints {
-			fmt.Fprintf(w, "  %13s  %12s  %12s  %9d  %7.f  %6d  %5d  %s\n", v.sum, v.avg, v.max, v.cacheFactor, float64(v.cacheCount)/float64(v.count)*100, v.cacheCount, v.count, v.key)
+			fmt.Fprintf(w, "  %15s  %12s  %12s  %9d  %7.f  %6d  %5d  %s\n", v.sum, v.avg, v.max, v.cacheFactor, float64(v.cacheCount)/float64(v.count)*100, v.cacheCount, v.count, v.key)
 		} else {
-			fmt.Fprintf(w, "  %13s  %12s  %12s  %5d  %s\n", v.sum, v.avg, v.max, v.count, v.key)
+			fmt.Fprintf(w, "  %15s  %12s  %12s  %5d  %s\n", v.sum, v.avg, v.max, v.count, v.key)
 		}
 	}
 }

navigation/pagemenus.go

@@ -41,20 +41,11 @@ type MenuQueryProvider interface {
 	IsMenuCurrent(menuID string, inme *MenuEntry) bool
 }
 
-func PageMenusFromPage(p Page) (PageMenus, error) {
-	params := p.Params()
-
-	ms, ok := params["menus"]
-	if !ok {
-		ms, ok = params["menu"]
-	}
-
-	pm := PageMenus{}
-
-	if !ok {
+func PageMenusFromPage(ms any, p Page) (PageMenus, error) {
+	if ms == nil {
 		return nil, nil
 	}
-
+	pm := PageMenus{}
 	me := MenuEntry{}
 	SetPageValues(&me, p)
 

parser/lowercase_camel_json.go

@@ -46,6 +46,12 @@ type LowerCaseCamelJSONMarshaller struct {
 	Value any
 }
 
+var preserveUpperCaseKeyRe = regexp.MustCompile(`^"HTTP`)
+
+func preserveUpperCaseKey(match []byte) bool {
+	return preserveUpperCaseKeyRe.Match(match)
+}
+
 func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
 	marshalled, err := json.Marshal(c.Value)
 
@@ -59,7 +65,7 @@ func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
 
 			// Empty keys are valid JSON, only lowercase if we do not have an
 			// empty key.
-			if len(match) > 2 {
+			if len(match) > 2 && !preserveUpperCaseKey(match) {
 				// Decode first rune after the double quotes
 				r, width := utf8.DecodeRune(match[1:])
 				r = unicode.ToLower(r)

parser/pageparser/pagelexer.go

@@ -62,15 +62,19 @@ func (l *pageLexer) Input() []byte {
 	return l.input
 }
 
-type Config struct{}
+type Config struct {
+	NoFrontMatter    bool
+	NoSummaryDivider bool
+}
 
 // note: the input position here is normally 0 (start), but
 // can be set if position of first shortcode is known
 func newPageLexer(input []byte, stateStart stateFunc, cfg Config) *pageLexer {
 	lexer := &pageLexer{
-		input:      input,
-		stateStart: stateStart,
-		cfg:        cfg,
+		input:          input,
+		stateStart:     stateStart,
+		summaryDivider: summaryDivider,
+		cfg:            cfg,
 		lexerShortcodeState: lexerShortcodeState{
 			currLeftDelimItem:  tLeftDelimScNoMarkup,
 			currRightDelimItem: tRightDelimScNoMarkup,
@@ -295,6 +299,8 @@ func (s *sectionHandlers) skip() int {
 }
 
 func createSectionHandlers(l *pageLexer) *sectionHandlers {
+	handlers := make([]*sectionHandler, 0, 2)
+
 	shortCodeHandler := &sectionHandler{
 		l: l,
 		skipFunc: func(l *pageLexer) int {
@@ -330,31 +336,36 @@ func createSectionHandlers(l *pageLexer) *sectionHandlers {
 		},
 	}
 
-	summaryDividerHandler := &sectionHandler{
-		l: l,
-		skipFunc: func(l *pageLexer) int {
-			if l.summaryDividerChecked || l.summaryDivider == nil {
-				return -1
-			}
-			return l.index(l.summaryDivider)
-		},
-		lexFunc: func(origin stateFunc, l *pageLexer) (stateFunc, bool) {
-			if !l.hasPrefix(l.summaryDivider) {
-				return origin, false
-			}
+	handlers = append(handlers, shortCodeHandler)
 
-			l.summaryDividerChecked = true
-			l.pos += len(l.summaryDivider)
-			// This makes it a little easier to reason about later.
-			l.consumeSpace()
-			l.emit(TypeLeadSummaryDivider)
+	if !l.cfg.NoSummaryDivider {
+		summaryDividerHandler := &sectionHandler{
+			l: l,
+			skipFunc: func(l *pageLexer) int {
+				if l.summaryDividerChecked {
+					return -1
+				}
+				return l.index(l.summaryDivider)
+			},
+			lexFunc: func(origin stateFunc, l *pageLexer) (stateFunc, bool) {
+				if !l.hasPrefix(l.summaryDivider) {
+					return origin, false
+				}
+
+				l.summaryDividerChecked = true
+				l.pos += len(l.summaryDivider)
+				// This makes it a little easier to reason about later.
+				l.consumeSpace()
+				l.emit(TypeLeadSummaryDivider)
+
+				return origin, true
+			},
+		}
+
+		handlers = append(handlers, summaryDividerHandler)
 
-			return origin, true
-		},
 	}
 
-	handlers := []*sectionHandler{shortCodeHandler, summaryDividerHandler}
-
 	return &sectionHandlers{
 		l:           l,
 		handlers:    handlers,

parser/pageparser/pagelexer_intro.go

@@ -14,8 +14,6 @@
 package pageparser
 
 func lexIntroSection(l *pageLexer) stateFunc {
-	l.summaryDivider = summaryDivider
-
 LOOP:
 	for {
 		r := l.next()

parser/pageparser/pageparser.go

@@ -36,16 +36,11 @@ var _ Result = (*pageLexer)(nil)
 
 // ParseBytes parses the page in b according to the given Config.
 func ParseBytes(b []byte, cfg Config) (Items, error) {
-	l, err := parseBytes(b, cfg, lexIntroSection)
-	if err != nil {
-		return nil, err
+	startLexer := lexIntroSection
+	if cfg.NoFrontMatter {
+		startLexer = lexMainSection
 	}
-	return l.items, l.err
-}
-
-// ParseBytesMain parses b starting with the main section.
-func ParseBytesMain(b []byte, cfg Config) (Items, error) {
-	l, err := parseBytes(b, cfg, lexMainSection)
+	l, err := parseBytes(b, cfg, startLexer)
 	if err != nil {
 		return nil, err
 	}

parser/pageparser/pageparser_test.go

@@ -101,3 +101,14 @@ func BenchmarkHasShortcode(b *testing.B) {
 		}
 	})
 }
+
+func TestSummaryDividerStartingFromMain(t *testing.T) {
+	c := qt.New(t)
+
+	input := `aaa <!--more--> bbb`
+	items, err := collectStringMain(input)
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(items, qt.HasLen, 4)
+	c.Assert(items[1].Type, qt.Equals, TypeLeadSummaryDivider)
+}

releaser/releaser.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -92,6 +92,8 @@ func (r *ReleaseHandler) Run() error {
 	mainVersion := newVersion
 	mainVersion.PatchLevel = 0
 
+	r.gitPull()
+
 	defer r.gitPush()
 
 	if r.step == 1 {
@@ -178,6 +180,12 @@ func (r ReleaseHandler) calculateVersions() (hugo.Version, hugo.Version) {
 	return newVersion, finalVersion
 }
 
+func (r *ReleaseHandler) gitPull() {
+	if _, err := r.git("pull", "origin", "HEAD"); err != nil {
+		log.Fatal("pull failed:", err)
+	}
+}
+
 func (r *ReleaseHandler) gitPush() {
 	if r.skipPush {
 		return

resources/page/pagemeta/page_frontmatter.go

@@ -101,6 +101,7 @@ type PageConfig struct {
 	Cascade []map[string]any
 	Sitemap config.SitemapConfig
 	Build   BuildConfig
+	Menus   []string
 
 	// User defined params.
 	Params maps.Params
@@ -151,13 +152,10 @@ func (p *PageConfig) Compile(basePath string, pagesFromData bool, ext string, lo
 		p.Path = path.Join(basePath, p.Path)
 	}
 
-	if pagesFromData {
-		// Note that NormalizePathStringBasic will make sure that we don't preserve the unnormalized path.
-		// We do that when we create pages from the file system; mostly for backward compatibility,
-		// but also because people tend to use use the filename to name their resources (with spaces and all),
-		// and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
-		p.Path = paths.NormalizePathStringBasic(p.Path)
+	if p.Params == nil {
+		p.Params = make(maps.Params)
 	}
+	maps.PrepareParams(p.Params)
 
 	if p.Content.Markup == "" && p.Content.MediaType == "" {
 		if ext == "" {
@@ -190,6 +188,18 @@ func (p *PageConfig) Compile(basePath string, pagesFromData bool, ext string, lo
 		p.Content.Markup = p.ContentMediaType.SubType
 	}
 
+	if pagesFromData {
+		if p.Kind == "" {
+			p.Kind = kinds.KindPage
+		}
+
+		// Note that NormalizePathStringBasic will make sure that we don't preserve the unnormalized path.
+		// We do that when we create pages from the file system; mostly for backward compatibility,
+		// but also because people tend to use use the filename to name their resources (with spaces and all),
+		// and this isn't relevant when creating resources from an API where it's easy to add textual meta data.
+		p.Path = paths.NormalizePathStringBasic(p.Path)
+	}
+
 	if p.Cascade != nil {
 		cascade, err := page.DecodeCascade(logger, p.Cascade)
 		if err != nil {

resources/page/site.go

@@ -139,8 +139,15 @@ type Site interface {
 // Sites represents an ordered list of sites (languages).
 type Sites []Site
 
-// First is a convenience method to get the first Site, i.e. the main language.
+// Deprecated: Use .Sites.Default instead.
 func (s Sites) First() Site {
+	hugo.Deprecate(".Sites.First", "Use .Sites.Default instead.", "v0.127.0")
+	return s.Default()
+}
+
+// Default is a convenience method to get the site corresponding to the default
+// content language.
+func (s Sites) Default() Site {
 	if len(s) == 0 {
 		return nil
 	}
@@ -165,7 +172,7 @@ func (s *siteWrapper) Key() string {
 	return s.s.Language().Lang
 }
 
-// // Deprecated: Use .Site.Params instead.
+// Deprecated: Use .Site.Params instead.
 func (s *siteWrapper) Social() map[string]string {
 	return s.s.Social()
 }

resources/page/site_integration_test.go

@@ -0,0 +1,44 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package page_test
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+// Issue 12513
+func TestPageSiteSitesDefault(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ['page','rss','section','sitemap','taxonomy','term']
+defaultContentLanguage = 'de'
+defaultContentLanguageInSubdir = true
+[languages.en]
+languageName = 'English'
+weight = 1
+[languages.de]
+languageName = 'Deutsch'
+weight = 2
+-- layouts/index.html --
+{{ .Site.Sites.Default.Language.LanguageName }}
+`
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/de/index.html", "Deutsch")
+}

resources/resource_cache.go

@@ -36,6 +36,11 @@ func newResourceCache(rs *Spec, memCache *dynacache.Cache) *ResourceCache {
 			"/res1",
 			dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
 		),
+		CacheResourceRemote: dynacache.GetOrCreatePartition[string, resource.Resource](
+			memCache,
+			"/resr",
+			dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
+		),
 		cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](
 			memCache,
 			"/ress",
@@ -53,6 +58,7 @@ type ResourceCache struct {
 	sync.RWMutex
 
 	cacheResource               *dynacache.Partition[string, resource.Resource]
+	CacheResourceRemote         *dynacache.Partition[string, resource.Resource]
 	cacheResources              *dynacache.Partition[string, resource.Resources]
 	cacheResourceTransformation *dynacache.Partition[string, *resourceAdapterInner]
 

resources/resource_factories/create/create.go

@@ -23,6 +23,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bep/logg"
+	"github.com/gohugoio/httpcache"
+	hhttpcache "github.com/gohugoio/hugo/cache/httpcache"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs/glob"
 	"github.com/gohugoio/hugo/identity"
@@ -31,7 +34,9 @@ import (
 
 	"github.com/gohugoio/hugo/cache/dynacache"
 	"github.com/gohugoio/hugo/cache/filecache"
+	"github.com/gohugoio/hugo/common/hcontext"
 	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/common/tasks"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
 )
@@ -39,19 +44,76 @@ import (
 // Client contains methods to create Resource objects.
 // tasks to Resource objects.
 type Client struct {
-	rs               *resources.Spec
-	httpClient       *http.Client
-	cacheGetResource *filecache.Cache
+	rs                   *resources.Spec
+	httpClient           *http.Client
+	httpCacheConfig      hhttpcache.ConfigCompiled
+	cacheGetResource     *filecache.Cache
+	resourceIDDispatcher hcontext.ContextDispatcher[string]
+
+	// Set when watching.
+	remoteResourceChecker *tasks.RunEvery
+	remoteResourceLogger  logg.LevelLogger
 }
 
+type contextKey string
+
 // New creates a new Client with the given specification.
 func New(rs *resources.Spec) *Client {
+	fileCache := rs.FileCaches.GetResourceCache()
+	resourceIDDispatcher := hcontext.NewContextDispatcher[string](contextKey("resourceID"))
+	httpCacheConfig := rs.Cfg.GetConfigSection("httpCacheCompiled").(hhttpcache.ConfigCompiled)
+	var remoteResourceChecker *tasks.RunEvery
+	if rs.Cfg.Watching() && !httpCacheConfig.IsPollingDisabled() {
+		remoteResourceChecker = &tasks.RunEvery{
+			HandleError: func(name string, err error) {
+				rs.Logger.Warnf("Failed to check remote resource: %s", err)
+			},
+			RunImmediately: false,
+		}
+
+		if err := remoteResourceChecker.Start(); err != nil {
+			panic(err)
+		}
+
+		rs.BuildClosers.Add(remoteResourceChecker)
+	}
+
+	httpTimeout := 2 * time.Minute // Need to cover retries.
+	if httpTimeout < (rs.Cfg.Timeout() + 30*time.Second) {
+		httpTimeout = rs.Cfg.Timeout() + 30*time.Second
+	}
+
 	return &Client{
-		rs: rs,
+		rs:                    rs,
+		httpCacheConfig:       httpCacheConfig,
+		resourceIDDispatcher:  resourceIDDispatcher,
+		remoteResourceChecker: remoteResourceChecker,
+		remoteResourceLogger:  rs.Logger.InfoCommand("remote"),
 		httpClient: &http.Client{
-			Timeout: time.Minute,
+			Timeout: httpTimeout,
+			Transport: &httpcache.Transport{
+				Cache: fileCache.AsHTTPCache(),
+				CacheKey: func(req *http.Request) string {
+					return resourceIDDispatcher.Get(req.Context())
+				},
+				Around: func(req *http.Request, key string) func() {
+					return fileCache.NamedLock(key)
+				},
+				AlwaysUseCachedResponse: func(req *http.Request, key string) bool {
+					return !httpCacheConfig.For(req.URL.String())
+				},
+				ShouldCache: func(req *http.Request, resp *http.Response, key string) bool {
+					return shouldCache(resp.StatusCode)
+				},
+				MarkCachedResponses: true,
+				EnableETagPair:      true,
+				Transport: &transport{
+					Cfg:    rs.Cfg,
+					Logger: rs.Logger,
+				},
+			},
 		},
-		cacheGetResource: rs.FileCaches.GetResourceCache(),
+		cacheGetResource: fileCache,
 	}
 }
 

resources/resource_factories/create/create_integration_test.go

@@ -134,8 +134,7 @@ mediaTypes = ['text/plain']
 		// This is hard to get stable on GitHub Actions, it sometimes succeeds due to timing issues.
 		if err != nil {
 			b.AssertLogContains("Got Err")
-			b.AssertLogContains("Retry timeout")
-			b.AssertLogContains("ContentLength:0")
+			b.AssertLogContains("retry timeout")
 		}
 	})
 }

resources/resource_factories/create/remote.go

@@ -14,22 +14,27 @@
 package create
 
 import (
-	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"math/rand"
 	"mime"
 	"net/http"
-	"net/http/httputil"
 	"net/url"
 	"path"
 	"strings"
 	"time"
 
+	gmaps "maps"
+
+	"github.com/gohugoio/httpcache"
 	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/common/tasks"
 	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources"
@@ -92,6 +97,60 @@ var temporaryHTTPStatusCodes = map[int]bool{
 	504: true,
 }
 
+func (c *Client) configurePollingIfEnabled(uri, optionsKey string, getRes func() (*http.Response, error)) {
+	if c.remoteResourceChecker == nil {
+		return
+	}
+
+	// Set up polling for changes to this resource.
+	pollingConfig := c.httpCacheConfig.PollConfigFor(uri)
+	if pollingConfig.IsZero() || pollingConfig.Config.Disable {
+		return
+	}
+
+	if c.remoteResourceChecker.Has(optionsKey) {
+		return
+	}
+
+	var lastChange time.Time
+	c.remoteResourceChecker.Add(optionsKey,
+		tasks.Func{
+			IntervalLow:  pollingConfig.Config.Low,
+			IntervalHigh: pollingConfig.Config.High,
+			F: func(interval time.Duration) (time.Duration, error) {
+				start := time.Now()
+				defer func() {
+					duration := time.Since(start)
+					c.rs.Logger.Debugf("Polled remote resource for changes in %13s. Interval: %4s (low: %4s high: %4s) resource: %q ", duration, interval, pollingConfig.Config.Low, pollingConfig.Config.High, uri)
+				}()
+				// TODO(bep) figure out a ways to remove unused tasks.
+				res, err := getRes()
+				if err != nil {
+					return pollingConfig.Config.High, err
+				}
+				// The caching is delayed until the body is read.
+				io.Copy(io.Discard, res.Body)
+				res.Body.Close()
+				x1, x2 := res.Header.Get(httpcache.XETag1), res.Header.Get(httpcache.XETag2)
+				if x1 != x2 {
+					lastChange = time.Now()
+					c.remoteResourceLogger.Logf("detected change in remote resource %q", uri)
+					c.rs.Rebuilder.SignalRebuild(identity.StringIdentity(optionsKey))
+				}
+
+				if time.Since(lastChange) < 10*time.Second {
+					// The user is typing, check more often.
+					return 0, nil
+				}
+
+				// Increase the interval to avoid hammering the server.
+				interval += 1 * time.Second
+
+				return interval, nil
+			},
+		})
+}
+
 // FromRemote expects one or n-parts of a URL to a resource
 // If you provide multiple parts they will be joined together to the final URL.
 func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) {
@@ -101,168 +160,139 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
 	}
 
 	method := "GET"
-	if s, ok := maps.LookupEqualFold(optionsm, "method"); ok {
+	if s, _, ok := maps.LookupEqualFold(optionsm, "method"); ok {
 		method = strings.ToUpper(s.(string))
 	}
 	isHeadMethod := method == "HEAD"
 
-	resourceID := calculateResourceID(uri, optionsm)
+	optionsm = gmaps.Clone(optionsm)
+	userKey, optionsKey := remoteResourceKeys(uri, optionsm)
 
-	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
+	// A common pattern is to use the key in the options map as
+	// a way to control cache eviction,
+	// so make sure we use any user provided key as the file cache key,
+	// but the auto generated and more stable key for everything else.
+	filecacheKey := userKey
+
+	return c.rs.ResourceCache.CacheResourceRemote.GetOrCreate(optionsKey, func(key string) (resource.Resource, error) {
 		options, err := decodeRemoteOptions(optionsm)
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err)
 		}
+
 		if err := c.validateFromRemoteArgs(uri, options); err != nil {
 			return nil, err
 		}
 
-		var (
-			start          time.Time
-			nextSleep      = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
-			nextSleepLimit = time.Duration(5) * time.Second
-		)
+		getRes := func() (*http.Response, error) {
+			ctx := context.Background()
+			ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey)
 
-		for {
-			b, retry, err := func() ([]byte, bool, error) {
-				req, err := options.NewRequest(uri)
-				if err != nil {
-					return nil, false, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
-				}
-
-				res, err := c.httpClient.Do(req)
-				if err != nil {
-					return nil, false, err
-				}
-				defer res.Body.Close()
-
-				if res.StatusCode != http.StatusNotFound {
-					if res.StatusCode < 200 || res.StatusCode > 299 {
-						return nil, temporaryHTTPStatusCodes[res.StatusCode], toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
-					}
-				}
-
-				b, err := httputil.DumpResponse(res, true)
-				if err != nil {
-					return nil, false, toHTTPError(err, res, !isHeadMethod)
-				}
-
-				return b, false, nil
-			}()
+			req, err := options.NewRequest(uri)
 			if err != nil {
-				if retry {
-					if start.IsZero() {
-						start = time.Now()
-					} else if d := time.Since(start) + nextSleep; d >= c.rs.Cfg.Timeout() {
-						c.rs.Logger.Errorf("Retry timeout (configured to %s) fetching remote resource.", c.rs.Cfg.Timeout())
-						return nil, err
-					}
-					time.Sleep(nextSleep)
-					if nextSleep < nextSleepLimit {
-						nextSleep *= 2
-					}
-					continue
-				}
-				return nil, err
+				return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
 			}
 
-			return hugio.ToReadCloser(bytes.NewReader(b)), nil
+			req = req.WithContext(ctx)
 
+			return c.httpClient.Do(req)
 		}
-	})
-	if err != nil {
-		return nil, err
-	}
-	defer httpResponse.Close()
 
-	res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode == http.StatusNotFound {
-		// Not found. This matches how looksup for local resources work.
-		return nil, nil
-	}
-
-	var (
-		body      []byte
-		mediaType media.Type
-	)
-	// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
-	// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
-	if !isHeadMethod && res.Body != nil {
-		body, err = io.ReadAll(res.Body)
+		res, err := getRes()
 		if err != nil {
-			return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
+			return nil, err
 		}
-	}
+		defer res.Body.Close()
 
-	filename := path.Base(rURL.Path)
-	if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
-		if _, ok := params["filename"]; ok {
-			filename = params["filename"]
+		c.configurePollingIfEnabled(uri, optionsKey, getRes)
+
+		if res.StatusCode == http.StatusNotFound {
+			// Not found. This matches how lookups for local resources work.
+			return nil, nil
 		}
-	}
 
-	contentType := res.Header.Get("Content-Type")
-
-	// For HEAD requests we have no body to work with, so we need to use the Content-Type header.
-	if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
-		var found bool
-		mediaType, found = c.rs.MediaTypes().GetByType(contentType)
-		if !found {
-			// A media type not configured in Hugo, just create one from the content type string.
-			mediaType, _ = media.FromString(contentType)
+		if res.StatusCode < 200 || res.StatusCode > 299 {
+			return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
 		}
-	}
 
-	if mediaType.IsZero() {
-
-		var extensionHints []string
-
-		// mime.ExtensionsByType gives a long list of extensions for text/plain,
-		// just use ".txt".
-		if strings.HasPrefix(contentType, "text/plain") {
-			extensionHints = []string{".txt"}
-		} else {
-			exts, _ := mime.ExtensionsByType(contentType)
-			if exts != nil {
-				extensionHints = exts
+		var (
+			body      []byte
+			mediaType media.Type
+		)
+		// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
+		// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
+		if !isHeadMethod && res.Body != nil {
+			body, err = io.ReadAll(res.Body)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
 			}
 		}
 
-		// Look for a file extension. If it's .txt, look for a more specific.
-		if extensionHints == nil || extensionHints[0] == ".txt" {
-			if ext := path.Ext(filename); ext != "" {
-				extensionHints = []string{ext}
+		filename := path.Base(rURL.Path)
+		if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+			if _, ok := params["filename"]; ok {
+				filename = params["filename"]
 			}
 		}
 
-		// Now resolve the media type primarily using the content.
-		mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
+		contentType := res.Header.Get("Content-Type")
 
-	}
+		// For HEAD requests we have no body to work with, so we need to use the Content-Type header.
+		if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
+			var found bool
+			mediaType, found = c.rs.MediaTypes().GetByType(contentType)
+			if !found {
+				// A media type not configured in Hugo, just create one from the content type string.
+				mediaType, _ = media.FromString(contentType)
+			}
+		}
 
-	if mediaType.IsZero() {
-		return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
-	}
+		if mediaType.IsZero() {
 
-	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
-	data := responseToData(res, false)
+			var extensionHints []string
 
-	return c.rs.NewResource(
-		resources.ResourceSourceDescriptor{
-			MediaType:     mediaType,
-			Data:          data,
-			GroupIdentity: identity.StringIdentity(resourceID),
-			LazyPublish:   true,
-			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
-				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
-			},
-			TargetPath: resourceID,
-		})
+			// mime.ExtensionsByType gives a long list of extensions for text/plain,
+			// just use ".txt".
+			if strings.HasPrefix(contentType, "text/plain") {
+				extensionHints = []string{".txt"}
+			} else {
+				exts, _ := mime.ExtensionsByType(contentType)
+				if exts != nil {
+					extensionHints = exts
+				}
+			}
+
+			// Look for a file extension. If it's .txt, look for a more specific.
+			if extensionHints == nil || extensionHints[0] == ".txt" {
+				if ext := path.Ext(filename); ext != "" {
+					extensionHints = []string{ext}
+				}
+			}
+
+			// Now resolve the media type primarily using the content.
+			mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
+
+		}
+
+		if mediaType.IsZero() {
+			return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
+		}
+
+		userKey = filename[:len(filename)-len(path.Ext(filename))] + "_" + userKey + mediaType.FirstSuffix.FullSuffix
+		data := responseToData(res, false)
+
+		return c.rs.NewResource(
+			resources.ResourceSourceDescriptor{
+				MediaType:     mediaType,
+				Data:          data,
+				GroupIdentity: identity.StringIdentity(optionsKey),
+				LazyPublish:   true,
+				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+					return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+				},
+				TargetPath: userKey,
+			})
+	})
 }
 
 func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
@@ -277,11 +307,17 @@ func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) e
 	return nil
 }
 
-func calculateResourceID(uri string, optionsm map[string]any) string {
-	if key, found := maps.LookupEqualFold(optionsm, "key"); found {
-		return identity.HashString(key)
+func remoteResourceKeys(uri string, optionsm map[string]any) (string, string) {
+	var userKey string
+	if key, k, found := maps.LookupEqualFold(optionsm, "key"); found {
+		userKey = identity.HashString(key)
+		delete(optionsm, k)
 	}
-	return identity.HashString(uri, optionsm)
+	optionsKey := identity.HashString(uri, optionsm)
+	if userKey == "" {
+		userKey = optionsKey
+	}
+	return userKey, optionsKey
 }
 
 func addDefaultHeaders(req *http.Request) {
@@ -350,3 +386,71 @@ func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) {
 
 	return options, nil
 }
+
+var _ http.RoundTripper = (*transport)(nil)
+
+type transport struct {
+	Cfg    config.AllProvider
+	Logger loggers.Logger
+}
+
+func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	defer func() {
+		if resp != nil && resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusNotModified {
+			t.Logger.Debugf("Fetched remote resource: %s", req.URL.String())
+		}
+	}()
+
+	var (
+		start          time.Time
+		nextSleep      = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
+		nextSleepLimit = time.Duration(5) * time.Second
+		retry          bool
+	)
+
+	for {
+		resp, retry, err = func() (*http.Response, bool, error) {
+			resp2, err := http.DefaultTransport.RoundTrip(req)
+			if err != nil {
+				return resp2, false, err
+			}
+
+			if resp2.StatusCode != http.StatusNotFound && resp2.StatusCode != http.StatusNotModified {
+				if resp2.StatusCode < 200 || resp2.StatusCode > 299 {
+					return resp2, temporaryHTTPStatusCodes[resp2.StatusCode], nil
+				}
+			}
+			return resp2, false, nil
+		}()
+
+		if retry {
+			if start.IsZero() {
+				start = time.Now()
+			} else if d := time.Since(start) + nextSleep; d >= t.Cfg.Timeout() {
+				msg := "<nil>"
+				if resp != nil {
+					msg = resp.Status
+				}
+				err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD")
+				return resp, err
+			}
+			time.Sleep(nextSleep)
+			if nextSleep < nextSleepLimit {
+				nextSleep *= 2
+			}
+			continue
+		}
+
+		return
+	}
+}
+
+// We need to send the redirect responses back to the HTTP client from RoundTrip,
+// but we don't want to cache them.
+func shouldCache(statusCode int) bool {
+	switch statusCode {
+	case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+		return false
+	}
+	return true
+}

resources/resource_factories/create/remote_test.go

@@ -115,15 +115,21 @@ func TestOptionsNewRequest(t *testing.T) {
 	c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"foo"})
 }
 
-func TestCalculateResourceID(t *testing.T) {
+func TestRemoteResourceKeys(t *testing.T) {
 	t.Parallel()
 
 	c := qt.New(t)
 
-	c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675")
-	c.Assert(calculateResourceID("foo", map[string]any{"bar": "baz"}), qt.Equals, "7294498335241413323")
+	check := func(uri string, optionsm map[string]any, expect1, expect2 string) {
+		got1, got2 := remoteResourceKeys(uri, optionsm)
+		c.Assert(got1, qt.Equals, expect1)
+		c.Assert(got2, qt.Equals, expect2)
+	}
 
-	c.Assert(calculateResourceID("foo", map[string]any{"key": "1234", "bar": "baz"}), qt.Equals, "14904296279238663669")
-	c.Assert(calculateResourceID("asdf", map[string]any{"key": "1234", "bar": "asdf"}), qt.Equals, "14904296279238663669")
-	c.Assert(calculateResourceID("asdf", map[string]any{"key": "12345", "bar": "asdf"}), qt.Equals, "12191037851845371770")
+	check("foo", nil, "5917621528921068675", "5917621528921068675")
+	check("foo", map[string]any{"bar": "baz"}, "7294498335241413323", "7294498335241413323")
+	check("foo", map[string]any{"key": "1234", "bar": "baz"}, "14904296279238663669", "7294498335241413323")
+	check("foo", map[string]any{"key": "12345", "bar": "baz"}, "12191037851845371770", "7294498335241413323")
+	check("asdf", map[string]any{"key": "1234", "bar": "asdf"}, "14904296279238663669", "3787889110563790121")
+	check("asdf", map[string]any{"key": "12345", "bar": "asdf"}, "12191037851845371770", "3787889110563790121")
 }

resources/resource_spec.go

@@ -29,6 +29,7 @@ import (
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/paths"
+	"github.com/gohugoio/hugo/common/types"
 
 	"github.com/gohugoio/hugo/identity"
 
@@ -53,6 +54,8 @@ func NewSpec(
 	logger loggers.Logger,
 	errorHandler herrors.ErrorSender,
 	execHelper *hexec.Exec,
+	buildClosers types.CloseAdder,
+	rebuilder identity.SignalRebuilder,
 ) (*Spec, error) {
 	conf := s.Cfg.GetConfig().(*allconfig.Config)
 	imgConfig := conf.Imaging
@@ -87,10 +90,12 @@ func NewSpec(
 	}
 
 	rs := &Spec{
-		PathSpec:    s,
-		Logger:      logger,
-		ErrorSender: errorHandler,
-		imaging:     imaging,
+		PathSpec:     s,
+		Logger:       logger,
+		ErrorSender:  errorHandler,
+		BuildClosers: buildClosers,
+		Rebuilder:    rebuilder,
+		imaging:      imaging,
 		ImageCache: newImageCache(
 			fileCaches.ImageCache(),
 			memCache,
@@ -111,8 +116,10 @@ func NewSpec(
 type Spec struct {
 	*helpers.PathSpec
 
-	Logger      loggers.Logger
-	ErrorSender herrors.ErrorSender
+	Logger       loggers.Logger
+	ErrorSender  herrors.ErrorSender
+	BuildClosers types.CloseAdder
+	Rebuilder    identity.SignalRebuilder
 
 	TextTemplates tpl.TemplateParseFinder
 

testscripts/commands/gen.txt

@@ -1,6 +1,6 @@
 # Test the gen commands.
 # Note that adding new commands will require updating the NUM_COMMANDS value.
-env NUM_COMMANDS=42
+env NUM_COMMANDS=43
 
 hugo gen -h
 stdout 'A collection of several useful generators\.'

tpl/resources/resources.go

@@ -369,7 +369,7 @@ func (ns *Namespace) ToCSS(args ...any) (resource.Resource, error) {
 	}
 
 	if m != nil {
-		if t, found := maps.LookupEqualFold(m, "transpiler"); found {
+		if t, _, found := maps.LookupEqualFold(m, "transpiler"); found {
 			switch t {
 			case transpilerDart, transpilerLibSass:
 				transpiler = cast.ToString(t)
@@ -440,7 +440,6 @@ func (ns *Namespace) Babel(args ...any) (resource.Resource, error) {
 	var options babel.Options
 	if m != nil {
 		options, err = babel.DecodeOptions(m)
-
 		if err != nil {
 			return nil, err
 		}

tpl/tplimpl/embedded/templates/_default/_markup/render-image.html

@@ -1,7 +1,8 @@
 {{- $u := urls.Parse .Destination -}}
 {{- $src := $u.String -}}
 {{- if not $u.IsAbs -}}
-  {{- with or (.PageInner.Resources.Get $u.Path) (resources.Get $u.Path) -}}
+  {{- $path := strings.TrimPrefix "./" $u.Path }}
+  {{- with or (.PageInner.Resources.Get $path) (resources.Get $path) -}}
     {{- $src = .RelPermalink -}}
     {{- with $u.RawQuery -}}
       {{- $src = printf "%s?%s" $src . -}}

tpl/tplimpl/embedded/templates/_default/_markup/render-link.html

@@ -3,10 +3,11 @@
 {{- if strings.HasPrefix $u.String "#" }}
   {{- $href = printf "%s#%s" .PageInner.RelPermalink $u.Fragment }}
 {{- else if not $u.IsAbs -}}
+  {{- $path := strings.TrimPrefix "./" $u.Path }}
   {{- with or
-    ($.PageInner.GetPage $u.Path)
-    ($.PageInner.Resources.Get $u.Path)
-    (resources.Get $u.Path)
+    ($.PageInner.GetPage $path)
+    ($.PageInner.Resources.Get $path)
+    (resources.Get $path)
   -}}
     {{- $href = .RelPermalink -}}
     {{- with $u.RawQuery -}}

tpl/tplimpl/render_hook_integration_test.go

@@ -51,7 +51,8 @@ title: s1/p1
 title: s1/p2
 ---
 [500](a.txt) // global resource
-[600](b.txt) // page resource
+[510](b.txt) // page resource
+[520](./b.txt) // page resource
 -- content/s1/p2/b.txt --
 irrelevant
 -- content/s1/p3.md --
@@ -125,12 +126,14 @@ title: s1/p3
 
 	b.AssertFileContent("public/s1/p2/index.html",
 		`<a href="/a.txt">500</a>`,
-		`<a href="/s1/p2/b.txt">600</a>`,
+		`<a href="/s1/p2/b.txt">510</a>`,
+		`<a href="/s1/p2/b.txt">520</a>`,
 	)
 }
 
 // Issue 12203
 // Issue 12468
+// Issue 12514
 func TestEmbeddedImageRenderHook(t *testing.T) {
 	t.Parallel()
 
@@ -145,7 +148,9 @@ block = false
 [markup.goldmark.renderHooks.image]
 enableDefault = true
 -- content/p1/index.md --
-![alt](pixel.png?a=b&c=d#fragment)
+![alt1](./pixel.png)
+
+![alt2](pixel.png?a=b&c=d#fragment)
 {.foo #bar}
 -- content/p1/pixel.png --
 iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==
@@ -154,10 +159,16 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
 `
 
 	b := hugolib.Test(t, files)
-	b.AssertFileContent("public/p1/index.html", `<img alt="alt" src="/dir/p1/pixel.png?a=b&c=d#fragment">`)
+	b.AssertFileContent("public/p1/index.html",
+		`<img alt="alt1" src="/dir/p1/pixel.png">`,
+		`<img alt="alt2" src="/dir/p1/pixel.png?a=b&c=d#fragment">`,
+	)
 
 	files = strings.Replace(files, "block = false", "block = true", -1)
 
 	b = hugolib.Test(t, files)
-	b.AssertFileContent("public/p1/index.html", `<img alt="alt" class="foo" id="bar" src="/dir/p1/pixel.png?a=b&c=d#fragment">`)
+	b.AssertFileContent("public/p1/index.html",
+		`<img alt="alt1" src="/dir/p1/pixel.png">`,
+		`<img alt="alt2" class="foo" id="bar" src="/dir/p1/pixel.png?a=b&c=d#fragment">`,
+	)
 }