Compare commits

..

1 Commits

Author SHA1 Message Date
Bjørn Erik Pedersen 8d7a77737c Avoid escaping HTML chars inside hugo_stats.json
Fixes #11371
2023-08-21 10:48:28 +02:00
1728 changed files with 48499 additions and 52662 deletions
+2 -2
View File
@@ -4,7 +4,7 @@ parameters:
defaults: &defaults
resource_class: large
docker:
- image: bepsays/ci-hugoreleaser:1.22100.20600
- image: bepsays/ci-hugoreleaser:1.22000.20100
environment: &buildenv
GOMODCACHE: /root/project/gomodcache
version: 2
@@ -60,7 +60,7 @@ jobs:
environment:
<<: [*buildenv]
docker:
- image: bepsays/ci-hugoreleaser-linux-arm64:1.22100.20600
- image: bepsays/ci-hugoreleaser-linux-arm64:1.22000.20100
steps:
- *restore-cache
- &attach-workspace
+2 -2
View File
@@ -14,7 +14,7 @@ jobs:
test:
strategy:
matrix:
go-version: [1.21.x]
go-version: [1.20.x]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
@@ -30,7 +30,7 @@ jobs:
**/go.sum
**/go.mod
- name: Install Ruby
uses: ruby/setup-ruby@036ef458ddccddb148a2b9fb67e95a22fdbf728b
uses: ruby/setup-ruby@ee2113536afb7f793eed4ce60e8d3b26db912da4
with:
ruby-version: '2.7'
bundler-cache: true #
+8 -14
View File
@@ -16,7 +16,7 @@ jobs:
test:
strategy:
matrix:
go-version: [1.20.x,1.21.x]
go-version: [1.19.x,1.20.x]
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
@@ -32,9 +32,9 @@ jobs:
**/go.sum
**/go.mod
- name: Install Ruby
uses: ruby/setup-ruby@036ef458ddccddb148a2b9fb67e95a22fdbf728b
uses: ruby/setup-ruby@ee2113536afb7f793eed4ce60e8d3b26db912da4
with:
ruby-version: '2.7'
ruby-version: '2.7'
bundler-cache: true #
- name: Install Python
uses: actions/setup-python@3105fb18c05ddd93efea5f9e0bef7a03a6e9e7df
@@ -58,13 +58,13 @@ jobs:
brew install pandoc
- if: matrix.os == 'windows-latest'
run: |
choco install pandoc
Choco-Install -PackageName pandoc
- run: pandoc -v
- if: matrix.os == 'windows-latest'
run: |
choco install mingw
Choco-Install -PackageName mingw -ArgumentList "--version","12.2.0","--allow-downgrade"
- if: matrix.os == 'ubuntu-latest'
name: Install dart-sass Linux
name: Install dart-sass-embedded Linux
run: |
echo "Install Dart Sass version ${SASS_VERSION} ..."
curl -LJO "https://github.com/sass/dart-sass/releases/download/${SASS_VERSION}/dart-sass-${SASS_VERSION}-linux-x64.tar.gz";
@@ -73,7 +73,7 @@ jobs:
echo "$GOBIN"
echo "$GITHUB_WORKSPACE/dart-sass/" >> $GITHUB_PATH
- if: matrix.os == 'macos-latest'
name: Install dart-sass MacOS
name: Install dart-sass-embedded MacOS
run: |
echo "Install Dart Sass version ${SASS_VERSION} ..."
curl -LJO "https://github.com/sass/dart-sass/releases/download/${SASS_VERSION}/dart-sass-${SASS_VERSION}-macos-x64.tar.gz";
@@ -81,18 +81,12 @@ jobs:
tar -xvf "dart-sass-${SASS_VERSION}-macos-x64.tar.gz";
echo "$GITHUB_WORKSPACE/dart-sass/" >> $GITHUB_PATH
- if: matrix.os == 'windows-latest'
name: Install dart-sass Windows
name: Install dart-sass-embedded Windows
run: |
echo "Install Dart Sass version ${env:SASS_VERSION} ..."
curl -LJO "https://github.com/sass/dart-sass/releases/download/${env:SASS_VERSION}/dart-sass-${env:SASS_VERSION}-windows-x64.zip";
Expand-Archive -Path "dart-sass-${env:SASS_VERSION}-windows-x64.zip" -DestinationPath .;
echo "$env:GITHUB_WORKSPACE/dart-sass/" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf-8 -Append
- if: matrix.os == 'ubuntu-latest'
name: Install staticcheck
run: go install honnef.co/go/tools/cmd/staticcheck@latest
- if: matrix.os == 'ubuntu-latest'
name: Run staticcheck
run: staticcheck ./...
- if: matrix.os != 'windows-latest'
name: Check
run: |
+2
View File
@@ -122,6 +122,8 @@ cd hugo
go install
```
>Note: Some Go tools may not be fully updated to support Go Modules yet. One example would be LiteIDE. Follow [this workaround](https://github.com/visualfc/liteide/issues/986#issuecomment-428117702) for how to continue to work with Hugo below `GOPATH`.
For some convenient build and test targets, you also will want to install Mage:
```bash
+2 -2
View File
@@ -2,7 +2,7 @@
# Twitter: https://twitter.com/gohugoio
# Website: https://gohugo.io/
FROM golang:1.21-alpine AS build
FROM golang:1.19-alpine AS build
# Optionally set HUGO_BUILD_TAGS to "extended" or "nodeploy" when building like so:
# docker build --build-arg HUGO_BUILD_TAGS=extended .
@@ -26,7 +26,7 @@ RUN mage hugo && mage install
# ---
FROM alpine:3.18
FROM alpine:3.16
COPY --from=build /go/bin/hugo /usr/bin/hugo
+79 -81
View File
@@ -33,7 +33,7 @@ A fast and flexible static site generator built with love by [bep], [spf13], and
[![Tests on Linux, MacOS and Windows](https://github.com/gohugoio/hugo/workflows/Test/badge.svg)](https://github.com/gohugoio/hugo/actions?query=workflow%3ATest)
[![Go Report Card](https://goreportcard.com/badge/github.com/gohugoio/hugo)](https://goreportcard.com/report/github.com/gohugoio/hugo)
[Website] | [Installation] | [Documentation] | [Support] | [Contributing] | <a rel="me" href="https://fosstodon.org/@gohugoio">Mastodon</a>
[Website] | [Installation] | [Documentation] | [Support] | [Contributing] | [Twitter]
## Overview
@@ -84,8 +84,8 @@ Hugo is available in two editions: standard and extended. With the extended edit
Prerequisites to build Hugo from source:
- Standard edition: Go 1.20 or later
- Extended edition: Go 1.20 or later, and GCC
- Standard edition: Go 1.19 or later
- Extended edition: Go 1.19 or later, and GCC
Build the standard edition:
@@ -142,40 +142,40 @@ Hugo stands on the shoulders of great open source libraries. Run `hugo env --log
```text
cloud.google.com/go/compute/metadata="v0.2.3"
cloud.google.com/go/iam="v1.1.3"
cloud.google.com/go/storage="v1.31.0"
cloud.google.com/go="v0.110.8"
github.com/Azure/azure-sdk-for-go/sdk/azcore="v1.7.0"
cloud.google.com/go/iam="v1.1.0"
cloud.google.com/go/storage="v1.30.1"
cloud.google.com/go="v0.110.2"
github.com/Azure/azure-sdk-for-go/sdk/azcore="v1.6.1"
github.com/Azure/azure-sdk-for-go/sdk/azidentity="v1.3.0"
github.com/Azure/azure-sdk-for-go/sdk/internal="v1.3.0"
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob="v1.1.0"
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob="v1.0.0"
github.com/Azure/go-autorest/autorest/to="v0.4.0"
github.com/AzureAD/microsoft-authentication-library-for-go="v1.0.0"
github.com/BurntSushi/locker="v0.0.0-20171006230638-a6e239ea1c69"
github.com/PuerkitoBio/purell="v1.1.1"
github.com/PuerkitoBio/urlesc="v0.0.0-20170810143723-de5bf2ad4578"
github.com/alecthomas/chroma/v2="v2.11.1"
github.com/alecthomas/chroma/v2="v2.7.0"
github.com/armon/go-radix="v1.0.0"
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream="v1.4.11"
github.com/aws/aws-sdk-go-v2/config="v1.18.32"
github.com/aws/aws-sdk-go-v2/credentials="v1.13.31"
github.com/aws/aws-sdk-go-v2/feature/ec2/imds="v1.13.7"
github.com/aws/aws-sdk-go-v2/feature/s3/manager="v1.11.76"
github.com/aws/aws-sdk-go-v2/internal/configsources="v1.1.37"
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2="v2.4.31"
github.com/aws/aws-sdk-go-v2/internal/ini="v1.3.38"
github.com/aws/aws-sdk-go-v2/internal/v4a="v1.1.0"
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding="v1.9.12"
github.com/aws/aws-sdk-go-v2/service/internal/checksum="v1.1.32"
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url="v1.9.31"
github.com/aws/aws-sdk-go-v2/service/internal/s3shared="v1.15.0"
github.com/aws/aws-sdk-go-v2/service/s3="v1.38.1"
github.com/aws/aws-sdk-go-v2/service/sso="v1.13.1"
github.com/aws/aws-sdk-go-v2/service/ssooidc="v1.15.1"
github.com/aws/aws-sdk-go-v2/service/sts="v1.21.1"
github.com/aws/aws-sdk-go-v2="v1.20.0"
github.com/aws/aws-sdk-go="v1.48.2"
github.com/aws/smithy-go="v1.14.0"
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream="v1.4.10"
github.com/aws/aws-sdk-go-v2/config="v1.18.27"
github.com/aws/aws-sdk-go-v2/credentials="v1.13.26"
github.com/aws/aws-sdk-go-v2/feature/ec2/imds="v1.13.4"
github.com/aws/aws-sdk-go-v2/feature/s3/manager="v1.11.70"
github.com/aws/aws-sdk-go-v2/internal/configsources="v1.1.34"
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2="v2.4.28"
github.com/aws/aws-sdk-go-v2/internal/ini="v1.3.35"
github.com/aws/aws-sdk-go-v2/internal/v4a="v1.0.26"
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding="v1.9.11"
github.com/aws/aws-sdk-go-v2/service/internal/checksum="v1.1.29"
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url="v1.9.28"
github.com/aws/aws-sdk-go-v2/service/internal/s3shared="v1.14.3"
github.com/aws/aws-sdk-go-v2/service/s3="v1.35.0"
github.com/aws/aws-sdk-go-v2/service/sso="v1.12.12"
github.com/aws/aws-sdk-go-v2/service/ssooidc="v1.14.12"
github.com/aws/aws-sdk-go-v2/service/sts="v1.19.2"
github.com/aws/aws-sdk-go-v2="v1.18.1"
github.com/aws/aws-sdk-go="v1.44.284"
github.com/aws/smithy-go="v1.13.5"
github.com/bep/clocks="v0.5.0"
github.com/bep/debounce="v1.2.0"
github.com/bep/gitmap="v1.1.2"
@@ -183,45 +183,45 @@ github.com/bep/goat="v0.5.0"
github.com/bep/godartsass/v2="v2.0.0"
github.com/bep/godartsass="v1.2.0"
github.com/bep/golibsass="v1.1.1"
github.com/bep/gowebp="v0.3.0"
github.com/bep/gowebp="v0.2.0"
github.com/bep/lazycache="v0.2.0"
github.com/bep/logg="v0.3.0"
github.com/bep/logg="v0.2.0"
github.com/bep/mclib="v1.20400.20402"
github.com/bep/overlayfs="v0.6.0"
github.com/bep/simplecobra="v0.3.2"
github.com/bep/tmc="v0.5.1"
github.com/clbanning/mxj/v2="v2.7.0"
github.com/clbanning/mxj/v2="v2.5.7"
github.com/cli/safeexec="v1.0.1"
github.com/cpuguy83/go-md2man/v2="v2.0.2"
github.com/disintegration/gift="v1.2.1"
github.com/dlclark/regexp2="v1.10.0"
github.com/dustin/go-humanize="v1.0.1"
github.com/evanw/esbuild="v0.19.7"
github.com/fatih/color="v1.16.0"
github.com/frankban/quicktest="v1.14.6"
github.com/fsnotify/fsnotify="v1.7.0"
github.com/getkin/kin-openapi="v0.120.0"
github.com/evanw/esbuild="v0.18.5"
github.com/fatih/color="v1.15.0"
github.com/frankban/quicktest="v1.14.5"
github.com/fsnotify/fsnotify="v1.6.0"
github.com/getkin/kin-openapi="v0.118.0"
github.com/ghodss/yaml="v1.0.0"
github.com/go-openapi/jsonpointer="v0.19.6"
github.com/go-openapi/swag="v0.22.4"
github.com/go-openapi/swag="v0.22.3"
github.com/gobuffalo/flect="v1.0.2"
github.com/gobwas/glob="v0.2.3"
github.com/gohugoio/go-i18n/v2="v2.1.3-0.20230805085216-e63c13218d0e"
github.com/gohugoio/go-i18n/v2="v2.1.3-0.20210430103248-4c28c89f8013"
github.com/gohugoio/locales="v0.14.0"
github.com/gohugoio/localescompressed="v1.0.1"
github.com/golang-jwt/jwt/v4="v4.5.0"
github.com/golang/groupcache="v0.0.0-20210331224755-41bb18bfe9da"
github.com/golang/protobuf="v1.5.3"
github.com/google/go-cmp="v0.6.0"
github.com/google/s2a-go="v0.1.7"
github.com/google/uuid="v1.4.0"
github.com/google/go-cmp="v0.5.9"
github.com/google/s2a-go="v0.1.4"
github.com/google/uuid="v1.3.0"
github.com/google/wire="v0.5.0"
github.com/googleapis/enterprise-certificate-proxy="v0.3.2"
github.com/googleapis/gax-go/v2="v2.12.0"
github.com/gorilla/websocket="v1.5.1"
github.com/hairyhenderson/go-codeowners="v0.4.0"
github.com/googleapis/enterprise-certificate-proxy="v0.2.5"
github.com/googleapis/gax-go/v2="v2.11.0"
github.com/gorilla/websocket="v1.5.0"
github.com/hairyhenderson/go-codeowners="v0.3.0"
github.com/hashicorp/golang-lru/v2="v2.0.1"
github.com/invopop/yaml="v0.2.0"
github.com/invopop/yaml="v0.1.0"
github.com/jdkato/prose="v1.2.1"
github.com/jmespath/go-jmespath="v0.4.0"
github.com/josharian/intern="v1.0.0"
@@ -230,9 +230,9 @@ github.com/kr/text="v0.2.0"
github.com/kylelemons/godebug="v1.1.0"
github.com/kyokomi/emoji/v2="v2.2.12"
github.com/mailru/easyjson="v0.7.7"
github.com/marekm4/color-extractor="v1.2.1"
github.com/marekm4/color-extractor="v1.2.0"
github.com/mattn/go-colorable="v0.1.13"
github.com/mattn/go-isatty="v0.0.20"
github.com/mattn/go-isatty="v0.0.19"
github.com/mattn/go-runewidth="v0.0.9"
github.com/mitchellh/hashstructure="v1.1.0"
github.com/mitchellh/mapstructure="v1.5.0"
@@ -240,53 +240,51 @@ github.com/mohae/deepcopy="v0.0.0-20170929034955-c48cc78d4826"
github.com/muesli/smartcrop="v0.3.0"
github.com/niklasfasching/go-org="v1.7.0"
github.com/olekukonko/tablewriter="v0.0.5"
github.com/pelletier/go-toml/v2="v2.1.0"
github.com/perimeterx/marshmallow="v1.1.5"
github.com/pelletier/go-toml/v2="v2.0.8"
github.com/perimeterx/marshmallow="v1.1.4"
github.com/pkg/browser="v0.0.0-20210911075715-681adbf594b8"
github.com/pkg/errors="v0.9.1"
github.com/rogpeppe/go-internal="v1.11.0"
github.com/rogpeppe/go-internal="v1.10.1-0.20230508101108-a4f6fabd84c5"
github.com/russross/blackfriday/v2="v2.1.0"
github.com/rwcarlsen/goexif="v0.0.0-20190401172101-9e8deecbddbd"
github.com/sanity-io/litter="v1.5.5"
github.com/sass/dart-sass/compiler="1.63.2"
github.com/sass/dart-sass/implementation="1.63.2"
github.com/sass/dart-sass/protocol="2.0.0"
github.com/sass/dart-sass/compiler="1.63.6"
github.com/sass/dart-sass/implementation="1.63.6"
github.com/sass/dart-sass/protocol="2.1.0"
github.com/sass/libsass="3.6.5"
github.com/spf13/afero="v1.10.0"
github.com/spf13/afero="v1.9.5"
github.com/spf13/cast="v1.5.1"
github.com/spf13/cobra="v1.7.0"
github.com/spf13/fsync="v0.9.0"
github.com/spf13/pflag="v1.0.5"
github.com/tdewolff/minify/v2="v2.20.7"
github.com/tdewolff/parse/v2="v2.7.5"
github.com/webmproject/libwebp="v1.3.2"
github.com/yuin/goldmark-emoji="v1.0.2"
github.com/yuin/goldmark="v1.6.0"
github.com/tdewolff/minify/v2="v2.12.7"
github.com/tdewolff/parse/v2="v2.6.6"
github.com/webmproject/libwebp="v1.2.4"
github.com/yuin/goldmark="v1.5.4"
go.opencensus.io="v0.24.0"
go.uber.org/atomic="v1.11.0"
go.uber.org/automaxprocs="v1.5.3"
gocloud.dev="v0.34.0"
golang.org/x/crypto="v0.15.0"
golang.org/x/exp="v0.0.0-20221031165847-c99f073a8326"
golang.org/x/image="v0.13.0"
golang.org/x/mod="v0.14.0"
golang.org/x/net="v0.18.0"
golang.org/x/oauth2="v0.13.0"
golang.org/x/sync="v0.5.0"
golang.org/x/sys="v0.14.0"
golang.org/x/text="v0.14.0"
golang.org/x/time="v0.3.0"
golang.org/x/tools="v0.15.0"
go.uber.org/automaxprocs="v1.5.2"
gocloud.dev="v0.30.0"
golang.org/x/crypto="v0.10.0"
golang.org/x/exp="v0.0.0-20230321023759-10a507213a29"
golang.org/x/image="v0.8.0"
golang.org/x/mod="v0.10.0"
golang.org/x/net="v0.11.0"
golang.org/x/oauth2="v0.9.0"
golang.org/x/sync="v0.3.0"
golang.org/x/sys="v0.9.0"
golang.org/x/text="v0.10.0"
golang.org/x/tools="v0.9.3"
golang.org/x/xerrors="v0.0.0-20220907171357-04be3eba64a2"
google.golang.org/api="v0.151.0"
google.golang.org/genproto/googleapis/api="v0.0.0-20231016165738-49dd2c1f3d0b"
google.golang.org/genproto/googleapis/rpc="v0.0.0-20231030173426-d783a09b4405"
google.golang.org/genproto="v0.0.0-20231016165738-49dd2c1f3d0b"
google.golang.org/grpc="v1.59.0"
google.golang.org/protobuf="v1.31.0"
google.golang.org/api="v0.128.0"
google.golang.org/appengine="v1.6.7"
google.golang.org/genproto/googleapis/api="v0.0.0-20230530153820-e85fd2cbaebc"
google.golang.org/genproto/googleapis/rpc="v0.0.0-20230530153820-e85fd2cbaebc"
google.golang.org/genproto="v0.0.0-20230530153820-e85fd2cbaebc"
google.golang.org/grpc="v1.56.0"
google.golang.org/protobuf="v1.30.0"
gopkg.in/yaml.v2="v2.4.0"
gopkg.in/yaml.v3="v3.0.1"
howett.net/plist="v1.0.0"
software.sslmate.com/src/go-pkcs12="v0.2.0"
```
</details>
-550
View File
@@ -1,550 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dynacache
import (
"context"
"fmt"
"math"
"path"
"regexp"
"runtime"
"sync"
"time"
"github.com/bep/lazycache"
"github.com/bep/logg"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/rungroup"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/resource"
)
const minMaxSize = 10
// New creates a new cache.
func New(opts Options) *Cache {
if opts.CheckInterval == 0 {
opts.CheckInterval = time.Second * 2
}
if opts.MaxSize == 0 {
opts.MaxSize = 100000
}
if opts.Log == nil {
panic("nil Log")
}
if opts.MinMaxSize == 0 {
opts.MinMaxSize = 30
}
stats := &stats{
opts: opts,
adjustmentFactor: 1.0,
currentMaxSize: opts.MaxSize,
availableMemory: config.GetMemoryLimit(),
}
infol := opts.Log.InfoCommand("dynacache")
c := &Cache{
partitions: make(map[string]PartitionManager),
opts: opts,
stats: stats,
infol: infol,
}
c.stop = c.start()
return c
}
// Options for the cache.
type Options struct {
Log loggers.Logger
CheckInterval time.Duration
MaxSize int
MinMaxSize int
Running bool
}
// Options for a partition.
type OptionsPartition struct {
// When to clear the this partition.
ClearWhen ClearWhen
// Weight is a number between 1 and 100 that indicates how, in general, how big this partition may get.
Weight int
}
func (o OptionsPartition) WeightFraction() float64 {
return float64(o.Weight) / 100
}
func (o OptionsPartition) CalculateMaxSize(maxSizePerPartition int) int {
return int(math.Floor(float64(maxSizePerPartition) * o.WeightFraction()))
}
// A dynamic partitioned cache.
type Cache struct {
mu sync.RWMutex
partitions map[string]PartitionManager
opts Options
infol logg.LevelLogger
stats *stats
stopOnce sync.Once
stop func()
}
// ClearMatching clears all partition for which the predicate returns true.
func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
NumWorkers: len(c.partitions),
Handle: func(ctx context.Context, partition PartitionManager) error {
partition.clearMatching(predicate)
return nil
},
})
for _, p := range c.partitions {
g.Enqueue(p)
}
g.Wait()
}
// ClearOnRebuild prepares the cache for a new rebuild taking the given changeset into account.
func (c *Cache) ClearOnRebuild(changeset ...identity.Identity) {
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
NumWorkers: len(c.partitions),
Handle: func(ctx context.Context, partition PartitionManager) error {
partition.clearOnRebuild(changeset...)
return nil
},
})
for _, p := range c.partitions {
g.Enqueue(p)
}
g.Wait()
// Clear any entries marked as stale above.
g = rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
NumWorkers: len(c.partitions),
Handle: func(ctx context.Context, partition PartitionManager) error {
partition.clearStale()
return nil
},
})
for _, p := range c.partitions {
g.Enqueue(p)
}
g.Wait()
}
type keysProvider interface {
Keys() []string
}
// Keys returns a list of keys in all partitions.
func (c *Cache) Keys(predicate func(s string) bool) []string {
if predicate == nil {
predicate = func(s string) bool { return true }
}
var keys []string
for pn, g := range c.partitions {
pkeys := g.(keysProvider).Keys()
for _, k := range pkeys {
p := path.Join(pn, k)
if predicate(p) {
keys = append(keys, p)
}
}
}
return keys
}
func calculateMaxSizePerPartition(maxItemsTotal, totalWeightQuantity, numPartitions int) int {
if numPartitions == 0 {
panic("numPartitions must be > 0")
}
if totalWeightQuantity == 0 {
panic("totalWeightQuantity must be > 0")
}
avgWeight := float64(totalWeightQuantity) / float64(numPartitions)
return int(math.Floor(float64(maxItemsTotal) / float64(numPartitions) * (100.0 / avgWeight)))
}
// Stop stops the cache.
func (c *Cache) Stop() {
c.stopOnce.Do(func() {
c.stop()
})
}
func (c *Cache) adjustCurrentMaxSize() {
c.mu.RLock()
defer c.mu.RUnlock()
if len(c.partitions) == 0 {
return
}
var m runtime.MemStats
runtime.ReadMemStats(&m)
s := c.stats
s.memstatsCurrent = m
// fmt.Printf("\n\nAvailable = %v\nAlloc = %v\nTotalAlloc = %v\nSys = %v\nNumGC = %v\nMaxSize = %d\nAdjustmentFactor=%f\n\n", helpers.FormatByteCount(s.availableMemory), helpers.FormatByteCount(m.Alloc), helpers.FormatByteCount(m.TotalAlloc), helpers.FormatByteCount(m.Sys), m.NumGC, c.stats.currentMaxSize, s.adjustmentFactor)
if s.availableMemory >= s.memstatsCurrent.Alloc {
if s.adjustmentFactor <= 1.0 {
s.adjustmentFactor += 0.2
}
} else {
// We're low on memory.
s.adjustmentFactor -= 0.4
}
if s.adjustmentFactor <= 0 {
s.adjustmentFactor = 0.05
}
if !s.adjustCurrentMaxSize() {
return
}
totalWeight := 0
for _, pm := range c.partitions {
totalWeight += pm.getOptions().Weight
}
maxSizePerPartition := calculateMaxSizePerPartition(c.stats.currentMaxSize, totalWeight, len(c.partitions))
evicted := 0
for _, p := range c.partitions {
evicted += p.adjustMaxSize(p.getOptions().CalculateMaxSize(maxSizePerPartition))
}
if evicted > 0 {
c.infol.
WithFields(
logg.Fields{
{Name: "evicted", Value: evicted},
{Name: "numGC", Value: m.NumGC},
{Name: "limit", Value: helpers.FormatByteCount(c.stats.availableMemory)},
{Name: "alloc", Value: helpers.FormatByteCount(m.Alloc)},
{Name: "totalAlloc", Value: helpers.FormatByteCount(m.TotalAlloc)},
},
).Logf("adjusted partitions' max size")
}
}
func (c *Cache) start() func() {
ticker := time.NewTicker(c.opts.CheckInterval)
quit := make(chan struct{})
go func() {
for {
select {
case <-ticker.C:
c.adjustCurrentMaxSize()
case <-quit:
ticker.Stop()
return
}
}
}()
return func() {
close(quit)
}
}
var partitionNameRe = regexp.MustCompile(`^\/[a-zA-Z0-9]{4}(\/[a-zA-Z0-9]+)?(\/[a-zA-Z0-9]+)?`)
// GetOrCreatePartition gets or creates a partition with the given name.
func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts OptionsPartition) *Partition[K, V] {
if c == nil {
panic("nil Cache")
}
if opts.Weight < 1 || opts.Weight > 100 {
panic("invalid Weight, must be between 1 and 100")
}
if partitionNameRe.FindString(name) != name {
panic(fmt.Sprintf("invalid partition name %q", name))
}
c.mu.RLock()
p, found := c.partitions[name]
c.mu.RUnlock()
if found {
return p.(*Partition[K, V])
}
c.mu.Lock()
defer c.mu.Unlock()
// Double check.
p, found = c.partitions[name]
if found {
return p.(*Partition[K, V])
}
// At this point, we don't know the the number of partitions or their configuration, but
// this will be re-adjusted later.
const numberOfPartitionsEstimate = 10
maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate)
// Create a new partition and cache it.
partition := &Partition[K, V]{
c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}),
maxSize: maxSize,
trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name),
opts: opts,
}
c.partitions[name] = partition
return partition
}
// Partition is a partition in the cache.
type Partition[K comparable, V any] struct {
c *lazycache.Cache[K, V]
zero V
trace logg.LevelLogger
opts OptionsPartition
maxSize int
}
// GetOrCreate gets or creates a value for the given key.
func (p *Partition[K, V]) GetOrCreate(key K, create func(key K) (V, error)) (V, error) {
v, _, err := p.c.GetOrCreate(key, create)
return v, err
}
// GetOrCreateWitTimeout gets or creates a value for the given key and times out if the create function
// takes too long.
func (p *Partition[K, V]) GetOrCreateWitTimeout(key K, duration time.Duration, create func(key K) (V, error)) (V, error) {
resultch := make(chan V, 1)
errch := make(chan error, 1)
go func() {
v, _, err := p.c.GetOrCreate(key, create)
if err != nil {
errch <- err
return
}
resultch <- v
}()
select {
case v := <-resultch:
return v, nil
case err := <-errch:
return p.zero, err
case <-time.After(duration):
return p.zero, &herrors.TimeoutError{
Duration: duration,
}
}
}
func (p *Partition[K, V]) clearMatching(predicate func(k, v any) bool) {
p.c.DeleteFunc(func(key K, v V) bool {
if predicate(key, v) {
p.trace.Log(
logg.StringFunc(
func() string {
return fmt.Sprintf("clearing cache key %v", key)
},
),
)
return true
}
return false
})
}
func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
opts := p.getOptions()
if opts.ClearWhen == ClearNever {
return
}
if opts.ClearWhen == ClearOnRebuild {
// Clear all.
p.Clear()
return
}
depsFinder := identity.NewFinder(identity.FinderConfig{})
shouldDelete := func(key K, v V) bool {
// We always clear elements marked as stale.
if resource.IsStaleAny(v) {
return true
}
// Now check if this entry has changed based on the changeset
// based on filesystem events.
if len(changeset) == 0 {
// Nothing changed.
return false
}
var probablyDependent bool
identity.WalkIdentitiesShallow(v, func(level int, id2 identity.Identity) bool {
for _, id := range changeset {
if r := depsFinder.Contains(id, id2, -1); r > 0 {
// It's probably dependent, evict from cache.
probablyDependent = true
return true
}
}
return false
})
return probablyDependent
}
// First pass.
// Second pass needs to be done in a separate loop to catch any
// elements marked as stale in the other partitions.
p.c.DeleteFunc(func(key K, v V) bool {
if shouldDelete(key, v) {
p.trace.Log(
logg.StringFunc(
func() string {
return fmt.Sprintf("first pass: clearing cache key %v", key)
},
),
)
resource.MarkStale(v)
return true
}
return false
})
}
func (p *Partition[K, V]) Keys() []K {
var keys []K
p.c.DeleteFunc(func(key K, v V) bool {
keys = append(keys, key)
return false
})
return keys
}
func (p *Partition[K, V]) clearStale() {
p.c.DeleteFunc(func(key K, v V) bool {
isStale := resource.IsStaleAny(v)
if isStale {
p.trace.Log(
logg.StringFunc(
func() string {
return fmt.Sprintf("second pass: clearing cache key %v", key)
},
),
)
}
return isStale
})
}
// adjustMaxSize adjusts the max size of the and returns the number of items evicted.
func (p *Partition[K, V]) adjustMaxSize(newMaxSize int) int {
if newMaxSize < minMaxSize {
newMaxSize = minMaxSize
}
p.maxSize = newMaxSize
// fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize)
return p.c.Resize(newMaxSize)
}
func (p *Partition[K, V]) getMaxSize() int {
return p.maxSize
}
func (p *Partition[K, V]) getOptions() OptionsPartition {
return p.opts
}
func (p *Partition[K, V]) Clear() {
p.c.DeleteFunc(func(key K, v V) bool {
return true
})
}
func (p *Partition[K, V]) Get(ctx context.Context, key K) (V, bool) {
return p.c.Get(key)
}
type PartitionManager interface {
adjustMaxSize(addend int) int
getMaxSize() int
getOptions() OptionsPartition
clearOnRebuild(changeset ...identity.Identity)
clearMatching(predicate func(k, v any) bool)
clearStale()
}
const (
ClearOnRebuild ClearWhen = iota + 1
ClearOnChange
ClearNever
)
type ClearWhen int
type stats struct {
opts Options
memstatsCurrent runtime.MemStats
currentMaxSize int
availableMemory uint64
adjustmentFactor float64
}
func (s *stats) adjustCurrentMaxSize() bool {
newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor))
if newCurrentMaxSize < s.opts.MaxSize {
newCurrentMaxSize = int(s.opts.MinMaxSize)
}
changed := newCurrentMaxSize != s.currentMaxSize
s.currentMaxSize = newCurrentMaxSize
return changed
}
// CleanKey turns s into a format suitable for a cache key for this package.
// The key will be a Unix-styled path with a leading slash but no trailing slash.
func CleanKey(s string) string {
return path.Clean(paths.ToSlashPreserveLeading(s))
}
-175
View File
@@ -1,175 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dynacache
import (
"path/filepath"
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/resource"
)
var (
_ resource.StaleInfo = (*testItem)(nil)
_ identity.Identity = (*testItem)(nil)
)
type testItem struct {
name string
isStale bool
}
func (t testItem) IsStale() bool {
return t.isStale
}
func (t testItem) IdentifierBase() string {
return t.name
}
func TestCache(t *testing.T) {
t.Parallel()
c := qt.New(t)
cache := New(Options{
Log: loggers.NewDefault(),
})
c.Cleanup(func() {
cache.Stop()
})
opts := OptionsPartition{Weight: 30}
c.Assert(cache, qt.Not(qt.IsNil))
p1 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", opts)
c.Assert(p1, qt.Not(qt.IsNil))
p2 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", opts)
c.Assert(func() { GetOrCreatePartition[string, testItem](cache, "foo bar", opts) }, qt.PanicMatches, ".*invalid partition name.*")
c.Assert(func() { GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", OptionsPartition{Weight: 1234}) }, qt.PanicMatches, ".*invalid Weight.*")
c.Assert(p2, qt.Equals, p1)
p3 := GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", opts)
c.Assert(p3, qt.Not(qt.IsNil))
c.Assert(p3, qt.Not(qt.Equals), p1)
c.Assert(func() { New(Options{}) }, qt.PanicMatches, ".*nil Log.*")
}
func TestCalculateMaxSizePerPartition(t *testing.T) {
t.Parallel()
c := qt.New(t)
c.Assert(calculateMaxSizePerPartition(1000, 500, 5), qt.Equals, 200)
c.Assert(calculateMaxSizePerPartition(1000, 250, 5), qt.Equals, 400)
c.Assert(func() { calculateMaxSizePerPartition(1000, 250, 0) }, qt.PanicMatches, ".*must be > 0.*")
c.Assert(func() { calculateMaxSizePerPartition(1000, 0, 1) }, qt.PanicMatches, ".*must be > 0.*")
}
func TestCleanKey(t *testing.T) {
c := qt.New(t)
c.Assert(CleanKey("a/b/c"), qt.Equals, "/a/b/c")
c.Assert(CleanKey("/a/b/c"), qt.Equals, "/a/b/c")
c.Assert(CleanKey("a/b/c/"), qt.Equals, "/a/b/c")
c.Assert(CleanKey(filepath.FromSlash("/a/b/c/")), qt.Equals, "/a/b/c")
}
func newTestCache(t *testing.T) *Cache {
cache := New(
Options{
Log: loggers.NewDefault(),
},
)
p1 := GetOrCreatePartition[string, testItem](cache, "/aaaa/bbbb", OptionsPartition{Weight: 30, ClearWhen: ClearOnRebuild})
p2 := GetOrCreatePartition[string, testItem](cache, "/aaaa/cccc", OptionsPartition{Weight: 30, ClearWhen: ClearOnChange})
p1.GetOrCreate("clearOnRebuild", func(string) (testItem, error) {
return testItem{}, nil
})
p2.GetOrCreate("clearBecauseStale", func(string) (testItem, error) {
return testItem{
isStale: true,
}, nil
})
p2.GetOrCreate("clearBecauseIdentityChanged", func(string) (testItem, error) {
return testItem{
name: "changed",
}, nil
})
p2.GetOrCreate("clearNever", func(string) (testItem, error) {
return testItem{
isStale: false,
}, nil
})
t.Cleanup(func() {
cache.Stop()
})
return cache
}
func TestClear(t *testing.T) {
t.Parallel()
c := qt.New(t)
predicateAll := func(string) bool {
return true
}
cache := newTestCache(t)
c.Assert(cache.Keys(predicateAll), qt.HasLen, 4)
cache.ClearOnRebuild()
// Stale items are always cleared.
c.Assert(cache.Keys(predicateAll), qt.HasLen, 2)
cache = newTestCache(t)
cache.ClearOnRebuild(identity.StringIdentity("changed"))
c.Assert(cache.Keys(nil), qt.HasLen, 1)
cache = newTestCache(t)
cache.ClearMatching(func(k, v any) bool {
return k.(string) == "clearOnRebuild"
})
c.Assert(cache.Keys(predicateAll), qt.HasLen, 3)
cache.adjustCurrentMaxSize()
}
func TestAdjustCurrentMaxSize(t *testing.T) {
t.Parallel()
c := qt.New(t)
cache := newTestCache(t)
alloc := cache.stats.memstatsCurrent.Alloc
cache.adjustCurrentMaxSize()
c.Assert(cache.stats.memstatsCurrent.Alloc, qt.Not(qt.Equals), alloc)
}
+6 -37
View File
@@ -24,7 +24,6 @@ import (
"time"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/helpers"
@@ -52,9 +51,6 @@ type Cache struct {
pruneAllRootDir string
nlocker *lockTracker
initOnce sync.Once
initErr error
}
type lockTracker struct {
@@ -107,23 +103,9 @@ func (l *lockedFile) Close() error {
return l.File.Close()
}
func (c *Cache) init() error {
c.initOnce.Do(func() {
// Create the base dir if it does not exist.
if err := c.Fs.MkdirAll("", 0o777); err != nil && !os.IsExist(err) {
c.initErr = err
}
})
return c.initErr
}
// WriteCloser returns a transactional writer into the cache.
// It's important that it's closed when done.
func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
id = cleanID(id)
c.nlocker.Lock(id)
@@ -147,12 +129,7 @@ func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
// it when done.
func (c *Cache) ReadOrCreate(id string,
read func(info ItemInfo, r io.ReadSeeker) error,
create func(info ItemInfo, w io.WriteCloser) error,
) (info ItemInfo, err error) {
if err := c.init(); err != nil {
return ItemInfo{}, err
}
create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
id = cleanID(id)
c.nlocker.Lock(id)
@@ -186,9 +163,6 @@ func (c *Cache) ReadOrCreate(id string,
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
id = cleanID(id)
c.nlocker.Lock(id)
@@ -223,9 +197,6 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
id = cleanID(id)
c.nlocker.Lock(id)
@@ -261,9 +232,6 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
// GetBytes gets the file content with the given id from the cache, nil if none found.
func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
id = cleanID(id)
c.nlocker.Lock(id)
@@ -282,9 +250,6 @@ func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
// Get gets the file with the given id from the cache, nil if none found.
func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
if err := c.init(); err != nil {
return ItemInfo{}, nil, err
}
id = cleanID(id)
c.nlocker.Lock(id)
@@ -382,7 +347,11 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
baseDir := v.DirCompiled
bfs := hugofs.NewBasePathFs(cfs, baseDir)
if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
return nil, err
}
bfs := afero.NewBasePathFs(cfs, baseDir)
var pruneAllRootDir string
if k == CacheKeyModules {
-6
View File
@@ -53,9 +53,6 @@ func (c *Cache) Prune(force bool) (int, error) {
if c.pruneAllRootDir != "" {
return c.pruneRootDir(force)
}
if err := c.init(); err != nil {
return 0, err
}
counter := 0
@@ -120,9 +117,6 @@ func (c *Cache) Prune(force bool) (int, error) {
}
func (c *Cache) pruneRootDir(force bool) (int, error) {
if err := c.init(); err != nil {
return 0, err
}
info, err := c.Fs.Stat(c.pruneAllRootDir)
if err != nil {
if herrors.IsNotExist(err) {
+11 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ import (
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
"testing"
@@ -85,8 +86,17 @@ dir = ":cacheDir/c"
cache := caches.Get("GetJSON")
c.Assert(cache, qt.Not(qt.IsNil))
bfs, ok := cache.Fs.(*afero.BasePathFs)
c.Assert(ok, qt.Equals, true)
filename, err := bfs.RealPath("key")
c.Assert(err, qt.IsNil)
cache = caches.Get("Images")
c.Assert(cache, qt.Not(qt.IsNil))
bfs, ok = cache.Fs.(*afero.BasePathFs)
c.Assert(ok, qt.Equals, true)
filename, _ = bfs.RealPath("key")
c.Assert(filename, qt.Equals, filepath.FromSlash("_gen/images/key"))
rf := func(s string) func() (io.ReadCloser, error) {
return func() (io.ReadCloser, error) {
+5 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@ package filecache_test
import (
"path/filepath"
"testing"
"time"
@@ -46,6 +47,7 @@ title: "Home"
_, err := b.H.BaseFs.ResourcesCache.Stat(filepath.Join("_gen", "images"))
b.Assert(err, qt.IsNil)
}
func TestPruneImages(t *testing.T) {
@@ -53,7 +55,6 @@ func TestPruneImages(t *testing.T) {
// TODO(bep)
t.Skip("skip flaky test on CI server")
}
t.Skip("skip flaky test")
files := `
-- hugo.toml --
baseURL = "https://example.com"
@@ -91,7 +92,7 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
// TODO(bep) we need a way to test full rebuilds.
// For now, just sleep a little so the cache elements expires.
time.Sleep(500 * time.Millisecond)
time.Sleep(300 * time.Millisecond)
b.RenameFile("assets/a/pixel.png", "assets/b/pixel2.png").Build()
@@ -103,4 +104,5 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
b.Assert(err, qt.Not(qt.IsNil))
_, err = b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir)
b.Assert(err, qt.IsNil)
}
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package namedmemcache provides a memory cache with a named lock. This is suitable
// for situations where creating the cached resource can be time consuming or otherwise
// resource hungry, or in situations where a "once only per key" is a requirement.
package namedmemcache
import (
"sync"
"github.com/BurntSushi/locker"
)
// Cache holds the cached values.
type Cache struct {
nlocker *locker.Locker
cache map[string]cacheEntry
mu sync.RWMutex
}
type cacheEntry struct {
value any
err error
}
// New creates a new cache.
func New() *Cache {
return &Cache{
nlocker: locker.NewLocker(),
cache: make(map[string]cacheEntry),
}
}
// Clear clears the cache state.
func (c *Cache) Clear() {
c.mu.Lock()
defer c.mu.Unlock()
c.cache = make(map[string]cacheEntry)
c.nlocker = locker.NewLocker()
}
// GetOrCreate tries to get the value with the given cache key, if not found
// create will be called and cached.
// This method is thread safe. It also guarantees that the create func for a given
// key is invoked only once for this cache.
func (c *Cache) GetOrCreate(key string, create func() (any, error)) (any, error) {
c.mu.RLock()
entry, found := c.cache[key]
c.mu.RUnlock()
if found {
return entry.value, entry.err
}
c.nlocker.Lock(key)
defer c.nlocker.Unlock(key)
// Create it.
value, err := create()
c.mu.Lock()
c.cache[key] = cacheEntry{value: value, err: err}
c.mu.Unlock()
return value, err
}
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package namedmemcache
import (
"fmt"
"sync"
"testing"
qt "github.com/frankban/quicktest"
)
func TestNamedCache(t *testing.T) {
t.Parallel()
c := qt.New(t)
cache := New()
counter := 0
create := func() (any, error) {
counter++
return counter, nil
}
for i := 0; i < 5; i++ {
v1, err := cache.GetOrCreate("a1", create)
c.Assert(err, qt.IsNil)
c.Assert(v1, qt.Equals, 1)
v2, err := cache.GetOrCreate("a2", create)
c.Assert(err, qt.IsNil)
c.Assert(v2, qt.Equals, 2)
}
cache.Clear()
v3, err := cache.GetOrCreate("a2", create)
c.Assert(err, qt.IsNil)
c.Assert(v3, qt.Equals, 3)
}
func TestNamedCacheConcurrent(t *testing.T) {
t.Parallel()
c := qt.New(t)
var wg sync.WaitGroup
cache := New()
create := func(i int) func() (any, error) {
return func() (any, error) {
return i, nil
}
}
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 100; j++ {
id := fmt.Sprintf("id%d", j)
v, err := cache.GetOrCreate(id, create(j))
c.Assert(err, qt.IsNil)
c.Assert(v, qt.Equals, j)
}
}()
}
wg.Wait()
}
+22 -21
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -38,7 +38,6 @@ import (
"github.com/gohugoio/hugo/common/hstrings"
"github.com/gohugoio/hugo/common/htime"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/config"
@@ -51,7 +50,9 @@ import (
"github.com/spf13/cobra"
)
var errHelp = errors.New("help requested")
var (
errHelp = errors.New("help requested")
)
// Execute executes a command.
func Execute(args []string) error {
@@ -181,9 +182,11 @@ func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commo
cfg: oldConf.cfg,
fs: fs,
}, nil
})
return cc, err
}
func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commonConfig, error) {
@@ -208,7 +211,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
if !cfg.IsSet("workingDir") {
cfg.Set("workingDir", dir)
} else {
if err := os.MkdirAll(cfg.GetString("workingDir"), 0o777); err != nil {
if err := os.MkdirAll(cfg.GetString("workingDir"), 0777); err != nil {
return nil, fmt.Errorf("failed to create workingDir: %w", err)
}
}
@@ -259,7 +262,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
publishDirStatic := cfg.GetString("publishDirStatic")
workingDir := cfg.GetString("workingDir")
absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
staticFs := hugofs.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
// Serve from both the static and dynamic fs,
// the first will take priority.
@@ -300,6 +303,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
})
return cc, err
}
func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
@@ -344,6 +348,7 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args
err := b.build()
return err
}()
if err != nil {
return err
}
@@ -405,14 +410,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
return err
}
r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5})
// We don't want to keep stale HugoSites in memory longer than needed.
r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{
MaxEntries: 1,
OnEvict: func(key int32, value *hugolib.HugoSites) {
value.Close()
},
})
r.commonConfigs = lazycache.New[int32, *commonConfig](lazycache.Options{MaxEntries: 5})
r.hugoSites = lazycache.New[int32, *hugolib.HugoSites](lazycache.Options{MaxEntries: 5})
return nil
}
@@ -435,26 +434,26 @@ func (r *rootCommand) createLogger(running bool) (loggers.Logger, error) {
}
} else {
if r.verbose {
hugo.Deprecate("--verbose", "use --logLevel info", "v0.114.0")
hugo.Deprecate("--verbose", "use --logLevel info", "v0.114.0")
helpers.Deprecated("--verbose", "use --logLevel info", false)
level = logg.LevelInfo
}
if r.debug {
hugo.Deprecate("--debug", "use --logLevel debug", "v0.114.0")
helpers.Deprecated("--debug", "use --logLevel debug", false)
level = logg.LevelDebug
}
}
optsLogger := loggers.Options{
DistinctLevel: logg.LevelWarn,
Level: level,
Stdout: r.Out,
Stderr: r.Out,
StoreErrors: running,
Distinct: true,
Level: level,
Stdout: r.Out,
Stderr: r.Out,
StoreErrors: running,
}
return loggers.New(optsLogger), nil
}
func (r *rootCommand) Reset() {
@@ -520,6 +519,7 @@ func applyLocalFlagsBuildConfig(cmd *cobra.Command, r *rootCommand) {
_ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{})
cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
_ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"})
}
// Flags needed to do a build (used by hugo and hugo server commands)
@@ -558,6 +558,7 @@ func applyLocalFlagsBuild(cmd *cobra.Command, r *rootCommand) {
cmd.Flags().StringSlice("disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)")
cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)")
_ = cmd.Flags().SetAnnotation("destination", cobra.BashCompSubdirsInDir, []string{})
}
func (r *rootCommand) timeTrack(start time.Time, name string) {
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -37,4 +37,5 @@ func newExec() (*simplecobra.Exec, error) {
}
return simplecobra.New(rootCmd)
}
+3 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ func newConfigCommand() *configCommand {
&configMountsCommand{},
},
}
}
type configCommand struct {
@@ -189,6 +190,7 @@ func (m *configModMounts) MarshalJSON() ([]byte, error) {
Dir: m.m.Dir(),
Mounts: mounts,
})
}
type configMountsCommand struct {
+3 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -134,7 +134,7 @@ func (c *convertCommand) convertAndSavePage(p page.Page, site *hugolib.Site, tar
}
}
if p.File() == nil {
if p.File().IsZero() {
// No content file.
return nil
}
@@ -209,7 +209,7 @@ func (c *convertCommand) convertContents(format metadecoders.Format) error {
var pagesBackedByFile page.Pages
for _, p := range site.AllPages() {
if p.File() == nil {
if p.File().IsZero() {
continue
}
pagesBackedByFile = append(pagesBackedByFile, p)
+3 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
//go:build !nodeploy
// +build !nodeploy
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@ import (
)
func newDeployCommand() simplecobra.Commander {
return &simpleCommand{
name: "deploy",
short: "Deploy your site to a Cloud provider.",
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
//go:build nodeploy
// +build nodeploy
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+7 -5
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -76,7 +76,7 @@ See https://xyproto.github.io/splash/docs/all.html for a preview of the availabl
},
withc: func(cmd *cobra.Command, r *rootCommand) {
cmd.PersistentFlags().StringVar(&style, "style", "friendly", "highlighter style (see https://xyproto.github.io/splash/docs/)")
cmd.PersistentFlags().StringVar(&highlightStyle, "highlightStyle", "", "style used for highlighting lines (see https://github.com/alecthomas/chroma)")
cmd.PersistentFlags().StringVar(&highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)")
cmd.PersistentFlags().StringVar(&linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)")
},
}
@@ -101,7 +101,7 @@ See https://xyproto.github.io/splash/docs/all.html for a preview of the availabl
}
if found, _ := helpers.Exists(genmandir, hugofs.Os); !found {
r.Println("Directory", genmandir, "does not exist, creating...")
if err := hugofs.Os.MkdirAll(genmandir, 0o777); err != nil {
if err := hugofs.Os.MkdirAll(genmandir, 0777); err != nil {
return err
}
}
@@ -136,7 +136,7 @@ url: %s
long: `Generate Markdown documentation for the Hugo CLI.
This command is, mostly, used to create up-to-date documentation
of Hugo's command-line interface for https://gohugo.io/.
It creates one Markdown file per command with front matter suitable
for rendering in Hugo.`,
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
@@ -150,7 +150,7 @@ url: %s
}
if found, _ := helpers.Exists(gendocdir, hugofs.Os); !found {
r.Println("Directory", gendocdir, "does not exist, creating...")
if err := hugofs.Os.MkdirAll(gendocdir, 0o777); err != nil {
if err := hugofs.Os.MkdirAll(gendocdir, 0777); err != nil {
return err
}
}
@@ -177,6 +177,7 @@ url: %s
cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
},
}
}
var docsHelperTarget string
@@ -240,6 +241,7 @@ url: %s
newDocsHelper(),
},
}
}
type genCommand struct {
+3 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -110,11 +110,12 @@ func flagsToCfgWithAdditionalConfigBase(cd *simplecobra.Commandeer, cfg config.P
})
return cfg
}
func mkdir(x ...string) {
p := filepath.Join(x...)
err := os.MkdirAll(p, 0o777) // before umask
err := os.MkdirAll(p, 0777) // before umask
if err != nil {
log.Fatal(err)
}
+4 -4
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -25,9 +25,9 @@ func init() {
// This message to show to Windows users if Hugo is opened from explorer.exe
cobra.MousetrapHelpText = `
Hugo is a command-line tool for generating static websites.
You need to open PowerShell and run Hugo from there.
Hugo is a command-line tool for generating static website.
You need to open cmd.exe and run Hugo from there.
Visit https://gohugo.io/ for more information.`
}
+95 -46
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@ import (
"runtime/trace"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bep/logg"
@@ -35,7 +34,6 @@ import (
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/terminal"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
@@ -85,6 +83,7 @@ func (c *hugoBuilder) withConf(fn func(conf *commonConfig)) {
c.confmu.Lock()
defer c.confmu.Unlock()
fn(c.conf)
}
type hugoBuilderErrState struct {
@@ -136,12 +135,46 @@ func (c *hugoBuilder) errCount() int {
// getDirList provides NewWatcher() with a list of directories to watch for changes.
func (c *hugoBuilder) getDirList() ([]string, error) {
var filenames []string
walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
c.r.logger.Errorln("walker: ", err)
return nil
}
if fi.IsDir() {
if fi.Name() == ".git" ||
fi.Name() == "node_modules" || fi.Name() == "bower_components" {
return filepath.SkipDir
}
filenames = append(filenames, fi.Meta().Filename)
}
return nil
}
h, err := c.hugo()
if err != nil {
return nil, err
}
watchFiles := h.PathSpec.BaseFs.WatchDirs()
for _, fi := range watchFiles {
if !fi.IsDir() {
filenames = append(filenames, fi.Meta().Filename)
continue
}
return helpers.UniqueStringsSorted(h.PathSpec.BaseFs.WatchFilenames()), nil
w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.r.logger, Info: fi, WalkFn: walkFn})
if err := w.Walk(); err != nil {
c.r.logger.Errorln("walker: ", err)
}
}
filenames = helpers.UniqueStringsSorted(filenames)
return filenames, nil
}
func (c *hugoBuilder) initCPUProfile() (func(), error) {
@@ -408,7 +441,7 @@ func (c *hugoBuilder) copyStatic() (map[string]uint64, error) {
}
func (c *hugoBuilder) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
infol := c.r.logger.InfoCommand("static")
infol := c.r.logger.InfoCommand("copy static")
publishDir := helpers.FilePathSeparator
if sourceFs.PublishFolder != "" {
@@ -434,11 +467,11 @@ func (c *hugoBuilder) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint
if syncer.Delete {
infol.Logf("removing all files from destination that don't exist in static dirs")
syncer.DeleteFilter = func(f fsync.FileInfo) bool {
syncer.DeleteFilter = func(f os.FileInfo) bool {
return f.IsDir() && strings.HasPrefix(f.Name(), ".")
}
}
start := time.Now()
infol.Logf("syncing static files to %s", publishDir)
// because we are using a baseFs (to get the union right).
// set sync src to root
@@ -446,10 +479,9 @@ func (c *hugoBuilder) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint
if err != nil {
return 0, err
}
loggers.TimeTrackf(infol, start, nil, "syncing static files to %s", publishDir)
// Sync runs Stat 2 times for every source file.
numFiles := fs.statCounter / 2
// Sync runs Stat 3 times for every source file (which sounds much)
numFiles := fs.statCounter / 3
return numFiles, err
}
@@ -620,31 +652,13 @@ func (c *hugoBuilder) handleBuildErr(err error, msg string) {
func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
staticSyncer *staticSyncer,
evs []fsnotify.Event,
configSet map[string]bool,
) {
configSet map[string]bool) {
defer func() {
c.errState.setWasErr(false)
}()
var isHandled bool
// Filter out ghost events (from deleted, renamed directories).
// This seems to be a bug in fsnotify, or possibly MacOS.
var n int
for _, ev := range evs {
keep := true
if ev.Has(fsnotify.Create) || ev.Has(fsnotify.Write) {
if _, err := os.Stat(ev.Name); err != nil {
keep = false
}
}
if keep {
evs[n] = ev
n++
}
}
evs = evs[:n]
for _, ev := range evs {
isConfig := configSet[ev.Name]
configChangeType := configChangeConfig
@@ -712,25 +726,48 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
return
}
c.r.logger.Debugln("Received System Events:", evs)
c.r.logger.Infoln("Received System Events:", evs)
staticEvents := []fsnotify.Event{}
dynamicEvents := []fsnotify.Event{}
filtered := []fsnotify.Event{}
h, err := c.hugo()
if err != nil {
c.r.logger.Errorln("Error getting the Hugo object:", err)
return
}
n = 0
for _, ev := range evs {
if h.ShouldSkipFileChangeEvent(ev) {
continue
}
evs[n] = ev
n++
// Check the most specific first, i.e. files.
contentMapped := h.ContentChanges.GetSymbolicLinkMappings(ev.Name)
if len(contentMapped) > 0 {
for _, mapped := range contentMapped {
filtered = append(filtered, fsnotify.Event{Name: mapped, Op: ev.Op})
}
continue
}
// Check for any symbolic directory mapping.
dir, name := filepath.Split(ev.Name)
contentMapped = h.ContentChanges.GetSymbolicLinkMappings(dir)
if len(contentMapped) == 0 {
filtered = append(filtered, ev)
continue
}
for _, mapped := range contentMapped {
mappedFilename := filepath.Join(mapped, name)
filtered = append(filtered, fsnotify.Event{Name: mappedFilename, Op: ev.Op})
}
}
evs = evs[:n]
evs = filtered
for _, ev := range evs {
ext := filepath.Ext(ev.Name)
@@ -751,7 +788,6 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
if istemp {
continue
}
if h.Deps.SourceSpec.IgnoreFile(ev.Name) {
continue
}
@@ -775,7 +811,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
continue
}
walkAdder := func(path string, f hugofs.FileMetaInfo) error {
walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error {
if f.IsDir() {
c.r.logger.Println("adding created directory to watchlist", path)
if err := watcher.Add(path); err != nil {
@@ -791,10 +827,11 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
}
// recursively add new directories to watch list
if ev.Has(fsnotify.Create) || ev.Has(fsnotify.Rename) {
// When mkdir -p is used, only the top directory triggers an event (at least on OSX)
if ev.Op&fsnotify.Create == fsnotify.Create {
c.withConf(func(conf *commonConfig) {
if s, err := conf.fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() {
_ = helpers.Walk(conf.fs.Source, ev.Name, walkAdder)
_ = helpers.SymbolicWalk(conf.fs.Source, ev.Name, walkAdder)
}
})
}
@@ -835,7 +872,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
return
}
path := h.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name)
path = h.RelURL(paths.ToSlashTrimLeading(path), false)
path = h.RelURL(helpers.ToSlashTrimLeading(path), false)
livereload.RefreshPath(path)
} else {
@@ -872,7 +909,7 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
// Nothing has changed.
return
} else if len(changed) == 1 {
pathToRefresh := h.PathSpec.RelURL(paths.ToSlashTrimLeading(changed[0]), false)
pathToRefresh := h.PathSpec.RelURL(helpers.ToSlashTrimLeading(changed[0]), false)
livereload.RefreshPath(pathToRefresh)
} else {
livereload.ForceRefresh()
@@ -907,6 +944,7 @@ func (c *hugoBuilder) hugo() (*hugolib.HugoSites, error) {
var err error
h, err = c.r.HugFromConfig(conf)
return err
}); err != nil {
return nil, err
}
@@ -962,7 +1000,6 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
}
if len(conf.configs.LoadingInfo.ConfigFiles) == 0 {
//lint:ignore ST1005 end user message.
return errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\nRun `hugo help new` for details.")
}
@@ -974,16 +1011,15 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
}
return nil
}
var rebuildCounter atomic.Uint64
}
func (c *hugoBuilder) printChangeDetected(typ string) {
msg := "\nChange"
if typ != "" {
msg += " of " + typ
}
msg += fmt.Sprintf(" detected, rebuilding site (#%d).", rebuildCounter.Add(1))
msg += " detected, rebuilding site."
c.r.logger.Println(msg)
const layout = "2006-01-02 15:04:05.000 -0700"
@@ -998,12 +1034,25 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error {
}
}
c.errState.setBuildErr(nil)
visited := c.visitedURLs.PeekAllSet()
h, err := c.hugo()
if err != nil {
return err
}
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
if c.fastRenderMode {
c.withConf(func(conf *commonConfig) {
// Make sure we always render the home pages
for _, l := range conf.configs.ConfigLangs() {
langPath := l.LanguagePrefix()
if langPath != "" {
langPath = langPath + "/"
}
home := h.PrependBasePath("/"+langPath, false)
visited[home] = true
}
})
}
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: visited, ErrRecovery: c.errState.wasErr()}, events...)
}
func (c *hugoBuilder) reloadConfig() error {
+11 -4
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import (
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -65,6 +66,7 @@ Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root
}
return c
}
type importCommand struct {
@@ -310,7 +312,7 @@ func (c *importCommand) convertJekyllPost(path, relPath, targetDir string, draft
targetFile := filepath.Join(targetDir, relPath)
targetParentDir := filepath.Dir(targetFile)
os.MkdirAll(targetParentDir, 0o777)
os.MkdirAll(targetParentDir, 0777)
contentBytes, err := os.ReadFile(path)
if err != nil {
@@ -396,6 +398,7 @@ func (c *importCommand) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyl
}
func (c *importCommand) importFromJekyll(args []string) error {
jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
if err != nil {
return newUserError("path error:", args[0])
@@ -426,7 +429,11 @@ func (c *importCommand) importFromJekyll(args []string) error {
c.r.Println("Importing...")
fileCount := 0
callback := func(path string, fi hugofs.FileMetaInfo) error {
callback := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
@@ -455,7 +462,7 @@ func (c *importCommand) importFromJekyll(args []string) error {
for jekyllPostDir, hasAnyPostInDir := range jekyllPostDirs {
if hasAnyPostInDir {
if err = helpers.Walk(hugofs.Os, filepath.Join(jekyllRoot, jekyllPostDir), callback); err != nil {
if err = helpers.SymbolicWalk(hugofs.Os, filepath.Join(jekyllRoot, jekyllPostDir), callback); err != nil {
return err
}
}
+10 -5
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@ import (
// newListCommand creates a new list command and its subcommands.
func newListCommand() *listCommand {
createRecord := func(workingDir string, p page.Page) []string {
return []string{
filepath.ToSlash(strings.TrimPrefix(p.File().Filename(), workingDir+string(os.PathSeparator))),
@@ -82,6 +83,7 @@ func newListCommand() *listCommand {
}
return nil
}
return &listCommand{
@@ -92,10 +94,11 @@ func newListCommand() *listCommand {
long: `List all of the drafts in your content directory.`,
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
shouldInclude := func(p page.Page) bool {
if !p.Draft() || p.File() == nil {
if !p.Draft() || p.File().IsZero() {
return false
}
return true
}
return list(cd, r, shouldInclude,
"buildDrafts", true,
@@ -110,10 +113,11 @@ func newListCommand() *listCommand {
long: `List all of the posts in your content directory which will be posted in the future.`,
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
shouldInclude := func(p page.Page) bool {
if !resource.IsFuture(p) || p.File() == nil {
if !resource.IsFuture(p) || p.File().IsZero() {
return false
}
return true
}
return list(cd, r, shouldInclude,
"buildFuture", true,
@@ -127,7 +131,7 @@ func newListCommand() *listCommand {
long: `List all of the posts in your content directory which has already expired.`,
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
shouldInclude := func(p page.Page) bool {
if !resource.IsExpired(p) || p.File() == nil {
if !resource.IsExpired(p) || p.File().IsZero() {
return false
}
return true
@@ -144,13 +148,14 @@ func newListCommand() *listCommand {
long: `List all of the posts in your content directory, include drafts, future and expired pages.`,
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
shouldInclude := func(p page.Page) bool {
return p.File() != nil
return !p.File().IsZero()
}
return list(cd, r, shouldInclude, "buildDrafts", true, "buildFuture", true, "buildExpired", true)
},
},
},
}
}
type listCommand struct {
+5 -9
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -69,7 +69,7 @@ so this may/will change in future versions of Hugo.
if err != nil {
return err
}
return npm.Pack(h.BaseFs.ProjectSourceFs, h.BaseFs.AssetsWithDuplicatesPreserved.Fs)
return npm.Pack(h.BaseFs.SourceFs, h.BaseFs.Assets.Dirs)
},
},
},
@@ -211,12 +211,7 @@ Install a specific version:
hugo mod get github.com/gohugoio/testshortcodes@v0.3.0
Install the latest versions of all direct module dependencies:
hugo mod get
hugo mod get ./... (recursive)
Install the latest versions of all module dependencies (direct and indirect):
Install the latest versions of all module dependencies:
hugo mod get -u
hugo mod get -u ./... (recursive)
@@ -286,6 +281,7 @@ Run "go help get" for more information. All flags available for "go get" is also
npmCommand,
},
}
}
type modCommands struct {
@@ -307,7 +303,7 @@ func (c *modCommands) Run(ctx context.Context, cd *simplecobra.Commandeer, args
if err != nil {
return err
}
// config := conf.configs.Base
//config := conf.configs.Base
return nil
}
+9 -8
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ import (
"strings"
"github.com/bep/simplecobra"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/create"
"github.com/gohugoio/hugo/create/skeletons"
@@ -63,7 +62,9 @@ Ensure you run this within the root directory of your site.`,
cmd.Flags().StringVarP(&contentType, "kind", "k", "", "content type to create")
cmd.Flags().String("editor", "", "edit new content with this editor, if provided")
cmd.Flags().BoolVarP(&force, "force", "f", false, "overwrite file if it already exists")
cmd.Flags().StringVar(&format, "format", "toml", "preferred file format (toml, yaml or json)")
applyLocalFlagsBuildConfig(cmd, r)
},
},
&simpleCommand{
@@ -119,15 +120,14 @@ according to your needs.`,
if len(args) < 1 {
return newUserError("theme name needs to be provided")
}
cfg := config.New()
cfg.Set("publishDir", "public")
conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, cfg))
h, err := r.Hugo(flagsToCfg(cd, nil))
if err != nil {
return err
}
sourceFs := conf.fs.Source
createpath := paths.AbsPathify(conf.configs.Base.WorkingDir, filepath.Join(conf.configs.Base.ThemesDir, args[0]))
ps := h.PathSpec
sourceFs := ps.Fs.Source
themesDir := h.Configs.LoadingInfo.BaseConfig.ThemesDir
createpath := ps.AbsPathify(filepath.Join(themesDir, args[0]))
r.Println("Creating new theme in", createpath)
err = skeletons.CreateTheme(createpath, sourceFs)
@@ -142,6 +142,7 @@ according to your needs.`,
}
return c
}
type newCommand struct {
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import (
// Note: This is a command only meant for internal use and must be run
// via "go run -tags release main.go release" on the actual code base that is in the release.
func newReleaseCommand() simplecobra.Commander {
var (
step int
skipPush bool
+42 -35
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -27,19 +27,20 @@ import (
"net/http"
"net/url"
"os"
"sync"
"sync/atomic"
"github.com/bep/mclib"
"os/signal"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/bep/mclib"
"github.com/bep/debounce"
"github.com/bep/simplecobra"
"github.com/fsnotify/fsnotify"
@@ -82,14 +83,10 @@ const (
)
func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder {
var visitedURLs *types.EvictingStringQueue
if s != nil && !s.disableFastRender {
visitedURLs = types.NewEvictingStringQueue(20)
}
return &hugoBuilder{
r: r,
s: s,
visitedURLs: visitedURLs,
visitedURLs: types.NewEvictingStringQueue(100),
fullRebuildSem: semaphore.NewWeighted(1),
debounce: debounce.New(4 * time.Second),
onConfigLoaded: func(reloaded bool) error {
@@ -123,6 +120,7 @@ func newServerCommand() *serverCommand {
},
withc: func(cmd *cobra.Command, r *rootCommand) {
cmd.Flags().BoolVar(&uninstall, "uninstall", false, "Uninstall the local CA (but do not delete it).")
},
},
},
@@ -221,7 +219,7 @@ func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
}
type fileServer struct {
baseURLs []urls.BaseURL
baseURLs []string
roots []string
errorTemplate func(err any) (io.Reader, error)
c *serverCommand
@@ -257,6 +255,12 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
r.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
}
// We're only interested in the path
u, err := url.Parse(baseURL)
if err != nil {
return nil, nil, "", "", fmt.Errorf("invalid baseURL: %w", err)
}
decorate := func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if f.c.showErrorInBrowser {
@@ -276,7 +280,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
port = lrport
}
})
lr := baseURL.URL()
lr := *u
lr.Host = fmt.Sprintf("%s:%d", lr.Hostname(), port)
fmt.Fprint(w, injectLiveReloadScript(r, lr))
@@ -307,7 +311,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
// This matches Netlify's behaviour and is needed for SPA behaviour.
// See https://docs.netlify.com/routing/redirects/rewrites-proxies/
if !redirect.Force {
path := filepath.Clean(strings.TrimPrefix(requestURI, baseURL.Path()))
path := filepath.Clean(strings.TrimPrefix(requestURI, u.Path))
if root != "" {
path = filepath.Join(root, path)
}
@@ -334,7 +338,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
switch redirect.Status {
case 404:
w.WriteHeader(404)
file, err := fs.Open(strings.TrimPrefix(redirect.To, baseURL.Path()))
file, err := fs.Open(strings.TrimPrefix(redirect.To, u.Path))
if err == nil {
defer file.Close()
io.Copy(w, file)
@@ -343,7 +347,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
}
return
case 200:
if r2 := f.rewriteRequest(r, strings.TrimPrefix(redirect.To, baseURL.Path())); r2 != nil {
if r2 := f.rewriteRequest(r, strings.TrimPrefix(redirect.To, u.Path)); r2 != nil {
requestURI = redirect.To
r = r2
}
@@ -381,10 +385,10 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
fileserver := decorate(http.FileServer(fs))
mu := http.NewServeMux()
if baseURL.Path() == "" || baseURL.Path() == "/" {
if u.Path == "" || u.Path == "/" {
mu.Handle("/", fileserver)
} else {
mu.Handle(baseURL.Path(), http.StripPrefix(baseURL.Path(), fileserver))
mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
}
if r.IsTestRun() {
var shutDownOnce sync.Once
@@ -397,7 +401,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
endpoint := net.JoinHostPort(f.c.serverInterface, strconv.Itoa(port))
return mu, listener, baseURL.String(), endpoint, nil
return mu, listener, u.String(), endpoint, nil
}
func (f *fileServer) rewriteRequest(r *http.Request, toPath string) *http.Request {
@@ -465,6 +469,7 @@ func (c *serverCommand) Name() string {
}
func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
// Watch runs its own server as part of the routine
if c.serverWatch {
@@ -671,7 +676,7 @@ func (c *serverCommand) createCertificates(conf *commonConfig) error {
// Create the directory if it doesn't exist.
if _, err := os.Stat(keyDir); os.IsNotExist(err) {
if err := os.MkdirAll(keyDir, 0o777); err != nil {
if err := os.MkdirAll(keyDir, 0777); err != nil {
return err
}
}
@@ -696,6 +701,7 @@ func (c *serverCommand) createCertificates(conf *commonConfig) error {
// Yes, this is unfortunate, but it's currently the only way to use Mkcert as a library.
os.Args = []string{"-cert-file", c.tlsCertFile, "-key-file", c.tlsKeyFile, hostname}
return mclib.RunMain()
}
func (c *serverCommand) verifyCert(rootPEM, certPEM []byte, name string) error {
@@ -825,9 +831,9 @@ func (c *serverCommand) partialReRender(urls ...string) error {
c.errState.setWasErr(false)
}()
c.errState.setBuildErr(nil)
visited := types.NewEvictingStringQueue(len(urls))
visited := make(map[string]bool)
for _, url := range urls {
visited.Add(url)
visited[url] = true
}
h, err := c.hugo()
@@ -840,7 +846,7 @@ func (c *serverCommand) partialReRender(urls ...string) error {
func (c *serverCommand) serve() error {
var (
baseURLs []urls.BaseURL
baseURLs []string
roots []string
h *hugolib.HugoSites
)
@@ -857,17 +863,18 @@ func (c *serverCommand) serve() error {
if isMultiHost {
for _, l := range conf.configs.ConfigLangs() {
baseURLs = append(baseURLs, l.BaseURL())
baseURLs = append(baseURLs, l.BaseURL().String())
roots = append(roots, l.Language().Lang)
}
} else {
l := conf.configs.GetFirstLanguageConfig()
baseURLs = []urls.BaseURL{l.BaseURL()}
baseURLs = []string{l.BaseURL().String()}
roots = []string{""}
}
return nil
})
if err != nil {
return err
}
@@ -939,11 +946,15 @@ func (c *serverCommand) serve() error {
servers = append(servers, srv)
if doLiveReload {
baseURL := baseURLs[i]
mu.HandleFunc(baseURL.Path()+"livereload.js", livereload.ServeJS)
mu.HandleFunc(baseURL.Path()+"livereload", livereload.Handler)
u, err := url.Parse(helpers.SanitizeURL(baseURLs[i]))
if err != nil {
return err
}
mu.HandleFunc(u.Path+"/livereload.js", livereload.ServeJS)
mu.HandleFunc(u.Path+"/livereload", livereload.Handler)
}
c.r.Printf("Web Server is available at %s (bind address %s) %s\n", serverURL, c.serverInterface, roots[i])
c.r.Printf("Web Server is available at %s (bind address %s)\n", serverURL, c.serverInterface)
wg1.Go(func() error {
if c.tlsCertFile != "" && c.tlsKeyFile != "" {
err = srv.ServeTLS(listener, c.tlsCertFile, c.tlsKeyFile)
@@ -960,12 +971,8 @@ func (c *serverCommand) serve() error {
if c.r.IsTestRun() {
// Write a .ready file to disk to signal ready status.
// This is where the test is run from.
var baseURLs []string
for _, baseURL := range srv.baseURLs {
baseURLs = append(baseURLs, baseURL.String())
}
testInfo := map[string]any{
"baseURLs": baseURLs,
"baseURLs": srv.baseURLs,
}
dir := os.Getenv("WORK")
@@ -976,7 +983,7 @@ func (c *serverCommand) serve() error {
if err != nil {
return err
}
err = os.WriteFile(readyFile, b, 0o777)
err = os.WriteFile(readyFile, b, 0777)
if err != nil {
return err
}
@@ -1160,7 +1167,7 @@ func cleanErrorLog(content string) string {
return strings.Join(keep, ": ")
}
func injectLiveReloadScript(src io.Reader, baseURL *url.URL) string {
func injectLiveReloadScript(src io.Reader, baseURL url.URL) string {
var b bytes.Buffer
chain := transform.Chain{livereloadinject.New(baseURL)}
chain.Apply(&b, src)
+3 -21
View File
@@ -16,28 +16,10 @@ package constants
// Error IDs.
// Do not change these values.
const (
ErrIDAmbigousDisableKindTaxonomy = "error-disable-taxonomy"
ErrIDAmbigousOutputKindTaxonomy = "error-output-taxonomy"
// IDs for remote errors in tpl/data.
ErrRemoteGetJSON = "error-remote-getjson"
ErrRemoteGetCSV = "error-remote-getcsv"
)
// Field/method names with special meaning.
const (
FieldRelPermalink = "RelPermalink"
FieldPermalink = "Permalink"
)
// IsFieldRelOrPermalink returns whether the given name is a RelPermalink or Permalink.
func IsFieldRelOrPermalink(name string) bool {
return name == FieldRelPermalink || name == FieldPermalink
}
// Resource transformations.
const (
ResourceTransformationFingerprint = "fingerprint"
)
// IsResourceTransformationLinkChange returns whether the given name is a resource transformation that changes the permalink based on the content.
func IsResourceTransformationPermalinkHash(name string) bool {
return name == ResourceTransformationFingerprint
}
-46
View File
@@ -1,46 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hcontext
import "context"
// ContextDispatcher is a generic interface for setting and getting values from a context.
type ContextDispatcher[T any] interface {
Set(ctx context.Context, value T) context.Context
Get(ctx context.Context) T
}
// NewContextDispatcher creates a new ContextDispatcher with the given key.
func NewContextDispatcher[T any, R comparable](key R) ContextDispatcher[T] {
return keyInContext[T, R]{
id: key,
}
}
type keyInContext[T any, R comparable] struct {
zero T
id R
}
func (f keyInContext[T, R]) Get(ctx context.Context) T {
v := ctx.Value(f.id)
if v == nil {
return f.zero
}
return v.(T)
}
func (f keyInContext[T, R]) Set(ctx context.Context, value T) context.Context {
return context.WithValue(ctx, f.id, value)
}
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -74,6 +74,7 @@ func ContainsMatcher(text string) func(m LineMatcher) int {
// ErrorContext contains contextual information about an error. This will
// typically be the lines surrounding some problem in a file.
type ErrorContext struct {
// If a match will contain the matched line and up to 2 lines before and after.
// Will be empty if no match.
Lines []string
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+11 -18
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,13 +15,14 @@
package herrors
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"runtime"
"runtime/debug"
"time"
"strconv"
)
// PrintStackTrace prints the current stacktrace to w.
@@ -48,22 +49,14 @@ func Recover(args ...any) {
}
}
// IsTimeoutError returns true if the given error is or contains a TimeoutError.
func IsTimeoutError(err error) bool {
return errors.Is(err, &TimeoutError{})
}
type TimeoutError struct {
Duration time.Duration
}
func (e *TimeoutError) Error() string {
return fmt.Sprintf("timeout after %s", e.Duration)
}
func (e *TimeoutError) Is(target error) bool {
_, ok := target.(*TimeoutError)
return ok
// GetGID the current goroutine id. Used only for debugging.
func GetGID() uint64 {
b := make([]byte, 64)
b = b[:runtime.Stack(b, false)]
b = bytes.TrimPrefix(b, []byte("goroutine "))
b = b[:bytes.IndexByte(b, ' ')]
n, _ := strconv.ParseUint(string(b), 10, 64)
return n
}
// IsFeatureNotAvailableError returns true if the given error is or contains a FeatureNotAvailableError.
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -42,4 +42,5 @@ func TestIsFeatureNotAvailableError(t *testing.T) {
c.Assert(IsFeatureNotAvailableError(ErrFeatureNotAvailable), qt.Equals, true)
c.Assert(IsFeatureNotAvailableError(&FeatureNotAvailableError{}), qt.Equals, true)
c.Assert(IsFeatureNotAvailableError(errors.New("asdf")), qt.Equals, false)
}
+10 -12
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@ package herrors
import (
"encoding/json"
"errors"
godartsassv1 "github.com/bep/godartsass"
"fmt"
"io"
"path/filepath"
godartsassv1 "github.com/bep/godartsass"
"github.com/bep/godartsass/v2"
"github.com/bep/golibsass/libsass/libsasserrors"
"github.com/gohugoio/hugo/common/paths"
@@ -29,6 +29,8 @@ import (
"github.com/pelletier/go-toml/v2"
"github.com/spf13/afero"
"github.com/tdewolff/parse/v2"
"errors"
)
// FileError represents an error when handling a file: Parsing a config file,
@@ -46,9 +48,6 @@ type FileError interface {
// UpdateContent updates the error with a new ErrorContext from the content of the file.
UpdateContent(r io.Reader, linematcher LineMatcherFn) FileError
// SetFilename sets the filename of the error.
SetFilename(filename string) FileError
}
// Unwrapper can unwrap errors created with fmt.Errorf.
@@ -61,11 +60,6 @@ var (
_ Unwrapper = (*fileError)(nil)
)
func (fe *fileError) SetFilename(filename string) FileError {
fe.position.Filename = filename
return fe
}
func (fe *fileError) UpdatePosition(pos text.Position) FileError {
oldFilename := fe.Position().Filename
if pos.Filename != "" && fe.fileType == "" {
@@ -121,6 +115,7 @@ func (fe *fileError) UpdateContent(r io.Reader, linematcher LineMatcherFn) FileE
}
return fe
}
type fileError struct {
@@ -186,6 +181,7 @@ func NewFileErrorFromName(err error, name string) FileError {
}
return &fileError{cause: err, fileType: fileType, position: pos}
}
// NewFileErrorFromPos will use the filename and line number from pos to create a new FileError, wrapping err.
@@ -196,6 +192,7 @@ func NewFileErrorFromPos(err error, pos text.Position) FileError {
_, fileType = paths.FileAndExtNoDelimiter(filepath.Clean(pos.Filename))
}
return &fileError{cause: err, fileType: fileType, position: pos}
}
func NewFileErrorFromFileInErr(err error, fs afero.Fs, linematcher LineMatcherFn) FileError {
@@ -252,6 +249,7 @@ func openFile(filename string, fs afero.Fs) (afero.File, string, error) {
}); ok {
realFilename = s.Filename()
}
}
f, err2 := fs.Open(filename)
+4 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,11 +14,12 @@
package herrors
import (
"errors"
"fmt"
"strings"
"testing"
"errors"
"github.com/gohugoio/hugo/common/text"
qt "github.com/frankban/quicktest"
@@ -47,6 +48,7 @@ func TestNewFileError(t *testing.T) {
c.Assert(errorContext.Lines, qt.DeepEquals, []string{"line 30", "line 31", "line 32", "line 33", "line 34"})
c.Assert(errorContext.LinesPos, qt.Equals, 2)
c.Assert(errorContext.ChromaLexer, qt.Equals, "go-html-template")
}
func TestNewFileErrorExtractFromMessage(t *testing.T) {
+3 -38
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2019 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -23,7 +23,6 @@ import (
"time"
"github.com/gohugoio/hugo/common/htime"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
)
@@ -189,20 +188,6 @@ func IsTime(tp reflect.Type) bool {
return false
}
// IsValid returns whether v is not nil and a valid value.
func IsValid(v reflect.Value) bool {
if !v.IsValid() {
return false
}
switch v.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
return !v.IsNil()
}
return true
}
// AsTime returns v as a time.Time if possible.
// The given location is only used if the value implements AsTimeProvider (e.g. go-toml local).
// A zero Time and false is returned if this isn't possible.
@@ -232,7 +217,7 @@ func CallMethodByName(cxt context.Context, name string, v reflect.Value) []refle
panic("not supported")
}
first := tp.In(0)
if IsContextType(first) {
if first.Implements(ContextInterface) {
args = append(args, reflect.ValueOf(cxt))
}
}
@@ -251,24 +236,4 @@ func indirectInterface(v reflect.Value) reflect.Value {
return v.Elem()
}
var contextInterface = reflect.TypeOf((*context.Context)(nil)).Elem()
var isContextCache = maps.NewCache[reflect.Type, bool]()
type k string
var contextTypeValue = reflect.TypeOf(context.WithValue(context.Background(), k("key"), 32))
// IsContextType returns whether tp is a context.Context type.
func IsContextType(tp reflect.Type) bool {
if tp == contextTypeValue {
return true
}
if tp == contextInterface {
return true
}
return isContextCache.GetOrCreate(tp, func() bool {
return tp.Implements(contextInterface)
})
}
var ContextInterface = reflect.TypeOf((*context.Context)(nil)).Elem()
-37
View File
@@ -14,7 +14,6 @@
package hreflect
import (
"context"
"reflect"
"testing"
"time"
@@ -41,42 +40,6 @@ func TestGetMethodByName(t *testing.T) {
c.Assert(GetMethodIndexByName(tp, "Foo"), qt.Equals, -1)
}
func TestIsContextType(t *testing.T) {
c := qt.New(t)
type k string
ctx := context.Background()
valueCtx := context.WithValue(ctx, k("key"), 32)
c.Assert(IsContextType(reflect.TypeOf(ctx)), qt.IsTrue)
c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue)
}
func BenchmarkIsContextType(b *testing.B) {
type k string
b.Run("value", func(b *testing.B) {
ctx := context.Background()
ctxs := make([]reflect.Type, b.N)
for i := 0; i < b.N; i++ {
ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i))
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if !IsContextType(ctxs[i]) {
b.Fatal("not context")
}
}
})
b.Run("background", func(b *testing.B) {
var ctxt reflect.Type = reflect.TypeOf(context.Background())
for i := 0; i < b.N; i++ {
if !IsContextType(ctxt) {
b.Fatal("not context")
}
}
})
}
func BenchmarkIsTruthFul(b *testing.B) {
v := reflect.ValueOf("Hugo")
+1 -29
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -99,31 +99,3 @@ var reCache = regexpCache{re: make(map[string]*regexp.Regexp)}
func GetOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) {
return reCache.getOrCompileRegexp(pattern)
}
// InSlice checks if a string is an element of a slice of strings
// and returns a boolean value.
func InSlice(arr []string, el string) bool {
for _, v := range arr {
if v == el {
return true
}
}
return false
}
// InSlicEqualFold checks if a string is an element of a slice of strings
// and returns a boolean value.
// It uses strings.EqualFold to compare.
func InSlicEqualFold(arr []string, el string) bool {
for _, v := range arr {
if strings.EqualFold(v, el) {
return true
}
}
return false
}
type Tuple struct {
First string
Second string
}
+3 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@ func TestStringEqualFold(t *testing.T) {
c.Assert(StringEqualFold(s1).EqualFold("b"), qt.Equals, false)
c.Assert(StringEqualFold(s1).Eq(s2), qt.Equals, true)
c.Assert(StringEqualFold(s1).Eq("b"), qt.Equals, false)
}
func TestGetOrCompileRegexp(t *testing.T) {
@@ -41,6 +42,7 @@ func TestGetOrCompileRegexp(t *testing.T) {
re, err := GetOrCompileRegexp(`\d+`)
c.Assert(err, qt.IsNil)
c.Assert(re.MatchString("123"), qt.Equals, true)
}
func BenchmarkGetOrCompileRegexp(b *testing.B) {
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+2 -7
View File
@@ -16,7 +16,6 @@ package hugio
import (
"fmt"
"io"
iofs "io/fs"
"path/filepath"
"github.com/spf13/afero"
@@ -61,16 +60,12 @@ func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool
return fmt.Errorf("%q is not a directory", from)
}
err = fs.MkdirAll(to, 0o777) // before umask
err = fs.MkdirAll(to, 0777) // before umask
if err != nil {
return err
}
d, err := fs.Open(from)
if err != nil {
return err
}
entries, _ := d.(iofs.ReadDirFile).ReadDir(-1)
entries, _ := afero.ReadDir(fs, from)
for _, entry := range entries {
fromFilename := filepath.Join(from, entry.Name())
toFilename := filepath.Join(to, entry.Name())
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
-20
View File
@@ -14,7 +14,6 @@
package hugio
import (
"bytes"
"io"
"strings"
)
@@ -58,22 +57,3 @@ func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser {
func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser {
return ReadSeekerNoOpCloser{strings.NewReader(content)}
}
// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser
// from the given bytes slice.
func NewReadSeekerNoOpCloserFromBytes(content []byte) ReadSeekerNoOpCloser {
return ReadSeekerNoOpCloser{bytes.NewReader(content)}
}
// NewReadSeekCloser creates a new ReadSeekCloser from the given ReadSeeker.
// The ReadSeeker will be seeked to the beginning before returned.
func NewOpenReadSeekCloser(r ReadSeekCloser) OpenReadSeekCloser {
return func() (ReadSeekCloser, error) {
r.Seek(0, io.SeekStart)
return r, nil
}
}
// OpenReadSeekCloser allows setting some other way (than reading from a filesystem)
// to open or create a ReadSeekCloser.
type OpenReadSeekCloser func() (ReadSeekCloser, error)
+8 -72
View File
@@ -22,21 +22,18 @@ import (
"sort"
"strings"
"sync"
"time"
godartsassv1 "github.com/bep/godartsass"
"github.com/bep/logg"
"github.com/mitchellh/mapstructure"
"time"
"github.com/bep/godartsass/v2"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/spf13/afero"
iofs "io/fs"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
)
@@ -81,22 +78,10 @@ func (i HugoInfo) Generator() template.HTML {
return template.HTML(fmt.Sprintf(`<meta name="generator" content="Hugo %s">`, CurrentVersion.String()))
}
// IsDevelopment reports whether the current running environment is "development".
func (i HugoInfo) IsDevelopment() bool {
return i.Environment == EnvironmentDevelopment
}
// IsProduction reports whether the current running environment is "production".
func (i HugoInfo) IsProduction() bool {
return i.Environment == EnvironmentProduction
}
// IsServer reports whether the built-in server is running.
func (i HugoInfo) IsServer() bool {
return i.conf.Running()
}
// IsExtended reports whether the Hugo binary is the extended version.
func (i HugoInfo) IsExtended() bool {
return IsExtended
}
@@ -114,7 +99,6 @@ func (i HugoInfo) Deps() []*Dependency {
// ConfigProvider represents the config options that are relevant for HugoInfo.
type ConfigProvider interface {
Environment() string
Running() bool
WorkingDir() string
}
@@ -161,12 +145,7 @@ func GetExecEnviron(workDir string, cfg config.AllProvider, fs afero.Fs) []strin
config.SetEnvVars(&env, "HUGO_PUBLISHDIR", filepath.Join(workDir, cfg.BaseConfig().PublishDir))
if fs != nil {
var fis []iofs.DirEntry
d, err := fs.Open(files.FolderJSConfig)
if err == nil {
fis, err = d.(iofs.ReadDirFile).ReadDir(-1)
}
fis, err := afero.ReadDir(fs, files.FolderJSConfig)
if err == nil {
for _, fi := range fis {
key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_"))
@@ -191,10 +170,8 @@ type buildInfo struct {
*debug.BuildInfo
}
var (
bInfo *buildInfo
bInfoInit sync.Once
)
var bInfo *buildInfo
var bInfoInit sync.Once
func getBuildInfo() *buildInfo {
bInfoInit.Do(func() {
@@ -221,6 +198,7 @@ func getBuildInfo() *buildInfo {
bInfo.GoArch = s.Value
}
}
})
return bInfo
@@ -259,12 +237,12 @@ func GetDependencyListNonGo() []string {
deps = append(
deps,
formatDep("github.com/sass/libsass", "3.6.5"),
formatDep("github.com/webmproject/libwebp", "v1.3.2"),
formatDep("github.com/webmproject/libwebp", "v1.2.4"),
)
}
if dartSass := dartSassVersion(); dartSass.ProtocolVersion != "" {
dartSassPath := "github.com/sass/dart-sass-embedded"
var dartSassPath = "github.com/sass/dart-sass-embedded"
if IsDartSassV2() {
dartSassPath = "github.com/sass/dart-sass"
}
@@ -355,45 +333,3 @@ var (
func IsDartSassV2() bool {
return !strings.Contains(DartSassBinaryName, "embedded")
}
// Deprecate informs about a deprecation starting at the given version.
//
// A deprecation typically needs a simple change in the template, but doing so will make the template incompatible with older versions.
// Theme maintainers generally want
// 1. No warnings or errors in the console when building a Hugo site.
// 2. Their theme to work for at least the last few Hugo versions.
func Deprecate(item, alternative string, version string) {
level := deprecationLogLevelFromVersion(version)
DeprecateLevel(item, alternative, version, level)
}
// DeprecateLevel informs about a deprecation logging at the given level.
func DeprecateLevel(item, alternative, version string, level logg.Level) {
var msg string
if level == logg.LevelError {
msg = fmt.Sprintf("%s was deprecated in Hugo %s and will be removed in Hugo %s. %s", item, version, CurrentVersion.Next().ReleaseVersion(), alternative)
} else {
msg = fmt.Sprintf("%s was deprecated in Hugo %s and will be removed in a future release. %s", item, version, alternative)
}
loggers.Log().Logger().WithLevel(level).WithField(loggers.FieldNameCmd, "deprecated").Logf(msg)
}
// We ususally do about one minor version a month.
// We want people to run at least the current and previous version without any warnings.
// We want people who don't update Hugo that often to see the warnings and errors before we remove the feature.
func deprecationLogLevelFromVersion(ver string) logg.Level {
from := MustParseVersion(ver)
to := CurrentVersion
minorDiff := to.Minor - from.Minor
switch {
case minorDiff >= 12:
// Start failing the build after about a year.
return logg.LevelError
case minorDiff >= 6:
// Start printing warnings after about six months.
return logg.LevelWarn
default:
return logg.LevelInfo
}
}
+2 -26
View File
@@ -17,14 +17,13 @@ import (
"fmt"
"testing"
"github.com/bep/logg"
qt "github.com/frankban/quicktest"
)
func TestHugoInfo(t *testing.T) {
c := qt.New(t)
conf := testConfig{environment: "production", workingDir: "/mywork", running: false}
conf := testConfig{environment: "production", workingDir: "/mywork"}
hugoInfo := NewInfo(conf, nil)
c.Assert(hugoInfo.Version(), qt.Equals, CurrentVersion.Version())
@@ -39,34 +38,15 @@ func TestHugoInfo(t *testing.T) {
}
c.Assert(hugoInfo.Environment, qt.Equals, "production")
c.Assert(string(hugoInfo.Generator()), qt.Contains, fmt.Sprintf("Hugo %s", hugoInfo.Version()))
c.Assert(hugoInfo.IsDevelopment(), qt.Equals, false)
c.Assert(hugoInfo.IsProduction(), qt.Equals, true)
c.Assert(hugoInfo.IsExtended(), qt.Equals, IsExtended)
c.Assert(hugoInfo.IsServer(), qt.Equals, false)
devHugoInfo := NewInfo(testConfig{environment: "development", running: true}, nil)
c.Assert(devHugoInfo.IsDevelopment(), qt.Equals, true)
devHugoInfo := NewInfo(testConfig{environment: "development"}, nil)
c.Assert(devHugoInfo.IsProduction(), qt.Equals, false)
c.Assert(devHugoInfo.IsServer(), qt.Equals, true)
}
func TestDeprecationLogLevelFromVersion(t *testing.T) {
c := qt.New(t)
c.Assert(deprecationLogLevelFromVersion("0.55.0"), qt.Equals, logg.LevelError)
ver := CurrentVersion
c.Assert(deprecationLogLevelFromVersion(ver.String()), qt.Equals, logg.LevelInfo)
ver.Minor -= 1
c.Assert(deprecationLogLevelFromVersion(ver.String()), qt.Equals, logg.LevelInfo)
ver.Minor -= 6
c.Assert(deprecationLogLevelFromVersion(ver.String()), qt.Equals, logg.LevelWarn)
ver.Minor -= 6
c.Assert(deprecationLogLevelFromVersion(ver.String()), qt.Equals, logg.LevelError)
}
type testConfig struct {
environment string
running bool
workingDir string
}
@@ -74,10 +54,6 @@ func (c testConfig) Environment() string {
return c.environment
}
func (c testConfig) Running() bool {
return c.running
}
func (c testConfig) WorkingDir() string {
return c.workingDir
}
+4 -5
View File
@@ -67,11 +67,8 @@ func (h VersionString) String() string {
// Compare implements the compare.Comparer interface.
func (h VersionString) Compare(other any) int {
return compareVersions(h.Version(), other)
}
func (h VersionString) Version() Version {
return MustParseVersion(h.String())
v := MustParseVersion(h.String())
return compareVersions(v, other)
}
// Eq implements the compare.Eqer interface.
@@ -267,6 +264,7 @@ func compareFloatWithVersion(v1 float64, v2 Version) int {
if v1maj > v2.Major {
return 1
}
if v1maj < v2.Major {
@@ -278,6 +276,7 @@ func compareFloatWithVersion(v1 float64, v2 Version) int {
}
return -1
}
func GoMinorVersion() int {
+1 -1
View File
@@ -17,7 +17,7 @@ package hugo
// This should be the only one.
var CurrentVersion = Version{
Major: 0,
Minor: 123,
Minor: 117,
PatchLevel: 0,
Suffix: "-DEV",
}
+3 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -27,9 +27,10 @@ import (
"github.com/fatih/color"
)
var bold = color.New(color.Bold)
// levelColor mapping.
var levelColor = [...]*color.Color{
logg.LevelTrace: color.New(color.FgWhite),
logg.LevelDebug: color.New(color.FgWhite),
logg.LevelInfo: color.New(color.FgBlue),
logg.LevelWarn: color.New(color.FgYellow),
@@ -38,7 +39,6 @@ var levelColor = [...]*color.Color{
// levelString mapping.
var levelString = [...]string{
logg.LevelTrace: "TRACE",
logg.LevelDebug: "DEBUG",
logg.LevelInfo: "INFO ",
logg.LevelWarn: "WARN ",
+18 -5
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -69,7 +69,7 @@ func (h *logLevelCounter) HandleLog(e *logg.Entry) error {
return nil
}
var errStop = fmt.Errorf("stop")
var stopError = fmt.Errorf("stop")
type logOnceHandler struct {
threshold logg.Level
@@ -87,7 +87,7 @@ func (h *logOnceHandler) HandleLog(e *logg.Entry) error {
defer h.mu.Unlock()
hash := identity.HashUint64(e.Level, e.Message, e.Fields)
if h.seen[hash] {
return errStop
return stopError
}
h.seen[hash] = true
return nil
@@ -107,7 +107,7 @@ type stopHandler struct {
func (h *stopHandler) HandleLog(e *logg.Entry) error {
for _, handler := range h.handlers {
if err := handler.HandleLog(e); err != nil {
if err == errStop {
if err == stopError {
return nil
}
return err
@@ -124,13 +124,26 @@ func (h *suppressStatementsHandler) HandleLog(e *logg.Entry) error {
for _, field := range e.Fields {
if field.Name == FieldNameStatementID {
if h.statements[field.Value.(string)] {
return errStop
return stopError
}
}
}
return nil
}
// replacer creates a new log handler that does string replacement in log messages.
func replacer(repl *strings.Replacer) logg.Handler {
return logg.HandlerFunc(func(e *logg.Entry) error {
e.Message = repl.Replace(e.Message)
for i, field := range e.Fields {
if s, ok := field.Value.(string); ok {
e.Fields[i].Value = repl.Replace(s)
}
}
return nil
})
}
// whiteSpaceTrimmer creates a new log handler that trims whitespace from log messages and string fields.
func whiteSpaceTrimmer() logg.Handler {
return logg.HandlerFunc(func(e *logg.Entry) error {
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -81,7 +81,7 @@ func (h *noColoursHandler) HandleLog(e *logg.Entry) error {
if strings.HasPrefix(field.Name, reservedFieldNamePrefix) {
continue
}
fmt.Fprintf(w, " %s %v", field.Name, field.Value)
fmt.Fprintf(w, " %s %q", field.Name, field.Value)
}
fmt.Fprintln(w)
+18 -69
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -37,13 +37,13 @@ var (
// Options defines options for the logger.
type Options struct {
Level logg.Level
Stdout io.Writer
Stderr io.Writer
DistinctLevel logg.Level
StoreErrors bool
HandlerPost func(e *logg.Entry) error
SuppressStatements map[string]bool
Level logg.Level
Stdout io.Writer
Stderr io.Writer
Distinct bool
StoreErrors bool
HandlerPost func(e *logg.Entry) error
SuppresssStatements map[string]bool
}
// New creates a new logger with the given options.
@@ -68,24 +68,11 @@ func New(opts Options) Logger {
errorsw := &strings.Builder{}
logCounters := newLogLevelCounter()
handlers := []logg.Handler{
whiteSpaceTrimmer(),
logHandler,
logCounters,
}
if opts.Level == logg.LevelTrace {
// Trace is used during development only, and it's useful to
// only see the trace messages.
handlers = append(handlers,
logg.HandlerFunc(func(e *logg.Entry) error {
if e.Level != logg.LevelTrace {
return logg.ErrStopLogEntry
}
return nil
}),
)
}
handlers = append(handlers, whiteSpaceTrimmer(), logHandler)
if opts.HandlerPost != nil {
var hookHandler logg.HandlerFunc = func(e *logg.Entry) error {
opts.HandlerPost(e)
@@ -105,13 +92,13 @@ func New(opts Options) Logger {
logHandler = multi.New(handlers...)
var logOnce *logOnceHandler
if opts.DistinctLevel != 0 {
logOnce = newLogOnceHandler(opts.DistinctLevel)
if opts.Distinct {
logOnce = newLogOnceHandler(logg.LevelWarn)
logHandler = newStopHandler(logOnce, logHandler)
}
if opts.SuppressStatements != nil && len(opts.SuppressStatements) > 0 {
logHandler = newStopHandler(newSuppressStatementsHandler(opts.SuppressStatements), logHandler)
if opts.SuppresssStatements != nil && len(opts.SuppresssStatements) > 0 {
logHandler = newStopHandler(newSuppressStatementsHandler(opts.SuppresssStatements), logHandler)
}
logger := logg.New(
@@ -140,7 +127,6 @@ func New(opts Options) Logger {
out: opts.Stdout,
level: opts.Level,
logger: logger,
tracel: l.WithLevel(logg.LevelTrace),
debugl: l.WithLevel(logg.LevelDebug),
infol: l.WithLevel(logg.LevelInfo),
warnl: l.WithLevel(logg.LevelWarn),
@@ -151,20 +137,10 @@ func New(opts Options) Logger {
// NewDefault creates a new logger with the default options.
func NewDefault() Logger {
opts := Options{
DistinctLevel: logg.LevelWarn,
Level: logg.LevelWarn,
Stdout: os.Stdout,
Stderr: os.Stdout,
}
return New(opts)
}
func NewTrace() Logger {
opts := Options{
DistinctLevel: logg.LevelWarn,
Level: logg.LevelTrace,
Stdout: os.Stdout,
Stderr: os.Stdout,
Distinct: true,
Level: logg.LevelWarn,
Stdout: os.Stdout,
Stderr: os.Stdout,
}
return New(opts)
}
@@ -174,7 +150,6 @@ func LevelLoggerToWriter(l logg.LevelLogger) io.Writer {
}
type Logger interface {
Debug() logg.LevelLogger
Debugf(format string, v ...any)
Debugln(v ...any)
Error() logg.LevelLogger
@@ -199,7 +174,6 @@ type Logger interface {
Warnf(format string, v ...any)
Warnln(v ...any)
Deprecatef(fail bool, format string, v ...any)
Trace(s logg.StringFunc)
}
type logAdapter struct {
@@ -209,17 +183,12 @@ type logAdapter struct {
out io.Writer
level logg.Level
logger logg.Logger
tracel logg.LevelLogger
debugl logg.LevelLogger
infol logg.LevelLogger
warnl logg.LevelLogger
errorl logg.LevelLogger
}
func (l *logAdapter) Debug() logg.LevelLogger {
return l.debugl
}
func (l *logAdapter) Debugf(format string, v ...any) {
l.debugl.Logf(format, v...)
}
@@ -325,10 +294,6 @@ func (l *logAdapter) Errorsf(id, format string, v ...any) {
l.errorl.WithField(FieldNameStatementID, id).Logf(format, v...)
}
func (l *logAdapter) Trace(s logg.StringFunc) {
l.tracel.Log(s)
}
func (l *logAdapter) sprint(v ...any) string {
return strings.TrimRight(fmt.Sprintln(v...), "\n")
}
@@ -350,19 +315,3 @@ func (w logWriter) Write(p []byte) (n int, err error) {
w.l.Log(logg.String(string(p)))
return len(p), nil
}
func TimeTrackf(l logg.LevelLogger, start time.Time, fields logg.Fields, format string, a ...any) {
elapsed := time.Since(start)
if fields != nil {
l = l.WithFields(fields)
}
l.WithField("duration", elapsed).Logf(format, a...)
}
func TimeTrackfn(fn func() (logg.LevelLogger, error)) error {
start := time.Now()
l, err := fn()
elapsed := time.Since(start)
l.WithField("duration", elapsed).Logf("")
return err
}
+12 -10
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -29,10 +29,10 @@ func TestLogDistinct(t *testing.T) {
c := qt.New(t)
opts := loggers.Options{
DistinctLevel: logg.LevelWarn,
StoreErrors: true,
Stdout: io.Discard,
Stderr: io.Discard,
Distinct: true,
StoreErrors: true,
Stdout: io.Discard,
Stderr: io.Discard,
}
l := loggers.New(opts)
@@ -85,6 +85,7 @@ func TestOptionStoreErrors(t *testing.T) {
c.Assert(sb.String(), qt.Contains, "error 1")
c.Assert(sb.String(), qt.Contains, "ERROR")
}
func TestLogCount(t *testing.T) {
@@ -109,7 +110,7 @@ func TestSuppressStatements(t *testing.T) {
opts := loggers.Options{
StoreErrors: true,
SuppressStatements: map[string]bool{
SuppresssStatements: map[string]bool{
"error-1": true,
},
}
@@ -123,16 +124,17 @@ func TestSuppressStatements(t *testing.T) {
c.Assert(errorsStr, qt.Not(qt.Contains), "error 1")
c.Assert(errorsStr, qt.Contains, "error 2")
c.Assert(l.LoggCount(logg.LevelError), qt.Equals, 1)
}
func TestReset(t *testing.T) {
c := qt.New(t)
opts := loggers.Options{
StoreErrors: true,
DistinctLevel: logg.LevelWarn,
Stdout: io.Discard,
Stderr: io.Discard,
StoreErrors: true,
Distinct: true,
Stdout: io.Discard,
Stderr: io.Discard,
}
l := loggers.New(opts)
+5 -6
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
// Some functions in this file (see comments) is based on the Go source code,
// copyright The Go Authors and governed by a BSD-style license.
//
@@ -21,7 +21,7 @@ import (
"github.com/bep/logg"
)
func InitGlobalLogger(level logg.Level, panicOnWarnings bool) {
func InitGlobalLogger(panicOnWarnings bool) {
logMu.Lock()
defer logMu.Unlock()
var logHookLast func(e *logg.Entry) error
@@ -31,9 +31,8 @@ func InitGlobalLogger(level logg.Level, panicOnWarnings bool) {
log = New(
Options{
Level: level,
DistinctLevel: logg.LevelInfo,
HandlerPost: logHookLast,
Distinct: true,
HandlerPost: logHookLast,
},
)
}
@@ -50,5 +49,5 @@ func Log() Logger {
var log Logger
func init() {
InitGlobalLogger(logg.LevelWarn, false)
InitGlobalLogger(false)
}
-90
View File
@@ -1,90 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package maps
import "sync"
// Cache is a simple thread safe cache backed by a map.
type Cache[K comparable, T any] struct {
m map[K]T
sync.RWMutex
}
// NewCache creates a new Cache.
func NewCache[K comparable, T any]() *Cache[K, T] {
return &Cache[K, T]{m: make(map[K]T)}
}
// Delete deletes the given key from the cache.
func (c *Cache[K, T]) Get(key K) (T, bool) {
c.RLock()
v, found := c.m[key]
c.RUnlock()
return v, found
}
// GetOrCreate gets the value for the given key if it exists, or creates it if not.
func (c *Cache[K, T]) GetOrCreate(key K, create func() T) T {
c.RLock()
v, found := c.m[key]
c.RUnlock()
if found {
return v
}
c.Lock()
defer c.Unlock()
v, found = c.m[key]
if found {
return v
}
v = create()
c.m[key] = v
return v
}
// Set sets the given key to the given value.
func (c *Cache[K, T]) Set(key K, value T) {
c.Lock()
c.m[key] = value
c.Unlock()
}
// SliceCache is a simple thread safe cache backed by a map.
type SliceCache[T any] struct {
m map[string][]T
sync.RWMutex
}
func NewSliceCache[T any]() *SliceCache[T] {
return &SliceCache[T]{m: make(map[string][]T)}
}
func (c *SliceCache[T]) Get(key string) ([]T, bool) {
c.RLock()
v, found := c.m[key]
c.RUnlock()
return v, found
}
func (c *SliceCache[T]) Append(key string, values ...T) {
c.Lock()
c.m[key] = append(c.m[key], values...)
c.Unlock()
}
func (c *SliceCache[T]) Reset() {
c.Lock()
c.m = make(map[string][]T)
c.Unlock()
}
+9 -8
View File
@@ -29,7 +29,7 @@ func ToStringMapE(in any) (map[string]any, error) {
case Params:
return vv, nil
case map[string]string:
m := map[string]any{}
var m = map[string]any{}
for k, v := range vv {
m[k] = v
}
@@ -192,20 +192,21 @@ func (KeyRenamer) keyPath(k1, k2 string) string {
}
func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]any) {
for k, v := range m {
keyPath := r.keyPath(parentKeyPath, k)
switch vv := v.(type) {
for key, val := range m {
keyPath := r.keyPath(parentKeyPath, key)
switch val.(type) {
case map[any]any:
r.renamePath(keyPath, cast.ToStringMap(vv))
val = cast.ToStringMap(val)
r.renamePath(keyPath, val.(map[string]any))
case map[string]any:
r.renamePath(keyPath, vv)
r.renamePath(keyPath, val.(map[string]any))
}
newKey := r.getNewKey(keyPath)
if newKey != "" {
delete(m, k)
m[newKey] = v
delete(m, key)
m[newKey] = val
}
}
}
+6 -4
View File
@@ -61,7 +61,7 @@ func SetParams(dst, src Params) {
// IsZero returns true if p is considered empty.
func (p Params) IsZero() bool {
if len(p) == 0 {
if p == nil || len(p) == 0 {
return true
}
@@ -74,6 +74,7 @@ func (p Params) IsZero() bool {
}
return false
}
// MergeParamsWithStrategy transfers values from src to dst for new keys using the merge strategy given.
@@ -92,7 +93,7 @@ func MergeParams(dst, src Params) {
func (p Params) merge(ps ParamsMergeStrategy, pp Params) {
ns, found := p.GetMergeStrategy()
ms := ns
var ms = ns
if !found && ps != "" {
ms = ps
}
@@ -247,7 +248,7 @@ const (
// CleanConfigStringMapString removes any processing instructions from m,
// m will never be modified.
func CleanConfigStringMapString(m map[string]string) map[string]string {
if len(m) == 0 {
if m == nil || len(m) == 0 {
return m
}
if _, found := m[MergeStrategyKey]; !found {
@@ -266,7 +267,7 @@ func CleanConfigStringMapString(m map[string]string) map[string]string {
// CleanConfigStringMap is the same as CleanConfigStringMapString but for
// map[string]any.
func CleanConfigStringMap(m map[string]any) map[string]any {
if len(m) == 0 {
if m == nil || len(m) == 0 {
return m
}
if _, found := m[MergeStrategyKey]; !found {
@@ -290,6 +291,7 @@ func CleanConfigStringMap(m map[string]any) map[string]any {
}
return m2
}
func toMergeStrategy(v any) ParamsMergeStrategy {
+2 -3
View File
@@ -32,9 +32,8 @@ func TestPara(t *testing.T) {
t.Skipf("skip para test, CPU count is %d", runtime.NumCPU())
}
// TODO(bep)
if htesting.IsCI() {
t.Skip("skip para test when running on CI")
if !htesting.IsCI() {
t.Skip("skip para test when not running on CI")
}
c := qt.New(t)
+52 -170
View File
@@ -16,18 +16,14 @@ package paths
import (
"errors"
"fmt"
"net/url"
"path"
"path/filepath"
"regexp"
"strings"
"unicode"
)
// FilePathSeparator as defined by os.Separator.
const (
FilePathSeparator = string(filepath.Separator)
slash = "/"
)
const FilePathSeparator = string(filepath.Separator)
// filepathPathBridge is a bridge for common functionality in filepath vs path
type filepathPathBridge interface {
@@ -76,30 +72,6 @@ func AbsPathify(workingDir, inPath string) string {
return filepath.Join(workingDir, inPath)
}
// AddTrailingSlash adds a trailing Unix styled slash (/) if not already
// there.
func AddTrailingSlash(path string) string {
if !strings.HasSuffix(path, "/") {
path += "/"
}
return path
}
// AddLeadingSlash adds a leading Unix styled slash (/) if not already
// there.
func AddLeadingSlash(path string) string {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return path
}
// AddTrailingAndLeadingSlash adds a leading and trailing Unix styled slash (/) if not already
// there.
func AddLeadingAndTrailingSlash(path string) string {
return AddTrailingSlash(AddLeadingSlash(path))
}
// MakeTitle converts the path given to a suitable title, trimming whitespace
// and replacing hyphens with whitespace.
func MakeTitle(inpath string) string {
@@ -122,6 +94,43 @@ func makePathRelative(inPath string, possibleDirectories ...string) (string, err
return inPath, errors.New("can't extract relative path, unknown prefix")
}
// Should be good enough for Hugo.
var isFileRe = regexp.MustCompile(`.*\..{1,6}$`)
// GetDottedRelativePath expects a relative path starting after the content directory.
// It returns a relative path with dots ("..") navigating up the path structure.
func GetDottedRelativePath(inPath string) string {
inPath = path.Clean(filepath.ToSlash(inPath))
if inPath == "." {
return "./"
}
if !isFileRe.MatchString(inPath) && !strings.HasSuffix(inPath, "/") {
inPath += "/"
}
if !strings.HasPrefix(inPath, "/") {
inPath = "/" + inPath
}
dir, _ := filepath.Split(inPath)
sectionCount := strings.Count(dir, "/")
if sectionCount == 0 || dir == "/" {
return "./"
}
var dottedPath string
for i := 1; i < sectionCount; i++ {
dottedPath += "../"
}
return dottedPath
}
// ExtNoDelimiter takes a path and returns the extension, excluding the delimiter, i.e. "md".
func ExtNoDelimiter(in string) string {
return strings.TrimPrefix(Ext(in), ".")
@@ -158,6 +167,12 @@ func Filename(in string) (name string) {
return
}
// PathNoExt takes a path, strips out the extension,
// and returns the name of the file.
func PathNoExt(in string) string {
return strings.TrimSuffix(in, path.Ext(in))
}
// FileAndExt returns the filename and any extension of a file path as
// two separate strings.
//
@@ -237,125 +252,16 @@ func prettifyPath(in string, b filepathPathBridge) string {
return b.Join(b.Dir(in), name, "index"+ext)
}
// CommonDir returns the common directory of the given paths.
func CommonDir(path1, path2 string) string {
if path1 == "" || path2 == "" {
return ""
}
p1 := strings.Split(path1, "/")
p2 := strings.Split(path2, "/")
var common []string
for i := 0; i < len(p1) && i < len(p2); i++ {
if p1[i] == p2[i] {
common = append(common, p1[i])
} else {
break
}
}
return strings.Join(common, "/")
type NamedSlice struct {
Name string
Slice []string
}
// Sanitize sanitizes string to be used in Hugo's file paths and URLs, allowing only
// a predefined set of special Unicode characters.
//
// Spaces will be replaced with a single hyphen.
//
// This function is the core function used to normalize paths in Hugo.
//
// Note that this is the first common step for URL/path sanitation,
// the final URL/path may end up looking differently if the user has stricter rules defined (e.g. removePathAccents=true).
func Sanitize(s string) string {
var willChange bool
for i, r := range s {
willChange = !isAllowedPathCharacter(s, i, r)
if willChange {
break
}
func (n NamedSlice) String() string {
if len(n.Slice) == 0 {
return n.Name
}
if !willChange {
// Prevent allocation when nothing changes.
return s
}
target := make([]rune, 0, len(s))
var (
prependHyphen bool
wasHyphen bool
)
for i, r := range s {
isAllowed := isAllowedPathCharacter(s, i, r)
if isAllowed {
// track explicit hyphen in input; no need to add a new hyphen if
// we just saw one.
wasHyphen = r == '-'
if prependHyphen {
// if currently have a hyphen, don't prepend an extra one
if !wasHyphen {
target = append(target, '-')
}
prependHyphen = false
}
target = append(target, r)
} else if len(target) > 0 && !wasHyphen && unicode.IsSpace(r) {
prependHyphen = true
}
}
return string(target)
}
func isAllowedPathCharacter(s string, i int, r rune) bool {
if r == ' ' {
return false
}
// Check for the most likely first (faster).
isAllowed := unicode.IsLetter(r) || unicode.IsDigit(r)
isAllowed = isAllowed || r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' || r == '-' || r == '@'
isAllowed = isAllowed || unicode.IsMark(r)
isAllowed = isAllowed || (r == '%' && i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2]))
return isAllowed
}
// From https://golang.org/src/net/url/url.go
func ishex(c byte) bool {
switch {
case '0' <= c && c <= '9':
return true
case 'a' <= c && c <= 'f':
return true
case 'A' <= c && c <= 'F':
return true
}
return false
}
var slashFunc = func(r rune) bool {
return r == '/'
}
// Dir behaves like path.Dir without the path.Clean step.
//
// The returned path ends in a slash only if it is the root "/".
func Dir(s string) string {
dir, _ := path.Split(s)
if len(dir) > 1 && dir[len(dir)-1] == '/' {
return dir[:len(dir)-1]
}
return dir
}
// FieldsSlash cuts s into fields separated with '/'.
func FieldsSlash(s string) []string {
f := strings.FieldsFunc(s, slashFunc)
return f
return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ","))
}
// DirFile holds the result from path.Split.
@@ -368,27 +274,3 @@ type DirFile struct {
func (df DirFile) String() string {
return fmt.Sprintf("%s|%s", df.Dir, df.File)
}
// PathEscape escapes unicode letters in pth.
// Use URLEscape to escape full URLs including scheme, query etc.
// This is slightly faster for the common case.
// Note, there is a url.PathEscape function, but that also
// escapes /.
func PathEscape(pth string) string {
u, err := url.Parse(pth)
if err != nil {
panic(err)
}
return u.EscapedPath()
}
// ToSlashTrimLeading is just a filepath.ToSlash with an added / prefix trimmer.
func ToSlashTrimLeading(s string) string {
return strings.TrimPrefix(filepath.ToSlash(s), "/")
}
// ToSlashPreserveLeading converts the path given to a forward slash separated path
// and preserves the leading slash if present trimming any trailing slash.
func ToSlashPreserveLeading(s string) string {
return "/" + strings.Trim(filepath.ToSlash(s), "/")
}
+39 -75
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -75,6 +75,44 @@ func TestMakePathRelative(t *testing.T) {
}
}
func TestGetDottedRelativePath(t *testing.T) {
// on Windows this will receive both kinds, both country and western ...
for _, f := range []func(string) string{filepath.FromSlash, func(s string) string { return s }} {
doTestGetDottedRelativePath(f, t)
}
}
func doTestGetDottedRelativePath(urlFixer func(string) string, t *testing.T) {
type test struct {
input, expected string
}
data := []test{
{"", "./"},
{urlFixer("/"), "./"},
{urlFixer("post"), "../"},
{urlFixer("/post"), "../"},
{urlFixer("post/"), "../"},
{urlFixer("tags/foo.html"), "../"},
{urlFixer("/tags/foo.html"), "../"},
{urlFixer("/post/"), "../"},
{urlFixer("////post/////"), "../"},
{urlFixer("/foo/bar/index.html"), "../../"},
{urlFixer("/foo/bar/foo/"), "../../../"},
{urlFixer("/foo/bar/foo"), "../../../"},
{urlFixer("foo/bar/foo/"), "../../../"},
{urlFixer("foo/bar/foo/bar"), "../../../../"},
{"404.html", "./"},
{"404.xml", "./"},
{"/404.html", "./"},
}
for i, d := range data {
output := GetDottedRelativePath(d.input)
if d.expected != output {
t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, output)
}
}
}
func TestMakeTitle(t *testing.T) {
type test struct {
input, expected string
@@ -188,77 +226,3 @@ func TestFileAndExt(t *testing.T) {
}
}
}
func TestSanitize(t *testing.T) {
c := qt.New(t)
tests := []struct {
input string
expected string
}{
{" Foo bar ", "Foo-bar"},
{"Foo.Bar/foo_Bar-Foo", "Foo.Bar/foo_Bar-Foo"},
{"fOO,bar:foobAR", "fOObarfoobAR"},
{"FOo/BaR.html", "FOo/BaR.html"},
{"FOo/Ba---R.html", "FOo/Ba---R.html"}, /// See #10104
{"FOo/Ba R.html", "FOo/Ba-R.html"},
{"трям/трям", "трям/трям"},
{"은행", "은행"},
{"Банковский кассир", "Банковский-кассир"},
// Issue #1488
{"संस्कृत", "संस्कृत"},
{"a%C3%B1ame", "a%C3%B1ame"}, // Issue #1292
{"this+is+a+test", "this+is+a+test"}, // Issue #1290
{"~foo", "~foo"}, // Issue #2177
}
for _, test := range tests {
c.Assert(Sanitize(test.input), qt.Equals, test.expected)
}
}
func BenchmarkSanitize(b *testing.B) {
const (
allAlowedPath = "foo/bar"
spacePath = "foo bar"
)
// This should not allocate any memory.
b.Run("All allowed", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := Sanitize(allAlowedPath)
if got != allAlowedPath {
b.Fatal(got)
}
}
})
// This will allocate some memory.
b.Run("Spaces", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := Sanitize(spacePath)
if got != "foo-bar" {
b.Fatal(got)
}
}
})
}
func TestDir(t *testing.T) {
c := qt.New(t)
c.Assert(Dir("/a/b/c/d"), qt.Equals, "/a/b/c")
c.Assert(Dir("/a"), qt.Equals, "/")
c.Assert(Dir("/"), qt.Equals, "/")
c.Assert(Dir(""), qt.Equals, "")
}
func TestFieldsSlash(t *testing.T) {
c := qt.New(t)
c.Assert(FieldsSlash("a/b/c"), qt.DeepEquals, []string{"a", "b", "c"})
c.Assert(FieldsSlash("/a/b/c"), qt.DeepEquals, []string{"a", "b", "c"})
c.Assert(FieldsSlash("/a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"})
c.Assert(FieldsSlash("a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"})
c.Assert(FieldsSlash("/"), qt.DeepEquals, []string{})
c.Assert(FieldsSlash(""), qt.DeepEquals, []string{})
}
-494
View File
@@ -1,494 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package paths
import (
"path"
"path/filepath"
"runtime"
"strings"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/hugofs/files"
)
var defaultPathParser PathParser
// PathParser parses a path into a Path.
type PathParser struct {
// Maps the language code to its index in the languages/sites slice.
LanguageIndex map[string]int
}
// Parse parses component c with path s into Path using the default path parser.
func Parse(c, s string) *Path {
return defaultPathParser.Parse(c, s)
}
// NormalizePathString returns a normalized path string using the very basic Hugo rules.
func NormalizePathStringBasic(s string) string {
// All lower case.
s = strings.ToLower(s)
// Replace spaces with hyphens.
s = strings.ReplaceAll(s, " ", "-")
return s
}
// Parse parses component c with path s into Path using Hugo's content path rules.
func (parser PathParser) Parse(c, s string) *Path {
p, err := parser.parse(c, s)
if err != nil {
panic(err)
}
return p
}
func (pp *PathParser) parse(component, s string) (*Path, error) {
ss := NormalizePathStringBasic(s)
p, err := pp.doParse(component, ss)
if err != nil {
return nil, err
}
if s != ss {
var err error
// Preserve the original case for titles etc.
p.unnormalized, err = pp.doParse(component, s)
if err != nil {
return nil, err
}
} else {
p.unnormalized = p
}
return p, nil
}
func (pp *PathParser) doParse(component, s string) (*Path, error) {
p := &Path{
component: component,
posContainerLow: -1,
posContainerHigh: -1,
posSectionHigh: -1,
posIdentifierLanguage: -1,
}
hasLang := pp.LanguageIndex != nil
hasLang = hasLang && (component == files.ComponentFolderContent || component == files.ComponentFolderLayouts)
if runtime.GOOS == "windows" {
s = path.Clean(filepath.ToSlash(s))
if s == "." {
s = ""
}
}
if s == "" {
s = "/"
}
// Leading slash, no trailing slash.
if !strings.HasPrefix(s, "/") {
s = "/" + s
}
if s != "/" && s[len(s)-1] == '/' {
s = s[:len(s)-1]
}
p.s = s
slashCount := 0
for i := len(s) - 1; i >= 0; i-- {
c := s[i]
switch c {
case '.':
if p.posContainerHigh == -1 {
var high int
if len(p.identifiers) > 0 {
high = p.identifiers[len(p.identifiers)-1].Low - 1
} else {
high = len(p.s)
}
id := types.LowHigh{Low: i + 1, High: high}
if len(p.identifiers) == 0 {
p.identifiers = append(p.identifiers, id)
} else if len(p.identifiers) == 1 {
// Check for a valid language.
s := p.s[id.Low:id.High]
if hasLang {
if _, found := pp.LanguageIndex[s]; found {
p.posIdentifierLanguage = 1
p.identifiers = append(p.identifiers, id)
}
}
}
}
case '/':
slashCount++
if p.posContainerHigh == -1 {
p.posContainerHigh = i + 1
} else if p.posContainerLow == -1 {
p.posContainerLow = i + 1
}
if i > 0 {
p.posSectionHigh = i
}
}
}
isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes
isContent := isContentComponent && files.IsContentExt(p.Ext())
if isContent {
id := p.identifiers[len(p.identifiers)-1]
b := p.s[p.posContainerHigh : id.Low-1]
switch b {
case "index":
p.bundleType = PathTypeLeaf
case "_index":
p.bundleType = PathTypeBranch
default:
p.bundleType = PathTypeContentSingle
}
if slashCount == 2 && p.IsLeafBundle() {
p.posSectionHigh = 0
}
}
return p, nil
}
func ModifyPathBundleTypeResource(p *Path) {
if p.IsContent() {
p.bundleType = PathTypeContentResource
} else {
p.bundleType = PathTypeFile
}
}
type PathType int
const (
// A generic resource, e.g. a JSON file.
PathTypeFile PathType = iota
// All below are content files.
// A resource of a content type with front matter.
PathTypeContentResource
// E.g. /blog/my-post.md
PathTypeContentSingle
// All bewlow are bundled content files.
// Leaf bundles, e.g. /blog/my-post/index.md
PathTypeLeaf
// Branch bundles, e.g. /blog/_index.md
PathTypeBranch
)
type Path struct {
s string
posContainerLow int
posContainerHigh int
posSectionHigh int
component string
bundleType PathType
identifiers []types.LowHigh
posIdentifierLanguage int
trimLeadingSlash bool
unnormalized *Path
}
// TrimLeadingSlash returns a copy of the Path with the leading slash removed.
func (p Path) TrimLeadingSlash() *Path {
p.trimLeadingSlash = true
return &p
}
func (p *Path) norm(s string) string {
if p.trimLeadingSlash {
s = strings.TrimPrefix(s, "/")
}
return s
}
// IdentifierBase satifies identity.Identity.
func (p *Path) IdentifierBase() string {
return p.Base()[1:]
}
// Component returns the component for this path (e.g. "content").
func (p *Path) Component() string {
return p.component
}
// Container returns the base name of the container directory for this path.
func (p *Path) Container() string {
if p.posContainerLow == -1 {
return ""
}
return p.norm(p.s[p.posContainerLow : p.posContainerHigh-1])
}
// ContainerDir returns the container directory for this path.
// For content bundles this will be the parent directory.
func (p *Path) ContainerDir() string {
if p.posContainerLow == -1 || !p.IsBundle() {
return p.Dir()
}
return p.norm(p.s[:p.posContainerLow-1])
}
// Section returns the first path element (section).
func (p *Path) Section() string {
if p.posSectionHigh <= 0 {
return ""
}
return p.norm(p.s[1:p.posSectionHigh])
}
// IsContent returns true if the path is a content file (e.g. mypost.md).
// Note that this will also return true for content files in a bundle.
func (p *Path) IsContent() bool {
return p.BundleType() >= PathTypeContentResource
}
// isContentPage returns true if the path is a content file (e.g. mypost.md),
// but nof if inside a leaf bundle.
func (p *Path) isContentPage() bool {
return p.BundleType() >= PathTypeContentSingle
}
// Name returns the last element of path.
func (p *Path) Name() string {
if p.posContainerHigh > 0 {
return p.s[p.posContainerHigh:]
}
return p.s
}
// Name returns the last element of path withhout any extension.
func (p *Path) NameNoExt() string {
if i := p.identifierIndex(0); i != -1 {
return p.s[p.posContainerHigh : p.identifiers[i].Low-1]
}
return p.s[p.posContainerHigh:]
}
// Name returns the last element of path withhout any language identifier.
func (p *Path) NameNoLang() string {
i := p.identifierIndex(p.posIdentifierLanguage)
if i == -1 {
return p.Name()
}
return p.s[p.posContainerHigh:p.identifiers[i].Low-1] + p.s[p.identifiers[i].High:]
}
// BaseNameNoIdentifier returns the logcical base name for a resource without any idenifier (e.g. no extension).
// For bundles this will be the containing directory's name, e.g. "blog".
func (p *Path) BaseNameNoIdentifier() string {
if p.IsBundle() {
return p.Container()
}
return p.NameNoIdentifier()
}
// NameNoIdentifier returns the last element of path withhout any identifier (e.g. no extension).
func (p *Path) NameNoIdentifier() string {
if len(p.identifiers) > 0 {
return p.s[p.posContainerHigh : p.identifiers[len(p.identifiers)-1].Low-1]
}
return p.s[p.posContainerHigh:]
}
// Dir returns all but the last element of path, typically the path's directory.
func (p *Path) Dir() (d string) {
if p.posContainerHigh > 0 {
d = p.s[:p.posContainerHigh-1]
}
if d == "" {
d = "/"
}
d = p.norm(d)
return
}
// Path returns the full path.
func (p *Path) Path() (d string) {
return p.norm(p.s)
}
// Unmormalized returns the Path with the original case preserved.
func (p *Path) Unmormalized() *Path {
return p.unnormalized
}
// PathNoLang returns the Path but with any language identifier removed.
func (p *Path) PathNoLang() string {
return p.base(true, false)
}
// PathNoIdentifier returns the Path but with any identifier (ext, lang) removed.
func (p *Path) PathNoIdentifier() string {
return p.base(false, false)
}
// PathRel returns the path relativeto the given owner.
func (p *Path) PathRel(owner *Path) string {
ob := owner.Base()
if !strings.HasSuffix(ob, "/") {
ob += "/"
}
return strings.TrimPrefix(p.Path(), ob)
}
// BaseRel returns the base path relative to the given owner.
func (p *Path) BaseRel(owner *Path) string {
ob := owner.Base()
if ob == "/" {
ob = ""
}
return p.Base()[len(ob)+1:]
}
// For content files, Base returns the path without any identifiers (extension, language code etc.).
// Any 'index' as the last path element is ignored.
//
// For other files (Resources), any extension is kept.
func (p *Path) Base() string {
return p.base(!p.isContentPage(), p.IsBundle())
}
// BaseNoLeadingSlash returns the base path without the leading slash.
func (p *Path) BaseNoLeadingSlash() string {
return p.Base()[1:]
}
func (p *Path) base(preserveExt, isBundle bool) string {
if len(p.identifiers) == 0 {
return p.norm(p.s)
}
if preserveExt && len(p.identifiers) == 1 {
// Preserve extension.
return p.norm(p.s)
}
id := p.identifiers[len(p.identifiers)-1]
high := id.Low - 1
if isBundle {
high = p.posContainerHigh - 1
}
if high == 0 {
high++
}
if !preserveExt {
return p.norm(p.s[:high])
}
// For txt files etc. we want to preserve the extension.
id = p.identifiers[0]
return p.norm(p.s[:high] + p.s[id.Low-1:id.High])
}
func (p *Path) Ext() string {
return p.identifierAsString(0)
}
func (p *Path) Lang() string {
return p.identifierAsString(1)
}
func (p *Path) Identifier(i int) string {
return p.identifierAsString(i)
}
func (p *Path) Identifiers() []string {
ids := make([]string, len(p.identifiers))
for i, id := range p.identifiers {
ids[i] = p.s[id.Low:id.High]
}
return ids
}
func (p *Path) IsHTML() bool {
return files.IsHTML(p.Ext())
}
func (p *Path) BundleType() PathType {
return p.bundleType
}
func (p *Path) IsBundle() bool {
return p.bundleType >= PathTypeLeaf
}
func (p *Path) IsBranchBundle() bool {
return p.bundleType == PathTypeBranch
}
func (p *Path) IsLeafBundle() bool {
return p.bundleType == PathTypeLeaf
}
func (p *Path) identifierAsString(i int) string {
i = p.identifierIndex(i)
if i == -1 {
return ""
}
id := p.identifiers[i]
return p.s[id.Low:id.High]
}
func (p *Path) identifierIndex(i int) int {
if i < 0 || i >= len(p.identifiers) {
return -1
}
return i
}
// HasExt returns true if the Unix styled path has an extension.
func HasExt(p string) bool {
for i := len(p) - 1; i >= 0; i-- {
if p[i] == '.' {
return true
}
if p[i] == '/' {
return false
}
}
return false
}
-351
View File
@@ -1,351 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package paths
import (
"path/filepath"
"testing"
"github.com/gohugoio/hugo/hugofs/files"
qt "github.com/frankban/quicktest"
)
var testParser = &PathParser{
LanguageIndex: map[string]int{
"no": 0,
"en": 1,
},
}
func TestParse(t *testing.T) {
c := qt.New(t)
tests := []struct {
name string
path string
assert func(c *qt.C, p *Path)
}{
{
"Basic text file",
"/a/b.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b.txt")
c.Assert(p.Base(), qt.Equals, "/a/b.txt")
c.Assert(p.Container(), qt.Equals, "a")
c.Assert(p.Dir(), qt.Equals, "/a")
c.Assert(p.Ext(), qt.Equals, "txt")
c.Assert(p.IsContent(), qt.IsFalse)
},
},
{
"Basic text file, upper case",
"/A/B.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b.txt")
c.Assert(p.NameNoExt(), qt.Equals, "b")
c.Assert(p.NameNoIdentifier(), qt.Equals, "b")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b")
c.Assert(p.Base(), qt.Equals, "/a/b.txt")
c.Assert(p.Ext(), qt.Equals, "txt")
},
},
{
"Basic text file, 1 space in dir",
"/a b/c.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a-b/c.txt")
},
},
{
"Basic text file, 2 spaces in dir",
"/a b/c.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a--b/c.txt")
},
},
{
"Basic text file, 1 space in filename",
"/a/b c.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b-c.txt")
},
},
{
"Basic text file, 2 spaces in filename",
"/a/b c.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b--c.txt")
},
},
{
"Basic text file, mixed case and spaces, unnormalized",
"/a/Foo BAR.txt",
func(c *qt.C, p *Path) {
pp := p.Unmormalized()
c.Assert(pp, qt.IsNotNil)
c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "Foo BAR")
},
},
{
"Basic Markdown file",
"/a/b/c.md",
func(c *qt.C, p *Path) {
c.Assert(p.IsContent(), qt.IsTrue)
c.Assert(p.IsLeafBundle(), qt.IsFalse)
c.Assert(p.Name(), qt.Equals, "c.md")
c.Assert(p.Base(), qt.Equals, "/a/b/c")
c.Assert(p.Section(), qt.Equals, "a")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "c")
c.Assert(p.Path(), qt.Equals, "/a/b/c.md")
c.Assert(p.Dir(), qt.Equals, "/a/b")
c.Assert(p.Container(), qt.Equals, "b")
c.Assert(p.ContainerDir(), qt.Equals, "/a/b")
c.Assert(p.Ext(), qt.Equals, "md")
},
},
{
"Content resource",
"/a/b.md",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b.md")
c.Assert(p.Base(), qt.Equals, "/a/b")
c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b")
c.Assert(p.Section(), qt.Equals, "a")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b")
// Reclassify it as a content resource.
ModifyPathBundleTypeResource(p)
c.Assert(p.BundleType(), qt.Equals, PathTypeContentResource)
c.Assert(p.IsContent(), qt.IsTrue)
c.Assert(p.Name(), qt.Equals, "b.md")
c.Assert(p.Base(), qt.Equals, "/a/b.md")
},
},
{
"No ext",
"/a/b",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b")
c.Assert(p.NameNoExt(), qt.Equals, "b")
c.Assert(p.Base(), qt.Equals, "/a/b")
c.Assert(p.Ext(), qt.Equals, "")
},
},
{
"No ext, trailing slash",
"/a/b/",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b")
c.Assert(p.Base(), qt.Equals, "/a/b")
c.Assert(p.Ext(), qt.Equals, "")
},
},
{
"Identifiers",
"/a/b.a.b.no.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Name(), qt.Equals, "b.a.b.no.txt")
c.Assert(p.NameNoIdentifier(), qt.Equals, "b.a.b")
c.Assert(p.NameNoLang(), qt.Equals, "b.a.b.txt")
c.Assert(p.Identifiers(), qt.DeepEquals, []string{"txt", "no"})
c.Assert(p.Base(), qt.Equals, "/a/b.a.b.txt")
c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b.a.b.txt")
c.Assert(p.PathNoLang(), qt.Equals, "/a/b.a.b.txt")
c.Assert(p.Ext(), qt.Equals, "txt")
c.Assert(p.PathNoIdentifier(), qt.Equals, "/a/b.a.b")
},
},
{
"Home branch cundle",
"/_index.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/")
c.Assert(p.Path(), qt.Equals, "/_index.md")
c.Assert(p.Container(), qt.Equals, "")
c.Assert(p.ContainerDir(), qt.Equals, "/")
},
},
{
"Index content file in root",
"/a/index.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "a")
c.Assert(p.Container(), qt.Equals, "a")
c.Assert(p.Container(), qt.Equals, "a")
c.Assert(p.ContainerDir(), qt.Equals, "")
c.Assert(p.Dir(), qt.Equals, "/a")
c.Assert(p.Ext(), qt.Equals, "md")
c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md"})
c.Assert(p.IsBranchBundle(), qt.IsFalse)
c.Assert(p.IsBundle(), qt.IsTrue)
c.Assert(p.IsLeafBundle(), qt.IsTrue)
c.Assert(p.Lang(), qt.Equals, "")
c.Assert(p.NameNoExt(), qt.Equals, "index")
c.Assert(p.NameNoIdentifier(), qt.Equals, "index")
c.Assert(p.NameNoLang(), qt.Equals, "index.md")
c.Assert(p.Section(), qt.Equals, "")
},
},
{
"Index content file with lang",
"/a/b/index.no.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b")
c.Assert(p.Container(), qt.Equals, "b")
c.Assert(p.ContainerDir(), qt.Equals, "/a")
c.Assert(p.Dir(), qt.Equals, "/a/b")
c.Assert(p.Ext(), qt.Equals, "md")
c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md", "no"})
c.Assert(p.IsBranchBundle(), qt.IsFalse)
c.Assert(p.IsBundle(), qt.IsTrue)
c.Assert(p.IsLeafBundle(), qt.IsTrue)
c.Assert(p.Lang(), qt.Equals, "no")
c.Assert(p.NameNoExt(), qt.Equals, "index.no")
c.Assert(p.NameNoIdentifier(), qt.Equals, "index")
c.Assert(p.NameNoLang(), qt.Equals, "index.md")
c.Assert(p.PathNoLang(), qt.Equals, "/a/b/index.md")
c.Assert(p.Section(), qt.Equals, "a")
},
},
{
"Index branch content file",
"/a/b/_index.no.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b")
c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b")
c.Assert(p.Container(), qt.Equals, "b")
c.Assert(p.ContainerDir(), qt.Equals, "/a")
c.Assert(p.Ext(), qt.Equals, "md")
c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md", "no"})
c.Assert(p.IsBranchBundle(), qt.IsTrue)
c.Assert(p.IsBundle(), qt.IsTrue)
c.Assert(p.IsLeafBundle(), qt.IsFalse)
c.Assert(p.NameNoExt(), qt.Equals, "_index.no")
c.Assert(p.NameNoLang(), qt.Equals, "_index.md")
},
},
{
"Index root no slash",
"_index.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/")
c.Assert(p.Ext(), qt.Equals, "md")
c.Assert(p.Name(), qt.Equals, "_index.md")
},
},
{
"Index root",
"/_index.md",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/")
c.Assert(p.Ext(), qt.Equals, "md")
c.Assert(p.Name(), qt.Equals, "_index.md")
},
},
{
"Index first",
"/a/_index.md",
func(c *qt.C, p *Path) {
c.Assert(p.Section(), qt.Equals, "a")
},
},
{
"Index text file",
"/a/b/index.no.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b/index.txt")
c.Assert(p.Ext(), qt.Equals, "txt")
c.Assert(p.Identifiers(), qt.DeepEquals, []string{"txt", "no"})
c.Assert(p.IsLeafBundle(), qt.IsFalse)
c.Assert(p.PathNoIdentifier(), qt.Equals, "/a/b/index")
},
},
{
"Empty",
"",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/")
c.Assert(p.Ext(), qt.Equals, "")
c.Assert(p.Name(), qt.Equals, "")
c.Assert(p.Path(), qt.Equals, "/")
},
},
{
"Slash",
"/",
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/")
c.Assert(p.Ext(), qt.Equals, "")
c.Assert(p.Name(), qt.Equals, "")
},
},
{
"Trim Leading Slash bundle",
"foo/bar/index.no.md",
func(c *qt.C, p *Path) {
c.Assert(p.Path(), qt.Equals, "/foo/bar/index.no.md")
pp := p.TrimLeadingSlash()
c.Assert(pp.Path(), qt.Equals, "foo/bar/index.no.md")
c.Assert(pp.PathNoLang(), qt.Equals, "foo/bar/index.md")
c.Assert(pp.Base(), qt.Equals, "foo/bar")
c.Assert(pp.Dir(), qt.Equals, "foo/bar")
c.Assert(pp.ContainerDir(), qt.Equals, "foo")
c.Assert(pp.Container(), qt.Equals, "bar")
c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "bar")
},
},
{
"Trim Leading Slash file",
"foo/bar.txt",
func(c *qt.C, p *Path) {
c.Assert(p.Path(), qt.Equals, "/foo/bar.txt")
pp := p.TrimLeadingSlash()
c.Assert(pp.Path(), qt.Equals, "foo/bar.txt")
c.Assert(pp.PathNoLang(), qt.Equals, "foo/bar.txt")
c.Assert(pp.Base(), qt.Equals, "foo/bar.txt")
c.Assert(pp.Dir(), qt.Equals, "foo")
c.Assert(pp.ContainerDir(), qt.Equals, "foo")
c.Assert(pp.Container(), qt.Equals, "foo")
c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "bar")
},
},
{
"File separator",
filepath.FromSlash("/a/b/c.txt"),
func(c *qt.C, p *Path) {
c.Assert(p.Base(), qt.Equals, "/a/b/c.txt")
c.Assert(p.Ext(), qt.Equals, "txt")
c.Assert(p.Name(), qt.Equals, "c.txt")
c.Assert(p.Path(), qt.Equals, "/a/b/c.txt")
},
},
}
for _, test := range tests {
c.Run(test.name, func(c *qt.C) {
test.assert(c, testParser.Parse(files.ComponentFolderContent, test.path))
})
}
}
func TestHasExt(t *testing.T) {
c := qt.New(t)
c.Assert(HasExt("/a/b/c.txt"), qt.IsTrue)
c.Assert(HasExt("/a/b.c/d.txt"), qt.IsTrue)
c.Assert(HasExt("/a/b/c"), qt.IsFalse)
c.Assert(HasExt("/a/b.c/d"), qt.IsFalse)
}
-80
View File
@@ -1,80 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package paths_test
import (
"testing"
"github.com/gohugoio/hugo/hugolib"
)
func TestRemovePathAccents(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
disableKinds = ["taxonomy", "term"]
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
[languages]
[languages.en]
weight = 1
[languages.fr]
weight = 2
removePathAccents = true
-- content/διακριτικός.md --
-- content/διακριτικός.fr.md --
-- layouts/_default/single.html --
{{ .Language.Lang }}|Single.
-- layouts/_default/list.html --
List
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/en/διακριτικός/index.html", "en|Single")
b.AssertFileContent("public/fr/διακριτικος/index.html", "fr|Single")
}
func TestDisablePathToLower(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
disableKinds = ["taxonomy", "term"]
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
[languages]
[languages.en]
weight = 1
[languages.fr]
weight = 2
disablePathToLower = true
-- content/MySection/MyPage.md --
-- content/MySection/MyPage.fr.md --
-- content/MySection/MyBundle/index.md --
-- content/MySection/MyBundle/index.fr.md --
-- layouts/_default/single.html --
{{ .Language.Lang }}|Single.
-- layouts/_default/list.html --
{{ .Language.Lang }}|List.
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/en/mysection/index.html", "en|List")
b.AssertFileContent("public/en/mysection/mypage/index.html", "en|Single")
b.AssertFileContent("public/fr/MySection/index.html", "fr|List")
b.AssertFileContent("public/fr/MySection/MyPage/index.html", "fr|Single")
b.AssertFileContent("public/en/mysection/mybundle/index.html", "en|Single")
b.AssertFileContent("public/fr/MySection/MyBundle/index.html", "fr|Single")
}
-27
View File
@@ -1,27 +0,0 @@
// Code generated by "stringer -type=PathType"; DO NOT EDIT.
package paths
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[PathTypeFile-0]
_ = x[PathTypeContentResource-1]
_ = x[PathTypeContentSingle-2]
_ = x[PathTypeLeaf-3]
_ = x[PathTypeBranch-4]
}
const _PathType_name = "PathTypeFilePathTypeContentResourcePathTypeContentSinglePathTypeLeafPathTypeBranch"
var _PathType_index = [...]uint8{0, 12, 35, 56, 68, 82}
func (i PathType) String() string {
if i < 0 || i >= PathType(len(_PathType_index)-1) {
return "PathType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _PathType_name[_PathType_index[i]:_PathType_index[i+1]]
}
+9 -24
View File
@@ -51,10 +51,9 @@ var pb pathBridge
// MakePermalink combines base URL with content path to create full URL paths.
// Example
//
// base: http://spf13.com/
// path: post/how-i-blog
// result: http://spf13.com/post/how-i-blog
// base: http://spf13.com/
// path: post/how-i-blog
// result: http://spf13.com/post/how-i-blog
func MakePermalink(host, plink string) *url.URL {
base, err := url.Parse(host)
if err != nil {
@@ -71,8 +70,6 @@ func MakePermalink(host, plink string) *url.URL {
}
base.Path = path.Join(base.Path, p.Path)
base.Fragment = p.Fragment
base.RawQuery = p.RawQuery
// path.Join will strip off the last /, so put it back if it was there.
hadTrailingSlash := (plink == "" && strings.HasSuffix(host, "/")) || strings.HasSuffix(p.Path, "/")
@@ -120,19 +117,17 @@ func PrettifyURL(in string) string {
// PrettifyURLPath takes a URL path to a content and converts it
// to enable pretty URLs.
//
// /section/name.html becomes /section/name/index.html
// /section/name/ becomes /section/name/index.html
// /section/name/index.html becomes /section/name/index.html
// /section/name.html becomes /section/name/index.html
// /section/name/ becomes /section/name/index.html
// /section/name/index.html becomes /section/name/index.html
func PrettifyURLPath(in string) string {
return prettifyPath(in, pb)
}
// Uglify does the opposite of PrettifyURLPath().
//
// /section/name/index.html becomes /section/name.html
// /section/name/ becomes /section/name.html
// /section/name.html becomes /section/name.html
// /section/name/index.html becomes /section/name.html
// /section/name/ becomes /section/name.html
// /section/name.html becomes /section/name.html
func Uglify(in string) string {
if path.Ext(in) == "" {
if len(in) < 2 {
@@ -184,13 +179,3 @@ func UrlToFilename(s string) (string, bool) {
return p, true
}
// URLEscape escapes unicode letters.
func URLEscape(uri string) string {
// escape unicode letters
u, err := url.Parse(uri)
if err != nil {
panic(err)
}
return u.String()
}
-1
View File
@@ -31,7 +31,6 @@ func TestMakePermalink(t *testing.T) {
{"http://abc.com", "bar", "http://abc.com/bar"},
{"http://abc.com/foo/bar", "post/bar", "http://abc.com/foo/bar/post/bar"},
{"http://abc.com/foo/bar", "post/bar/", "http://abc.com/foo/bar/post/bar/"},
{"http://abc.com/foo", "post/bar?a=b#c", "http://abc.com/foo/post/bar?a=b#c"},
}
for i, d := range data {
-72
View File
@@ -1,72 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package predicate
// P is a predicate function that tests whether a value of type T satisfies some condition.
type P[T any] func(T) bool
// And returns a predicate that is a short-circuiting logical AND of this and the given predicates.
func (p P[T]) And(ps ...P[T]) P[T] {
return func(v T) bool {
for _, pp := range ps {
if !pp(v) {
return false
}
}
return p(v)
}
}
// Or returns a predicate that is a short-circuiting logical OR of this and the given predicates.
func (p P[T]) Or(ps ...P[T]) P[T] {
return func(v T) bool {
for _, pp := range ps {
if pp(v) {
return true
}
}
return p(v)
}
}
// Negate returns a predicate that is a logical negation of this predicate.
func (p P[T]) Negate() P[T] {
return func(v T) bool {
return !p(v)
}
}
// Filter returns a new slice holding only the elements of s that satisfy p.
// Filter modifies the contents of the slice s and returns the modified slice, which may have a smaller length.
func (p P[T]) Filter(s []T) []T {
var n int
for _, v := range s {
if p(v) {
s[n] = v
n++
}
}
return s[:n]
}
// FilterCopy returns a new slice holding only the elements of s that satisfy p.
func (p P[T]) FilterCopy(s []T) []T {
var result []T
for _, v := range s {
if p(v) {
result = append(result, v)
}
}
return result
}
-83
View File
@@ -1,83 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package predicate_test
import (
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/common/predicate"
)
func TestAdd(t *testing.T) {
c := qt.New(t)
var p predicate.P[int] = intP1
c.Assert(p(1), qt.IsTrue)
c.Assert(p(2), qt.IsFalse)
neg := p.Negate()
c.Assert(neg(1), qt.IsFalse)
c.Assert(neg(2), qt.IsTrue)
and := p.And(intP2)
c.Assert(and(1), qt.IsFalse)
c.Assert(and(2), qt.IsFalse)
c.Assert(and(10), qt.IsTrue)
or := p.Or(intP2)
c.Assert(or(1), qt.IsTrue)
c.Assert(or(2), qt.IsTrue)
c.Assert(or(10), qt.IsTrue)
c.Assert(or(11), qt.IsFalse)
}
func TestFilter(t *testing.T) {
c := qt.New(t)
var p predicate.P[int] = intP1
p = p.Or(intP2)
ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2}
c.Assert(p.Filter(ints), qt.DeepEquals, []int{1, 2, 1, 2})
c.Assert(ints, qt.DeepEquals, []int{1, 2, 1, 2, 1, 6, 7, 8, 2})
}
func TestFilterCopy(t *testing.T) {
c := qt.New(t)
var p predicate.P[int] = intP1
p = p.Or(intP2)
ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2}
c.Assert(p.FilterCopy(ints), qt.DeepEquals, []int{1, 2, 1, 2})
c.Assert(ints, qt.DeepEquals, []int{1, 2, 3, 4, 1, 6, 7, 8, 2})
}
var intP1 = func(i int) bool {
if i == 10 {
return true
}
return i == 1
}
var intP2 = func(i int) bool {
if i == 10 {
return true
}
return i == 2
}
-93
View File
@@ -1,93 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rungroup
import (
"context"
"golang.org/x/sync/errgroup"
)
// Group is a group of workers that can be used to enqueue work and wait for
// them to finish.
type Group[T any] interface {
Enqueue(T) error
Wait() error
}
type runGroup[T any] struct {
ctx context.Context
g *errgroup.Group
ch chan T
}
// Config is the configuration for a new Group.
type Config[T any] struct {
NumWorkers int
Handle func(context.Context, T) error
}
// Run creates a new Group with the given configuration.
func Run[T any](ctx context.Context, cfg Config[T]) Group[T] {
if cfg.NumWorkers <= 0 {
cfg.NumWorkers = 1
}
if cfg.Handle == nil {
panic("Handle must be set")
}
g, ctx := errgroup.WithContext(ctx)
// Buffered for performance.
ch := make(chan T, cfg.NumWorkers)
for i := 0; i < cfg.NumWorkers; i++ {
g.Go(func() error {
for {
select {
case <-ctx.Done():
return nil
case v, ok := <-ch:
if !ok {
return nil
}
if err := cfg.Handle(ctx, v); err != nil {
return err
}
}
}
})
}
return &runGroup[T]{
ctx: ctx,
g: g,
ch: ch,
}
}
// Enqueue enqueues a new item to be handled by the workers.
func (r *runGroup[T]) Enqueue(t T) error {
select {
case <-r.ctx.Done():
return nil
case r.ch <- t:
}
return nil
}
// Wait waits for all workers to finish and returns the first error.
func (r *runGroup[T]) Wait() error {
close(r.ch)
return r.g.Wait()
}
-44
View File
@@ -1,44 +0,0 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rungroup
import (
"context"
"testing"
qt "github.com/frankban/quicktest"
)
func TestNew(t *testing.T) {
c := qt.New(t)
var result int
adder := func(ctx context.Context, i int) error {
result += i
return nil
}
g := Run[int](
context.Background(),
Config[int]{
Handle: adder,
},
)
c.Assert(g, qt.IsNotNil)
g.Enqueue(32)
g.Enqueue(33)
c.Assert(g.Wait(), qt.IsNil)
c.Assert(result, qt.Equals, 65)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+2 -13
View File
@@ -35,11 +35,11 @@ func NewEvictingStringQueue(size int) *EvictingStringQueue {
}
// Add adds a new string to the tail of the queue if it's not already there.
func (q *EvictingStringQueue) Add(v string) *EvictingStringQueue {
func (q *EvictingStringQueue) Add(v string) {
q.mu.Lock()
if q.set[v] {
q.mu.Unlock()
return q
return
}
if len(q.set) == q.size {
@@ -50,17 +50,6 @@ func (q *EvictingStringQueue) Add(v string) *EvictingStringQueue {
q.set[v] = true
q.vals = append(q.vals, v)
q.mu.Unlock()
return q
}
func (q *EvictingStringQueue) Len() int {
if q == nil {
return 0
}
q.mu.Lock()
defer q.mu.Unlock()
return len(q.vals)
}
// Contains returns whether the queue contains v.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2022 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
-13
View File
@@ -92,18 +92,5 @@ type DevMarker interface {
DevOnly()
}
// Unwrapper is implemented by types that can unwrap themselves.
type Unwrapper interface {
// Unwrapv is for internal use only.
// It got its slightly odd name to prevent collisions with user types.
Unwrapv() any
}
// LowHigh is typically used to represent a slice boundary.
type LowHigh struct {
Low int
High int
}
// This is only used for debugging purposes.
var InvocationCounter atomic.Int64
+13 -15
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -23,12 +23,10 @@ import (
// A BaseURL in Hugo is normally on the form scheme://path, but the
// form scheme: is also valid (mailto:hugo@rules.com).
type BaseURL struct {
url *url.URL
WithPath string
WithPathNoTrailingSlash string
WithoutPath string
BasePath string
BasePathNoTrailingSlash string
url *url.URL
WithPath string
WithoutPath string
BasePath string
}
func (b BaseURL) String() string {
@@ -94,19 +92,19 @@ func NewBaseURLFromString(b string) (BaseURL, error) {
return BaseURL{}, err
}
return newBaseURLFromURL(u)
}
func newBaseURLFromURL(u *url.URL) (BaseURL, error) {
// A baseURL should always have a trailing slash, see #11669.
if !strings.HasSuffix(u.Path, "/") {
u.Path += "/"
}
baseURL := BaseURL{url: u, WithPath: u.String(), WithPathNoTrailingSlash: strings.TrimSuffix(u.String(), "/")}
baseURLNoPath := baseURL.URL()
baseURL := BaseURL{url: u, WithPath: u.String()}
var baseURLNoPath = baseURL.URL()
baseURLNoPath.Path = ""
baseURL.WithoutPath = baseURLNoPath.String()
baseURL.BasePath = u.Path
baseURL.BasePathNoTrailingSlash = strings.TrimSuffix(u.Path, "/")
basePath := u.Path
if basePath != "" && basePath != "/" {
baseURL.BasePath = basePath
}
return baseURL, nil
}
+7 -21
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -21,24 +21,17 @@ import (
func TestBaseURL(t *testing.T) {
c := qt.New(t)
b, err := NewBaseURLFromString("http://example.com/")
b, err := NewBaseURLFromString("http://example.com")
c.Assert(err, qt.IsNil)
c.Assert(b.String(), qt.Equals, "http://example.com/")
b, err = NewBaseURLFromString("http://example.com")
c.Assert(err, qt.IsNil)
c.Assert(b.String(), qt.Equals, "http://example.com/")
c.Assert(b.WithPathNoTrailingSlash, qt.Equals, "http://example.com")
c.Assert(b.BasePath, qt.Equals, "/")
c.Assert(b.String(), qt.Equals, "http://example.com")
p, err := b.WithProtocol("webcal://")
c.Assert(err, qt.IsNil)
c.Assert(p.String(), qt.Equals, "webcal://example.com/")
c.Assert(p.String(), qt.Equals, "webcal://example.com")
p, err = b.WithProtocol("webcal")
c.Assert(err, qt.IsNil)
c.Assert(p.String(), qt.Equals, "webcal://example.com/")
c.Assert(p.String(), qt.Equals, "webcal://example.com")
_, err = b.WithProtocol("mailto:")
c.Assert(err, qt.Not(qt.IsNil))
@@ -64,18 +57,11 @@ func TestBaseURL(t *testing.T) {
b, err = NewBaseURLFromString("")
c.Assert(err, qt.IsNil)
c.Assert(b.String(), qt.Equals, "/")
c.Assert(b.String(), qt.Equals, "")
// BaseURL with sub path
b, err = NewBaseURLFromString("http://example.com/sub")
c.Assert(err, qt.IsNil)
c.Assert(b.String(), qt.Equals, "http://example.com/sub/")
c.Assert(b.WithPathNoTrailingSlash, qt.Equals, "http://example.com/sub")
c.Assert(b.BasePath, qt.Equals, "/sub/")
c.Assert(b.BasePathNoTrailingSlash, qt.Equals, "/sub")
b, err = NewBaseURLFromString("http://example.com/sub/")
c.Assert(err, qt.IsNil)
c.Assert(b.String(), qt.Equals, "http://example.com/sub/")
c.Assert(b.String(), qt.Equals, "http://example.com/sub")
c.Assert(b.HostURL(), qt.Equals, "http://example.com")
}
-13
View File
@@ -52,16 +52,3 @@ func Eq(v1, v2 any) bool {
return v1 == v2
}
// ProbablyEq returns whether v1 is probably equal to v2.
func ProbablyEq(v1, v2 any) bool {
if Eq(v1, v2) {
return true
}
if peqer, ok := v1.(ProbablyEqer); ok {
return peqer.ProbablyEq(v2)
}
return false
}
+60 -79
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -27,10 +27,8 @@ import (
"time"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/privacy"
@@ -284,13 +282,12 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
disabledLangs := make(map[string]bool)
for _, lang := range c.DisableLanguages {
if lang == c.DefaultContentLanguage {
return fmt.Errorf("cannot disable default content language %q", lang)
}
disabledLangs[lang] = true
}
for lang, language := range c.Languages {
if !language.Disabled && disabledLangs[lang] {
language.Disabled = true
c.Languages[lang] = language
}
if language.Disabled {
disabledLangs[lang] = true
if lang == c.DefaultContentLanguage {
@@ -410,19 +407,15 @@ type ConfigCompiled struct {
}
// This may be set after the config is compiled.
func (c *ConfigCompiled) SetMainSections(sections []string) {
func (c *ConfigCompiled) SetMainSectionsIfNotSet(sections []string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.MainSections != nil {
return
}
c.MainSections = sections
}
// IsMainSectionsSet returns whether the main sections have been set.
func (c *ConfigCompiled) IsMainSectionsSet() bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.MainSections != nil
}
// This is set after the config is compiled by the server command.
func (c *ConfigCompiled) SetBaseURL(baseURL, baseURLLiveReload urls.BaseURL) {
c.BaseURL = baseURL
@@ -431,6 +424,7 @@ func (c *ConfigCompiled) SetBaseURL(baseURL, baseURLLiveReload urls.BaseURL) {
// RootConfig holds all the top-level configuration options in Hugo
type RootConfig struct {
// The base URL of the site.
// Note that the default value is empty, but Hugo requires a valid URL (e.g. "https://example.com/") to work properly.
// <docsmeta>{"identifiers": ["URL"] }</docsmeta>
@@ -653,16 +647,13 @@ type Configs struct {
LanguageConfigMap map[string]*Config
LanguageConfigSlice []*Config
IsMultihost bool
IsMultihost bool
Languages langs.Languages
LanguagesDefaultFirst langs.Languages
Modules modules.Modules
ModulesClient *modules.Client
// All below is set in Init.
Languages langs.Languages
LanguagesDefaultFirst langs.Languages
ContentPathParser paths.PathParser
configLangs []config.AllProvider
}
@@ -682,58 +673,6 @@ func (c *Configs) IsZero() bool {
}
func (c *Configs) Init() error {
var languages langs.Languages
defaultContentLanguage := c.Base.DefaultContentLanguage
for k, v := range c.LanguageConfigMap {
c.LanguageConfigSlice = append(c.LanguageConfigSlice, v)
languageConf := v.Languages[k]
language, err := langs.NewLanguage(k, defaultContentLanguage, v.TimeZone, languageConf)
if err != nil {
return err
}
languages = append(languages, language)
}
// Sort the sites by language weight (if set) or lang.
sort.Slice(languages, func(i, j int) bool {
li := languages[i]
lj := languages[j]
if li.Weight != lj.Weight {
return li.Weight < lj.Weight
}
return li.Lang < lj.Lang
})
for _, l := range languages {
c.LanguageConfigSlice = append(c.LanguageConfigSlice, c.LanguageConfigMap[l.Lang])
}
// Filter out disabled languages.
var n int
for _, l := range languages {
if !l.Disabled {
languages[n] = l
n++
}
}
languages = languages[:n]
var languagesDefaultFirst langs.Languages
for _, l := range languages {
if l.Lang == defaultContentLanguage {
languagesDefaultFirst = append(languagesDefaultFirst, l)
}
}
for _, l := range languages {
if l.Lang != defaultContentLanguage {
languagesDefaultFirst = append(languagesDefaultFirst, l)
}
}
c.Languages = languages
c.LanguagesDefaultFirst = languagesDefaultFirst
c.ContentPathParser = paths.PathParser{LanguageIndex: languagesDefaultFirst.AsIndexSet()}
c.configLangs = make([]config.AllProvider, len(c.Languages))
for i, l := range c.LanguagesDefaultFirst {
c.configLangs[i] = ConfigLanguage{
@@ -811,6 +750,7 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon
}
langConfigMap := make(map[string]*Config)
var langConfigs []*Config
languagesConfig := cfg.GetStringMap("languages")
var isMultiHost bool
@@ -844,7 +784,7 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon
// We accidentally allowed it in the past, so we need to support it a little longer,
// But log a warning.
if _, found := params[kk]; !found {
hugo.Deprecate(fmt.Sprintf("config: languages.%s.%s: custom params on the language top level", k, kk), fmt.Sprintf("Put the value below [languages.%s.params]. See https://gohugo.io/content-management/multilingual/#changes-in-hugo-01120", k), "v0.112.0")
helpers.Deprecated(fmt.Sprintf("config: languages.%s.%s: custom params on the language top level", k, kk), fmt.Sprintf("Put the value below [languages.%s.params]. See https://gohugo.io/content-management/multilingual/#changes-in-hugo-01120", k), false)
params[kk] = vv
}
}
@@ -907,24 +847,65 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon
}
}
var languages langs.Languages
defaultContentLanguage := all.DefaultContentLanguage
for k, v := range langConfigMap {
languageConf := v.Languages[k]
language, err := langs.NewLanguage(k, defaultContentLanguage, v.TimeZone, languageConf)
if err != nil {
return nil, err
}
languages = append(languages, language)
}
// Sort the sites by language weight (if set) or lang.
sort.Slice(languages, func(i, j int) bool {
li := languages[i]
lj := languages[j]
if li.Weight != lj.Weight {
return li.Weight < lj.Weight
}
return li.Lang < lj.Lang
})
for _, l := range languages {
langConfigs = append(langConfigs, langConfigMap[l.Lang])
}
var languagesDefaultFirst langs.Languages
for _, l := range languages {
if l.Lang == defaultContentLanguage {
languagesDefaultFirst = append(languagesDefaultFirst, l)
}
}
for _, l := range languages {
if l.Lang != defaultContentLanguage {
languagesDefaultFirst = append(languagesDefaultFirst, l)
}
}
bcfg.PublishDir = all.PublishDir
res.BaseConfig = bcfg
all.CommonDirs.CacheDir = bcfg.CacheDir
for _, l := range langConfigMap {
for _, l := range langConfigs {
l.CommonDirs.CacheDir = bcfg.CacheDir
}
cm := &Configs{
Base: all,
LanguageConfigMap: langConfigMap,
LoadingInfo: res,
IsMultihost: isMultiHost,
Base: all,
LanguageConfigMap: langConfigMap,
LanguageConfigSlice: langConfigs,
LoadingInfo: res,
IsMultihost: isMultiHost,
Languages: languages,
LanguagesDefaultFirst: languagesDefaultFirst,
}
return cm, nil
}
func decodeConfigFromParams(fs afero.Fs, logger loggers.Logger, bcfg config.BaseConfig, p config.Provider, target *Config, keys []string) error {
var decoderSetups []decodeWeight
if len(keys) == 0 {
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+2 -16
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package allconfig
import (
"time"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/langs"
@@ -42,15 +41,10 @@ func (c ConfigLanguage) LanguagesDefaultFirst() langs.Languages {
return c.m.LanguagesDefaultFirst
}
func (c ConfigLanguage) PathParser() paths.PathParser {
return c.m.ContentPathParser
}
func (c ConfigLanguage) LanguagePrefix() string {
if c.DefaultContentLanguageInSubdir() && c.DefaultContentLanguage() == c.Language().Lang {
return c.Language().Lang
}
if !c.IsMultiLingual() || c.DefaultContentLanguage() == c.Language().Lang {
return ""
}
@@ -125,10 +119,6 @@ func (c ConfigLanguage) Quiet() bool {
return c.m.Base.Internal.Quiet
}
func (c ConfigLanguage) Watching() bool {
return c.m.Base.Internal.Watch
}
// GetConfigSection is mostly used in tests. The switch statement isn't complete, but what's in use.
func (c ConfigLanguage) GetConfigSection(s string) any {
switch s {
@@ -150,7 +140,7 @@ func (c ConfigLanguage) GetConfigSection(s string) any {
return c.config.Permalinks
case "minify":
return c.config.Minify
case "allModules":
case "activeModules":
return c.m.Modules
case "deployment":
return c.config.Deployment
@@ -238,7 +228,3 @@ func (c ConfigLanguage) PaginatePath() string {
func (c ConfigLanguage) StaticDirs() []string {
return c.config.staticDirs()
}
func (c ConfigLanguage) EnableEmoji() bool {
return c.config.EnableEmoji
}
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import (
// This is is just some helpers used to create some JSON used in the Hugo docs.
func init() {
docsProvider := func() docshelper.DocProvider {
cfg := config.New()
for configRoot, v := range allDecoderSetups {
if v.internalOrDeprecated {
+5 -2
View File
@@ -10,6 +10,7 @@ import (
)
func TestDirsMount(t *testing.T) {
files := `
-- hugo.toml --
baseURL = "https://example.com"
@@ -43,7 +44,7 @@ Title: {{ .Title }}
hugolib.IntegrationTestConfig{T: t, TxtarString: files},
).Build()
// b.AssertFileContent("public/p1/index.html", "Title: p1")
//b.AssertFileContent("public/p1/index.html", "Title: p1")
sites := b.H.Sites
b.Assert(len(sites), qt.Equals, 2)
@@ -57,7 +58,7 @@ Title: {{ .Title }}
enConcp := sites[0].Conf
enConf := enConcp.GetConfig().(*allconfig.Config)
b.Assert(enConcp.BaseURL().String(), qt.Equals, "https://example.com/")
b.Assert(enConcp.BaseURL().String(), qt.Equals, "https://example.com")
modConf := enConf.Module
b.Assert(modConf.Mounts, qt.HasLen, 8)
b.Assert(modConf.Mounts[0].Source, qt.Equals, filepath.FromSlash("content/en"))
@@ -66,9 +67,11 @@ Title: {{ .Title }}
b.Assert(modConf.Mounts[1].Source, qt.Equals, filepath.FromSlash("content/sv"))
b.Assert(modConf.Mounts[1].Target, qt.Equals, "content")
b.Assert(modConf.Mounts[1].Lang, qt.Equals, "sv")
}
func TestConfigAliases(t *testing.T) {
files := `
-- hugo.toml --
baseURL = "https://example.com"
+18 -5
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -34,10 +34,10 @@ import (
hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/modules"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/gohugoio/hugo/tpl"
"github.com/spf13/afero"
)
//lint:ignore ST1005 end user message.
var ErrNoConfigFile = errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n")
func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
@@ -84,16 +84,19 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
return nil, fmt.Errorf("failed to create config: %w", err)
}
configs.Modules = moduleConfig.AllModules
configs.Modules = moduleConfig.ActiveModules
configs.ModulesClient = modulesClient
if err := configs.Init(); err != nil {
return nil, fmt.Errorf("failed to init config: %w", err)
}
loggers.InitGlobalLogger(d.Logger.Level(), configs.Base.PanicOnWarning)
// This is unfortunate, but these are global settings.
tpl.SetSecurityAllowActionJSTmpl(configs.Base.Security.GoTemplates.AllowActionJSTmpl)
loggers.InitGlobalLogger(configs.Base.PanicOnWarning)
return configs, nil
}
// ConfigSourceDescriptor describes where to find the config (e.g. config.toml etc.).
@@ -327,6 +330,7 @@ func (l *configLoader) envStringToVal(k, v string) any {
default:
return v
}
}
func (l *configLoader) loadConfigMain(d ConfigSourceDescriptor) (config.LoadConfigResult, modules.ModulesConfig, error) {
@@ -467,7 +471,7 @@ func (l *configLoader) loadModules(configs *Configs) (modules.ModulesConfig, *mo
ex := hexec.New(conf.Security)
hook := func(m *modules.ModulesConfig) error {
for _, tc := range m.AllModules {
for _, tc := range m.ActiveModules {
if len(tc.ConfigFilenames()) > 0 {
if tc.Watch() {
l.ModulesConfigFiles = append(l.ModulesConfigFiles, tc.ConfigFilenames()...)
@@ -567,6 +571,15 @@ func (l configLoader) deleteMergeStrategies() {
})
}
func (l configLoader) loadModulesConfig() (modules.Config, error) {
modConfig, err := modules.DecodeConfig(l.cfg)
if err != nil {
return modules.Config{}, err
}
return modConfig, nil
}
func (l configLoader) wrapFileError(err error, filename string) error {
fe := herrors.UnwrapFileError(err)
if fe != nil {
+16 -7
View File
@@ -85,22 +85,29 @@ var defaultBuild = BuildConfig{
BuildStats: BuildStats{},
CacheBusters: []CacheBuster{
{
Source: `assets/.*\.(js|ts|jsx|tsx)`,
Target: `(js|scripts|javascript)`,
},
{
Source: `assets/.*\.(css|sass|scss)$`,
Target: cssTargetCachebusterRe,
},
{
Source: `(postcss|tailwind)\.config\.js`,
Target: cssTargetCachebusterRe,
},
// This is deliberately coarse grained; it will cache bust resources with "json" in the cache key when js files changes, which is good.
{
Source: `assets/.*\.(.*)$`,
Target: `$1`,
},
},
}
// BuildConfig holds some build related configuration.
type BuildConfig struct {
// When to use the resource file cache.
// One of never, fallback, always. Default is fallback
UseResourceCacheWhen string
// When enabled, will duplicate bundled resource files across languages that
// doesn't have a translated version.
DuplicateResourceFiles bool
UseResourceCacheWhen string // never, fallback, always. Default is fallback
// When enabled, will collect and write a hugo_stats.json with some build
// related aggregated data (e.g. CSS class names).
@@ -366,6 +373,7 @@ func (c *CacheBuster) CompileConfig(logger loggers.Logger) error {
return match
}
}
return compileErr
}
@@ -408,6 +416,7 @@ func DecodeServer(cfg Provider) (Server, error) {
Status: 404,
},
}
}
return *s, nil
+11 -3
View File
@@ -148,13 +148,21 @@ func TestBuildConfigCacheBusters(t *testing.T) {
l := loggers.NewDefault()
c.Assert(conf.CompileConfig(l), qt.IsNil)
m, _ := conf.MatchCacheBuster(l, "tailwind.config.js")
m, err := conf.MatchCacheBuster(l, "assets/foo/main.js")
c.Assert(err, qt.IsNil)
c.Assert(m, qt.IsNotNil)
c.Assert(m("scripts"), qt.IsTrue)
c.Assert(m("asdf"), qt.IsFalse)
m, _ = conf.MatchCacheBuster(l, "tailwind.config.js")
c.Assert(m("css"), qt.IsTrue)
c.Assert(m("js"), qt.IsFalse)
m, _ = conf.MatchCacheBuster(l, "foo.bar")
c.Assert(m, qt.IsNil)
m, err = conf.MatchCacheBuster(l, "assets/foo.json")
c.Assert(err, qt.IsNil)
c.Assert(m, qt.IsNotNil)
c.Assert(m("json"), qt.IsTrue)
}
func TestBuildConfigCacheBusterstTailwindSetup(t *testing.T) {
+6 -4
View File
@@ -17,7 +17,6 @@ import (
"time"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/langs"
@@ -31,7 +30,6 @@ type AllProvider interface {
LanguagePrefix() string
BaseURL() urls.BaseURL
BaseURLLiveReload() urls.BaseURL
PathParser() paths.PathParser
Environment() string
IsMultihost() bool
IsMultiLingual() bool
@@ -56,7 +54,6 @@ type AllProvider interface {
BuildFuture() bool
BuildDrafts() bool
Running() bool
Watching() bool
PrintUnusedTemplates() bool
EnableMissingTranslationPlaceholders() bool
TemplateMetrics() bool
@@ -69,7 +66,6 @@ type AllProvider interface {
StaticDirs() []string
IgnoredErrors() map[string]bool
WorkingDir() string
EnableEmoji() bool
}
// Provider provides the configuration settings for Hugo.
@@ -98,3 +94,9 @@ func GetStringSlicePreserveString(cfg Provider, key string) []string {
sd := cfg.Get(key)
return types.ToStringSlicePreserveString(sd)
}
func setIfNotSet(cfg Provider, key string, value any) {
if !cfg.IsSet(key) {
cfg.Set(key, value)
}
}
-37
View File
@@ -18,12 +18,6 @@ import (
"runtime"
"strconv"
"strings"
"github.com/pbnjay/memory"
)
const (
gigabyte = 1 << 30
)
// GetNumWorkerMultiplier returns the base value used to calculate the number
@@ -39,37 +33,6 @@ func GetNumWorkerMultiplier() int {
return runtime.NumCPU()
}
// GetMemoryLimit returns the upper memory limit in bytes for Hugo's in-memory caches.
// Note that this does not represent "all of the memory" that Hugo will use,
// so it needs to be set to a lower number than the available system memory.
// It will read from the HUGO_MEMORYLIMIT (in Gigabytes) environment variable.
// If that is not set, it will set aside a quarter of the total system memory.
func GetMemoryLimit() uint64 {
if mem := os.Getenv("HUGO_MEMORYLIMIT"); mem != "" {
if v := stringToGibabyte(mem); v > 0 {
return v
}
}
// There is a FreeMemory function, but as the kernel in most situations
// will take whatever memory that is left and use for caching etc.,
// that value is not something that we can use.
m := memory.TotalMemory()
if m != 0 {
return uint64(m / 4)
}
return 2 * gigabyte
}
func stringToGibabyte(f string) uint64 {
if v, err := strconv.ParseFloat(f, 32); err == nil && v > 0 {
return uint64(v * gigabyte)
}
return 0
}
// SetEnvVars sets vars on the form key=value in the oldVars slice.
func SetEnvVars(oldVars *[]string, keyValues ...string) {
for i := 0; i < len(keyValues); i += 2 {
+2 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import (
)
func DecodeNamespace[S, C any](configSource any, buildConfig func(any) (C, any, error)) (*ConfigNamespace[S, C], error) {
// Calculate the hash of the input (not including any defaults applied later).
// This allows us to introduce new config options without breaking the hash.
h := identity.HashString(configSource)
+10 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ func TestNamespace(t *testing.T) {
c := qt.New(t)
c.Assert(true, qt.Equals, true)
// ns, err := config.DecodeNamespace[map[string]DocsMediaTypeConfig](in, defaultMediaTypesConfig, buildConfig)
//ns, err := config.DecodeNamespace[map[string]DocsMediaTypeConfig](in, defaultMediaTypesConfig, buildConfig)
ns, err := DecodeNamespace[[]*tstNsExt](
map[string]interface{}{"foo": "bar"},
@@ -46,15 +46,23 @@ func TestNamespace(t *testing.T) {
c.Assert(ns.SourceHash, qt.Equals, "14368731254619220105")
c.Assert(ns.Config, qt.DeepEquals, &tstNsExt{Foo: "bar"})
c.Assert(ns.Signature(), qt.DeepEquals, []*tstNsExt(nil))
}
type (
tstNsExt struct {
Foo string
}
tstNsInt struct {
Foo string
}
)
func (t *tstNsExt) Init() error {
t.Foo = strings.ToUpper(t.Foo)
return nil
}
func (t *tstNsInt) Compile(ext *tstNsExt) error {
t.Foo = ext.Foo + " qux"
return nil
}
+18 -2
View File
@@ -68,6 +68,9 @@ type Config struct {
// Allow inline shortcodes
EnableInlineShortcodes bool `json:"enableInlineShortcodes"`
// Go templates related security config.
GoTemplates GoTemplates `json:"goTemplates"`
}
// Exec holds os/exec policies.
@@ -93,6 +96,15 @@ type HTTP struct {
MediaTypes Whitelist `json:"mediaTypes"`
}
type GoTemplates struct {
// Enable to allow template actions inside bakcticks in ES6 template literals.
// This was blocked in Hugo 0.114.0 for security reasons and you now get errors on the form
// "... appears in a JS template literal" if you have this in your templates.
// See https://github.com/golang/go/issues/59234
AllowActionJSTmpl bool
}
// ToTOML converts c to TOML with [security] as the root.
func (c Config) ToTOML() string {
sec := c.ToSecurityMap()
@@ -115,6 +127,7 @@ func (c Config) CheckAllowedExec(name string) error {
}
}
return nil
}
func (c Config) CheckAllowedGetEnv(name string) error {
@@ -163,6 +176,7 @@ func (c Config) ToSecurityMap() map[string]any {
"security": m,
}
return sec
}
// DecodeConfig creates a privacy Config from a given Hugo configuration.
@@ -192,14 +206,15 @@ func DecodeConfig(cfg config.Provider) (Config, error) {
}
return sc, nil
}
func stringSliceToWhitelistHook() mapstructure.DecodeHookFuncType {
return func(
f reflect.Type,
t reflect.Type,
data any,
) (any, error) {
data any) (any, error) {
if t != reflect.TypeOf(Whitelist{}) {
return data, nil
}
@@ -207,6 +222,7 @@ func stringSliceToWhitelistHook() mapstructure.DecodeHookFuncType {
wl := types.ToStringSlicePreserveString(data)
return NewWhitelist(wl...)
}
}
+7 -1
View File
@@ -53,6 +53,7 @@ getEnv=["a", "b"]
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
c.Assert(pc.Funcs.Getenv.Accept("a"), qt.IsTrue)
c.Assert(pc.Funcs.Getenv.Accept("c"), qt.IsFalse)
})
c.Run("String whitelist", func(c *qt.C) {
@@ -79,6 +80,7 @@ osEnv="b"
c.Assert(pc.Exec.Allow.Accept("d"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("b"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
})
c.Run("Default exec.osEnv", func(c *qt.C) {
@@ -103,6 +105,7 @@ allow="a"
c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("PATH"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
})
c.Run("Enable inline shortcodes, legacy", func(c *qt.C) {
@@ -126,7 +129,9 @@ osEnv="b"
pc, err := DecodeConfig(cfg)
c.Assert(err, qt.IsNil)
c.Assert(pc.EnableInlineShortcodes, qt.IsTrue)
})
}
func TestToTOML(t *testing.T) {
@@ -135,7 +140,7 @@ func TestToTOML(t *testing.T) {
got := DefaultConfig.ToTOML()
c.Assert(got, qt.Equals,
"[security]\n enableInlineShortcodes = false\n\n [security.exec]\n allow = ['^(dart-)?sass(-embedded)?$', '^go$', '^npx$', '^postcss$']\n osEnv = ['(?i)^((HTTPS?|NO)_PROXY|PATH(EXT)?|APPDATA|TE?MP|TERM|GO\\w+|(XDG_CONFIG_)?HOME|USERPROFILE|SSH_AUTH_SOCK|DISPLAY|LANG)$']\n\n [security.funcs]\n getenv = ['^HUGO_', '^CI$']\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']",
"[security]\n enableInlineShortcodes = false\n\n [security.exec]\n allow = ['^(dart-)?sass(-embedded)?$', '^go$', '^npx$', '^postcss$']\n osEnv = ['(?i)^((HTTPS?|NO)_PROXY|PATH(EXT)?|APPDATA|TE?MP|TERM|GO\\w+|(XDG_CONFIG_)?HOME|USERPROFILE|SSH_AUTH_SOCK|DISPLAY|LANG)$']\n\n [security.funcs]\n getenv = ['^HUGO_', '^CI$']\n\n [security.goTemplates]\n AllowActionJSTmpl = false\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']",
)
}
@@ -164,4 +169,5 @@ func TestDecodeConfigDefault(t *testing.T) {
c.Assert(pc.Exec.OsEnv.Accept("a"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("MYSECRET"), qt.IsFalse)
}
+3 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ func GetTestConfigs(fs afero.Fs, cfg config.Provider) *allconfig.Configs {
// Make sure that the workingDir exists.
workingDir := cfg.GetString("workingDir")
if workingDir != "" {
if err := fs.MkdirAll(workingDir, 0o777); err != nil {
if err := fs.MkdirAll(workingDir, 0777); err != nil {
panic(err)
}
}
@@ -46,6 +46,7 @@ func GetTestConfigs(fs afero.Fs, cfg config.Provider) *allconfig.Configs {
panic(err)
}
return configs
}
func GetTestConfig(fs afero.Fs, cfg config.Provider) config.AllProvider {
+44 -52
View File
@@ -16,7 +16,6 @@ package create
import (
"bytes"
"errors"
"fmt"
"io"
"os"
@@ -26,9 +25,10 @@ import (
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/hstrings"
"github.com/gohugoio/hugo/common/paths"
"errors"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs"
@@ -53,7 +53,7 @@ draft: true
// NewContent creates a new content file in h (or a full bundle if the archetype is a directory)
// in targetPath.
func NewContent(h *hugolib.HugoSites, kind, targetPath string, force bool) error {
if _, err := h.BaseFs.Content.Fs.Stat(""); err != nil {
if h.BaseFs.Content.Dirs == nil {
return errors.New("no existing content directory configured for this project")
}
@@ -103,6 +103,7 @@ func NewContent(h *hugolib.HugoSites, kind, targetPath string, force bool) error
}
return b.buildFile()
}
filename, err := withBuildLock()
@@ -115,6 +116,7 @@ func NewContent(h *hugolib.HugoSites, kind, targetPath string, force bool) error
}
return nil
}
type contentBuilder struct {
@@ -126,12 +128,12 @@ type contentBuilder struct {
cf hugolib.ContentFactory
// Builder state
archetypeFi hugofs.FileMetaInfo
targetPath string
kind string
isDir bool
dirMap archetypeMap
force bool
archetypeFilename string
targetPath string
kind string
isDir bool
dirMap archetypeMap
force bool
}
func (b *contentBuilder) buildDir() error {
@@ -144,10 +146,7 @@ func (b *contentBuilder) buildDir() error {
var baseDir string
for _, fi := range b.dirMap.contentFiles {
targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().PathInfo.Path(), b.archetypeFi.Meta().PathInfo.Path()))
// ===> post/my-post/pages/bio.md
targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().Path, b.archetypeFilename))
abs, err := b.cf.CreateContentPlaceHolder(targetFilename, b.force)
if err != nil {
return err
@@ -171,6 +170,7 @@ func (b *contentBuilder) buildDir() error {
}
return false
})
}
if err := b.h.Build(hugolib.BuildCfg{NoBuildLock: true, SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil {
@@ -178,20 +178,22 @@ func (b *contentBuilder) buildDir() error {
}
for i, filename := range contentTargetFilenames {
if err := b.applyArcheType(filename, b.dirMap.contentFiles[i]); err != nil {
if err := b.applyArcheType(filename, b.dirMap.contentFiles[i].Meta().Path); err != nil {
return err
}
}
// Copy the rest as is.
for _, fi := range b.dirMap.otherFiles {
meta := fi.Meta()
for _, f := range b.dirMap.otherFiles {
meta := f.Meta()
filename := meta.Path
in, err := meta.Open()
if err != nil {
return fmt.Errorf("failed to open non-content file: %w", err)
}
targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(fi.Meta().Filename, b.archetypeFi.Meta().Filename))
targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(filename, b.archetypeFilename))
targetDir := filepath.Dir(targetFilename)
if err := b.sourceFs.MkdirAll(targetDir, 0o777); err != nil && !os.IsExist(err) {
@@ -223,7 +225,7 @@ func (b *contentBuilder) buildFile() (string, error) {
return "", err
}
usesSite, err := b.usesSiteVar(b.archetypeFi)
usesSite, err := b.usesSiteVar(b.archetypeFilename)
if err != nil {
return "", err
}
@@ -241,7 +243,7 @@ func (b *contentBuilder) buildFile() (string, error) {
return "", err
}
if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFi); err != nil {
if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFilename); err != nil {
return "", err
}
@@ -262,14 +264,15 @@ func (b *contentBuilder) setArcheTypeFilenameToUse(ext string) {
for _, p := range pathsToCheck {
fi, err := b.archeTypeFs.Stat(p)
if err == nil {
b.archetypeFi = fi.(hugofs.FileMetaInfo)
b.archetypeFilename = p
b.isDir = fi.IsDir()
return
}
}
}
func (b *contentBuilder) applyArcheType(contentFilename string, archetypeFi hugofs.FileMetaInfo) error {
func (b *contentBuilder) applyArcheType(contentFilename, archetypeFilename string) error {
p := b.h.GetContentPage(contentFilename)
if p == nil {
panic(fmt.Sprintf("[BUG] no Page found for %q", contentFilename))
@@ -281,39 +284,32 @@ func (b *contentBuilder) applyArcheType(contentFilename string, archetypeFi hugo
}
defer f.Close()
if archetypeFi == nil {
if archetypeFilename == "" {
return b.cf.ApplyArchetypeTemplate(f, p, b.kind, DefaultArchetypeTemplateTemplate)
}
return b.cf.ApplyArchetypeFi(f, p, b.kind, archetypeFi)
return b.cf.ApplyArchetypeFilename(f, p, b.kind, archetypeFilename)
}
func (b *contentBuilder) mapArcheTypeDir() error {
var m archetypeMap
seen := map[hstrings.Tuple]bool{}
walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
return err
}
walkFn := func(path string, fim hugofs.FileMetaInfo) error {
if fim.IsDir() {
if fi.IsDir() {
return nil
}
pi := fim.Meta().PathInfo
fil := fi.(hugofs.FileMetaInfo)
if pi.IsContent() {
pathLang := hstrings.Tuple{First: pi.PathNoIdentifier(), Second: fim.Meta().Lang}
if seen[pathLang] {
// Duplicate content file, e.g. page.md and page.html.
// In the regular build, we will filter out the duplicates, but
// for archetype folders these are ambiguous and we need to
// fail.
return fmt.Errorf("duplicate content file found in archetype folder: %q; having both e.g. %s.md and %s.html is ambigous", path, pi.BaseNameNoIdentifier(), pi.BaseNameNoIdentifier())
}
seen[pathLang] = true
m.contentFiles = append(m.contentFiles, fim)
if files.IsContentFile(path) {
m.contentFiles = append(m.contentFiles, fil)
if !m.siteUsed {
var err error
m.siteUsed, err = b.usesSiteVar(fim)
m.siteUsed, err = b.usesSiteVar(path)
if err != nil {
return err
}
@@ -321,7 +317,7 @@ func (b *contentBuilder) mapArcheTypeDir() error {
return nil
}
m.otherFiles = append(m.otherFiles, fim)
m.otherFiles = append(m.otherFiles, fil)
return nil
}
@@ -329,13 +325,13 @@ func (b *contentBuilder) mapArcheTypeDir() error {
walkCfg := hugofs.WalkwayConfig{
WalkFn: walkFn,
Fs: b.archeTypeFs,
Root: filepath.FromSlash(b.archetypeFi.Meta().PathInfo.Path()),
Root: b.archetypeFilename,
}
w := hugofs.NewWalkway(walkCfg)
if err := w.Walk(); err != nil {
return fmt.Errorf("failed to walk archetype dir %q: %w", b.archetypeFi.Meta().Filename, err)
return fmt.Errorf("failed to walk archetype dir %q: %w", b.archetypeFilename, err)
}
b.dirMap = m
@@ -374,21 +370,17 @@ func (b *contentBuilder) openInEditorIfConfigured(filename string) error {
return cmd.Run()
}
func (b *contentBuilder) usesSiteVar(fi hugofs.FileMetaInfo) (bool, error) {
if fi == nil {
func (b *contentBuilder) usesSiteVar(filename string) (bool, error) {
if filename == "" {
return false, nil
}
f, err := fi.Meta().Open()
bb, err := afero.ReadFile(b.archeTypeFs, filename)
if err != nil {
return false, err
}
defer f.Close()
bb, err := io.ReadAll(f)
if err != nil {
return false, fmt.Errorf("failed to read archetype file: %w", err)
return false, fmt.Errorf("failed to open archetype file: %w", err)
}
return bytes.Contains(bb, []byte(".Site")) || bytes.Contains(bb, []byte("site.")), nil
}
type archetypeMap struct {
+130 -1
View File
@@ -114,6 +114,58 @@ func TestNewContentFromFile(t *testing.T) {
}
}
func TestNewContentFromDir(t *testing.T) {
mm := afero.NewMemMapFs()
c := qt.New(t)
archetypeDir := filepath.Join("archetypes", "my-bundle")
c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil)
archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle")
c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil)
contentFile := `
File: %s
Site Lang: {{ .Site.Language.Lang }}
Name: {{ replace .Name "-" " " | title }}
i18n: {{ T "hugo" }}
`
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(initFs(mm), qt.IsNil)
cfg, fs := newTestCfg(c, mm)
conf := testconfig.GetTestConfigs(fs.Source, cfg)
h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs})
c.Assert(err, qt.IsNil)
c.Assert(len(h.Sites), qt.Equals, 2)
c.Assert(create.NewContent(h, "my-bundle", "post/my-post", false), qt.IsNil)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`)
// Content files should get the correct site context.
// TODO(bep) archetype check i18n
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: Bio`)
c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post", false), qt.IsNil)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Theme Post`, `i18n: Hugo Rocks!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
}
func TestNewContentFromDirSiteFunction(t *testing.T) {
mm := afero.NewMemMapFs()
c := qt.New(t)
@@ -154,6 +206,83 @@ site RegularPages: {{ len site.RegularPages }}
// Regular files should fall back to the default archetype (we have no regular file archetype).
c.Assert(create.NewContent(h, "my-bundle", "mypage.md", false), qt.IsNil)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "mypage.md")), `draft: true`)
}
func TestNewContentFromDirNoSite(t *testing.T) {
mm := afero.NewMemMapFs()
c := qt.New(t)
archetypeDir := filepath.Join("archetypes", "my-bundle")
c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil)
archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle")
c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil)
contentFile := `
File: %s
Name: {{ replace .Name "-" " " | title }}
i18n: {{ T "hugo" }}
`
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil)
c.Assert(initFs(mm), qt.IsNil)
cfg, fs := newTestCfg(c, mm)
conf := testconfig.GetTestConfigs(fs.Source, cfg)
h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs})
c.Assert(err, qt.IsNil)
c.Assert(len(h.Sites), qt.Equals, 2)
c.Assert(create.NewContent(h, "my-bundle", "post/my-post", false), qt.IsNil)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Name: My Post`, `i18n: Hugo Rocks!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Name: My Post`, `i18n: Hugo Rokkar!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Name: Bio`)
c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post", false), qt.IsNil)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Name: My Theme Post`, `i18n: Hugo Rocks!`)
cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
}
func TestNewContentForce(t *testing.T) {
mm := afero.NewMemMapFs()
c := qt.New(t)
archetypeDir := filepath.Join("archetypes", "my-bundle")
c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(""), 0o755), qt.IsNil)
c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(""), 0o755), qt.IsNil)
c.Assert(initFs(mm), qt.IsNil)
cfg, fs := newTestCfg(c, mm)
conf := testconfig.GetTestConfigs(fs.Source, cfg)
h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs})
c.Assert(err, qt.IsNil)
c.Assert(len(h.Sites), qt.Equals, 2)
// from file
c.Assert(create.NewContent(h, "post", "post/my-post.md", false), qt.IsNil)
c.Assert(create.NewContent(h, "post", "post/my-post.md", false), qt.IsNotNil)
c.Assert(create.NewContent(h, "post", "post/my-post.md", true), qt.IsNil)
// from dir
c.Assert(create.NewContent(h, "my-bundle", "post/my-post", false), qt.IsNil)
c.Assert(create.NewContent(h, "my-bundle", "post/my-post", false), qt.IsNotNil)
c.Assert(create.NewContent(h, "my-bundle", "post/my-post", true), qt.IsNil)
}
func initFs(fs afero.Fs) error {
@@ -179,7 +308,7 @@ func initFs(fs afero.Fs) error {
afero.WriteFile(fs, filename, []byte(`---
title: Test
---
`), 0o666)
`), 0666)
}
// create archetype files

Some files were not shown because too many files have changed in this diff Show More