diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91af1538e..430472ae8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -122,7 +122,6 @@ jobs: name: Test run: | mage -v test - go clean -i -testcache env: HUGO_BUILD_TAGS: extended,withdeploy - if: matrix.os == 'ubuntu-latest' diff --git a/.gitignore b/.gitignore index ddad69611..ed5bfdcb3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ imports.* dist/ public/ -.DS_Store \ No newline at end of file +.DS_Store +cache/filecache/_gen/ \ No newline at end of file diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 01c466ca6..c64f23914 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -16,6 +16,7 @@ package filecache import ( "bytes" "errors" + "fmt" "io" "os" "path/filepath" @@ -54,7 +55,11 @@ type Cache struct { nlocker *lockTracker + isResourceDir bool + baseDir string + initOnce sync.Once + isInited bool initErr error } @@ -109,9 +114,14 @@ func (l *lockedFile) Close() error { } func (c *Cache) init() error { + if c == nil { + panic("cache is nil") + } c.initOnce.Do(func() { + c.isInited = true // Create the base dir if it does not exist. if err := c.Fs.MkdirAll("", 0o777); err != nil && !os.IsExist(err) { + err = fmt.Errorf("failled to create base cache directory: %s", err) c.initErr = err } }) @@ -286,8 +296,32 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item return info, b, nil } +// SetBytes sets the file content with the given id in the cache. +func (c *Cache) SetBytes(id string, data []byte) error { + if err := c.init(); err != nil { + return err + } + id = cleanID(id) + + c.nlocker.Lock(id) + defer c.nlocker.Unlock(id) + + if c.maxAge == 0 { + // No caching. + return nil + } + + return c.writeReader(id, bytes.NewReader(data)) +} + // GetBytes gets the file content with the given id from the cache, nil if none found. -func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) { +func (c *Cache) GetBytes(id string) ([]byte, error) { + _, b, err := c.GetItemBytes(id) + return b, err +} + +// GetItemBytes gets the ItemInfo and file content with the given id from the cache, nil if none found. +func (c *Cache) GetItemBytes(id string) (ItemInfo, []byte, error) { if err := c.init(); err != nil { return ItemInfo{}, nil, err } @@ -417,6 +451,17 @@ func (c *Cache) GetString(id string) string { // Caches is a named set of caches. type Caches map[string]*Cache +func (f Caches) SetResourceFs(fs afero.Fs) { + for _, c := range f { + if c.isResourceDir { + if c.isInited { + panic("cannot set resource fs after init") + } + c.Fs = hugofs.NewBasePathFs(fs, c.baseDir) + } + } +} + // Get gets a named cache, nil if none found. func (f Caches) Get(name string) *Cache { return f[strings.ToLower(name)] @@ -424,24 +469,19 @@ func (f Caches) Get(name string) *Cache { // NewCaches creates a new set of file caches from the given // configuration. -func NewCaches(p *helpers.PathSpec) (Caches, error) { - dcfg := p.Cfg.GetConfigSection("caches").(Configs) - fs := p.Fs.Source +func NewCaches(dcfg Configs, sourceFs afero.Fs) (Caches, error) { + // dcfg := p.Cfg.GetConfigSection("caches").(Configs) + fs := sourceFs m := make(Caches) for k, v := range dcfg { var cfs afero.Fs - if v.IsResourceDir { - cfs = p.BaseFs.ResourcesCache + cfs = nil // Set later. TODO(bep) this needs to be cleanded up. } else { cfs = fs } - if cfs == nil { - panic("nil fs") - } - baseDir := v.DirCompiled bfs := hugofs.NewBasePathFs(cfs, baseDir) @@ -451,7 +491,11 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { pruneAllRootDir = "pkg" } - m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir) + c := NewCache(bfs, v.MaxAge, pruneAllRootDir) + c.isResourceDir = v.IsResourceDir + c.baseDir = baseDir + + m[k] = c } return m, nil diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go index b4e4efdc2..025b62b7f 100644 --- a/cache/filecache/filecache_config.go +++ b/cache/filecache/filecache_config.go @@ -40,13 +40,14 @@ var defaultCacheConfig = FileCacheConfig{ } const ( - CacheKeyGetJSON = "getjson" - CacheKeyGetCSV = "getcsv" - CacheKeyImages = "images" - CacheKeyAssets = "assets" - CacheKeyModules = "modules" - CacheKeyGetResource = "getresource" - CacheKeyMisc = "misc" + CacheKeyGetJSON = "getjson" + CacheKeyGetCSV = "getcsv" + CacheKeyImages = "images" + CacheKeyAssets = "assets" + CacheKeyModules = "modules" + CacheKeyModuleQueries = "modulequeries" + CacheKeyGetResource = "getresource" + CacheKeyMisc = "misc" ) type Configs map[string]FileCacheConfig @@ -68,6 +69,10 @@ var defaultCacheConfigs = Configs{ MaxAge: -1, Dir: ":cacheDir/modules", }, + CacheKeyModuleQueries: { + MaxAge: 24 * time.Hour, + Dir: ":cacheDir/modules", + }, CacheKeyGetJSON: defaultCacheConfig, CacheKeyGetCSV: defaultCacheConfig, CacheKeyImages: { @@ -127,6 +132,16 @@ func (f Caches) ModulesCache() *Cache { return f[CacheKeyModules] } +// ModuleQueriesCache gets the file cache for Hugo Module version queries. +// Returns nil if not found. +func (f Caches) ModuleQueriesCache() *Cache { + c, ok := f[CacheKeyModuleQueries] + if !ok { + panic("module queries cache not set") + } + return c +} + // AssetsCache gets the file cache for assets (processed resources, SCSS etc.). func (f Caches) AssetsCache() *Cache { return f[CacheKeyAssets] diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index c6d346dfc..8da26fa18 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -59,7 +59,7 @@ dir = "/path/to/c4" c.Assert(err, qt.IsNil) fs := afero.NewMemMapFs() decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches - c.Assert(len(decoded), qt.Equals, 7) + c.Assert(len(decoded), qt.Equals, 8) c2 := decoded["getcsv"] c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s") @@ -106,7 +106,7 @@ dir = "/path/to/c4" c.Assert(err, qt.IsNil) fs := afero.NewMemMapFs() decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches - c.Assert(len(decoded), qt.Equals, 7) + c.Assert(len(decoded), qt.Equals, 8) for _, v := range decoded { c.Assert(v.MaxAge, qt.Equals, time.Duration(0)) @@ -129,7 +129,7 @@ func TestDecodeConfigDefault(t *testing.T) { fs := afero.NewMemMapFs() decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches - c.Assert(len(decoded), qt.Equals, 7) + c.Assert(len(decoded), qt.Equals, 8) imgConfig := decoded[filecache.CacheKeyImages] jsonConfig := decoded[filecache.CacheKeyGetJSON] diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index b49ba7645..e009ce2a9 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -55,9 +55,12 @@ dir = ":resourceDir/_gen" for _, name := range []string{filecache.CacheKeyGetCSV, filecache.CacheKeyGetJSON, filecache.CacheKeyAssets, filecache.CacheKeyImages} { msg := qt.Commentf("cache: %s", name) - p := newPathsSpec(t, afero.NewMemMapFs(), configStr) - caches, err := filecache.NewCaches(p) + fs := afero.NewMemMapFs() + p := newPathsSpec(t, fs, configStr) + fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs) + caches, err := filecache.NewCaches(fileCachConfig, fs) c.Assert(err, qt.IsNil) + caches.SetResourceFs(fs) cache := caches[name] for i := range 10 { id := fmt.Sprintf("i%d", i) @@ -84,8 +87,9 @@ dir = ":resourceDir/_gen" } } - caches, err = filecache.NewCaches(p) + caches, err = filecache.NewCaches(fileCachConfig, fs) c.Assert(err, qt.IsNil) + caches.SetResourceFs(fs) cache = caches[name] // Touch one and then prune. cache.GetOrCreateBytes("i5", func() ([]byte, error) { diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index c1d79dfa8..ed669933f 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -78,9 +78,11 @@ dir = ":cacheDir/c" configStr = strings.Replace(configStr, "\\", winPathSep, -1) p := newPathsSpec(t, osfs, configStr) + fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs) - caches, err := filecache.NewCaches(p) + caches, err := filecache.NewCaches(fileCachConfig, p.Fs.Source) c.Assert(err, qt.IsNil) + caches.SetResourceFs(p.SourceFs) cache := caches.Get("GetJSON") c.Assert(cache, qt.Not(qt.IsNil)) @@ -149,7 +151,7 @@ dir = ":cacheDir/c" r.Close() c.Assert(string(b), qt.Equals, "Hugo is great!") - info, b, err = caches.ImageCache().GetBytes("mykey") + info, b, err = caches.ImageCache().GetItemBytes("mykey") c.Assert(err, qt.IsNil) c.Assert(info.Name, qt.Equals, "mykey") c.Assert(string(b), qt.Equals, "Hugo is great!") @@ -179,9 +181,10 @@ dir = "/cache/c" ` p := newPathsSpec(t, afero.NewMemMapFs(), configStr) - - caches, err := filecache.NewCaches(p) + fileCachConfig := p.Cfg.GetConfigSection("caches").(filecache.Configs) + caches, err := filecache.NewCaches(fileCachConfig, p.Fs.Source) c.Assert(err, qt.IsNil) + caches.SetResourceFs(p.Fs.Source) const cacheName = "getjson" diff --git a/common/hashing/hashing.go b/common/hashing/hashing.go index 793e25a8c..ad28bbce2 100644 --- a/common/hashing/hashing.go +++ b/common/hashing/hashing.go @@ -87,11 +87,13 @@ func XXHashFromString(s string) (uint64, error) { return h.Sum64(), nil } -// XxHashFromStringHexEncoded calculates the xxHash for the given string +// XxHashFromStringHexEncoded calculates the xxHash for the given strings // and returns the hash as a hex encoded string. -func XxHashFromStringHexEncoded(f string) string { +func XxHashFromStringHexEncoded(s ...string) string { h := xxhash.New() - h.WriteString(f) + for _, f := range s { + h.WriteString(f) + } hash := h.Sum(nil) return hex.EncodeToString(hash) } diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index ddfa70572..1efe502b1 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -822,6 +822,7 @@ type Configs struct { Modules modules.Modules ModulesClient *modules.Client + FileCaches filecache.Caches // All below is set in Init. Languages langs.Languages @@ -852,7 +853,7 @@ func (c *Configs) IsZero() bool { return c == nil || len(c.Languages) == 0 } -func (c *Configs) Init(logger loggers.Logger) error { +func (c *Configs) Init(sourceFs afero.Fs, logger loggers.Logger) error { var languages langs.Languages for _, f := range c.Base.Languages.Config.Sorted { @@ -1154,10 +1155,16 @@ func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadCon l.CommonDirs.CacheDir = bcfg.CacheDir } + caches, err := filecache.NewCaches(all.Caches, fs) + if err != nil { + return nil, fmt.Errorf("failed to create file caches from configuration: %w", err) + } + cm := &Configs{ Base: all, LanguageConfigMap: langConfigMap, LoadingInfo: res, + FileCaches: caches, IsMultihost: isMultihost, } diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index d27b1ba4f..c52d5da3d 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -25,9 +25,8 @@ import ( ) type ConfigLanguage struct { - config *Config - baseConfig config.BaseConfig - + config *Config + baseConfig config.BaseConfig m *Configs language *langs.Language languageIndex int @@ -123,6 +122,10 @@ func (c ConfigLanguage) BaseConfig() config.BaseConfig { return c.baseConfig } +func (c ConfigLanguage) FileCaches() any { + return c.m.FileCaches +} + func (c ConfigLanguage) Dirs() config.CommonDirs { return c.config.CommonDirs } diff --git a/config/allconfig/load.go b/config/allconfig/load.go index 1aa6a1281..b04d024b9 100644 --- a/config/allconfig/load.go +++ b/config/allconfig/load.go @@ -95,7 +95,7 @@ func LoadConfig(d ConfigSourceDescriptor) (configs *Configs, err error) { configs.Modules = moduleConfig.AllModules configs.ModulesClient = modulesClient - if err := configs.Init(d.Logger); err != nil { + if err := configs.Init(d.Fs, d.Logger); err != nil { return nil, fmt.Errorf("failed to init config: %w", err) } @@ -477,6 +477,7 @@ func (l *configLoader) loadModules(configs *Configs, ignoreModuleDoesNotExist bo PublishDir: publishDir, Environment: l.Environment, CacheDir: conf.Caches.CacheDirModules(), + ModuleQueriesCache: configs.FileCaches.ModuleQueriesCache(), ModuleConfig: conf.Module, IgnoreVendor: ignoreVendor, IgnoreModuleDoesNotExist: ignoreModuleDoesNotExist, diff --git a/config/configProvider.go b/config/configProvider.go index 69efc595c..c1a7afeb6 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -41,6 +41,7 @@ type AllProvider interface { Dirs() CommonDirs Quiet() bool DirsBase() CommonDirs + FileCaches() any ContentTypes() ContentTypesProvider GetConfigSection(string) any GetConfig() any diff --git a/deps/deps.go b/deps/deps.go index 33b224013..85e808527 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -260,10 +260,10 @@ func (d *Deps) Init() error { common = d.ResourceSpec.SpecCommon } - fileCaches, err := filecache.NewCaches(d.PathSpec) - if err != nil { - return fmt.Errorf("failed to create file caches from configuration: %w", err) - } + d.Cfg.BaseConfig() + + fileCaches := d.Cfg.FileCaches().(filecache.Caches) + fileCaches.SetResourceFs(d.BaseFs.ResourcesCache) resourceSpec, err := resources.NewSpec(d.PathSpec, common, d.WasmDispatchers, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState) if err != nil { diff --git a/magefile.go b/magefile.go index 266e0bfd4..36ebce06d 100644 --- a/magefile.go +++ b/magefile.go @@ -180,7 +180,25 @@ func Test386() error { // Run tests func Test() error { env := map[string]string{"GOFLAGS": testGoFlags()} - return runCmd(env, goexe, "test", "-p", "2", "./...", "-tags", buildTags()) + if isCI() { + // We have space issues on GitHub Actions (lots tests, incresing usage of Go generics). + // Test each package separately and clean up in between. + pkgs, err := hugoPackages() + if err != nil { + return err + } + for _, pkg := range pkgs { + if err := cmp.Or(CleanTest(), UninstallAll()); err != nil { + return err + } + if err := runCmd(env, goexe, "test", "-p", "2", pkg, "-tags", buildTags()); err != nil { + return err + } + } + return nil + } else { + return runCmd(env, goexe, "test", "-p", "2", "./...", "-tags", buildTags()) + } } // Run tests with race detector @@ -194,13 +212,6 @@ func TestRace() error { return err } for _, pkg := range pkgs { - /*slashCount := strings.Count(pkg, "/") - if slashCount > 1 { - continue - } - if pkg != "." { - pkg += "/..." - }*/ if err := cmp.Or(CleanTest(), UninstallAll()); err != nil { return err } @@ -209,7 +220,6 @@ func TestRace() error { } } return nil - } else { return runCmd(env, goexe, "test", "-p", "2", "-race", "./...", "-tags", buildTags()) } diff --git a/modules/client.go b/modules/client.go index c3d568188..dbad4d35c 100644 --- a/modules/client.go +++ b/modules/client.go @@ -30,8 +30,10 @@ import ( "time" "github.com/gohugoio/hugo/common/collections" + "github.com/gohugoio/hugo/common/hashing" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" @@ -45,8 +47,6 @@ import ( "golang.org/x/mod/module" - "github.com/gohugoio/hugo/common/hugio" - "github.com/spf13/afero" ) @@ -440,16 +440,31 @@ func isProbablyModule(path string) bool { } func (c *Client) downloadModuleVersion(path, version string) (*goModule, error) { - args := []string{"mod", "download", "-json", fmt.Sprintf("%s@%s", path, version)} - b := &bytes.Buffer{} + // If it's already cached, use it. + const keyPrefix = "downloadmoduleversion_" + cacheKey := keyPrefix + hashing.XxHashFromStringHexEncoded(c.ccfg.CacheDir, path, version) + ".json" + if b, _ := c.ccfg.ModuleQueriesCache.GetBytes(cacheKey); len(b) > 0 { + var cm goModule + if err := json.Unmarshal(b, &cm); err == nil && cm.Dir != "" { + if c.isDirNonEmpty(cm.Dir) { + return &cm, nil + } + } + } + // No valid cache, need to query. + args := []string{"mod", "download", "-json", fmt.Sprintf("%s@%s", path, version)} + + b := &bytes.Buffer{} err := c.runGo(context.Background(), b, args...) if err != nil { return nil, fmt.Errorf("failed to download module %s@%s: %w", path, version, err) } + jsonResult := b.Bytes() + m := &goModule{} - if err := json.NewDecoder(b).Decode(m); err != nil { + if err := json.NewDecoder(bytes.NewReader(jsonResult)).Decode(m); err != nil { return nil, fmt.Errorf("failed to decode module download result: %w", err) } @@ -457,9 +472,26 @@ func (c *Client) downloadModuleVersion(path, version string) (*goModule, error) return nil, errors.New(m.Error.Err) } + // Cache the successful result if it has a Dir field. + if m.Dir != "" { + _ = c.ccfg.ModuleQueriesCache.SetBytes(cacheKey, jsonResult) + } + return m, nil } +// isDirNonEmpty returns true if dir exists and contains at least one file or subdirectory. +func (c *Client) isDirNonEmpty(dir string) bool { + f, err := c.fs.Open(dir) + if err != nil { + return false + } + defer f.Close() + // Read just one entry to check if directory is non-empty. + _, err = f.Readdirnames(1) + return err == nil +} + func (c *Client) listGoMods() (goModules, error) { if c.GoModulesFilename == "" || !c.moduleConfig.hasModuleImport() { return nil, nil @@ -865,8 +897,15 @@ type ClientConfig struct { Exec *hexec.Exec - CacheDir string // Module cache - ModuleConfig Config + CacheDir string // Module cache + ModuleQueriesCache ModuleQueriesCache + ModuleConfig Config +} + +// ModuleQueriesCache is a cache for module version queries. +type ModuleQueriesCache interface { + GetBytes(id string) ([]byte, error) + SetBytes(id string, data []byte) error } func (c ClientConfig) shouldIgnoreVendor(path string) bool { diff --git a/modules/collect.go b/modules/collect.go index de803dbd4..add5ae69a 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -391,14 +391,11 @@ func (c *collector) addAndRecurse(owner *moduleAdapter) error { pk := pathVersionKey{path: tc.Path(), version: tc.Version()} seenInCurrent := seen[pk] if seenInCurrent { - // Only one import of the same module per project. - if owner.projectMod { - // In Hugo v0.150.0 we introduced direct dependencies, and it may be tempting to import the same version - // with different mount setups. We may allow that in the future, but we need to get some experience first. - // For now, we just warn. The user needs to add multiple mount points in the same import. - c.logger.Warnf("module with path %q is imported for the same version %q more than once", tc.Path(), tc.Version()) + // Only one import of the same module per project, unless this is the main project. + if !owner.projectMod { + c.logger.Warnf("module with path %q is imported for the same version %q more than once; this is currently only allowed in the main project's config.", tc.Path(), tc.Version()) + continue } - continue } seen[pk] = true diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 898cd4c31..e91b0ea7e 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -111,7 +111,7 @@ func (c *ResourceCache) getFromFile(key string) (filecache.ItemInfo, io.ReadClos var meta transformedResourceMetadata filenameMeta, filenameContent := c.getFilenames(key) - _, jsonContent, _ := c.fileCache.GetBytes(filenameMeta) + jsonContent, _ := c.fileCache.GetBytes(filenameMeta) if jsonContent == nil { return filecache.ItemInfo{}, nil, meta, false } diff --git a/resources/resource_transformers/babel/babel_integration_test.go b/resources/resource_transformers/babel/babel_integration_test.go index 4344af2be..c08114b6c 100644 --- a/resources/resource_transformers/babel/babel_integration_test.go +++ b/resources/resource_transformers/babel/babel_integration_test.go @@ -67,9 +67,9 @@ Transpiled3: {{ $transpiled.Permalink }} "scripts": {}, "devDependencies": { - "@babel/cli": "7.8.4", - "@babel/core": "7.9.0", - "@babel/preset-env": "7.9.5" + "@babel/cli": "7.28.6", + "@babel/core": "7.28.6", + "@babel/preset-env": "7.28.6" } }