testing: Revise usage of b.N and b.Loop() in benchmarks

common/hreflect/helpers_test.go

@@ -152,17 +152,18 @@ func TestIndirectInterface(t *testing.T) {
 }
 
 func BenchmarkIsContextType(b *testing.B) {
+	const size = 1000
 	type k string
 	b.Run("value", func(b *testing.B) {
 		ctx := context.Background()
-		ctxs := make([]reflect.Type, b.N)
+		ctxs := make([]reflect.Type, size)
-		for i := 0; b.Loop(); i++ {
+		for i := range size {
 			ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i))
 		}
 
-		b.ResetTimer()
+		for i := 0; b.Loop(); i++ {
-		for i := 0; i < b.N; i++ {
+			idx := i % size
-			if !IsContextType(ctxs[i]) {
+			if !IsContextType(ctxs[idx]) {
 				b.Fatal("not context")
 			}
 		}

common/hstrings/strings_test.go

@@ -71,6 +71,7 @@ func TestUniqueStringsSorted(t *testing.T) {
 	c.Assert(UniqueStringsSorted(nil), qt.IsNil)
 }
 
+// Note that these cannot use b.Loop() because of golang/go#27217.
 func BenchmarkUniqueStrings(b *testing.B) {
 	input := []string{"a", "b", "d", "e", "d", "h", "a", "i"}
 
@@ -84,15 +85,14 @@ func BenchmarkUniqueStrings(b *testing.B) {
 	})
 
 	b.Run("Reuse slice", func(b *testing.B) {
-		b.StopTimer()
 		inputs := make([][]string, b.N)
-		for i := 0; b.Loop(); i++ {
+		for i := 0; i < b.N; i++ {
 			inputc := make([]string, len(input))
 			copy(inputc, input)
 			inputs[i] = inputc
 		}
-		b.StartTimer()
+		b.ResetTimer()
-		for i := 0; b.Loop(); i++ {
+		for i := 0; i < b.N; i++ {
 			inputc := inputs[i]
 
 			result := UniqueStringsReuse(inputc)
@@ -103,14 +103,13 @@ func BenchmarkUniqueStrings(b *testing.B) {
 	})
 
 	b.Run("Reuse slice sorted", func(b *testing.B) {
-		b.StopTimer()
 		inputs := make([][]string, b.N)
 		for i := 0; i < b.N; i++ {
 			inputc := make([]string, len(input))
 			copy(inputc, input)
 			inputs[i] = inputc
 		}
-		b.StartTimer()
+		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			inputc := inputs[i]
 

compare/compare_strings_test.go

@@ -63,6 +63,7 @@ func TestLexicographicSort(t *testing.T) {
 	c.Assert(s, qt.DeepEquals, []string{"A", "b", "Ba", "ba", "ba", "Bz"})
 }
 
+// // Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkStringSort(b *testing.B) {
 	prototype := []string{"b", "Bz", "zz", "ba", "αβδ αβδ αβδ", "A", "Ba", "ba", "nnnnasdfnnn", "AAgæåz", "αβδC"}
 	b.Run("LessStrings", func(b *testing.B) {

hugolib/cascade_test.go

@@ -34,15 +34,11 @@ func BenchmarkCascade(b *testing.B) {
 		langs := allLangs[0:i]
 		b.Run(fmt.Sprintf("langs-%d", len(langs)), func(b *testing.B) {
 			c := qt.New(b)
-			b.StopTimer()
-			builders := make([]*sitesBuilder, b.N)
-			for i := 0; b.Loop(); i++ {
-				builders[i] = newCascadeTestBuilder(b, langs)
-			}
-			b.StartTimer()
 
-			for i := 0; b.Loop(); i++ {
+			for b.Loop() {
-				builder := builders[i]
+				b.StopTimer()
+				builder := newCascadeTestBuilder(b, langs)
+				b.StartTimer()
 				err := builder.BuildE(BuildCfg{})
 				c.Assert(err, qt.IsNil)
 				first := builder.H.Sites[0]
@@ -75,16 +71,13 @@ kind = '{section,term}'
 			T:           b,
 			TxtarString: files,
 		}
-		builders := make([]*IntegrationTestBuilder, b.N)
-
-		for i := range builders {
-			builders[i] = NewIntegrationTestBuilder(cfg)
-		}
-
 		b.ResetTimer()
 
-		for i := 0; i < b.N; i++ {
+		for b.Loop() {
-			builders[i].Build()
+			b.StopTimer()
+			builder := NewIntegrationTestBuilder(cfg)
+			b.StartTimer()
+			builder.Build()
 		}
 	})
 }

hugolib/hugo_smoke_test.go

@@ -743,14 +743,12 @@ func BenchmarkBaseline(b *testing.B) {
 		T:           b,
 		TxtarString: benchmarkBaselineFiles(false),
 	}
-	builders := make([]*IntegrationTestBuilder, b.N)
 
-	for i := range builders {
+	for b.Loop() {
-		builders[i] = NewIntegrationTestBuilder(cfg)
+		b.StopTimer()
-	}
+		builder := NewIntegrationTestBuilder(cfg)
-
+		b.StartTimer()
-	for i := 0; i < b.N; i++ {
+		builder.Build()
-		builders[i].Build()
 	}
 }
 

hugolib/page_permalink_test.go

@@ -64,7 +64,6 @@ func TestPermalink(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		test := test
 		t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) {
 			t.Parallel()
 			c := qt.New(t)

hugolib/pagecollections_test.go

@@ -37,17 +37,16 @@ categories:
 `
 
 func BenchmarkGetPage(b *testing.B) {
-	var (
+	cfg, fs := newTestCfg()
-		cfg, fs = newTestCfg()
-		r       = rand.New(rand.NewSource(time.Now().UnixNano()))
-	)
 
 	configs, err := loadTestConfigFromProvider(cfg)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	for i := range 10 {
+	const size = 10
+
+	for i := range size {
 		for j := range 100 {
 			writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), "CONTENT")
 		}
@@ -55,19 +54,18 @@ func BenchmarkGetPage(b *testing.B) {
 
 	s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Configs: configs}, BuildCfg{SkipRender: true})
 
-	pagePaths := make([]string, b.N)
+	pagePaths := make([]string, size)
-
+	for i := range size {
-	for i := 0; b.Loop(); i++ {
+		pagePaths[i] = fmt.Sprintf("sect%d", i)
-		pagePaths[i] = fmt.Sprintf("sect%d", r.Intn(10))
 	}
 
-	for i := 0; i < b.N; i++ {
+	for i := 0; b.Loop(); i++ {
 		home, _ := s.getPage(nil, "/")
 		if home == nil {
 			b.Fatal("Home is nil")
 		}
 
-		p, _ := s.getPage(nil, pagePaths[i])
+		p, _ := s.getPage(nil, pagePaths[i%size])
 		if p == nil {
 			b.Fatal("Section is nil")
 		}
@@ -113,20 +111,21 @@ func TestBenchmarkGetPageRegular(t *testing.T) {
 
 func BenchmarkGetPageRegular(b *testing.B) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	const size = 100
 
 	b.Run("From root", func(b *testing.B) {
 		s := createGetPageRegularBenchmarkSite(b)
 		c := qt.New(b)
 
-		pagePaths := make([]string, b.N)
+		pagePaths := make([]string, size)
 
-		for i := 0; i < b.N; i++ {
+		for i := range size {
-			pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100)))
+			pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", i))
 		}
 
 		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
+		for i := 0; b.Loop(); i++ {
-			page, _ := s.getPage(nil, pagePaths[i])
+			page, _ := s.getPage(nil, pagePaths[i%size])
 			c.Assert(page, qt.Not(qt.IsNil))
 		}
 	})
@@ -136,17 +135,15 @@ func BenchmarkGetPageRegular(b *testing.B) {
 		c := qt.New(b)
 		allPages := s.RegularPages()
 
-		pagePaths := make([]string, b.N)
+		pagePaths := make([]string, size)
-		pages := make([]page.Page, b.N)
+		pages := allPages[:size]
 
-		for i := 0; i < b.N; i++ {
+		for i := range size {
-			pagePaths[i] = fmt.Sprintf("page%d.md", r.Intn(100))
+			pagePaths[i] = fmt.Sprintf("page%d.md", i)
-			pages[i] = allPages[r.Intn(len(allPages)/3)]
 		}
 
-		b.ResetTimer()
+		for i := 0; b.Loop(); i++ {
-		for i := 0; i < b.N; i++ {
+			page, _ := s.getPage(pages[i%size], pagePaths[i%size])
-			page, _ := s.getPage(pages[i], pagePaths[i])
 			c.Assert(page, qt.Not(qt.IsNil))
 		}
 	})

hugolib/rebuild_test.go

@@ -1787,22 +1787,15 @@ func BenchmarkRebuildContentFileChange(b *testing.B) {
 		T:           b,
 		TxtarString: files,
 		Running:     true,
-		// Verbose:     true,
-		// LogLevel: logg.LevelInfo,
-	}
-	builders := make([]*IntegrationTestBuilder, b.N)
-
-	for i := range builders {
-		builders[i] = NewIntegrationTestBuilder(cfg)
-		builders[i].Build()
 	}
 
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
-		bb := builders[i]
+		b.StopTimer()
+		bb := NewIntegrationTestBuilder(cfg).Build()
+		b.StartTimer()
 		bb.EditFileReplaceFunc("content/mysect/p123/index.md", func(s string) string {
 			return s + "... Edited"
 		}).Build()
-		// fmt.Println(bb.LogString())
 	}
 }
 

hugolib/shortcode_test.go

@@ -242,6 +242,7 @@ CSV: {{< myShort >}}
 	)
 }
 
+// Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 	type input struct {
 		in           []byte
@@ -263,7 +264,7 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
 	cnt := 0
 	in := make([]input, b.N*len(data))
-	for b.Loop() {
+	for i := 0; i < b.N; i++ {
 		for _, this := range data {
 			replacements := make(map[string]shortcodeRenderer)
 			for k, v := range this.replacements {
@@ -279,7 +280,8 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
 	cnt = 0
 	ctx := context.Background()
-	for i := 0; b.Loop(); i++ {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
 		for j := range data {
 			currIn := in[cnt]
 			cnt++
@@ -335,14 +337,12 @@ title: "Markdown Shortcode"
 		T:           b,
 		TxtarString: files,
 	}
-	builders := make([]*IntegrationTestBuilder, b.N)
 
-	for i := range builders {
+	for b.Loop() {
-		builders[i] = NewIntegrationTestBuilder(cfg)
+		b.StopTimer()
-	}
+		builder := NewIntegrationTestBuilder(cfg)
-
+		b.StartTimer()
-	for i := 0; i < b.N; i++ {
+		builder.Build()
-		builders[i].Build()
 	}
 }
 

hugolib/site_benchmark_new_test.go

@@ -1,560 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
-	"fmt"
-	"math/rand"
-	"path"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/gohugoio/hugo/resources/page"
-
-	qt "github.com/frankban/quicktest"
-)
-
-type siteBenchmarkTestcase struct {
-	name   string
-	create func(t testing.TB) *sitesBuilder
-	check  func(s *sitesBuilder)
-}
-
-func getBenchmarkSiteDeepContent(b testing.TB) *sitesBuilder {
-	pageContent := func(size int) string {
-		return getBenchmarkTestDataPageContentForMarkdown(size, false, "", benchmarkMarkdownSnippets)
-	}
-
-	sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-			
-`)
-
-	createContent := func(dir, name string) {
-		sb.WithContent(filepath.Join("content", dir, name), pageContent(1))
-	}
-
-	createBundledFiles := func(dir string) {
-		sb.WithContent(filepath.Join("content", dir, "data.json"), `{ "hello": "world" }`)
-		for i := 1; i <= 3; i++ {
-			sb.WithContent(filepath.Join("content", dir, fmt.Sprintf("page%d.md", i)), pageContent(1))
-		}
-	}
-
-	for _, lang := range []string{"en", "fr", "no", "sv"} {
-		for level := 1; level <= 5; level++ {
-			sectionDir := path.Join(lang, strings.Repeat("section/", level))
-			createContent(sectionDir, "_index.md")
-			createBundledFiles(sectionDir)
-			for i := 1; i <= 3; i++ {
-				leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-				createContent(leafBundleDir, "index.md")
-				createBundledFiles(path.Join(leafBundleDir, "assets1"))
-				createBundledFiles(path.Join(leafBundleDir, "assets1", "assets2"))
-			}
-		}
-	}
-
-	return sb
-}
-
-func getBenchmarkTestDataPageContentForMarkdown(size int, toml bool, category, markdown string) string {
-	base := `---
-title: "My Page"
-%s
----
-
-My page content.
-`
-	if toml {
-		base = `+++
-title="My Page"
-%s
-+++
-
-My page content.
-`
-	}
-
-	var categoryKey string
-	if category != "" {
-		categoryKey = fmt.Sprintf("categories: [%s]", category)
-		if toml {
-			categoryKey = fmt.Sprintf("categories=[%s]", category)
-		}
-	}
-	base = fmt.Sprintf(base, categoryKey)
-
-	return base + strings.Repeat(markdown, size)
-}
-
-const benchmarkMarkdownSnippets = `
-
-## Links
-
-
-This is [an example](http://example.com/ "Title") inline link.
-
-[This link](http://example.net/) has no title attribute.
-
-This is [Relative](/all-is-relative).
-
-See my [About](/about/) page for details. 
-`
-
-func getBenchmarkSiteTestCases() []siteBenchmarkTestcase {
-	pageContentWithCategory := func(size int, category string) string {
-		return getBenchmarkTestDataPageContentForMarkdown(size, false, category, benchmarkMarkdownSnippets)
-	}
-
-	pageContent := func(size int) string {
-		return getBenchmarkTestDataPageContentForMarkdown(size, false, "", benchmarkMarkdownSnippets)
-	}
-
-	config := `
-baseURL = "https://example.com"
-`
-
-	benchmarks := []siteBenchmarkTestcase{
-		{
-			"Bundle with image", func(b testing.TB) *sitesBuilder {
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-				sb.WithContent("content/blog/mybundle/index.md", pageContent(1))
-				sb.WithSunset("content/blog/mybundle/sunset1.jpg")
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.AssertFileContent("public/blog/mybundle/index.html", "/blog/mybundle/sunset1.jpg")
-				s.CheckExists("public/blog/mybundle/sunset1.jpg")
-			},
-		},
-		{
-			"Bundle with JSON file", func(b testing.TB) *sitesBuilder {
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-				sb.WithContent("content/blog/mybundle/index.md", pageContent(1))
-				sb.WithContent("content/blog/mybundle/mydata.json", `{ "hello": "world" }`)
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.AssertFileContent("public/blog/mybundle/index.html", "Resources: application/json: /blog/mybundle/mydata.json")
-				s.CheckExists("public/blog/mybundle/mydata.json")
-			},
-		},
-		{
-			"Tags and categories", func(b testing.TB) *sitesBuilder {
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-title = "Tags and Cats"
-baseURL = "https://example.com"
-
-`)
-
-				const pageTemplate = `
----
-title: "Some tags and cats"
-categories: ["caGR", "cbGR"]
-tags: ["taGR", "tbGR"]
----
-
-Some content.
-			
-`
-				for i := 1; i <= 100; i++ {
-					content := strings.Replace(pageTemplate, "GR", strconv.Itoa(i/3), -1)
-					sb.WithContent(fmt.Sprintf("content/page%d.md", i), content)
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.AssertFileContent("public/page3/index.html", "/page3/|Permalink: https://example.com/page3/")
-				s.AssertFileContent("public/tags/ta3/index.html", "a3")
-			},
-		},
-		{
-			"Canonify URLs", func(b testing.TB) *sitesBuilder {
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-title = "Canon"
-baseURL = "https://example.com"
-canonifyURLs = true
-
-`)
-				for i := 1; i <= 100; i++ {
-					sb.WithContent(fmt.Sprintf("content/page%d.md", i), pageContent(i))
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.AssertFileContent("public/page8/index.html", "https://example.com/about/")
-			},
-		},
-
-		{
-			"Deep content tree", func(b testing.TB) *sitesBuilder {
-				return getBenchmarkSiteDeepContent(b)
-			},
-			func(s *sitesBuilder) {
-				s.CheckExists("public/blog/mybundle/index.html")
-				s.Assert(len(s.H.Sites), qt.Equals, 4)
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, len(s.H.Sites[1].RegularPages()))
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 30)
-			},
-		},
-		{
-			"TOML front matter", func(b testing.TB) *sitesBuilder {
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", config)
-				for i := 1; i <= 200; i++ {
-					content := getBenchmarkTestDataPageContentForMarkdown(1, true, "\"a\", \"b\", \"c\"", benchmarkMarkdownSnippets)
-					sb.WithContent(fmt.Sprintf("content/p%d.md", i), content)
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-			},
-		},
-		{
-			"Many HTML templates", func(b testing.TB) *sitesBuilder {
-				pageTemplateTemplate := `
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-    <title>{{ if not .IsPage }}{{ .Title }}{{ else }}{{ printf "Site: %s" site.Title }}{{ end }}</title>
-    <style>
-     body {
-       margin: 3rem;
-     }
-    </style>
-  </head>
-  <body>
-    <div class="page">{{ .Content }}</div>
-    <ul>
-    {{ with .Pages }}
-    {{ range . }}
-    <li><a href="{{ .RelPermalink }}">{{ .LinkTitle }} {{ if not .IsNode }} (Page){{ end }}</a></li>
-    {{ end }}
-    {{ end }}
-    </ul>
-  </body>
-</html>
-`
-
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-			
-`)
-
-				createContent := func(dir, name string) {
-					sb.WithContent(filepath.Join("content", dir, name), pageContent(1))
-				}
-
-				for _, lang := range []string{"en", "fr", "no", "sv"} {
-					sb.WithTemplatesAdded(fmt.Sprintf("_default/single.%s.html", lang), pageTemplateTemplate)
-					sb.WithTemplatesAdded(fmt.Sprintf("_default/list.%s.html", lang), pageTemplateTemplate)
-
-					for level := 1; level <= 5; level++ {
-						sectionDir := path.Join(lang, strings.Repeat("section/", level))
-						createContent(sectionDir, "_index.md")
-						for i := 1; i <= 3; i++ {
-							leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-							createContent(leafBundleDir, "index.md")
-						}
-					}
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.CheckExists("public/blog/mybundle/index.html")
-				s.Assert(len(s.H.Sites), qt.Equals, 4)
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, len(s.H.Sites[1].RegularPages()))
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 15)
-			},
-		},
-		{
-			"Page collections", func(b testing.TB) *sitesBuilder {
-				pageTemplateTemplate := `
-{{ if .IsNode }}
-{{ len .Paginator.Pages }}
-{{ end }}
-{{ len .Sections }}
-{{ len .Pages }}
-{{ len .RegularPages }}
-{{ len .Resources }}
-{{ len site.RegularPages }}
-{{ len site.Pages }}
-{{ with .NextInSection }}Next in section: {{ .RelPermalink }}{{ end }}
-{{ with .PrevInSection }}Prev in section: {{ .RelPermalink }}{{ end }}
-{{ with .Next }}Next: {{ .RelPermalink }}{{ end }}
-{{ with .Prev }}Prev: {{ .RelPermalink }}{{ end }}
-`
-
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-
-[languages]
-[languages.en]
-weight=1
-contentDir="content/en"
-[languages.fr]
-weight=2
-contentDir="content/fr"
-[languages.no]
-weight=3
-contentDir="content/no"
-[languages.sv]
-weight=4
-contentDir="content/sv"
-			
-`)
-
-				sb.WithTemplates("index.html", pageTemplateTemplate)
-				sb.WithTemplates("_default/single.html", pageTemplateTemplate)
-				sb.WithTemplates("_default/list.html", pageTemplateTemplate)
-
-				r := rand.New(rand.NewSource(99))
-
-				createContent := func(dir, name string) {
-					var content string
-					if strings.Contains(name, "_index") {
-						content = pageContent(1)
-					} else {
-						content = pageContentWithCategory(1, fmt.Sprintf("category%d", r.Intn(5)+1))
-					}
-
-					sb.WithContent(filepath.Join("content", dir, name), content)
-				}
-
-				createBundledFiles := func(dir string) {
-					sb.WithContent(filepath.Join("content", dir, "data.json"), `{ "hello": "world" }`)
-					for i := 1; i <= 3; i++ {
-						sb.WithContent(filepath.Join("content", dir, fmt.Sprintf("page%d.md", i)), pageContent(1))
-					}
-				}
-
-				for _, lang := range []string{"en", "fr", "no", "sv"} {
-					for level := 1; level <= r.Intn(5)+1; level++ {
-						sectionDir := path.Join(lang, strings.Repeat("section/", level))
-						createContent(sectionDir, "_index.md")
-						createBundledFiles(sectionDir)
-						for i := 1; i <= r.Intn(20)+1; i++ {
-							leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-							createContent(leafBundleDir, "index.md")
-							createBundledFiles(path.Join(leafBundleDir, "assets1"))
-							createBundledFiles(path.Join(leafBundleDir, "assets1", "assets2"))
-						}
-					}
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.CheckExists("public/blog/mybundle/index.html")
-				s.Assert(len(s.H.Sites), qt.Equals, 4)
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 26)
-			},
-		},
-		{
-			"List terms", func(b testing.TB) *sitesBuilder {
-				pageTemplateTemplate := `
-<ul>
-    {{ range (.GetTerms "categories") }}
-        <li><a href="{{ .Permalink }}">{{ .LinkTitle }}</a></li>
-   {{ end }}
-</ul>
-`
-
-				sb := newTestSitesBuilder(b).WithConfigFile("toml", `
-baseURL = "https://example.com"
-`)
-
-				sb.WithTemplates("_default/single.html", pageTemplateTemplate)
-				sb.WithTemplates("_default/list.html", "List")
-
-				r := rand.New(rand.NewSource(99))
-
-				createContent := func(dir, name string) {
-					var content string
-					if strings.Contains(name, "_index") {
-						// Empty
-					} else {
-						content = pageContentWithCategory(1, fmt.Sprintf("category%d", r.Intn(5)+1))
-					}
-					sb.WithContent(filepath.Join("content", dir, name), content)
-				}
-
-				for level := 1; level <= r.Intn(5)+1; level++ {
-					sectionDir := path.Join(strings.Repeat("section/", level))
-					createContent(sectionDir, "_index.md")
-					for i := 1; i <= r.Intn(33); i++ {
-						leafBundleDir := path.Join(sectionDir, fmt.Sprintf("bundle%d", i))
-						createContent(leafBundleDir, "index.md")
-					}
-				}
-
-				return sb
-			},
-			func(s *sitesBuilder) {
-				s.AssertFileContent("public/section/bundle8/index.html", `<a href="https://example.com/categories/category1/">`)
-				s.Assert(len(s.H.Sites), qt.Equals, 1)
-				s.Assert(len(s.H.Sites[0].RegularPages()), qt.Equals, 35)
-			},
-		},
-	}
-
-	return benchmarks
-}
-
-// Run the benchmarks below as tests. Mostly useful when adding new benchmark
-// variants.
-func TestBenchmarkSite(b *testing.T) {
-	benchmarks := getBenchmarkSiteTestCases()
-	for _, bm := range benchmarks {
-		if bm.name != "Deep content tree" {
-			continue
-		}
-		b.Run(bm.name, func(b *testing.T) {
-			s := bm.create(b)
-
-			err := s.BuildE(BuildCfg{})
-			if err != nil {
-				b.Fatal(err)
-			}
-			bm.check(s)
-		})
-	}
-}
-
-func TestBenchmarkSiteDeepContentEdit(t *testing.T) {
-	b := getBenchmarkSiteDeepContent(t).Running()
-	b.Build(BuildCfg{})
-
-	p := b.H.Sites[0].RegularPages()[12]
-
-	b.EditFiles(p.File().Filename(), fmt.Sprintf(`---
-title: %s
----
-
-Edited!!`, p.Title()))
-
-	counters := &buildCounters{}
-
-	b.Build(BuildCfg{testCounters: counters})
-
-	// We currently rebuild all the language versions of the same content file.
-	// We could probably optimize that case, but it's not trivial.
-	b.Assert(int(counters.contentRenderCounter.Load()), qt.Equals, 4)
-	b.AssertFileContent("public"+p.RelPermalink()+"index.html", "Edited!!")
-}
-
-func BenchmarkSiteNew(b *testing.B) {
-	rnd := rand.New(rand.NewSource(32))
-	benchmarks := getBenchmarkSiteTestCases()
-	for _, edit := range []bool{true, false} {
-		for _, bm := range benchmarks {
-			name := bm.name
-			if edit {
-				name = "Edit_" + name
-			} else {
-				name = "Regular_" + name
-			}
-			b.Run(name, func(b *testing.B) {
-				sites := make([]*sitesBuilder, b.N)
-				for i := 0; i < b.N; i++ {
-					sites[i] = bm.create(b)
-					if edit {
-						sites[i].Running()
-					}
-				}
-
-				b.ResetTimer()
-				for i := 0; b.Loop(); i++ {
-					if edit {
-						b.StopTimer()
-					}
-					s := sites[i]
-					err := s.BuildE(BuildCfg{})
-					if err != nil {
-						b.Fatal(err)
-					}
-					bm.check(s)
-
-					if edit {
-						if edit {
-							b.StartTimer()
-						}
-						// Edit a random page in a random language.
-						pages := s.H.Sites[rnd.Intn(len(s.H.Sites))].Pages()
-						var p page.Page
-						count := 0
-						for {
-							count++
-							if count > 100 {
-								panic("infinite loop")
-							}
-							p = pages[rnd.Intn(len(pages))]
-							if p.File() != nil {
-								break
-							}
-						}
-
-						s.EditFiles(p.File().Filename(), fmt.Sprintf(`---
-title: %s
----
-
-Edited!!`, p.Title()))
-
-						err := s.BuildE(BuildCfg{})
-						if err != nil {
-							b.Fatal(err)
-						}
-					}
-				}
-			})
-		}
-	}
-}

hugolib/sitesmatrix/sitematrix_integration_test.go

@@ -1124,13 +1124,11 @@ func BenchmarkSitesMatrixContent(b *testing.B) {
 	for _, numPages := range []int{10, 100} {
 		for _, multipleDimensions := range []bool{false, true} {
 			b.Run(fmt.Sprintf("n%d/md%t", numPages, multipleDimensions), func(b *testing.B) {
-				builders := make([]*hugolib.IntegrationTestBuilder, b.N)
+				for b.Loop() {
-				for i := 0; i < b.N; i++ {
+					b.StopTimer()
-					builders[i] = newSitesMatrixContentBenchmarkBuilder(b, numPages, true, true)
+					bb := newSitesMatrixContentBenchmarkBuilder(b, numPages, true, multipleDimensions)
-				}
+					b.StartTimer()
-				b.ResetTimer()
+					bb.Build()
-				for i := 0; i < b.N; i++ {
-					builders[i].Build()
 				}
 			})
 		}

hugolib/taxonomy_test.go

@@ -830,7 +830,9 @@ title: p1
 }
 
 func BenchmarkTaxonomiesGetTerms(b *testing.B) {
-	createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder {
+	createBuilder := func(b *testing.B, numPages int) *IntegrationTestBuilder {
+		b.StopTimer()
+
 		files := `
 -- hugo.toml --
 baseURL = "https://example.com"
@@ -859,26 +861,23 @@ GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }}
 			tags := tagsVariants[i%len(tagsVariants)]
 			files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags)
 		}
+
 		cfg := IntegrationTestConfig{
 			T:           b,
 			TxtarString: files,
 		}
-		builders := make([]*IntegrationTestBuilder, b.N)
 
-		for i := range builders {
+		bb := NewIntegrationTestBuilder(cfg)
-			builders[i] = NewIntegrationTestBuilder(cfg)
-		}
 
-		b.ResetTimer()
+		b.StartTimer()
 
-		return builders
+		return bb
 	}
 
 	for _, numPages := range []int{100, 1000, 10000, 20000} {
 		b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) {
-			builders := createBuilders(b, numPages)
 			for i := 0; b.Loop(); i++ {
-				builders[i].Build()
+				createBuilder(b, numPages).Build()
 			}
 		})
 	}

identity/identity_test.go

@@ -41,25 +41,14 @@ func BenchmarkIdentityManager(b *testing.B) {
 		}
 	})
 
-	b.Run("Add unique", func(b *testing.B) {
+	b.Run("Add many", func(b *testing.B) {
-		ids := createIds(b.N)
+		const size = 1000
+		ids := createIds(size)
 		im := identity.NewManager()
 
 		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
+		for i := 0; b.Loop(); i++ {
-			im.AddIdentity(ids[i])
+			im.AddIdentity(ids[i%size])
-		}
-
-		b.StopTimer()
-	})
-
-	b.Run("Add duplicates", func(b *testing.B) {
-		id := &testIdentity{base: "a", name: "b"}
-		im := identity.NewManager()
-
-		b.ResetTimer()
-		for b.Loop() {
-			im.AddIdentity(id)
 		}
 
 		b.StopTimer()

markup/goldmark/goldmark_integration_test.go

@@ -278,14 +278,12 @@ D.
 		T:           b,
 		TxtarString: files,
 	}
-	builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-	for i := range builders {
+	for b.Loop() {
-		builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
+		b.StopTimer()
-	}
+		bb := hugolib.NewIntegrationTestBuilder(cfg)
-
+		b.StartTimer()
-	for i := 0; i < b.N; i++ {
+		bb.Build()
-		builders[i].Build()
 	}
 }
 
@@ -335,16 +333,12 @@ FENCE
 			T:           b,
 			TxtarString: files,
 		}
-		builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-		for i := range builders {
+		for b.Loop() {
-			builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
+			b.StopTimer()
-		}
+			bb := hugolib.NewIntegrationTestBuilder(cfg)
-
+			b.StartTimer()
-		b.ResetTimer()
+			bb.Build()
-
-		for i := 0; i < b.N; i++ {
-			builders[i].Build()
 		}
 	}
 

markup/tableofcontents/tableofcontents_test.go

@@ -193,6 +193,7 @@ func TestTocMisc(t *testing.T) {
 	})
 }
 
+// Note that some of these cannot use b.Loop() because of golang/go#27217.
 func BenchmarkToc(b *testing.B) {
 	newTocs := func(n int) []*Fragments {
 		var tocs []*Fragments
@@ -204,22 +205,21 @@ func BenchmarkToc(b *testing.B) {
 
 	b.Run("Build", func(b *testing.B) {
 		var builders []Builder
-		for b.Loop() {
+		for i := 0; i < b.N; i++ {
 			builders = append(builders, newTestTocBuilder())
 		}
 		b.ResetTimer()
-
+		for i := 0; i < b.N; i++ {
-		for i := 0; b.Loop(); i++ {
 			b := builders[i]
 			b.Build()
 		}
 	})
 
 	b.Run("ToHTML", func(b *testing.B) {
-		tocs := newTocs(b.N)
+		const size = 1000
-		b.ResetTimer()
+		tocs := newTocs(size)
-		for i := 0; i < b.N; i++ {
+		for i := 0; b.Loop(); i++ {
-			toc := tocs[i]
+			toc := tocs[i%size]
 			toc.ToHTML(1, -1, false)
 		}
 	})

related/related_integration_test.go

@@ -175,13 +175,11 @@ keywords: ['k%d']
 		T:           b,
 		TxtarString: files,
 	}
-	builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-	for i := range builders {
+	for b.Loop() {
-		builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
+		b.StopTimer()
-	}
+		bb := hugolib.NewIntegrationTestBuilder(cfg)
-
+		b.StartTimer()
-	for i := 0; i < b.N; i++ {
+		bb.Build()
-		builders[i].Build()
 	}
 }

resources/image_test.go

@@ -466,13 +466,9 @@ func TestImageColorsLuminance(t *testing.T) {
 }
 
 func BenchmarkImageExif(b *testing.B) {
-	getImages := func(c *qt.C, b *testing.B, fs afero.Fs) []images.ImageResource {
+	getImage := func(i int, c *qt.C, b *testing.B, fs afero.Fs) images.ImageResource {
 		spec := newTestResourceSpec(specDescriptor{fs: fs, c: c})
-		imgs := make([]images.ImageResource, b.N)
+		return fetchResourceForSpec(spec, c, "sunset.jpg", strconv.Itoa(i)).(images.ImageResource)
-		for i := 0; i < b.N; i++ {
-			imgs[i] = fetchResourceForSpec(spec, c, "sunset.jpg", strconv.Itoa(i)).(images.ImageResource)
-		}
-		return imgs
 	}
 
 	getAndCheckExif := func(c *qt.C, image images.ImageResource) {
@@ -482,43 +478,45 @@ func BenchmarkImageExif(b *testing.B) {
 	}
 
 	b.Run("Cold cache", func(b *testing.B) {
-		b.StopTimer()
 		c := qt.New(b)
-		images := getImages(c, b, afero.NewMemMapFs())
+		fs := afero.NewMemMapFs()
-
-		b.StartTimer()
 		for i := 0; b.Loop(); i++ {
-			getAndCheckExif(c, images[i])
+			b.StopTimer()
+			image := getImage(i, c, b, fs)
+			b.StartTimer()
+			getAndCheckExif(c, image)
 		}
 	})
 
 	b.Run("Cold cache, 10", func(b *testing.B) {
-		b.StopTimer()
 		c := qt.New(b)
-		images := getImages(c, b, afero.NewMemMapFs())
+		fs := afero.NewMemMapFs()
-
-		b.StartTimer()
 		for i := 0; b.Loop(); i++ {
+			b.StopTimer()
+			image := getImage(i, c, b, fs)
+			b.StartTimer()
 			for range 10 {
-				getAndCheckExif(c, images[i])
+				getAndCheckExif(c, image)
 			}
 		}
 	})
 
 	b.Run("Warm cache", func(b *testing.B) {
-		b.StopTimer()
 		c := qt.New(b)
 		fs := afero.NewMemMapFs()
-		images := getImages(c, b, fs)
+		// Prime the cache
-		for i := 0; b.Loop(); i++ {
+		for i := 0; i < b.N; i++ {
-			getAndCheckExif(c, images[i])
+			image := getImage(i, c, b, fs)
+			getAndCheckExif(c, image)
 		}
 
-		images = getImages(c, b, fs)
+		// Start the real benchmark,
-
+		b.ResetTimer()
-		b.StartTimer()
+		for i := 0; i < b.N; i++ {
-		for i := 0; b.Loop(); i++ {
+			b.StopTimer()
-			getAndCheckExif(c, images[i])
+			image := getImage(i, c, b, fs)
+			b.StartTimer()
+			getAndCheckExif(c, image)
 		}
 	})
 }

tpl/collections/collections_test.go

@@ -195,7 +195,6 @@ func TestDictionary(t *testing.T) {
 		{[]any{5, "b"}, false},
 		{[]any{"a", "b", "c"}, false},
 	} {
-		test := test
 		c.Run(fmt.Sprint(i), func(c *qt.C) {
 			c.Parallel()
 			errMsg := qt.Commentf("[%d] %v", i, test.values)

tpl/collections/merge_test.go

@@ -137,8 +137,6 @@ func TestMerge(t *testing.T) {
 		{"all nil", []any{nil, nil}, nil, true},
 	} {
 
-		i := i
-
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 			errMsg := qt.Commentf("[%d] %v", i, test)

tpl/partials/partials_integration_test.go

@@ -237,14 +237,12 @@ ABCDE
 		T:           b,
 		TxtarString: files,
 	}
-	builders := make([]*hugolib.IntegrationTestBuilder, b.N)
 
-	for i := range builders {
+	for b.Loop() {
-		builders[i] = hugolib.NewIntegrationTestBuilder(cfg)
+		b.StopTimer()
-	}
+		bb := hugolib.NewIntegrationTestBuilder(cfg)
-
+		b.StartTimer()
-	for i := 0; i < b.N; i++ {
+		bb.Build()
-		builders[i].Build()
 	}
 }