mirror of
https://github.com/gohugoio/hugo.git
synced 2026-08-24 15:28:54 +00:00
Bjørn Erik Pedersen
c4eba92863
When the server returns a temporary HTTP error (e.g. 429 or 503) together with a Retry-After header, use that value as the next sleep duration instead of the default exponential backoff. The Retry-After value is also surfaced in the retry-timeout error message. Fixes #14828
319 lines
8.2 KiB
Go
319 lines
8.2 KiB
Go
// Copyright 2024 The Hugo Authors. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package create_test
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
qt "github.com/frankban/quicktest"
|
|
"github.com/gohugoio/hugo/htesting"
|
|
"github.com/gohugoio/hugo/hugolib"
|
|
)
|
|
|
|
func TestGetRemoteHead(t *testing.T) {
|
|
files := `
|
|
-- hugo.toml --
|
|
[security]
|
|
[security.http]
|
|
methods = ['(?i)GET|POST|HEAD']
|
|
urls = ['.*gohugo\.io.*']
|
|
-- layouts/home.html --
|
|
{{ $url := "https://gohugo.io/img/hugo.png" }}
|
|
{{ $opts := dict "method" "head" }}
|
|
{{ with try (resources.GetRemote $url $opts) }}
|
|
{{ with .Err }}
|
|
{{ errorf "Unable to get remote resource: %s" . }}
|
|
{{ else with .Value }}
|
|
Head Content: {{ .Content }}. Head Data: {{ .Data }}
|
|
{{ else }}
|
|
{{ errorf "Unable to get remote resource: %s" $url }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
|
|
b := hugolib.Test(t, files)
|
|
|
|
b.AssertFileContent("public/index.html",
|
|
"Head Content: .",
|
|
"Head Data: map[ContentLength:18210 ContentType:image/png Headers:map[] Status:200 OK StatusCode:200 TransferEncoding:[]]",
|
|
)
|
|
}
|
|
|
|
func TestGetRemoteResponseHeaders(t *testing.T) {
|
|
files := `
|
|
-- hugo.toml --
|
|
[security]
|
|
[security.http]
|
|
methods = ['(?i)GET|POST|HEAD']
|
|
urls = ['.*gohugo\.io.*']
|
|
-- layouts/home.html --
|
|
{{ $url := "https://gohugo.io/img/hugo.png" }}
|
|
{{ $opts := dict "method" "head" "responseHeaders" (slice "X-Frame-Options" "Server") }}
|
|
{{ with try (resources.GetRemote $url $opts) }}
|
|
{{ with .Err }}
|
|
{{ errorf "Unable to get remote resource: %s" . }}
|
|
{{ else with .Value }}
|
|
Response Headers: {{ .Data.Headers }}
|
|
{{ else }}
|
|
{{ errorf "Unable to get remote resource: %s" $url }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
|
|
b := hugolib.Test(t, files)
|
|
|
|
b.AssertFileContent("public/index.html",
|
|
"Response Headers: map[Server:[Netlify] X-Frame-Options:[DENY]]",
|
|
)
|
|
}
|
|
|
|
func TestGetRemoteRetry(t *testing.T) {
|
|
t.Parallel()
|
|
htesting.SkipSlowTestUnlessCI(t)
|
|
|
|
temporaryHTTPCodes := []int{408, 429, 500, 502, 503, 504}
|
|
numPages := 20
|
|
|
|
handler := func(w http.ResponseWriter, r *http.Request) {
|
|
if rand.Intn(3) == 0 {
|
|
w.WriteHeader(temporaryHTTPCodes[rand.Intn(len(temporaryHTTPCodes))])
|
|
return
|
|
}
|
|
w.Header().Add("Content-Type", "text/plain")
|
|
w.Write([]byte("Response for " + r.URL.Path + "."))
|
|
}
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(handler))
|
|
t.Cleanup(func() { srv.Close() })
|
|
|
|
filesTemplate := `
|
|
-- hugo.toml --
|
|
disableKinds = ["home", "taxonomy", "term"]
|
|
timeout = "TIMEOUT"
|
|
[security]
|
|
[security.http]
|
|
urls = ['.*']
|
|
mediaTypes = ['text/plain']
|
|
-- layouts/single.html --
|
|
{{ $url := printf "%s%s" "URL" .RelPermalink}}
|
|
{{ $opts := dict }}
|
|
{{ with try (resources.GetRemote $url $opts) }}
|
|
{{ with .Err }}
|
|
{{ errorf "Got Err: %s" . }}
|
|
{{ with .Cause }}{{ errorf "Data: %s" .Data }}{{ end }}
|
|
{{ else with .Value }}
|
|
Content: {{ .Content }}
|
|
{{ else }}
|
|
{{ errorf "Unable to get remote resource: %s" $url }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
|
|
for i := range numPages {
|
|
filesTemplate += fmt.Sprintf("-- content/post/p%d.md --\n", i)
|
|
}
|
|
|
|
filesTemplate = strings.ReplaceAll(filesTemplate, "URL", srv.URL)
|
|
|
|
t.Run("OK", func(t *testing.T) {
|
|
files := strings.ReplaceAll(filesTemplate, "TIMEOUT", "60s")
|
|
b := hugolib.Test(t, files)
|
|
|
|
for i := range numPages {
|
|
b.AssertFileContent(fmt.Sprintf("public/post/p%d/index.html", i), fmt.Sprintf("Content: Response for /post/p%d/.", i))
|
|
}
|
|
})
|
|
|
|
t.Run("Timeout", func(t *testing.T) {
|
|
files := strings.ReplaceAll(filesTemplate, "TIMEOUT", "100ms")
|
|
b, err := hugolib.TestE(t, files)
|
|
// This is hard to get stable on GitHub Actions, it sometimes succeeds due to timing issues.
|
|
if err != nil {
|
|
b.AssertLogContains("Got Err")
|
|
b.AssertLogContains("retry timeout")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestGetRemotePerRequestTimeout(t *testing.T) {
|
|
t.Parallel()
|
|
htesting.SkipSlowTestUnlessCI(t)
|
|
|
|
// A server that always sleeps longer than the per-request timeout.
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(2 * time.Second)
|
|
w.Header().Add("Content-Type", "text/plain")
|
|
w.Write([]byte("too late"))
|
|
}))
|
|
t.Cleanup(func() { srv.Close() })
|
|
|
|
files := `
|
|
-- hugo.toml --
|
|
timeout = "30s"
|
|
[security]
|
|
[security.http]
|
|
urls = ['.*']
|
|
mediaTypes = ['text/plain']
|
|
-- layouts/home.html --
|
|
{{ $url := "URL" }}
|
|
{{ $opts := dict "timeout" "200ms" }}
|
|
{{ with try (resources.GetRemote $url $opts) }}
|
|
{{ with .Err }}
|
|
Err: {{ . }}
|
|
{{ else with .Value }}
|
|
Content: {{ .Content }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
files = strings.ReplaceAll(files, "URL", srv.URL)
|
|
|
|
b := hugolib.Test(t, files)
|
|
|
|
// The per-request timeout of 200ms should fire well before the global 30s timeout.
|
|
b.AssertFileContent("public/index.html", "Err:")
|
|
}
|
|
|
|
// Issue 14828.
|
|
func TestGetRemoteRetryAfterIssue14828(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("Honored", func(t *testing.T) {
|
|
var (
|
|
mu sync.Mutex
|
|
reqTimes []time.Time
|
|
)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
reqTimes = append(reqTimes, time.Now())
|
|
n := len(reqTimes)
|
|
mu.Unlock()
|
|
|
|
if n == 1 {
|
|
w.Header().Set("Retry-After", "2")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
w.Header().Add("Content-Type", "text/plain")
|
|
w.Write([]byte("OK"))
|
|
}))
|
|
t.Cleanup(func() { srv.Close() })
|
|
|
|
files := `
|
|
-- hugo.toml --
|
|
timeout = "30s"
|
|
[security]
|
|
[security.http]
|
|
urls = ['.*']
|
|
mediaTypes = ['text/plain']
|
|
-- layouts/home.html --
|
|
{{ $url := "URL" }}
|
|
{{ with try (resources.GetRemote $url) }}
|
|
{{ with .Err }}
|
|
{{ errorf "Got Err: %s" . }}
|
|
{{ else with .Value }}
|
|
Content: {{ .Content }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
files = strings.ReplaceAll(files, "URL", srv.URL)
|
|
|
|
b := hugolib.Test(t, files)
|
|
b.AssertFileContent("public/index.html", "Content: OK")
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
b.Assert(len(reqTimes) >= 2, qt.IsTrue, qt.Commentf("expected at least 2 requests, got %d", len(reqTimes)))
|
|
// Default exponential backoff caps the initial sleep at 1100ms.
|
|
// A Retry-After of 2s should produce a gap well above that.
|
|
gap := reqTimes[1].Sub(reqTimes[0])
|
|
b.Assert(gap >= 1500*time.Millisecond, qt.IsTrue, qt.Commentf("expected gap of at least 1500ms (Retry-After: 2), got %s", gap))
|
|
})
|
|
|
|
t.Run("TimeoutMessage", func(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Retry-After", "1")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}))
|
|
t.Cleanup(func() { srv.Close() })
|
|
|
|
files := `
|
|
-- hugo.toml --
|
|
timeout = "200ms"
|
|
[security]
|
|
[security.http]
|
|
urls = ['.*']
|
|
-- layouts/home.html --
|
|
{{ $url := "URL" }}
|
|
{{ with try (resources.GetRemote $url) }}
|
|
{{ with .Err }}
|
|
{{ errorf "Got Err: %s" . }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
files = strings.ReplaceAll(files, "URL", srv.URL)
|
|
|
|
b, _ := hugolib.TestE(t, files)
|
|
b.AssertLogContains("Retry-After: 1s")
|
|
})
|
|
}
|
|
|
|
// Issue 14611.
|
|
func TestGetRemotePerRequestTimeoutBodyRead(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Server sends headers immediately but streams body with a delay.
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Add("Content-Type", "text/plain")
|
|
w.WriteHeader(200)
|
|
if f, ok := w.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
time.Sleep(300 * time.Millisecond)
|
|
w.Write([]byte("Hello from remote."))
|
|
}))
|
|
t.Cleanup(func() { srv.Close() })
|
|
|
|
files := `
|
|
-- hugo.toml --
|
|
timeout = "30s"
|
|
[security]
|
|
[security.http]
|
|
urls = ['.*']
|
|
mediaTypes = ['text/plain']
|
|
-- layouts/home.html --
|
|
{{ $url := "URL" }}
|
|
{{ $opts := dict "timeout" "5s" }}
|
|
{{ with try (resources.GetRemote $url $opts) }}
|
|
{{ with .Err }}
|
|
Err: {{ . }}
|
|
{{ else with .Value }}
|
|
Content: {{ .Content }}
|
|
{{ end }}
|
|
{{ end }}
|
|
`
|
|
files = strings.ReplaceAll(files, "URL", srv.URL)
|
|
|
|
b := hugolib.Test(t, files)
|
|
|
|
b.AssertFileContent("public/index.html", "Content: Hello from remote.")
|
|
}
|