From c4eba92863bbb988b23e63af40a22d6661b0ced6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 29 Apr 2026 14:10:51 +0200 Subject: [PATCH] resources: Honor Retry-After header in resources.GetRemote retries When the server returns a temporary HTTP error (e.g. 429 or 503) together with a Retry-After header, use that value as the next sleep duration instead of the default exponential backoff. The Retry-After value is also surfaced in the retry-timeout error message. Fixes #14828 --- .../create/create_integration_test.go | 87 +++++++++++++++++++ resources/resource_factories/create/remote.go | 36 +++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/resources/resource_factories/create/create_integration_test.go b/resources/resource_factories/create/create_integration_test.go index 897055432..c2c234bcc 100644 --- a/resources/resource_factories/create/create_integration_test.go +++ b/resources/resource_factories/create/create_integration_test.go @@ -19,9 +19,11 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" + qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugolib" ) @@ -189,6 +191,91 @@ mediaTypes = ['text/plain'] b.AssertFileContent("public/index.html", "Err:") } +// Issue 14828. +func TestGetRemoteRetryAfterIssue14828(t *testing.T) { + t.Parallel() + + t.Run("Honored", func(t *testing.T) { + var ( + mu sync.Mutex + reqTimes []time.Time + ) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + reqTimes = append(reqTimes, time.Now()) + n := len(reqTimes) + mu.Unlock() + + if n == 1 { + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Add("Content-Type", "text/plain") + w.Write([]byte("OK")) + })) + t.Cleanup(func() { srv.Close() }) + + files := ` +-- hugo.toml -- +timeout = "30s" +[security] +[security.http] +urls = ['.*'] +mediaTypes = ['text/plain'] +-- layouts/home.html -- +{{ $url := "URL" }} +{{ with try (resources.GetRemote $url) }} + {{ with .Err }} + {{ errorf "Got Err: %s" . }} + {{ else with .Value }} + Content: {{ .Content }} + {{ end }} +{{ end }} +` + files = strings.ReplaceAll(files, "URL", srv.URL) + + b := hugolib.Test(t, files) + b.AssertFileContent("public/index.html", "Content: OK") + + mu.Lock() + defer mu.Unlock() + b.Assert(len(reqTimes) >= 2, qt.IsTrue, qt.Commentf("expected at least 2 requests, got %d", len(reqTimes))) + // Default exponential backoff caps the initial sleep at 1100ms. + // A Retry-After of 2s should produce a gap well above that. + gap := reqTimes[1].Sub(reqTimes[0]) + b.Assert(gap >= 1500*time.Millisecond, qt.IsTrue, qt.Commentf("expected gap of at least 1500ms (Retry-After: 2), got %s", gap)) + }) + + t.Run("TimeoutMessage", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(func() { srv.Close() }) + + files := ` +-- hugo.toml -- +timeout = "200ms" +[security] +[security.http] +urls = ['.*'] +-- layouts/home.html -- +{{ $url := "URL" }} +{{ with try (resources.GetRemote $url) }} + {{ with .Err }} + {{ errorf "Got Err: %s" . }} + {{ end }} +{{ end }} +` + files = strings.ReplaceAll(files, "URL", srv.URL) + + b, _ := hugolib.TestE(t, files) + b.AssertLogContains("Retry-After: 1s") + }) +} + // Issue 14611. func TestGetRemotePerRequestTimeoutBodyRead(t *testing.T) { t.Parallel() diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go index 7c6104e9c..8d43ef31c 100644 --- a/resources/resource_factories/create/remote.go +++ b/resources/resource_factories/create/remote.go @@ -24,6 +24,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "time" @@ -110,6 +111,28 @@ var temporaryHTTPStatusCodes = map[int]bool{ 504: true, } +// parseRetryAfter returns the duration to wait per the Retry-After header in +// resp, or 0 if the header is absent or unparseable. Per RFC 7231 the value +// may be either delta-seconds or an HTTP-date. +func parseRetryAfter(resp *http.Response) time.Duration { + if resp == nil { + return 0 + } + h := strings.TrimSpace(resp.Header.Get("Retry-After")) + if h == "" { + return 0 + } + if n, err := strconv.Atoi(h); err == nil && n >= 0 { + return time.Duration(n) * time.Second + } + if t, err := http.ParseTime(h); err == nil { + if d := time.Until(t); d > 0 { + return d + } + } + return 0 +} + func (c *Client) configurePollingIfEnabled(uri, optionsKey string, getRes func() (*http.Response, context.CancelFunc, error)) { if c.remoteResourceChecker == nil { return @@ -473,17 +496,26 @@ func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error }() if retry { + sleep := nextSleep + retryAfter := parseRetryAfter(resp) + if retryAfter > 0 { + sleep = retryAfter + } if start.IsZero() { start = time.Now() - } else if d := time.Since(start) + nextSleep; d >= t.Cfg.Timeout() { + } + if d := time.Since(start) + sleep; d >= t.Cfg.Timeout() { msg := "" if resp != nil { msg = resp.Status } + if retryAfter > 0 { + msg = fmt.Sprintf("%s (server requested Retry-After: %s)", msg, retryAfter) + } err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD", nil) return resp, err } - time.Sleep(nextSleep) + time.Sleep(sleep) if nextSleep < nextSleepLimit { nextSleep *= 2 }