mirror of
https://github.com/gohugoio/hugo.git
synced 2026-08-24 15:28:54 +00:00
Add per-request timeout option to resources.GetRemote
This commit is contained in:
committed by
GitHub
parent
b96d58a144
commit
2d691c7e47
@@ -46,11 +46,12 @@ key
|
|||||||
method
|
method
|
||||||
: (`string`) The action to perform on the requested resource, typically one of `GET`, `POST`, or `HEAD`.
|
: (`string`) The action to perform on the requested resource, typically one of `GET`, `POST`, or `HEAD`.
|
||||||
|
|
||||||
|
timeout
|
||||||
|
: (`string`) Cancels the request if it does not complete within this duration (e.g. "30s").
|
||||||
|
|
||||||
responseHeaders
|
responseHeaders
|
||||||
: {{< new-in 0.143.0 />}}
|
: {{< new-in 0.143.0 />}}
|
||||||
: (`[]string`) The headers to extract from the server's response, accessible through the resource's [`Data.Headers`] method. Header name matching is case-insensitive.
|
: (`[]string`) The headers to extract from the server's response, accessible through the resource's [`Data.Headers`] method. Header name matching is case-insensitive.[`Data.Headers`]: /methods/resource/data/#headers
|
||||||
|
|
||||||
[`Data.Headers`]: /methods/resource/data/#headers
|
|
||||||
|
|
||||||
## Options examples
|
## Options examples
|
||||||
|
|
||||||
@@ -110,6 +111,20 @@ To extract specific headers from the server's response:
|
|||||||
{{ $resource := resources.GetRemote $url $opts }}
|
{{ $resource := resources.GetRemote $url $opts }}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To set a per-request timeout (e.g. when fetching many feeds where a few slow ones should not stall the build):
|
||||||
|
|
||||||
|
```go-html-template
|
||||||
|
{{ $url := "https://example.org/feed.rss" }}
|
||||||
|
{{ $opts := dict "timeout" "10s" }}
|
||||||
|
{{ with try (resources.GetRemote $url $opts) }}
|
||||||
|
{{ with .Err }}
|
||||||
|
{{ warnf "Failed to fetch feed: %s" . }}
|
||||||
|
{{ else with .Value }}
|
||||||
|
{{ $data = . | transform.Unmarshal }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
```
|
||||||
|
|
||||||
## Remote data
|
## Remote data
|
||||||
|
|
||||||
When retrieving remote data, use the [`transform.Unmarshal`] function to [unmarshal](g) the response.
|
When retrieving remote data, use the [`transform.Unmarshal`] function to [unmarshal](g) the response.
|
||||||
@@ -224,4 +239,5 @@ Note that the entry above is:
|
|||||||
|
|
||||||
[`try`]: /functions/go-template/try
|
[`try`]: /functions/go-template/try
|
||||||
[configure file caches]: /configuration/caches/
|
[configure file caches]: /configuration/caches/
|
||||||
|
|
||||||
[error handling]: #error-handling
|
[error handling]: #error-handling
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/gohugoio/hugo/htesting"
|
"github.com/gohugoio/hugo/htesting"
|
||||||
"github.com/gohugoio/hugo/hugolib"
|
"github.com/gohugoio/hugo/hugolib"
|
||||||
@@ -175,3 +176,47 @@ mediaTypes = ['text/plain']
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetRemotePerRequestTimeout(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
htesting.SkipSlowTestUnlessCI(t)
|
||||||
|
|
||||||
|
// A server that always sleeps longer than the per-request timeout.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
w.Header().Add("Content-Type", "text/plain")
|
||||||
|
w.Write([]byte("too late"))
|
||||||
|
}))
|
||||||
|
t.Cleanup(func() { srv.Close() })
|
||||||
|
|
||||||
|
files := `
|
||||||
|
-- hugo.toml --
|
||||||
|
timeout = "30s"
|
||||||
|
[security]
|
||||||
|
[security.http]
|
||||||
|
urls = ['.*']
|
||||||
|
mediaTypes = ['text/plain']
|
||||||
|
-- layouts/home.html --
|
||||||
|
{{ $url := "URL" }}
|
||||||
|
{{ $opts := dict "timeout" "200ms" }}
|
||||||
|
{{ with try (resources.GetRemote $url $opts) }}
|
||||||
|
{{ with .Err }}
|
||||||
|
Err: {{ . }}
|
||||||
|
{{ else with .Value }}
|
||||||
|
Content: {{ .Content }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
`
|
||||||
|
files = strings.ReplaceAll(files, "URL", srv.URL)
|
||||||
|
|
||||||
|
b := hugolib.NewIntegrationTestBuilder(
|
||||||
|
hugolib.IntegrationTestConfig{
|
||||||
|
T: t,
|
||||||
|
TxtarString: files,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
b.Build()
|
||||||
|
|
||||||
|
// The per-request timeout of 200ms should fire well before the global 30s timeout.
|
||||||
|
b.AssertFileContent("public/index.html", "Err:")
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cast"
|
||||||
|
|
||||||
"github.com/gohugoio/httpcache"
|
"github.com/gohugoio/httpcache"
|
||||||
"github.com/gohugoio/hugo/common/hashing"
|
"github.com/gohugoio/hugo/common/hashing"
|
||||||
"github.com/gohugoio/hugo/common/hmaps"
|
"github.com/gohugoio/hugo/common/hmaps"
|
||||||
@@ -177,6 +179,19 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
|
|||||||
isHeadMethod := method == "HEAD"
|
isHeadMethod := method == "HEAD"
|
||||||
|
|
||||||
optionsm = maps.Clone(optionsm)
|
optionsm = maps.Clone(optionsm)
|
||||||
|
|
||||||
|
// Extract timeout before computing cache keys: it only affects fetch behaviour,
|
||||||
|
// not the cached content, so it must not influence the cache key.
|
||||||
|
var perRequestTimeout time.Duration
|
||||||
|
if v, k, ok := hmaps.LookupEqualFold(optionsm, "timeout"); ok {
|
||||||
|
d, err := cast.ToDurationE(v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid timeout for resource %s: %w", uri, err)
|
||||||
|
}
|
||||||
|
perRequestTimeout = d
|
||||||
|
delete(optionsm, k)
|
||||||
|
}
|
||||||
|
|
||||||
userKey, optionsKey := remoteResourceKeys(uri, optionsm)
|
userKey, optionsKey := remoteResourceKeys(uri, optionsm)
|
||||||
|
|
||||||
// A common pattern is to use the key in the options map as
|
// A common pattern is to use the key in the options map as
|
||||||
@@ -197,6 +212,11 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
|
|||||||
|
|
||||||
getRes := func() (*http.Response, error) {
|
getRes := func() (*http.Response, error) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
if perRequestTimeout > 0 {
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, perRequestTimeout)
|
||||||
|
defer cancel()
|
||||||
|
}
|
||||||
ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey)
|
ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey)
|
||||||
|
|
||||||
req, err := options.NewRequest(uri)
|
req, err := options.NewRequest(uri)
|
||||||
|
|||||||
Reference in New Issue
Block a user