diff --git a/deps/deps.go b/deps/deps.go index 188e530a0..8e7780cb2 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -435,10 +435,28 @@ type DepsCfg struct { // Build triggered by the IntegrationTest framework. IsIntegrationTest bool + // TestCfg holds configuration used only in tests. + // It is a programming error to set this when IsIntegrationTest is not set, + // and doing so will panic. + TestCfg TestConfig + // ChangesFromBuild for changes passed back to the server/watch process. ChangesFromBuild chan []identity.Identity } +// TestConfig holds configuration used only in tests. +// See DepsCfg.TestCfg. +type TestConfig struct { + // WarpcMemory, if set, overrides the memory limit in MiB for the WASM based + // image processors (WebP and AVIF). Used to provoke memory allocation failures. + WarpcMemory int +} + +// IsZero reports whether c holds no test configuration. +func (c TestConfig) IsZero() bool { + return c == TestConfig{} +} + // BuildState are state used during a build. type BuildState struct { counter uint64 diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index e476ff632..d83a5d640 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -947,7 +947,7 @@ func (s *IntegrationTestBuilder) initBuilder() error { // In the full setup, this channel is created in the commands package. changesFromBuild := make(chan []identity.Identity, 10) - depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true} + depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true, TestCfg: deps.TestConfig{WarpcMemory: s.Cfg.WarpcMemory}} sites, err := NewHugoSites(depsCfg) if err != nil { initErr = err @@ -1226,4 +1226,8 @@ type IntegrationTestConfig struct { // The config to pass to Build. BuildCfg BuildCfg + + // WarpcMemory, if set, overrides the memory limit in MiB for the WASM based + // image processors (WebP and AVIF). Used to provoke memory allocation failures. + WarpcMemory int } diff --git a/hugolib/site.go b/hugolib/site.go index b92289027..409a3e30d 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -197,6 +197,10 @@ func (s *Site) Debug() { // NewHugoSites creates HugoSites from the given config. func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { + if !cfg.TestCfg.IsZero() && !cfg.IsIntegrationTest { + panic("DepsCfg.TestCfg must only be set in integration tests") + } + conf := cfg.Configs.GetFirstLanguageConfig() rolesSorted := cfg.Configs.Base.Roles.Config.Sorted versionsSorted := cfg.Configs.Base.Versions.Config.Sorted @@ -257,6 +261,11 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { compilationCacheDir := filepath.Join(conf.Dirs().CacheDir, "_warpc") + imageWasmMemory := 384 // 384 MiB (4096 MiB Max) + if m := cfg.TestCfg.WarpcMemory; m > 0 { + imageWasmMemory = m + } + firstSiteDeps := &deps.Deps{ Fs: cfg.Fs, Log: logger, @@ -280,7 +289,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { warpc.Options{ CompilationCacheDir: compilationCacheDir, PoolSize: poolSizeWebP, - Memory: 384, // 384 MiB (4096 MiB Max) + Memory: imageWasmMemory, Infof: logger.InfoCommand("webp").Logf, Warnf: logger.WarnCommand("webp").Logf, }, @@ -288,7 +297,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { warpc.Options{ CompilationCacheDir: compilationCacheDir, PoolSize: poolSizeAvif, - Memory: 384, // 384 MiB (4096 MiB Max) + Memory: imageWasmMemory, Infof: logger.InfoCommand("avif").Logf, Warnf: logger.WarnCommand("avif").Logf, }, diff --git a/internal/warpc/avif_integration_test.go b/internal/warpc/avif_integration_test.go index eda14269a..3b3b4f907 100644 --- a/internal/warpc/avif_integration_test.go +++ b/internal/warpc/avif_integration_test.go @@ -66,3 +66,30 @@ gif:{{ $gif.RelPermalink }} AssertLoopCount(0). AssertFrameDurations(durations) } + +// See issue 14985. +func TestAvifEncodeOutOfMemory(t *testing.T) { + files := ` +-- assets/gopher.png -- +sourcefilename: ../../resources/testdata/bw-gopher.png +-- layouts/home.html -- +{{ $img := resources.Get "gopher.png" }} +{{ $r := try ($img.Resize "3000x3000 avif") }} +{{ with $r.Err }}BigErr: {{ . }}|{{ else }}BigOK|{{ end }} +{{ $small := $img.Resize "32x32 avif" }} +SmallAfter: {{ $small.RelPermalink }}| +` + + b := hugolib.Test(t, files, hugolib.TestOptWithConfig(func(c *hugolib.IntegrationTestConfig) { + c.WarpcMemory = 8 + })) + + // The big resize must fail gracefully (caught by try) ... + b.AssertFileContent("public/index.html", + "BigErr:", + "out of memory allocating", + "for blob data", + // ... and the dispatcher must still process images afterwards. + "SmallAfter: /gopher_", + ) +} diff --git a/internal/warpc/genavif/avif.c b/internal/warpc/genavif/avif.c index 3e4aaebf8..f572f7cf5 100644 --- a/internal/warpc/genavif/avif.c +++ b/internal/warpc/genavif/avif.c @@ -243,6 +243,23 @@ void write_output_message(const OutputMessage *msg) json_value_free(root_value); } +// drain_bytes discards n bytes from stream. Used to keep the protocol aligned +// after an error that prevents the blob from being consumed normally. +static void drain_bytes(FILE *stream, size_t n) +{ + uint8_t buf[4096]; + while (n > 0) + { + size_t want = n < sizeof(buf) ? n : sizeof(buf); + size_t got = fread(buf, 1, want, stream); + if (got == 0) + { + break; + } + n -= got; + } +} + void handle_commands(FILE *stream) { @@ -278,7 +295,15 @@ void handle_commands(FILE *stream) blob_data = malloc((size_t)blob_size); if (blob_data == NULL) { - fprintf(stderr, "[%d] Error allocating memory for blob data\n", blob_id); + // Out of memory. Drain the blob from the input stream so the next + // command stays aligned, then report the error to the client instead + // of leaving the stream corrupted with no response. + drain_bytes(stream, (size_t)blob_size); + OutputMessage err_output = {0}; + err_output.header = input.header; + snprintf(err_output.header.err, sizeof(err_output.header.err), + "out of memory allocating %u bytes for blob data", blob_size); + write_output_message(&err_output); goto cleanup; } read_bytes = fread(blob_data, 1, (size_t)blob_size, stream); diff --git a/internal/warpc/genwebp/webp.c b/internal/warpc/genwebp/webp.c index 5e551d9f0..999c4f490 100644 --- a/internal/warpc/genwebp/webp.c +++ b/internal/warpc/genwebp/webp.c @@ -524,6 +524,23 @@ void write_output_message(const OutputMessage *msg) json_value_free(root_value); } +// drain_bytes discards n bytes from stream. Used to keep the protocol aligned +// after an error that prevents the blob from being consumed normally. +static void drain_bytes(FILE *stream, size_t n) +{ + uint8_t buf[4096]; + while (n > 0) + { + size_t want = n < sizeof(buf) ? n : sizeof(buf); + size_t got = fread(buf, 1, want, stream); + if (got == 0) + { + break; + } + n -= got; + } +} + void handle_commands(FILE *stream) { @@ -559,7 +576,15 @@ void handle_commands(FILE *stream) blob_data = malloc((size_t)blob_size); if (blob_data == NULL) { - fprintf(stderr, "[%d] Error allocating memory for blob data\n", blob_id); + // Out of memory. Drain the blob from the input stream so the next + // command stays aligned, then report the error to the client instead + // of leaving the stream corrupted with no response. + drain_bytes(stream, (size_t)blob_size); + OutputMessage err_output = {0}; + err_output.header = input.header; + snprintf(err_output.header.err, sizeof(err_output.header.err), + "out of memory allocating %u bytes for blob data", blob_size); + write_output_message(&err_output); goto cleanup; } read_bytes = fread(blob_data, 1, (size_t)blob_size, stream); diff --git a/internal/warpc/warpc.go b/internal/warpc/warpc.go index b0c12d88f..b7763e782 100644 --- a/internal/warpc/warpc.go +++ b/internal/warpc/warpc.go @@ -419,7 +419,11 @@ func (d *dispatcher[Q, R]) pendingCall(id uint32) *call[Q, R] { defer d.mu.Unlock() c, ok := d.pending[id] if !ok { - panic(fmt.Errorf("call with ID %d not found", id)) + // The WASM module wrote a response for an ID we never sent. This means it + // broke the RPC protocol, e.g. a corrupted stream after an error path that + // failed to drain its input or write a response. This is a bug in the + // module and should be reported. + panic(fmt.Errorf("received response for unknown call ID %d: WASM module violated the RPC protocol", id)) } return c } diff --git a/internal/warpc/wasm/avif.wasm b/internal/warpc/wasm/avif.wasm index bcd6a1144..4fb7b2995 100755 Binary files a/internal/warpc/wasm/avif.wasm and b/internal/warpc/wasm/avif.wasm differ diff --git a/internal/warpc/wasm/webp.wasm b/internal/warpc/wasm/webp.wasm index b5e34f1ea..dc7f4bfb3 100755 Binary files a/internal/warpc/wasm/webp.wasm and b/internal/warpc/wasm/webp.wasm differ diff --git a/internal/warpc/webp_integration_test.go b/internal/warpc/webp_integration_test.go index 7093947ae..857b1c3e8 100644 --- a/internal/warpc/webp_integration_test.go +++ b/internal/warpc/webp_integration_test.go @@ -121,6 +121,33 @@ sourcefilename: ../../resources/testdata/giphy.gif b.ImageHelper("public/anim_hu_58eb49733894e7ce.gif").AssertFormat("gif").AssertIsAnimated(true).AssertLoopCount(0).AssertFrameDurations(animFrameDurations) } +// See issue 14985. +func TestWebPEncodeOutOfMemory(t *testing.T) { + files := ` +-- assets/gopher.png -- +sourcefilename: ../../resources/testdata/bw-gopher.png +-- layouts/home.html -- +{{ $img := resources.Get "gopher.png" }} +{{ $r := try ($img.Resize "3000x3000 webp") }} +{{ with $r.Err }}BigErr: {{ . }}|{{ else }}BigOK|{{ end }} +{{ $small := $img.Resize "32x32 webp" }} +SmallAfter: {{ $small.RelPermalink }}| +` + + b := hugolib.Test(t, files, hugolib.TestOptWithConfig(func(c *hugolib.IntegrationTestConfig) { + c.WarpcMemory = 8 + })) + + // The big resize must fail gracefully (caught by try) ... + b.AssertFileContent("public/index.html", + "BigErr:", + "out of memory allocating", + "for blob data", + // ... and the dispatcher must still process images afterwards. + "SmallAfter: /gopher_", + ) +} + func BenchmarkWebp(b *testing.B) { files := ` -- content/p1/sunrise.webp --