images: Recover from memory alloc errors in WASM image processors

Fixes #14985

deps/deps.go

@@ -435,10 +435,28 @@ type DepsCfg struct {
 	// Build triggered by the IntegrationTest framework.
 	IsIntegrationTest bool
 
+	// TestCfg holds configuration used only in tests.
+	// It is a programming error to set this when IsIntegrationTest is not set,
+	// and doing so will panic.
+	TestCfg TestConfig
+
 	// ChangesFromBuild for changes passed back to the server/watch process.
 	ChangesFromBuild chan []identity.Identity
 }
 
+// TestConfig holds configuration used only in tests.
+// See DepsCfg.TestCfg.
+type TestConfig struct {
+	// WarpcMemory, if set, overrides the memory limit in MiB for the WASM based
+	// image processors (WebP and AVIF). Used to provoke memory allocation failures.
+	WarpcMemory int
+}
+
+// IsZero reports whether c holds no test configuration.
+func (c TestConfig) IsZero() bool {
+	return c == TestConfig{}
+}
+
 // BuildState are state used during a build.
 type BuildState struct {
 	counter uint64

hugolib/integrationtest_builder.go

@@ -947,7 +947,7 @@ func (s *IntegrationTestBuilder) initBuilder() error {
 		// In the full setup, this channel is created in the commands package.
 		changesFromBuild := make(chan []identity.Identity, 10)
 
-		depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true}
+		depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true, TestCfg: deps.TestConfig{WarpcMemory: s.Cfg.WarpcMemory}}
 		sites, err := NewHugoSites(depsCfg)
 		if err != nil {
 			initErr = err
@@ -1226,4 +1226,8 @@ type IntegrationTestConfig struct {
 
 	// The config to pass to Build.
 	BuildCfg BuildCfg
+
+	// WarpcMemory, if set, overrides the memory limit in MiB for the WASM based
+	// image processors (WebP and AVIF). Used to provoke memory allocation failures.
+	WarpcMemory int
 }

hugolib/site.go

@@ -197,6 +197,10 @@ func (s *Site) Debug() {
 
 // NewHugoSites creates HugoSites from the given config.
 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
+	if !cfg.TestCfg.IsZero() && !cfg.IsIntegrationTest {
+		panic("DepsCfg.TestCfg must only be set in integration tests")
+	}
+
 	conf := cfg.Configs.GetFirstLanguageConfig()
 	rolesSorted := cfg.Configs.Base.Roles.Config.Sorted
 	versionsSorted := cfg.Configs.Base.Versions.Config.Sorted
@@ -257,6 +261,11 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 
 	compilationCacheDir := filepath.Join(conf.Dirs().CacheDir, "_warpc")
 
+	imageWasmMemory := 384 // 384 MiB (4096 MiB Max)
+	if m := cfg.TestCfg.WarpcMemory; m > 0 {
+		imageWasmMemory = m
+	}
+
 	firstSiteDeps := &deps.Deps{
 		Fs:   cfg.Fs,
 		Log:  logger,
@@ -280,7 +289,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 			warpc.Options{
 				CompilationCacheDir: compilationCacheDir,
 				PoolSize:            poolSizeWebP,
-				Memory:              384, // 384 MiB (4096 MiB Max)
+				Memory:              imageWasmMemory,
 				Infof:               logger.InfoCommand("webp").Logf,
 				Warnf:               logger.WarnCommand("webp").Logf,
 			},
@@ -288,7 +297,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 			warpc.Options{
 				CompilationCacheDir: compilationCacheDir,
 				PoolSize:            poolSizeAvif,
-				Memory:              384, // 384 MiB (4096 MiB Max)
+				Memory:              imageWasmMemory,
 				Infof:               logger.InfoCommand("avif").Logf,
 				Warnf:               logger.WarnCommand("avif").Logf,
 			},

internal/warpc/avif_integration_test.go

@@ -66,3 +66,30 @@ gif:{{ $gif.RelPermalink }}
 		AssertLoopCount(0).
 		AssertFrameDurations(durations)
 }
+
+// See issue 14985.
+func TestAvifEncodeOutOfMemory(t *testing.T) {
+	files := `
+-- assets/gopher.png --
+sourcefilename: ../../resources/testdata/bw-gopher.png
+-- layouts/home.html --
+{{ $img := resources.Get "gopher.png" }}
+{{ $r := try ($img.Resize "3000x3000 avif") }}
+{{ with $r.Err }}BigErr: {{ . }}|{{ else }}BigOK|{{ end }}
+{{ $small := $img.Resize "32x32 avif" }}
+SmallAfter: {{ $small.RelPermalink }}|
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptWithConfig(func(c *hugolib.IntegrationTestConfig) {
+		c.WarpcMemory = 8
+	}))
+
+	// The big resize must fail gracefully (caught by try) ...
+	b.AssertFileContent("public/index.html",
+		"BigErr:",
+		"out of memory allocating",
+		"for blob data",
+		// ... and the dispatcher must still process images afterwards.
+		"SmallAfter: /gopher_",
+	)
+}

internal/warpc/genavif/avif.c

@@ -243,6 +243,23 @@ void write_output_message(const OutputMessage *msg)
     json_value_free(root_value);
 }
 
+// drain_bytes discards n bytes from stream. Used to keep the protocol aligned
+// after an error that prevents the blob from being consumed normally.
+static void drain_bytes(FILE *stream, size_t n)
+{
+    uint8_t buf[4096];
+    while (n > 0)
+    {
+        size_t want = n < sizeof(buf) ? n : sizeof(buf);
+        size_t got = fread(buf, 1, want, stream);
+        if (got == 0)
+        {
+            break;
+        }
+        n -= got;
+    }
+}
+
 void handle_commands(FILE *stream)
 {
 
@@ -278,7 +295,15 @@ void handle_commands(FILE *stream)
         blob_data = malloc((size_t)blob_size);
         if (blob_data == NULL)
         {
-            fprintf(stderr, "[%d] Error allocating memory for blob data\n", blob_id);
+            // Out of memory. Drain the blob from the input stream so the next
+            // command stays aligned, then report the error to the client instead
+            // of leaving the stream corrupted with no response.
+            drain_bytes(stream, (size_t)blob_size);
+            OutputMessage err_output = {0};
+            err_output.header = input.header;
+            snprintf(err_output.header.err, sizeof(err_output.header.err),
+                     "out of memory allocating %u bytes for blob data", blob_size);
+            write_output_message(&err_output);
             goto cleanup;
         }
         read_bytes = fread(blob_data, 1, (size_t)blob_size, stream);

internal/warpc/genwebp/webp.c

@@ -524,6 +524,23 @@ void write_output_message(const OutputMessage *msg)
     json_value_free(root_value);
 }
 
+// drain_bytes discards n bytes from stream. Used to keep the protocol aligned
+// after an error that prevents the blob from being consumed normally.
+static void drain_bytes(FILE *stream, size_t n)
+{
+    uint8_t buf[4096];
+    while (n > 0)
+    {
+        size_t want = n < sizeof(buf) ? n : sizeof(buf);
+        size_t got = fread(buf, 1, want, stream);
+        if (got == 0)
+        {
+            break;
+        }
+        n -= got;
+    }
+}
+
 void handle_commands(FILE *stream)
 {
 
@@ -559,7 +576,15 @@ void handle_commands(FILE *stream)
         blob_data = malloc((size_t)blob_size);
         if (blob_data == NULL)
         {
-            fprintf(stderr, "[%d] Error allocating memory for blob data\n", blob_id);
+            // Out of memory. Drain the blob from the input stream so the next
+            // command stays aligned, then report the error to the client instead
+            // of leaving the stream corrupted with no response.
+            drain_bytes(stream, (size_t)blob_size);
+            OutputMessage err_output = {0};
+            err_output.header = input.header;
+            snprintf(err_output.header.err, sizeof(err_output.header.err),
+                     "out of memory allocating %u bytes for blob data", blob_size);
+            write_output_message(&err_output);
             goto cleanup;
         }
         read_bytes = fread(blob_data, 1, (size_t)blob_size, stream);

internal/warpc/warpc.go

@@ -419,7 +419,11 @@ func (d *dispatcher[Q, R]) pendingCall(id uint32) *call[Q, R] {
 	defer d.mu.Unlock()
 	c, ok := d.pending[id]
 	if !ok {
-		panic(fmt.Errorf("call with ID %d not found", id))
+		// The WASM module wrote a response for an ID we never sent. This means it
+		// broke the RPC protocol, e.g. a corrupted stream after an error path that
+		// failed to drain its input or write a response. This is a bug in the
+		// module and should be reported.
+		panic(fmt.Errorf("received response for unknown call ID %d: WASM module violated the RPC protocol", id))
 	}
 	return c
 }

internal/warpc/webp_integration_test.go

@@ -121,6 +121,33 @@ sourcefilename: ../../resources/testdata/giphy.gif
 	b.ImageHelper("public/anim_hu_58eb49733894e7ce.gif").AssertFormat("gif").AssertIsAnimated(true).AssertLoopCount(0).AssertFrameDurations(animFrameDurations)
 }
 
+// See issue 14985.
+func TestWebPEncodeOutOfMemory(t *testing.T) {
+	files := `
+-- assets/gopher.png --
+sourcefilename: ../../resources/testdata/bw-gopher.png
+-- layouts/home.html --
+{{ $img := resources.Get "gopher.png" }}
+{{ $r := try ($img.Resize "3000x3000 webp") }}
+{{ with $r.Err }}BigErr: {{ . }}|{{ else }}BigOK|{{ end }}
+{{ $small := $img.Resize "32x32 webp" }}
+SmallAfter: {{ $small.RelPermalink }}|
+`
+
+	b := hugolib.Test(t, files, hugolib.TestOptWithConfig(func(c *hugolib.IntegrationTestConfig) {
+		c.WarpcMemory = 8
+	}))
+
+	// The big resize must fail gracefully (caught by try) ...
+	b.AssertFileContent("public/index.html",
+		"BigErr:",
+		"out of memory allocating",
+		"for blob data",
+		// ... and the dispatcher must still process images afterwards.
+		"SmallAfter: /gopher_",
+	)
+}
+
 func BenchmarkWebp(b *testing.B) {
 	files := `
 -- content/p1/sunrise.webp --