mirror of
https://github.com/gohugoio/hugo.git
synced 2026-08-24 07:18:54 +00:00
Misc webp performance work
*- Dynamic pool sizing for WebP and Katex based on CPU count
- WebP decode: use RGB (3 bytes) for opaque images instead of RGBA (4 bytes)
- Add new imaging.config with new encoder method option; this is default 4 (what we had before), if have tested with value 2 for a 30% speedup on my MacBook, but that does sacrifice quality in some cases (quality/speed trade-off 0-6)
- Track hasAlpha in decode response to enable RGB optimization
- Scale image processing workers (1->2) on high-core machines
- Add WebP benchmark test
On my MacBook Pro M1 Pro, I now get these numbers compared to Hugo v0.152.0:
```
goos: darwin
goarch: arm64
pkg: github.com/gohugoio/hugo/internal/warpc
cpu: Apple M1 Pro
│ cmp152.bench │ fix-webpbench.bench │
│ sec/op │ sec/op vs base │
Webp-10 162.6m ± 32% 145.3m ± 21% -10.60% (p=0.026 n=6)
│ cmp152.bench │ fix-webpbench.bench │
│ B/op │ B/op vs base │
Webp-10 41.69Mi ± 0% 123.79Mi ± 0% +196.92% (p=0.002 n=6)
│ cmp152.bench │ fix-webpbench.bench │
│ allocs/op │ allocs/op vs base │
Webp-10 9.963k ± 3% 18.797k ± 2% +88.67% (p=0.002 n=6)
```
For general image resize benchmarks compared to Hugo v0.152.0 (note that that's not the result of this branch):
```bash
goos: darwin
goarch: arm64
pkg: github.com/gohugoio/hugo/resources/images
cpu: Apple M1 Pro
│ cmp152.bench │ fix-webpbench.bench │
│ sec/op │ sec/op vs base │
ImageResize-10 84.12m ± ∞ ¹ 40.70m ± ∞ ¹ -51.61% (p=0.029 n=4)
¹ need >= 6 samples for confidence interval at level 0.95
│ cmp152.bench │ fix-webpbench.bench │
│ B/op │ B/op vs base │
ImageResize-10 32.37Mi ± ∞ ¹ 29.25Mi ± ∞ ¹ -9.64% (p=0.029 n=4)
¹ need >= 6 samples for confidence interval at level 0.95
│ cmp152.bench │ fix-webpbench.bench │
│ allocs/op │ allocs/op vs base │
ImageResize-10 11.34k ± ∞ ¹ 12.03k ± ∞ ¹ +6.07% (p=0.029 n=4)
¹ need >= 6 samples for confidence interval at level 0.95
```
Closes #14370
This commit is contained in:
Bjørn Erik Pedersen
committed by
GitHub
GitHub
parent
5ba03bf657
commit
e569dd59a0
+11
-3
@@ -201,6 +201,15 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
|
||||
rolesSorted := cfg.Configs.Base.Roles.Config.Sorted
|
||||
versionsSorted := cfg.Configs.Base.Versions.Config.Sorted
|
||||
|
||||
var (
|
||||
poolSizeKatex = 2
|
||||
poolSizeWebP = 1
|
||||
)
|
||||
if n := config.GetNumWorkerMultiplier(); n > 1 {
|
||||
poolSizeKatex = min(n, 8)
|
||||
poolSizeWebP = max(2, n/2)
|
||||
}
|
||||
|
||||
var logger loggers.Logger
|
||||
if cfg.TestLogger != nil {
|
||||
logger = cfg.TestLogger
|
||||
@@ -261,15 +270,14 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
|
||||
warpc.Options{
|
||||
CompilationCacheDir: compilationCacheDir,
|
||||
|
||||
// Katex is relatively slow.
|
||||
PoolSize: 8,
|
||||
PoolSize: poolSizeKatex,
|
||||
Infof: logger.InfoCommand("katex").Logf,
|
||||
Warnf: logger.WarnCommand("katex").Logf,
|
||||
},
|
||||
// WebP options.
|
||||
warpc.Options{
|
||||
CompilationCacheDir: compilationCacheDir,
|
||||
PoolSize: 1,
|
||||
PoolSize: poolSizeWebP,
|
||||
Memory: 384, // 384 MiB (4096 MiB Max)
|
||||
Infof: logger.InfoCommand("webp").Logf,
|
||||
Warnf: logger.WarnCommand("webp").Logf,
|
||||
|
||||
@@ -68,6 +68,7 @@ typedef struct
|
||||
int loopCount;
|
||||
int frameCount;
|
||||
int *frameDurations;
|
||||
bool hasAlpha;
|
||||
|
||||
} InputParams;
|
||||
|
||||
@@ -79,6 +80,7 @@ typedef struct
|
||||
int preset; // preset to use; resolved from hint.
|
||||
|
||||
bool useSharpYuv; // use sharp YUV for better quality.
|
||||
int method; // quality/speed trade-off (0=fast, 6=slower-better). Default is 2.
|
||||
|
||||
} InputOptions;
|
||||
|
||||
@@ -316,6 +318,7 @@ static uint8_t initEncoderConfig(WebPConfig *config, InputOptions opts)
|
||||
}
|
||||
|
||||
config->use_sharp_yuv = opts.useSharpYuv ? 1 : 0;
|
||||
config->method = opts.method;
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -410,6 +413,11 @@ InputMessage parse_input_message(const char *line)
|
||||
msg.data.options.hint[sizeof(msg.data.options.hint) - 1] = '\0';
|
||||
}
|
||||
msg.data.options.useSharpYuv = json_object_get_number(options_object, "useSharpYuv") != 0;
|
||||
msg.data.options.method = (int)json_object_get_number(options_object, "method");
|
||||
if (msg.data.options.method < 0 || msg.data.options.method > 6)
|
||||
{
|
||||
msg.data.options.method = 2; // default
|
||||
}
|
||||
if (msg.data.options.quality < 0 || msg.data.options.quality > 100)
|
||||
{
|
||||
msg.data.options.quality = 75; // default
|
||||
@@ -493,6 +501,7 @@ void write_output_message(const OutputMessage *msg)
|
||||
json_object_set_number(params_object, "width", msg->data.params.width);
|
||||
json_object_set_number(params_object, "height", msg->data.params.height);
|
||||
json_object_set_number(params_object, "stride", msg->data.params.stride);
|
||||
json_object_set_boolean(params_object, "hasAlpha", msg->data.params.hasAlpha);
|
||||
if (msg->data.params.frameDurations != NULL)
|
||||
{
|
||||
JSON_Value *durations_value = json_value_init_array();
|
||||
@@ -611,6 +620,7 @@ void handle_commands(FILE *stream)
|
||||
output.data.params.stride = anim_info.canvas_width * 4;
|
||||
output.data.params.frameCount = anim_info.frame_count;
|
||||
output.data.params.loopCount = anim_info.loop_count;
|
||||
output.data.params.hasAlpha = true; // Animated WebP always decoded as RGBA
|
||||
|
||||
int *frameDurations = malloc(sizeof(int) * anim_info.frame_count);
|
||||
if (frameDurations == NULL)
|
||||
@@ -680,7 +690,22 @@ void handle_commands(FILE *stream)
|
||||
}
|
||||
else
|
||||
{
|
||||
uint8_t *output_buffer = WebPDecodeRGBA(blob_data, (size_t)blob_size, &output.data.params.width, &output.data.params.height);
|
||||
uint8_t *output_buffer;
|
||||
int bytesPerPixel;
|
||||
|
||||
output.data.params.hasAlpha = config.input.has_alpha;
|
||||
|
||||
if (config.input.has_alpha)
|
||||
{
|
||||
output_buffer = WebPDecodeRGBA(blob_data, (size_t)blob_size, &output.data.params.width, &output.data.params.height);
|
||||
bytesPerPixel = 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
output_buffer = WebPDecodeRGB(blob_data, (size_t)blob_size, &output.data.params.width, &output.data.params.height);
|
||||
bytesPerPixel = 3;
|
||||
}
|
||||
|
||||
if (output_buffer == NULL)
|
||||
{
|
||||
strncpy(output.header.err, "Failed to decode WebP", sizeof(output.header.err) - 1);
|
||||
@@ -688,7 +713,7 @@ void handle_commands(FILE *stream)
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
output.data.params.stride = output.data.params.width * 4;
|
||||
output.data.params.stride = output.data.params.width * bytesPerPixel;
|
||||
|
||||
write_output_message(&output);
|
||||
|
||||
|
||||
Binary file not shown.
+33
-7
@@ -34,9 +34,10 @@ var (
|
||||
)
|
||||
|
||||
type CommonImageProcessingParams struct {
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
Stride int `json:"stride,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
Stride int `json:"stride,omitempty"`
|
||||
HasAlpha bool `json:"hasAlpha,omitempty"`
|
||||
|
||||
// For animated images.
|
||||
FrameDurations []int `json:"frameDurations,omitempty"`
|
||||
@@ -132,7 +133,7 @@ func (d *WebpCodec) Decode(r io.Reader) (image.Image, error) {
|
||||
}
|
||||
|
||||
if len(out.Data.Params.FrameDurations) > 0 {
|
||||
// Animated WebP.
|
||||
// Animated WebP (always RGBA).
|
||||
img := &WEBP{
|
||||
frameDurations: out.Data.Params.FrameDurations,
|
||||
loopCount: out.Data.Params.LoopCount,
|
||||
@@ -153,9 +154,35 @@ func (d *WebpCodec) Decode(r io.Reader) (image.Image, error) {
|
||||
return img, nil
|
||||
}
|
||||
|
||||
var pix []byte
|
||||
var nrgbaStride int
|
||||
|
||||
if out.Data.Params.HasAlpha {
|
||||
// RGBA data (4 bytes per pixel).
|
||||
pix = destination.Bytes()
|
||||
nrgbaStride = stride
|
||||
} else {
|
||||
// RGB data (3 bytes per pixel) - convert to NRGBA.
|
||||
rgbData := destination.Bytes()
|
||||
nrgbaStride = w * 4
|
||||
pix = make([]byte, nrgbaStride*h)
|
||||
for y := 0; y < h; y++ {
|
||||
srcIdx := y * stride
|
||||
dstIdx := y * nrgbaStride
|
||||
for x := 0; x < w; x++ {
|
||||
pix[dstIdx+0] = rgbData[srcIdx+0] // R
|
||||
pix[dstIdx+1] = rgbData[srcIdx+1] // G
|
||||
pix[dstIdx+2] = rgbData[srcIdx+2] // B
|
||||
pix[dstIdx+3] = 0xFF // A (fully opaque)
|
||||
srcIdx += 3
|
||||
dstIdx += 4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
img := &image.NRGBA{
|
||||
Pix: destination.Bytes(),
|
||||
Stride: stride,
|
||||
Pix: pix,
|
||||
Stride: nrgbaStride,
|
||||
Rect: image.Rect(0, 0, w, h),
|
||||
}
|
||||
|
||||
@@ -294,7 +321,6 @@ func (d *WebpCodec) Encode(w io.Writer, img image.Image, opts map[string]any) er
|
||||
}
|
||||
|
||||
opts = maps.Clone(opts)
|
||||
opts["useSharpYuv"] = true // Use sharp (and slow) RGB->YUV conversion.
|
||||
|
||||
message := Message[WebpInput]{
|
||||
Header: Header{
|
||||
|
||||
@@ -120,3 +120,56 @@ sourcefilename: ../../resources/testdata/giphy.gif
|
||||
b.ImageHelper("public/anim_hu_edc2f24aaad2cee6.webp").AssertFormat("webp").AssertIsAnimated(true).AssertLoopCount(0).AssertFrameDurations(animFrameDurations)
|
||||
b.ImageHelper("public/anim_hu_58eb49733894e7ce.gif").AssertFormat("gif").AssertIsAnimated(true).AssertLoopCount(0).AssertFrameDurations(animFrameDurations)
|
||||
}
|
||||
|
||||
func BenchmarkWebp(b *testing.B) {
|
||||
files := `
|
||||
-- content/p1/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p1/index.md --
|
||||
-- content/p2/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p2/index.md --
|
||||
-- content/p3/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p3/index.md --
|
||||
-- content/p4/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p4/index.md --
|
||||
-- content/p5/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p5/index.md --
|
||||
-- content/p6/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p6/index.md --
|
||||
-- content/p7/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p7/index.md --
|
||||
-- content/p8/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p8/index.md --
|
||||
-- content/p9/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p9/index.md --
|
||||
-- content/p10/sunrise.webp --
|
||||
sourcefilename: ../../resources/testdata/sunrise.webp
|
||||
-- content/p10/index.md --
|
||||
-- layouts/home.html --
|
||||
Home.
|
||||
-- layouts/page.html --
|
||||
{{ $image := .Resources.Get "sunrise.webp" }}
|
||||
{{ $resized := $image.Fit "400x300 webp" }}
|
||||
Resized RelPermalink: {{ $resized.RelPermalink }}|
|
||||
`
|
||||
|
||||
cfg := hugolib.IntegrationTestConfig{
|
||||
T: b,
|
||||
TxtarString: files,
|
||||
}
|
||||
|
||||
for b.Loop() {
|
||||
b.StopTimer()
|
||||
builder := hugolib.NewIntegrationTestBuilder(cfg).Init()
|
||||
b.StartTimer()
|
||||
builder.Build()
|
||||
}
|
||||
}
|
||||
|
||||
+9
-7
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/gohugoio/hugo/common/hashing"
|
||||
"github.com/gohugoio/hugo/common/hugo"
|
||||
"github.com/gohugoio/hugo/common/paths"
|
||||
"github.com/gohugoio/hugo/config"
|
||||
|
||||
"github.com/gohugoio/gift"
|
||||
|
||||
@@ -394,14 +395,15 @@ func (i *imageResource) processOptions(options []string) (images.ImageResource,
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// Serialize image processing. The imaging library spins up its own set of Go routines,
|
||||
// so there is not much to gain from adding more load to the mix. That
|
||||
// can even have negative effect in low resource scenarios.
|
||||
// Note that this only effects the non-cached scenario. Once the processed
|
||||
// image is written to disk, everything is fast, fast fast.
|
||||
const imageProcWorkers = 1
|
||||
var imageProcSem chan bool
|
||||
|
||||
var imageProcSem = make(chan bool, imageProcWorkers)
|
||||
func init() {
|
||||
imageProcWorkers := 1
|
||||
if n := config.GetNumWorkerMultiplier(); n > 4 {
|
||||
imageProcWorkers = 2
|
||||
}
|
||||
imageProcSem = make(chan bool, imageProcWorkers)
|
||||
}
|
||||
|
||||
func (i *imageResource) doWithImageConfig(conf images.ImageConfig, f func(src image.Image) (image.Image, error)) (images.ImageResource, error) {
|
||||
img, err := i.getSpec().ImageCache.getOrCreate(i, conf, func() (*imageResource, image.Image, error) {
|
||||
|
||||
@@ -129,6 +129,8 @@ func (d *Codec) EncodeTo(conf ImageConfig, w io.Writer, img image.Image) error {
|
||||
"compression": conf.Compression,
|
||||
"quality": conf.Quality,
|
||||
"hint": conf.Hint,
|
||||
"useSharpYuv": conf.UseSharpYuv,
|
||||
"method": conf.Method,
|
||||
}
|
||||
return d.webp.Encode(w, img, opts)
|
||||
default:
|
||||
|
||||
@@ -131,11 +131,13 @@ func ImageFormatFromMediaSubType(sub string) (Format, bool) {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultJPEGQuality = 75
|
||||
defaultResampleFilter = "box"
|
||||
defaultBgColor = "#ffffff"
|
||||
defaultHint = "photo"
|
||||
defaultCompression = "lossy"
|
||||
defaultJPEGQuality = 75
|
||||
defaultResampleFilter = "box"
|
||||
defaultBgColor = "#ffffff"
|
||||
defaultHint = "photo"
|
||||
defaultCompression = "lossy"
|
||||
defaultWebpUseSharpYuv = true
|
||||
defaultWebpMethod = 4
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -145,6 +147,10 @@ var (
|
||||
"hint": defaultHint,
|
||||
"quality": defaultJPEGQuality,
|
||||
"compression": defaultCompression,
|
||||
"webp": map[string]any{
|
||||
"useSharpYuv": defaultWebpUseSharpYuv,
|
||||
"method": defaultWebpMethod,
|
||||
},
|
||||
}
|
||||
|
||||
defaultImageConfig *config.ConfigNamespace[ImagingConfig, ImagingConfigInternal]
|
||||
@@ -171,6 +177,11 @@ func DecodeConfig(in map[string]any) (*config.ConfigNamespace[ImagingConfig, Ima
|
||||
// Merge in the defaults.
|
||||
hmaps.MergeShallow(m, defaultImaging)
|
||||
|
||||
// Deep merge webp defaults.
|
||||
if webp, ok := m["webp"].(map[string]any); ok {
|
||||
hmaps.MergeShallow(webp, defaultImaging["webp"].(map[string]any))
|
||||
}
|
||||
|
||||
var i ImagingConfigInternal
|
||||
if err := mapstructure.Decode(m, &i.Imaging); err != nil {
|
||||
return i, nil, err
|
||||
@@ -304,7 +315,7 @@ func DecodeImageConfig(options []string, defaults *config.ConfigNamespace[Imagin
|
||||
}
|
||||
|
||||
if c.Hint == "" {
|
||||
c.Hint = "photo"
|
||||
c.Hint = defaults.Config.Imaging.Webp.Hint
|
||||
}
|
||||
|
||||
if c.Action != "" && c.Anchor == -1 {
|
||||
@@ -378,6 +389,10 @@ type ImageConfig struct {
|
||||
|
||||
Compression string
|
||||
|
||||
// WebP-specific options.
|
||||
UseSharpYuv bool
|
||||
Method int
|
||||
|
||||
Width int
|
||||
Height int
|
||||
|
||||
@@ -442,7 +457,8 @@ type ImagingConfig struct {
|
||||
// Currently only used when encoding to Webp.
|
||||
// Default is "photo".
|
||||
// Valid values are "picture", "photo", "drawing", "icon", or "text".
|
||||
Hint string
|
||||
// Moved to WebpConfig in v0.155.0, but kept here for backwards compatibility.
|
||||
Hint string `json:"-"`
|
||||
|
||||
// The anchor to use in Fill. Default is "smart", i.e. Smart Crop.
|
||||
Anchor string
|
||||
@@ -452,6 +468,7 @@ type ImagingConfig struct {
|
||||
|
||||
Exif ExifConfig
|
||||
Meta MetaConfig
|
||||
Webp WebpConfig
|
||||
}
|
||||
|
||||
var validMetaSources = map[string]bool{
|
||||
@@ -503,6 +520,26 @@ func (cfg *ImagingConfig) init() error {
|
||||
}
|
||||
}
|
||||
|
||||
// WebP config with backwards compatibility for root-level Hint.
|
||||
cfg.Webp.Hint = strings.ToLower(cfg.Webp.Hint)
|
||||
if cfg.Webp.Hint == "" {
|
||||
// Fall back to root-level hint for backwards compatibility.
|
||||
if cfg.Hint != "" {
|
||||
cfg.Webp.Hint = cfg.Hint
|
||||
} else {
|
||||
cfg.Webp.Hint = defaultHint
|
||||
}
|
||||
}
|
||||
if cfg.Webp.Hint != "" && !hints[cfg.Webp.Hint] {
|
||||
return fmt.Errorf("invalid webp hint %q; must be one of picture, photo, drawing, icon, or text", cfg.Webp.Hint)
|
||||
}
|
||||
if cfg.Webp.Method == 0 {
|
||||
cfg.Webp.Method = defaultWebpMethod
|
||||
}
|
||||
if cfg.Webp.Method < 0 || cfg.Webp.Method > 6 {
|
||||
return fmt.Errorf("webp method must be between 0 and 6, got %d", cfg.Webp.Method)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -549,3 +586,19 @@ type MetaConfig struct {
|
||||
// Default is ["exif", "iptc"] (XMP is excluded for performance reasons).
|
||||
Sources []string
|
||||
}
|
||||
|
||||
// WebpConfig holds WebP-specific encoding configuration.
|
||||
type WebpConfig struct {
|
||||
// Hint about what type of image this is.
|
||||
// Valid values are "picture", "photo", "drawing", "icon", or "text".
|
||||
// Default is "photo".
|
||||
Hint string
|
||||
|
||||
// Use sharp (and slow) RGB->YUV conversion.
|
||||
// Default is true.
|
||||
UseSharpYuv bool
|
||||
|
||||
// Quality/speed trade-off (0=fast, 6=slower-better).
|
||||
// Default is 2.
|
||||
Method int
|
||||
}
|
||||
|
||||
@@ -321,9 +321,11 @@ func GetDefaultImageConfig(defaults *config.ConfigNamespace[ImagingConfig, Imagi
|
||||
}
|
||||
return ImageConfig{
|
||||
Anchor: -1, // The real values start at 0.
|
||||
Hint: "photo",
|
||||
Hint: defaults.Config.Imaging.Webp.Hint,
|
||||
Quality: defaults.Config.Imaging.Quality,
|
||||
Compression: defaults.Config.Imaging.Compression,
|
||||
UseSharpYuv: defaults.Config.Imaging.Webp.UseSharpYuv,
|
||||
Method: defaults.Config.Imaging.Webp.Method,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -153,7 +153,7 @@ func RunGolden(opts GoldenImageTestOpts) *hugolib.IntegrationTestBuilder {
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(len(entries1), qt.Equals, len(entries2))
|
||||
for i, e1 := range entries1 {
|
||||
if shouldSkip != nil && shouldSkip(e1) {
|
||||
if shouldSkip(e1) {
|
||||
continue
|
||||
}
|
||||
c.Assert(filepath.Ext(e1.Name()), qt.Not(qt.Equals), "")
|
||||
|
||||
@@ -33,6 +33,7 @@ func (f overlayFilter) Draw(dst draw.Image, src image.Image, options *gift.Optio
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to decode image: %s", err))
|
||||
}
|
||||
|
||||
gift.New().Draw(dst, src)
|
||||
gift.New().DrawAt(dst, overlaySrc, image.Pt(f.x, f.y), gift.OverOperator)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user