From abb92804112a08d6646d9c71a2ff64ba925f1cc7 Mon Sep 17 00:00:00 2001 From: Sjoerd Simons Date: Sat, 18 Jun 2022 19:58:09 +0200 Subject: [PATCH] Use parallel gzip instead of gzip for compression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit golangs compress/gzip isn't a parallel implementation, so it's quite a bit slower on most modern servers then pgzip. The below benchmark run shows that publishing a debian bullseye mirror snapshot (amd64, arm64, armhf, source) shows a gain of about 35% in publishing time (when skipping bz2 using MR #1081) ``` hyperfine -w 1 -m 3 -L aptly aptly-nobz2,aptly-nobz2-pgzip -p "{aptly} -config aptly.conf publish drop bullseye || true" "{aptly} -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye" Benchmark 1: aptly-nobz2 -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye Time (mean ± σ): 35.548 s ± 0.378 s [User: 39.465 s, System: 10.046 s] Range (min … max): 35.149 s … 35.902 s 3 runs Benchmark 2: aptly-nobz2-pgzip -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye Time (mean ± σ): 26.592 s ± 0.069 s [User: 42.207 s, System: 9.676 s] Range (min … max): 26.521 s … 26.660 s 3 runs Summary 'aptly-nobz2-pgzip -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ran 1.34 ± 0.01 times faster than 'aptly-nobz2 -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ``` Signed-off-by: Sjoerd Simons --- go.mod | 1 + go.sum | 2 ++ utils/compress.go | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 18ef50b91..4c2e85add 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/jlaffaye/ftp v0.0.0-20180404123514-2403248fa8cc // indirect github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d github.com/klauspost/compress v1.13.6 + github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.2 // indirect diff --git a/go.sum b/go.sum index 6f1460dd5..370f953dd 100644 --- a/go.sum +++ b/go.sum @@ -203,6 +203,8 @@ github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKM github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= diff --git a/utils/compress.go b/utils/compress.go index aef8f1861..a4521b7e9 100644 --- a/utils/compress.go +++ b/utils/compress.go @@ -1,10 +1,11 @@ package utils import ( - "compress/gzip" "io" "os" "os/exec" + + "github.com/klauspost/pgzip" ) // CompressFile compresses file specified by source to .gz & .bz2 @@ -19,7 +20,7 @@ func CompressFile(source *os.File, onlyGzip bool) error { } defer gzFile.Close() - gzWriter := gzip.NewWriter(gzFile) + gzWriter := pgzip.NewWriter(gzFile) defer gzWriter.Close() source.Seek(0, 0)