diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 0c3464eaafd0..be9f47cf7a2c 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -30,7 +30,7 @@ on: required: false env: - GO_VERSION: "1.26" + GO_VERSION: "1.27rc2" SETUP_BUILDX_VERSION: "edge" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 7a4bf4141958..5d674b6aa87f 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -24,7 +24,7 @@ on: - 'frontend/dockerfile/docs/**' env: - GO_VERSION: "1.26" + GO_VERSION: "1.27rc2" SETUP_BUILDX_VERSION: "edge" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" SCOUT_VERSION: "1.20.2" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 1ebc1d72adba..582e5b6899d5 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -22,7 +22,7 @@ on: - 'frontend/dockerfile/docs/**' env: - GO_VERSION: "1.26" + GO_VERSION: "1.27rc2" SETUP_BUILDX_VERSION: "edge" SETUP_BUILDKIT_TAG: "moby/buildkit:latest" SCOUT_VERSION: "1.20.2" diff --git a/.github/workflows/test-os.yml b/.github/workflows/test-os.yml index cd4bd088dc2f..632e7235a901 100644 --- a/.github/workflows/test-os.yml +++ b/.github/workflows/test-os.yml @@ -22,7 +22,7 @@ on: - 'frontend/dockerfile/docs/**' env: - GO_VERSION: "1.26" + GO_VERSION: "1.27rc2" SETUP_BUILDX_VERSION: "edge" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" DESTDIR: "./bin" @@ -139,7 +139,7 @@ jobs: name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: "${{ env.GO_VERSION }}" + go-version: "1.27.0-rc.2" cache: false - name: Download artifacts diff --git a/.golangci.yml b/.golangci.yml index fd11475517ad..00619ca702ad 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -91,6 +91,8 @@ linters: staticcheck: checks: - all + # Produces false positives for shared callers of platform-specific unsupported stubs. + - -SA4023 testifylint: disable: - empty diff --git a/Dockerfile b/Dockerfile index 96bbaff06a48..5872431ce280 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ ARG EXPORT_BASE=alpine ARG ALPINE_VERSION=3.23 ARG UBUNTU_VERSION=24.04 -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG XX_VERSION=1.9.0 ARG BUILDKIT_DEBUG diff --git a/cache/manager_test.go b/cache/manager_test.go index 8ba7a43bccbf..ad94d0d1c191 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -2664,9 +2664,15 @@ func mapToBlob(m map[string]string, compress bool) ([]byte, ocispecs.Descriptor, if !compress { return mapToBlobWithCompression(m, nil) } - return mapToBlobWithCompression(m, func(w io.Writer) (io.WriteCloser, string, error) { - return gzip.NewWriter(w), ocispecs.MediaTypeImageLayerGzip, nil - }) + return mapToBlobWithCompression(m, gzipBlobWriter) +} + +func gzipBlobWriter(w io.Writer) (io.WriteCloser, string, error) { + // Go 1.27 changed compress/flate output in CL 707355 for golang/go#75532. + // Use BuildKit's pinned Go 1.26 encoder when calculating expected digests. + compressor, _ := compression.Gzip.Compress(context.Background(), compression.New(compression.Gzip)) + wc, err := compressor(w, ocispecs.MediaTypeImageLayerGzip) + return wc, ocispecs.MediaTypeImageLayerGzip, err } func mapToBlobWithCompression(m map[string]string, compress func(io.Writer) (io.WriteCloser, string, error)) ([]byte, ocispecs.Descriptor, error) { @@ -2718,7 +2724,11 @@ func fileToBlob(file *os.File, compress bool) ([]byte, ocispecs.Descriptor, erro var dest io.WriteCloser = &iohelper.NopWriteCloser{Writer: buf} if compress { - dest = gzip.NewWriter(buf) + var err error + dest, _, err = gzipBlobWriter(buf) + if err != nil { + return nil, ocispecs.Descriptor{}, err + } } tw := tar.NewWriter(io.MultiWriter(sha.Hash(), dest)) diff --git a/client/client_export_image_test.go b/client/client_export_image_test.go index 902e1dfaecc6..484eedf1c58d 100644 --- a/client/client_export_image_test.go +++ b/client/client_export_image_test.go @@ -215,7 +215,9 @@ func testBuildExportWithForeignLayer(t *testing.T, sb integration.Sandbox) { ctx := namespaces.WithNamespace(sb.Context(), "buildkit") - resolver := docker.NewResolver(docker.ResolverOptions{PlainHTTP: true}) + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchAllHosts)), + }) name, desc, err := resolver.Resolve(ctx, target) require.NoError(t, err) @@ -262,7 +264,9 @@ func testBuildExportWithForeignLayer(t *testing.T, sb integration.Sandbox) { ctx := namespaces.WithNamespace(sb.Context(), "buildkit") - resolver := docker.NewResolver(docker.ResolverOptions{PlainHTTP: true}) + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchAllHosts)), + }) name, desc, err := resolver.Resolve(ctx, target) require.NoError(t, err) diff --git a/client/policy_test.go b/client/policy_test.go index b53cf8ff32a7..ab23190ef807 100644 --- a/client/policy_test.go +++ b/client/policy_test.go @@ -696,12 +696,12 @@ func testSourceMetaPolicySession(t *testing.T, sb integration.Sandbox) { source: func() (*opspb.SourceOp, sourceresolver.Opt) { p := platforms.DefaultSpec() return &opspb.SourceOp{ - Identifier: "docker-image://docker.io/library/alpine:latest", - }, sourceresolver.Opt{ - ImageOpt: &sourceresolver.ResolveImageOpt{ - Platform: &p, - }, - } + Identifier: "docker-image://docker.io/library/alpine:latest", + }, sourceresolver.Opt{ + ImageOpt: &sourceresolver.ResolveImageOpt{ + Platform: &p, + }, + } }, callbacks: []policysession.PolicyCallback{ func(ctx context.Context, req *policysession.CheckPolicyRequest) (*policysession.DecisionResponse, *pb.ResolveSourceMetaRequest, error) { diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index cce8b96c6939..50e4e896c06a 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -33,7 +33,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/examples/buildkit1/buildkit.go b/examples/buildkit1/buildkit.go index 8045dbf57dcd..b327c73e747b 100644 --- a/examples/buildkit1/buildkit.go +++ b/examples/buildkit1/buildkit.go @@ -33,7 +33,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/examples/buildkit2/buildkit.go b/examples/buildkit2/buildkit.go index d201b8b962f5..88f9d3369a79 100644 --- a/examples/buildkit2/buildkit.go +++ b/examples/buildkit2/buildkit.go @@ -33,7 +33,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/examples/buildkit3/buildkit.go b/examples/buildkit3/buildkit.go index 8d19f9bf1013..bc1879a9bc0d 100644 --- a/examples/buildkit3/buildkit.go +++ b/examples/buildkit3/buildkit.go @@ -34,7 +34,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/examples/buildkit4/buildkit.go b/examples/buildkit4/buildkit.go index 69889a3b459b..582c7d615f4c 100644 --- a/examples/buildkit4/buildkit.go +++ b/examples/buildkit4/buildkit.go @@ -37,7 +37,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/examples/nested-llb/main.go b/examples/nested-llb/main.go index 2c3ee1924169..b9b67f73ce63 100644 --- a/examples/nested-llb/main.go +++ b/examples/nested-llb/main.go @@ -32,7 +32,7 @@ func main() { } func goBuildBase() llb.State { - goAlpine := llb.Image("docker.io/library/golang:1.26-alpine") + goAlpine := llb.Image("docker.io/library/golang:1.27rc2-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnvUnix). AddEnv("GOPATH", "/go"). diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index 9435f3e8ccd9..eb2940bf010a 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile-upstream:master -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 ARG XX_VERSION=1.9.0 diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index d79ee40957c5..f922bf117f1a 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -327,12 +327,12 @@ func metadataMount(def *opspb.Definition) (*executor.Mount, func(), error) { } return &executor.Mount{ - Src: &bind{dir}, - Dest: "/run/config/buildkit/metadata", - Readonly: true, - }, func() { - os.RemoveAll(dir) - }, nil + Src: &bind{dir}, + Dest: "/run/config/buildkit/metadata", + Readonly: true, + }, func() { + os.RemoveAll(dir) + }, nil } type bind struct { diff --git a/go.mod b/go.mod index 89f3be5b0a0d..910497a000ec 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/containerd/nydus-snapshotter v0.15.15 github.com/containerd/platforms v1.0.0-rc.4 github.com/containerd/stargz-snapshotter v0.18.2 - github.com/containerd/stargz-snapshotter/estargz v0.18.2 + github.com/containerd/stargz-snapshotter/estargz v0.18.3-0.20260607065519-4daea594a1da github.com/containerd/typeurl/v2 v2.3.0 github.com/containernetworking/plugins v1.9.1 github.com/coreos/go-systemd/v22 v22.7.0 diff --git a/go.sum b/go.sum index 56fe52a74274..ee34a5ba1035 100644 --- a/go.sum +++ b/go.sum @@ -163,8 +163,8 @@ github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew1 github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc= github.com/containerd/stargz-snapshotter v0.18.2 h1:Ev/sxfQUjwzJQ9eqy3XzttcQ3osMIqkQgMYlcET+10M= github.com/containerd/stargz-snapshotter v0.18.2/go.mod h1:iS0a4lgCFjGbdBJNrm1jwvaMFGGnQ6PZ5Sd09i060h8= -github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= -github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= +github.com/containerd/stargz-snapshotter/estargz v0.18.3-0.20260607065519-4daea594a1da h1:r8f4sAf5mAeHWsO1VYjKHSwmj1yHpa69+JGDZvF/C68= +github.com/containerd/stargz-snapshotter/estargz v0.18.3-0.20260607065519-4daea594a1da/go.mod h1:Ri93bhwN/gKJeEuEpFle/31ij7qC8La07W4ZFJQPIlM= github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uihg= github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs= github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ= diff --git a/hack/dockerfiles/archutil.Dockerfile b/hack/dockerfiles/archutil.Dockerfile index cad6e26b2fb5..941b3edf529f 100644 --- a/hack/dockerfiles/archutil.Dockerfile +++ b/hack/dockerfiles/archutil.Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile-upstream:master # check=error=true -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 ARG DEBIAN_VERSION=trixie diff --git a/hack/dockerfiles/docs-dockerfile.Dockerfile b/hack/dockerfiles/docs-dockerfile.Dockerfile index b11993f4bce6..b4c6d4d6f33c 100644 --- a/hack/dockerfiles/docs-dockerfile.Dockerfile +++ b/hack/dockerfiles/docs-dockerfile.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest diff --git a/hack/dockerfiles/docs.Dockerfile b/hack/dockerfiles/docs.Dockerfile index caee9db72739..112ec8e4c8e6 100644 --- a/hack/dockerfiles/docs.Dockerfile +++ b/hack/dockerfiles/docs.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS golatest diff --git a/hack/dockerfiles/generated-files.Dockerfile b/hack/dockerfiles/generated-files.Dockerfile index 286a3565bb7c..a439a8d410de 100644 --- a/hack/dockerfiles/generated-files.Dockerfile +++ b/hack/dockerfiles/generated-files.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile-upstream:master -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG DEBIAN_VERSION=trixie ARG PROTOC_VERSION=3.14.0 ARG PROTOC_GOOGLEAPIS_VERSION=2af421884dd468d565137215c946ebe4e245ae26 diff --git a/hack/dockerfiles/govulncheck.Dockerfile b/hack/dockerfiles/govulncheck.Dockerfile index b1fa26ab5e3c..7b98bfd5487a 100644 --- a/hack/dockerfiles/govulncheck.Dockerfile +++ b/hack/dockerfiles/govulncheck.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG GOVULNCHECK_VERSION=v1.3.0 ARG FORMAT="text" diff --git a/hack/dockerfiles/lint.Dockerfile b/hack/dockerfiles/lint.Dockerfile index 848312a913d6..60bdf31575ab 100644 --- a/hack/dockerfiles/lint.Dockerfile +++ b/hack/dockerfiles/lint.Dockerfile @@ -1,10 +1,11 @@ # syntax=docker/dockerfile-upstream:master -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 ARG XX_VERSION=1.9.0 ARG PROTOLINT_VERSION=0.56.4 ARG GOLANGCI_LINT_VERSION=v2.12.2 +ARG STATICCHECK_VERSION=v0.8.0-rc.1 ARG GOLANGCI_FROM_SOURCE=false ARG GOPLS_VERSION=v0.38.0 # GOPLS_ANALYZERS defines gopls analyzers to be run. disabled by default: deprecated simplifyrange unusedfunc unusedvariable @@ -18,7 +19,9 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM golang-base AS golangci-build WORKDIR /src ARG GOLANGCI_LINT_VERSION +ARG STATICCHECK_VERSION ADD https://github.com/golangci/golangci-lint.git#${GOLANGCI_LINT_VERSION} . +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/ go get honnef.co/go/tools@${STATICCHECK_VERSION} RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/ go mod download RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/ mkdir -p out && go build -o /out/golangci-lint ./cmd/golangci-lint diff --git a/hack/dockerfiles/vendor.Dockerfile b/hack/dockerfiles/vendor.Dockerfile index 45dc8660d51e..5e086cbeb4f5 100644 --- a/hack/dockerfiles/vendor.Dockerfile +++ b/hack/dockerfiles/vendor.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile-upstream:master -ARG GO_VERSION=1.26 +ARG GO_VERSION=1.27rc2 ARG ALPINE_VERSION=3.23 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base diff --git a/solver/llbsolver/ops/exec_binfmt.go b/solver/llbsolver/ops/exec_binfmt.go index 460ec4014b54..5e8d0fdb389b 100644 --- a/solver/llbsolver/ops/exec_binfmt.go +++ b/solver/llbsolver/ops/exec_binfmt.go @@ -71,12 +71,12 @@ func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) { ret = true return []mount.Mount{{ - Type: "bind", - Source: filepath.Join(tmpdir, qemuMountName), - Options: []string{"ro", "bind"}, - }}, func() error { - return os.RemoveAll(tmpdir) - }, nil + Type: "bind", + Source: filepath.Join(tmpdir, qemuMountName), + Options: []string{"ro", "bind"}, + }}, func() error { + return os.RemoveAll(tmpdir) + }, nil } func (m *staticEmulatorMount) IdentityMapping() *user.IdentityMapping { diff --git a/util/archutil/generate.go b/util/archutil/generate.go index c7a0bc41108d..995031f9463a 100644 --- a/util/archutil/generate.go +++ b/util/archutil/generate.go @@ -4,13 +4,15 @@ package main import ( "bytes" - "compress/gzip" + "context" "flag" "html/template" "io" "os" "path/filepath" + "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/util/compression/go126flate" "github.com/pkg/errors" ) @@ -32,7 +34,12 @@ func main() { defer f.Close() buf := &bytes.Buffer{} - gz, err := gzip.NewWriterLevel(newHexStringWriter(buf), gzip.BestCompression) + // Go 1.27 changed compress/flate output in CL 707355 for + // golang/go#75532. Use BuildKit's Go 1.26-compatible gzip writer so + // the checked-in archutil binaries do not depend on the Go toolchain. + compressor, _ := compression.Gzip.Compress(context.Background(), + compression.New(compression.Gzip).SetLevel(go126flate.BestCompression)) + gz, err := compressor(newHexStringWriter(buf), "") if err != nil { panic(err) } diff --git a/util/compression/go126flate/LICENSE b/util/compression/go126flate/LICENSE new file mode 100644 index 000000000000..2a7cf70da6e4 --- /dev/null +++ b/util/compression/go126flate/LICENSE @@ -0,0 +1,27 @@ +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/util/compression/go126flate/deflate.go b/util/compression/go126flate/deflate.go new file mode 100644 index 000000000000..1d173de887d2 --- /dev/null +++ b/util/compression/go126flate/deflate.go @@ -0,0 +1,747 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "errors" + "fmt" + "io" + "math" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + BestCompression = 9 + DefaultCompression = -1 + + // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman + // entropy encoding. This mode is useful in compressing data that has + // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) + // that lacks an entropy encoder. Compression gains are achieved when + // certain bytes in the input stream occur more frequently than others. + // + // Note that HuffmanOnly produces a compressed output that is + // RFC 1951 compliant. That is, any valid DEFLATE decompressor will + // continue to be able to decompress this output. + HuffmanOnly = -2 +) + +const ( + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + + // The LZ77 step produces a sequence of literal tokens and + // pair tokens. The offset is also known as distance. The underlying wire + // format limits the range of lengths and offsets. For example, there are + // 256 legitimate lengths: those in the range [3, 258]. This package's + // compressor uses a higher minimum match length, enabling optimizations + // such as finding matches via 32-bit loads and compares. + baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 + minMatchLength = 4 // The smallest match length that the compressor actually emits + maxMatchLength = 258 // The largest match length + baseMatchOffset = 1 // The smallest match offset + maxMatchOffset = 1 << 15 // The largest match offset + + // The maximum number of tokens we put into a single flate block, just to + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 17 // After 17 performance degrades + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + maxHashOffset = 1 << 24 + + skipNever = math.MaxInt32 +) + +type compressionLevel struct { + level, good, lazy, nice, chain, fastSkipHashing int +} + +var levels = []compressionLevel{ + {0, 0, 0, 0, 0, 0}, // NoCompression. + {1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go. + // For levels 2-3 we don't bother trying with lazy matches. + {2, 4, 0, 16, 8, 5}, + {3, 4, 0, 32, 32, 6}, + // Levels 4-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 4, 4, 16, 16, skipNever}, + {5, 8, 16, 32, 32, skipNever}, + {6, 8, 16, 128, 128, skipNever}, + {7, 8, 32, 128, 256, skipNever}, + {8, 32, 128, 258, 1024, skipNever}, + {9, 32, 258, 258, 4096, skipNever}, +} + +type compressor struct { + compressionLevel + + w *huffmanBitWriter + bulkHasher func([]byte, []uint32) + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + bestSpeed *deflateFast // Encoder for BestSpeed + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + sync bool // requesting flush + + // queued output tokens + tokens []token + + // deflate state + length int + offset int + maxInsertIndex int + err error + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + // These are large and do not contain pointers, so put them + // near the end of the struct so the GC has to scan less. + chainHead int + hashHead [hashSize]uint32 + hashPrev [windowSize]uint32 + hashOffset int + + // hashMatch must be able to contain hashes for the maximum match length. + hashMatch [maxMatchLength - 1]uint32 +} + +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + d.hashOffset += windowSize + if d.hashOffset > maxHashOffset { + delta := d.hashOffset - 1 + d.hashOffset -= delta + d.chainHead -= delta + + // Iterate over slices instead of arrays to avoid copying + // the entire table onto the stack (Issue #18625). + for i, v := range d.hashPrev[:] { + if int(v) > delta { + d.hashPrev[i] = uint32(int(v) - delta) + } else { + d.hashPrev[i] = 0 + } + } + for i, v := range d.hashHead[:] { + if int(v) > delta { + d.hashHead[i] = uint32(int(v) - delta) + } else { + d.hashHead[i] = 0 + } + } + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tokens []token, index int) error { + if index > 0 { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tokens, false, window) + return d.w.err + } + return nil +} + +// fillWindow will fill the current window with the supplied +// dictionary and calculate all hashes. +// This is much faster than doing a full encode. +// Should only be used after a reset. +func (d *compressor) fillWindow(b []byte) { + // Do not fill window if we are in store-only mode. + if d.compressionLevel.level < 2 { + return + } + if d.index != 0 || d.windowEnd != 0 { + panic("internal error: fillWindow called with stale data") + } + + // If we are given too much, cut it. + if len(b) > windowSize { + b = b[len(b)-windowSize:] + } + // Add all to window. + n := copy(d.window, b) + + // Calculate 256 hashes at the time (more L1 cache hits) + loops := (n + 256 - minMatchLength) / 256 + for j := 0; j < loops; j++ { + index := j * 256 + end := index + 256 + minMatchLength - 1 + if end > n { + end = n + } + toCheck := d.window[index:end] + dstSize := len(toCheck) - minMatchLength + 1 + + if dstSize <= 0 { + continue + } + + dst := d.hashMatch[:dstSize] + d.bulkHasher(toCheck, dst) + for i, val := range dst { + di := i + index + hh := &d.hashHead[val&hashMask] + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = *hh + // Set the head of the hash chain to us. + *hh = uint32(di + d.hashOffset) + } + } + // Update window information. + d.windowEnd = n + d.index = n +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + wEnd := win[pos+length] + wPos := win[pos:] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLen(win[i:], wPos, minMatchLook) + + if n > length && (n > minMatchLength || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.hashPrev[i&windowMask]) - d.hashOffset + if i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +const hashmul = 0x1e35a7bd + +// hash4 returns a hash representation of the first 4 bytes +// of the supplied slice. +// The caller must ensure that len(b) >= 4. +func hash4(b []byte) uint32 { + return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits) +} + +// bulkHash4 will compute hashes using the same +// algorithm as hash4. +func bulkHash4(b []byte, dst []uint32) { + if len(b) < minMatchLength { + return + } + hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 + dst[0] = (hb * hashmul) >> (32 - hashBits) + end := len(b) - minMatchLength + 1 + for i := 1; i < end; i++ { + hb = (hb << 8) | uint32(b[i+3]) + dst[i] = (hb * hashmul) >> (32 - hashBits) + } +} + +// matchLen returns the number of matching bytes in a and b +// up to length 'max'. Both slices must be at least 'max' +// bytes in size. +func matchLen(a, b []byte, max int) int { + a = a[:max] + b = b[:len(a)] + for i, av := range a { + if b[i] != av { + return i + } + } + return max +} + +// encSpeed will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) encSpeed() { + // We only compress if we have maxStoreBlockSize. + if d.windowEnd < maxStoreBlockSize { + if !d.sync { + return + } + + // Handle small sizes. + if d.windowEnd < 128 { + switch { + case d.windowEnd == 0: + return + case d.windowEnd <= 16: + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + default: + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + } + d.windowEnd = 0 + d.bestSpeed.reset() + return + } + + } + // Encode the block. + d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd]) + + // If we removed less than 1/16th, Huffman compress the block. + if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) { + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + } else { + d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd]) + } + d.err = d.w.err + d.windowEnd = 0 +} + +func (d *compressor) initDeflate() { + d.window = make([]byte, 2*windowSize) + d.hashOffset = 1 + d.tokens = make([]token, 0, maxFlateBlockTokens+1) + d.length = minMatchLength - 1 + d.offset = 0 + d.byteAvailable = false + d.index = 0 + d.chainHead = -1 + d.bulkHasher = bulkHash4 +} + +func (d *compressor) deflate() { + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + +Loop: + for { + if d.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + break Loop + } + if d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1]))) + d.byteAvailable = false + } + if len(d.tokens) > 0 { + if d.err = d.writeBlock(d.tokens, d.index); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + break Loop + } + } + if d.index < d.maxInsertIndex { + // Update the hash + hash := hash4(d.window[d.index : d.index+minMatchLength]) + hh := &d.hashHead[hash&hashMask] + d.chainHead = int(*hh) + d.hashPrev[d.index&windowMask] = uint32(d.chainHead) + *hh = uint32(d.index + d.hashOffset) + } + prevLength := d.length + prevOffset := d.offset + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && + (d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 || + d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.fastSkipHashing != skipNever && d.length >= minMatchLength || + d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + if d.fastSkipHashing != skipNever { + d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset))) + } else { + d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset))) + } + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + if d.fastSkipHashing != skipNever { + newIndex = d.index + d.length + } else { + newIndex = d.index + prevLength - 1 + } + index := d.index + for index++; index < newIndex; index++ { + if index < d.maxInsertIndex { + hash := hash4(d.window[index : index+minMatchLength]) + // Get previous value with the same hash. + // Our chain should point to the previous value. + hh := &d.hashHead[hash&hashMask] + d.hashPrev[index&windowMask] = *hh + // Set the head of the hash chain to us. + *hh = uint32(index + d.hashOffset) + } + } + d.index = index + + if d.fastSkipHashing == skipNever { + d.byteAvailable = false + d.length = minMatchLength - 1 + } + } else { + // For matches this long, we don't bother inserting each individual + // item into the table. + d.index += d.length + } + if len(d.tokens) == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(d.tokens, d.index); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + } else { + if d.fastSkipHashing != skipNever || d.byteAvailable { + i := d.index - 1 + if d.fastSkipHashing != skipNever { + i = d.index + } + d.tokens = append(d.tokens, literalToken(uint32(d.window[i]))) + if len(d.tokens) == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, i+1); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + } + d.index++ + if d.fastSkipHashing == skipNever { + d.byteAvailable = true + } + } + } +} + +func (d *compressor) fillStore(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) store() { + if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + d.windowEnd = 0 + } +} + +// storeHuff compresses and stores the currently added data +// when the d.window is full or we are at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeHuff() { + if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { + return + } + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + d.windowEnd = 0 +} + +func (d *compressor) write(b []byte) (n int, err error) { + if d.err != nil { + return 0, d.err + } + n = len(b) + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + if d.err != nil { + return 0, d.err + } + } + return n, nil +} + +func (d *compressor) syncFlush() error { + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store + case level == HuffmanOnly: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).storeHuff + case level == BestSpeed: + d.compressionLevel = levels[level] + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).encSpeed + d.bestSpeed = newDeflateFast() + d.tokens = make([]token, maxStoreBlockSize) + case level == DefaultCompression: + level = 6 + fallthrough + case 2 <= level && level <= 9: + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflate + default: + return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) + } + return nil +} + +func (d *compressor) reset(w io.Writer) { + d.w.reset(w) + d.sync = false + d.err = nil + switch d.compressionLevel.level { + case NoCompression: + d.windowEnd = 0 + case BestSpeed: + d.windowEnd = 0 + d.tokens = d.tokens[:0] + d.bestSpeed.reset() + default: + d.chainHead = -1 + clear(d.hashHead[:]) + clear(d.hashPrev[:]) + d.hashOffset = 1 + d.index, d.windowEnd = 0, 0 + d.blockStart, d.byteAvailable = 0, false + d.tokens = d.tokens[:0] + d.length = minMatchLength - 1 + d.offset = 0 + d.maxInsertIndex = 0 + } +} + +func (d *compressor) close() error { + if d.err == errWriterClosed { + return nil + } + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + if d.w.err != nil { + return d.w.err + } + d.err = errWriterClosed + return nil +} + +// NewWriter returns a new [Writer] compressing data at the given level. +// Following zlib, levels range from 1 ([BestSpeed]) to 9 ([BestCompression]); +// higher levels typically run slower but compress more. Level 0 +// ([NoCompression]) does not attempt any compression; it only adds the +// necessary DEFLATE framing. +// Level -1 ([DefaultCompression]) uses the default compression level. +// Level -2 ([HuffmanOnly]) will use Huffman compression only, giving +// a very fast compression for all types of input, but sacrificing considerable +// compression efficiency. +// +// If level is in the range [-2, 9] then the error returned will be nil. +// Otherwise the error returned will be non-nil. +func NewWriter(w io.Writer, level int) (*Writer, error) { + var dw Writer + if err := dw.d.init(w, level); err != nil { + return nil, err + } + return &dw, nil +} + +// NewWriterDict is like [NewWriter] but initializes the new +// [Writer] with a preset dictionary. The returned [Writer] behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a reader initialized with the +// same dictionary (see [NewReaderDict]). +func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { + dw := &dictWriter{w} + zw, err := NewWriter(dw, level) + if err != nil { + return nil, err + } + zw.d.fillWindow(dict) + zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. + return zw, nil +} + +type dictWriter struct { + w io.Writer +} + +func (w *dictWriter) Write(b []byte) (n int, err error) { + return w.w.Write(b) +} + +var errWriterClosed = errors.New("flate: closed writer") + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see [NewWriter]). +type Writer struct { + d compressor + dict []byte +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err error) { + return w.d.write(data) +} + +// Flush flushes any pending data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// Calling Flush when there is no pending data still causes the [Writer] +// to emit a sync marker of at least 4 bytes. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() error { + // For more about flushing: + // https://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() error { + return w.d.close() +} + +// Reset discards the writer's state and makes it equivalent to +// the result of [NewWriter] or [NewWriterDict] called with dst +// and w's level and dictionary. +func (w *Writer) Reset(dst io.Writer) { + if dw, ok := w.d.w.writer.(*dictWriter); ok { + // w was created with NewWriterDict + dw.w = dst + w.d.reset(dw) + w.d.fillWindow(w.dict) + } else { + // w was created with NewWriter + w.d.reset(dst) + } +} diff --git a/util/compression/go126flate/deflatefast.go b/util/compression/go126flate/deflatefast.go new file mode 100644 index 000000000000..dd25cff8fde3 --- /dev/null +++ b/util/compression/go126flate/deflatefast.go @@ -0,0 +1,309 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import "math" + +// This encoding algorithm, which prioritizes speed over output size, is +// based on Snappy's LZ77-style encoder: github.com/golang/snappy + +const ( + tableBits = 14 // Bits used in the table. + tableSize = 1 << tableBits // Size of the table. + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. + + // Reset the buffer offset when reaching this. + // Offsets are stored between blocks as int32 values. + // Since the offset we are checking against is at the beginning + // of the buffer, we need to subtract the current and input + // buffer to not risk overflowing the int32. + bufferReset = math.MaxInt32 - maxStoreBlockSize*2 +) + +func load32(b []byte, i int32) uint32 { + b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func load64(b []byte, i int32) uint64 { + b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func hash(u uint32) uint32 { + return (u * 0x1e35a7bd) >> tableShift +} + +// These constants are defined by the Snappy implementation so that its +// assembly implementation can fast-path some 16-bytes-at-a-time copies. They +// aren't necessary in the pure Go implementation, as we don't use those same +// optimizations, but using the same thresholds doesn't really hurt. +const ( + inputMargin = 16 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin +) + +type tableEntry struct { + val uint32 // Value at destination + offset int32 +} + +// deflateFast maintains the table for matches, +// and the previous byte block for cross block matching. +type deflateFast struct { + table [tableSize]tableEntry + prev []byte // Previous block, zero length if unknown. + cur int32 // Current match offset. +} + +func newDeflateFast() *deflateFast { + return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)} +} + +// encode encodes a block given in src and appends tokens +// to dst and returns the result. +func (e *deflateFast) encode(dst []token, src []byte) []token { + // Ensure that e.cur doesn't wrap. + if e.cur >= bufferReset { + e.shiftOffsets() + } + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + e.cur += maxStoreBlockSize + e.prev = e.prev[:0] + return emitLiteral(dst, src) + } + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := int32(0) + s := int32(0) + cv := load32(src, s) + nextHash := hash(cv) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := int32(32) + + nextS := s + var candidate tableEntry + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = e.table[nextHash&tableMask] + now := load32(src, nextS) + e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv} + nextHash = hash(now) + + offset := s - (candidate.offset - e.cur) + if offset > maxMatchOffset || cv != candidate.val { + // Out of range or not matched. + cv = now + continue + } + break + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + dst = emitLiteral(dst, src[nextEmit:s]) + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + // + s += 4 + t := candidate.offset - e.cur + 4 + l := e.matchLen(s, t, src) + + // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) + dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))) + s += l + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load64(src, s-1) + prevHash := hash(uint32(x)) + e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)} + x >>= 8 + currHash := hash(uint32(x)) + candidate = e.table[currHash&tableMask] + e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)} + + offset := s - (candidate.offset - e.cur) + if offset > maxMatchOffset || uint32(x) != candidate.val { + cv = uint32(x >> 8) + nextHash = hash(cv) + s++ + break + } + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + dst = emitLiteral(dst, src[nextEmit:]) + } + e.cur += int32(len(src)) + e.prev = e.prev[:len(src)] + copy(e.prev, src) + return dst +} + +func emitLiteral(dst []token, lit []byte) []token { + for _, v := range lit { + dst = append(dst, literalToken(uint32(v))) + } + return dst +} + +// matchLen returns the match length between src[s:] and src[t:]. +// t can be negative to indicate the match is starting in e.prev. +// We assume that src[s-4:s] and src[t-4:t] already match. +func (e *deflateFast) matchLen(s, t int32, src []byte) int32 { + s1 := int(s) + maxMatchLength - 4 + if s1 > len(src) { + s1 = len(src) + } + + // If we are inside the current block + if t >= 0 { + b := src[t:] + a := src[s:s1] + b = b[:len(a)] + // Extend the match to be as long as possible. + for i := range a { + if a[i] != b[i] { + return int32(i) + } + } + return int32(len(a)) + } + + // We found a match in the previous block. + tp := int32(len(e.prev)) + t + if tp < 0 { + return 0 + } + + // Extend the match to be as long as possible. + a := src[s:s1] + b := e.prev[tp:] + if len(b) > len(a) { + b = b[:len(a)] + } + a = a[:len(b)] + for i := range b { + if a[i] != b[i] { + return int32(i) + } + } + + // If we reached our limit, we matched everything we are + // allowed to in the previous block and we return. + n := int32(len(b)) + if int(s+n) == s1 { + return n + } + + // Continue looking for more matches in the current block. + a = src[s+n : s1] + b = src[:len(a)] + for i := range a { + if a[i] != b[i] { + return int32(i) + n + } + } + return int32(len(a)) + n +} + +// Reset resets the encoding history. +// This ensures that no matches are made to the previous block. +func (e *deflateFast) reset() { + e.prev = e.prev[:0] + // Bump the offset, so all matches will fail distance check. + // Nothing should be >= e.cur in the table. + e.cur += maxMatchOffset + + // Protect against e.cur wraparound. + if e.cur >= bufferReset { + e.shiftOffsets() + } +} + +// shiftOffsets will shift down all match offset. +// This is only called in rare situations to prevent integer overflow. +// +// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121. +func (e *deflateFast) shiftOffsets() { + if len(e.prev) == 0 { + // We have no history; just clear the table. + clear(e.table[:]) + e.cur = maxMatchOffset + 1 + return + } + + // Shift down everything in the table that isn't already too far away. + for i := range e.table[:] { + v := e.table[i].offset - e.cur + maxMatchOffset + 1 + if v < 0 { + // We want to reset e.cur to maxMatchOffset + 1, so we need to shift + // all table entries down by (e.cur - (maxMatchOffset + 1)). + // Because we ignore matches > maxMatchOffset, we can cap + // any negative offsets at 0. + v = 0 + } + e.table[i].offset = v + } + e.cur = maxMatchOffset + 1 +} diff --git a/util/compression/go126flate/doc.go b/util/compression/go126flate/doc.go new file mode 100644 index 000000000000..32958954cc97 --- /dev/null +++ b/util/compression/go126flate/doc.go @@ -0,0 +1,20 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package go126flate contains the DEFLATE writer from Go 1.26.5. +// +// The sources are copied from Go commit +// c19862e5f8415b4f24b189d065ed739517c548ba. Go 1.27 changed the encoded +// output while improving compression speed in CL 707355 for golang/go#75532. +// BuildKit keeps this writer to preserve the digests of gzip-compressed layers. +package go126flate + +// maxNumLit comes from RFC 1951 section 3.2.7. It normally lives in the Go +// flate decoder, which is intentionally not included in this writer-only copy. +const maxNumLit = 286 + +// InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) Error() string { return "flate: internal error: " + string(e) } diff --git a/util/compression/go126flate/huffman_bit_writer.go b/util/compression/go126flate/huffman_bit_writer.go new file mode 100644 index 000000000000..62b48cc1ce5a --- /dev/null +++ b/util/compression/go126flate/huffman_bit_writer.go @@ -0,0 +1,695 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "io" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 + + // bufferFlushSize indicates the buffer size + // after which bytes are flushed to the writer. + // Should preferably be a multiple of 6, since + // we accumulate 6 bytes between writes to the buffer. + bufferFlushSize = 240 + + // bufferSize is the actual output byte buffer size. + // It must have additional headroom for a flush + // which can contain up to 8 bytes. + bufferSize = bufferFlushSize + 8 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, +} + +var offsetBase = []uint32{ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + // writer is the underlying writer. + // Do not use it directly; use the write method, which ensures + // that Write errors are sticky. + writer io.Writer + + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. Data is always written + // sequentially into the bytes array. + bits uint64 + nbits uint + bytes [bufferSize]byte + codegenFreq [codegenCodeCount]int32 + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err error +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + writer: w, + literalFreq: make([]int32, maxNumLit), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxNumLit+offsetCodeCount+1), + literalEncoding: newHuffmanEncoder(maxNumLit), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + } +} + +func (w *huffmanBitWriter) reset(writer io.Writer) { + w.writer = writer + w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + n++ + } + w.bits = 0 + w.write(w.bytes[:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) write(b []byte) { + if w.err != nil { + return + } + _, w.err = w.writer.Write(b) +} + +func (w *huffmanBitWriter) writeBits(b int32, nb uint) { + if w.err != nil { + return + } + w.bits |= uint64(b) << w.nbits + w.nbits += nb + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n >= bufferFlushSize { + w.write(w.bytes[:n]) + n = 0 + } + w.nbytes = n + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits&7 != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if n != 0 { + w.write(w.bytes[:n]) + } + w.nbytes = 0 + w.write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +// litenc, offenc The literal and offset encoder to use +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { + clear(w.codegenFreq[:]) + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + cgnl := codegen[:numLiterals] + for i := range cgnl { + cgnl[i] = uint8(litEnc.codes[i].len) + } + + cgnl = codegen[numLiterals : numLiterals+numOffsets] + for i := range cgnl { + cgnl[i] = uint8(offEnc.codes[i].len) + } + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := 6 + if n > count { + n = count + } + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := 138 + if n > count { + n = count + } + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +// dynamicSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + header := 3 + 5 + 5 + 4 + (3 * numCodegens) + + w.codegenEncoding.bitLength(w.codegenFreq[:]) + + int(w.codegenFreq[16])*2 + + int(w.codegenFreq[17])*3 + + int(w.codegenFreq[18])*7 + size = header + + litEnc.bitLength(w.literalFreq) + + offEnc.bitLength(w.offsetFreq) + + extraBits + + return size, numCodegens +} + +// fixedSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) fixedSize(extraBits int) int { + return 3 + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits +} + +// storedSize calculates the stored size, including header. +// The function returns the size in bits and whether the block +// fits inside a single block. +func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { + if in == nil { + return 0, false + } + if len(in) <= maxStoreBlockSize { + return (len(in) + 5) * 8, true + } + return 0, false +} + +func (w *huffmanBitWriter) writeCode(c hcode) { + if w.err != nil { + return + } + w.bits |= uint64(c.code) << w.nbits + w.nbits += uint(c.len) + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n >= bufferFlushSize { + w.write(w.bytes[:n]) + n = 0 + } + w.nbytes = n + } +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := uint(w.codegenEncoding.codes[codegenOrder[i]].len) + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + w.writeCode(w.codegenEncoding.codes[uint32(codeWord)]) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +// writeBlock will write a block of tokens with the smallest encoding. +// The original input can be supplied, and if the huffman encoded data +// is larger than the original bytes, the data will be written as a +// stored block. +// If the input is nil, the tokens will always be Huffman encoded. +func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + + tokens = append(tokens, endBlockMarker) + numLiterals, numOffsets := w.indexTokens(tokens) + + var extraBits int + storedSize, storable := w.storedSize(input) + if storable { + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode]) + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + var size = w.fixedSize(extraBits) + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storable && storedSize < size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + + // Write the tokens. + w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes) +} + +// writeBlockDynamic encodes a block using a dynamic Huffman table. +// This should be used if the symbols used have a disproportionate +// histogram distribution. +// If input is supplied and the compression savings are below 1/16th of the +// input size the block is stored. +func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + + tokens = append(tokens, endBlockMarker) + numLiterals, numOffsets := w.indexTokens(tokens) + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0) + + // Store bytes, if we don't get a reasonable improvement. + if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Write Huffman table. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + + // Write the tokens. + w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes) +} + +// indexTokens indexes a slice of tokens, and updates +// literalFreq and offsetFreq, and generates literalEncoding +// and offsetEncoding. +// The number of literal and offset tokens is returned. +func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) { + clear(w.literalFreq) + clear(w.offsetFreq) + + for _, t := range tokens { + if t < matchType { + w.literalFreq[t.literal()]++ + continue + } + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + + // get the number of literals + numLiterals = len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets = len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + return +} + +// writeTokens writes a slice of tokens to the output. +// codes for literal and offset encoding must be supplied. +func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { + if w.err != nil { + return + } + for _, t := range tokens { + if t < matchType { + w.writeCode(leCodes[t.literal()]) + continue + } + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(leCodes[lengthCode+lengthCodesStart]) + extraLengthBits := uint(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(oeCodes[offsetCode]) + extraOffsetBits := uint(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + } +} + +// huffOffset is a static offset encoder used for huffman only encoding. +// It can be reused since we will not be encoding offset values. +var huffOffset *huffmanEncoder + +func init() { + offsetFreq := make([]int32, offsetCodeCount) + offsetFreq[0] = 1 + huffOffset = newHuffmanEncoder(offsetCodeCount) + huffOffset.generate(offsetFreq, 15) +} + +// writeBlockHuff encodes a block of bytes as either +// Huffman encoded literals or uncompressed bytes if the +// results only gains very little from compression. +func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) { + if w.err != nil { + return + } + + // Clear histogram + clear(w.literalFreq) + + // Add everything as literals + histogram(input, w.literalFreq) + + w.literalFreq[endBlockMarker] = 1 + + const numLiterals = endBlockMarker + 1 + w.offsetFreq[0] = 1 + const numOffsets = 1 + + w.literalEncoding.generate(w.literalFreq, 15) + + // Figure out smallest code. + // Always use dynamic Huffman or Store + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0) + + // Store bytes, if we don't get a reasonable improvement. + if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Huffman. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + encoding := w.literalEncoding.codes[:257] + n := w.nbytes + for _, t := range input { + // Bitwriting inlined, ~30% speedup + c := encoding[t] + w.bits |= uint64(c.code) << w.nbits + w.nbits += uint(c.len) + if w.nbits < 48 { + continue + } + // Store 6 bytes + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n < bufferFlushSize { + continue + } + w.write(w.bytes[:n]) + if w.err != nil { + return // Return early in the event of write failures + } + n = 0 + } + w.nbytes = n + w.writeCode(encoding[endBlockMarker]) +} + +// histogram accumulates a histogram of b in h. +// +// len(h) must be >= 256, and h's elements must be all zeroes. +func histogram(b []byte, h []int32) { + h = h[:256] + for _, t := range b { + h[t]++ + } +} diff --git a/util/compression/go126flate/huffman_code.go b/util/compression/go126flate/huffman_code.go new file mode 100644 index 000000000000..1af0b06e8fd4 --- /dev/null +++ b/util/compression/go126flate/huffman_code.go @@ -0,0 +1,347 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "math" + "math/bits" + "sort" +) + +// hcode is a huffman code with a bit code and bit length. +type hcode struct { + code, len uint16 +} + +type huffmanEncoder struct { + codes []hcode + freqcache []literalNode + bitCount [17]int32 + lns byLiteral // stored to avoid repeated allocation in generate + lfs byFreq // stored to avoid repeated allocation in generate +} + +type literalNode struct { + literal uint16 + freq int32 +} + +// A levelInfo describes the state of the constructed tree for a given depth. +type levelInfo struct { + // Our level. for better printing + level int32 + + // The frequency of the last node at this level + lastFreq int32 + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 +} + +// set sets the code and length of an hcode. +func (h *hcode) set(code uint16, length uint16) { + h.len = length + h.code = code +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{codes: make([]hcode, size)} +} + +// Generates a HuffmanCode corresponding to the fixed literal table. +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxNumLit) + codes := h.codes + var ch uint16 + for ch = 0; ch < maxNumLit; ch++ { + var bits uint16 + var size uint16 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size} + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codes := h.codes + for ch := range codes { + codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5} + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int { + var total int + for i, f := range freq { + if f != 0 { + total += int(f) * int(h.codes[i].len) + } + } + return total +} + +const maxBitsLimit = 16 + +// bitCounts computes the number of literals assigned to each bit size in the Huffman encoding. +// It is only called when list.length >= 3. +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list is an array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency and has as its last element a special element with frequency +// MaxInt32. +// +// maxBits is the maximum number of bits that should be used to encode any literal. +// It must be less than 16. +// +// bitCounts returns an integer slice in which slice[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + if maxBits >= maxBitsLimit { + panic("flate: maxBits too large") + } + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + if maxBits > n-1 { + maxBits = n - 1 + } + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + var levels [maxBitsLimit]levelInfo + // leafCounts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leafCounts[i][j] is the number of literals at the left + // of the level j ancestor. + var leafCounts [maxBitsLimit][maxBitsLimit]int32 + + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = levelInfo{ + level: level, + lastFreq: list[1].freq, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + } + leafCounts[level][level] = 2 + if level == 1 { + levels[level].nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[maxBits].needed = 2*n - 4 + + level := maxBits + for { + l := &levels[level] + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leaves and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.needed = 0 + levels[level+1].nextPairFreq = math.MaxInt32 + level++ + continue + } + + prevFreq := l.lastFreq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := leafCounts[level][level] + 1 + l.lastFreq = l.nextCharFreq + // Lower leafCounts are the same of the previous node. + leafCounts[level][level] = n + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastFreq = l.nextPairFreq + // Take leaf counts from the lower level, except counts[level] remains the same. + copy(leafCounts[level][:level], leafCounts[level-1][:level]) + levels[l.level-1].needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if l.level == maxBits { + // All done! + break + } + levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq + level++ + } else { + // If we stole from below, move down temporarily to replenish it. + for levels[level-1].needed > 0 { + level-- + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if leafCounts[maxBits][maxBits] != n { + panic("leafCounts[maxBits][maxBits] != n") + } + + bitCount := h.bitCount[:maxBits+1] + bits := 1 + counts := &leafCounts[maxBits] + for level := maxBits; level > 0; level-- { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = counts[level] - counts[level-1] + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + + h.lns.sort(chunk) + for _, node := range chunk { + h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)} + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq is an array of frequencies, in which freq[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + if h.freqcache == nil { + // Allocate a reusable buffer with the longest possible frequency table. + // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit. + // The largest of these is maxNumLit, so we allocate for that case. + h.freqcache = make([]literalNode, maxNumLit+1) + } + list := h.freqcache[:len(freq)+1] + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + h.codes[i].len = 0 + } + } + + list = list[:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codes[node.literal].set(uint16(i), 1) + } + return + } + h.lfs.sort(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type byLiteral []literalNode + +func (s *byLiteral) sort(a []literalNode) { + *s = byLiteral(a) + sort.Sort(s) +} + +func (s byLiteral) Len() int { return len(s) } + +func (s byLiteral) Less(i, j int) bool { + return s[i].literal < s[j].literal +} + +func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type byFreq []literalNode + +func (s *byFreq) sort(a []literalNode) { + *s = byFreq(a) + sort.Sort(s) +} + +func (s byFreq) Len() int { return len(s) } + +func (s byFreq) Less(i, j int) bool { + if s[i].freq == s[j].freq { + return s[i].literal < s[j].literal + } + return s[i].freq < s[j].freq +} + +func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func reverseBits(number uint16, bitLength byte) uint16 { + return bits.Reverse16(number << (16 - bitLength)) +} diff --git a/util/compression/go126flate/token.go b/util/compression/go126flate/token.go new file mode 100644 index 000000000000..2f56c253050a --- /dev/null +++ b/util/compression/go126flate/token.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset. +func offsetCode(off uint32) uint32 { + if off < uint32(len(offsetCodes)) { + return offsetCodes[off] + } + if off>>7 < uint32(len(offsetCodes)) { + return offsetCodes[off>>7] + 14 + } + return offsetCodes[off>>14] + 28 +} diff --git a/util/compression/go126gzip.go b/util/compression/go126gzip.go new file mode 100644 index 000000000000..dba942c5e87b --- /dev/null +++ b/util/compression/go126gzip.go @@ -0,0 +1,92 @@ +package compression + +import ( + "encoding/binary" + "hash/crc32" + "io" + + "github.com/moby/buildkit/util/compression/go126flate" + "github.com/pkg/errors" +) + +const go126GzipDefaultCompression = go126flate.DefaultCompression + +// go126GzipWriter writes the fixed gzip framing used for image layers around +// the Go 1.26 DEFLATE stream. Image layers do not use optional gzip headers. +type go126GzipWriter struct { + w io.Writer + level int + wroteHeader bool + closed bool + buf [10]byte + compressor *go126flate.Writer + digest uint32 + size uint32 + err error +} + +func newGo126GzipWriterLevel(w io.Writer, level int) (io.WriteCloser, error) { + if level < go126flate.HuffmanOnly || level > go126flate.BestCompression { + return nil, errors.Errorf("gzip: invalid compression level: %d", level) + } + return &go126GzipWriter{w: w, level: level}, nil +} + +func (z *go126GzipWriter) writeHeader() error { + z.wroteHeader = true + z.buf = [10]byte{0: 0x1f, 1: 0x8b, 2: 8, 9: 255} + switch z.level { + case go126flate.BestCompression: + z.buf[8] = 2 + case go126flate.BestSpeed: + z.buf[8] = 4 + } + if _, err := z.w.Write(z.buf[:10]); err != nil { + return err + } + compressor, err := go126flate.NewWriter(z.w, z.level) + if err != nil { + return errors.WithStack(err) + } + z.compressor = compressor + return nil +} + +func (z *go126GzipWriter) Write(p []byte) (int, error) { + if z.err != nil { + return 0, z.err + } + if !z.wroteHeader { + z.err = z.writeHeader() + if z.err != nil { + return 0, z.err + } + } + z.size += uint32(len(p)) + z.digest = crc32.Update(z.digest, crc32.IEEETable, p) + n, err := z.compressor.Write(p) + z.err = err + return n, err +} + +func (z *go126GzipWriter) Close() error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if !z.wroteHeader { + if z.err = z.writeHeader(); z.err != nil { + return z.err + } + } + if z.err = z.compressor.Close(); z.err != nil { + return z.err + } + binary.LittleEndian.PutUint32(z.buf[:4], z.digest) + binary.LittleEndian.PutUint32(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[:8]) + return z.err +} diff --git a/util/compression/go126gzip_test.go b/util/compression/go126gzip_test.go new file mode 100644 index 000000000000..ceec921e20c7 --- /dev/null +++ b/util/compression/go126gzip_test.go @@ -0,0 +1,46 @@ +package compression + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "fmt" + "io" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGo126GzipWriter(t *testing.T) { + data := bytes.Repeat([]byte("BuildKit gzip compatibility data\x00with repetition\n"), 4096) + // These digests were produced by compress/gzip from Go 1.26.5. + expected := map[int]string{ + -1: "b9738544854e7f362743dabbe03fac154d2b62cb7e705781dcc4456583bec7dc", + 0: "0069b765e18ce96a96cee8d1ea6f7968177081057cb34da7f7b8d5c0d9e19628", + 1: "e356453bad442c4eb03a0b87efd9e335d983ebb3c1eca0a64b2e1e22e628438e", + 6: "b9738544854e7f362743dabbe03fac154d2b62cb7e705781dcc4456583bec7dc", + 9: "79f9d5915049926bfe979ec91245ab7b9ac469172e5a539c135e571e3b2b4a04", + } + + for level, digest := range expected { + t.Run(strconv.Itoa(level), func(t *testing.T) { + var compressed bytes.Buffer + w, err := gzipWriter(Config{Level: &level})(&compressed) + require.NoError(t, err) + _, err = w.Write(data) + require.NoError(t, err) + require.NoError(t, w.Close()) + + sum := sha256.Sum256(compressed.Bytes()) + require.Equal(t, digest, fmt.Sprintf("%x", sum)) + + r, err := gzip.NewReader(bytes.NewReader(compressed.Bytes())) + require.NoError(t, err) + uncompressed, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + require.Equal(t, data, uncompressed) + }) + } +} diff --git a/util/compression/gzip.go b/util/compression/gzip.go index b6135e782cd4..c8f0180ba571 100644 --- a/util/compression/gzip.go +++ b/util/compression/gzip.go @@ -1,7 +1,6 @@ package compression import ( - "compress/gzip" "context" "io" @@ -57,10 +56,13 @@ func (c gzipType) String() string { func gzipWriter(comp Config) func(io.Writer) (io.WriteCloser, error) { return func(dest io.Writer) (io.WriteCloser, error) { - level := gzip.DefaultCompression + level := go126GzipDefaultCompression if comp.Level != nil { level = *comp.Level } - return gzip.NewWriterLevel(dest, level) + // Go 1.27 changed compress/flate output in CL 707355 for + // golang/go#75532. Keep Go 1.26 encoding so existing compatibility + // versions continue to produce the same compressed layer digests. + return newGo126GzipWriterLevel(dest, level) } } diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go index a9e1b72ba781..55235c15ce5b 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go @@ -37,10 +37,8 @@ import ( "sync" "sync/atomic" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" "github.com/klauspost/compress/zstd" digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/errgroup" ) type GzipHelperFunc func(io.Reader) (io.ReadCloser, error) @@ -235,15 +233,15 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { } writers := make([]*Writer, len(tarParts)) payloads := make([]*os.File, len(tarParts)) - var mu sync.Mutex - var eg errgroup.Group + var wg sync.WaitGroup + errCh := make(chan error, len(tarParts)) // buffered to avoid goroutine leaks for i, parts := range tarParts { - i, parts := i, parts // builds verifiable stargz sub-blobs - eg.Go(func() error { + wg.Go(func() { esgzFile, err := layerFiles.TempFile("", "esgzdata") if err != nil { - return err + errCh <- err + return } sw := NewWriterWithCompressor(esgzFile, opts.compression) sw.ChunkSize = opts.chunkSize @@ -255,18 +253,20 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { sw.needsOpenGzEntries[f] = struct{}{} } if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { - return err + errCh <- err + return } - mu.Lock() writers[i] = sw payloads[i] = esgzFile - mu.Unlock() - return nil }) } - if err := eg.Wait(); err != nil { - rErr = err - return nil, err + wg.Wait() + close(errCh) + + for err := range errCh { + if err != nil { + return nil, err + } } tocAndFooter, tocDgst, err := closeWithCombine(writers...) if err != nil { @@ -656,7 +656,7 @@ func (tf *tempFiles) cleanupAll() error { } } tf.files = nil - return errorutil.Aggregate(allErr) + return errors.Join(allErr...) } func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go deleted file mode 100644 index 6de78b02dcdd..000000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errorutil - -import ( - "errors" - "fmt" - "strings" -) - -// Aggregate combines a list of errors into a single new error. -func Aggregate(errs []error) error { - switch len(errs) { - case 0: - return nil - case 1: - return errs[0] - default: - points := make([]string, len(errs)+1) - points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) - for i, err := range errs { - points[i+1] = fmt.Sprintf("* %s", err) - } - return errors.New(strings.Join(points, "\n\t")) - } -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go index 693730420c86..628f3382a53d 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go @@ -38,7 +38,6 @@ import ( "sync" "time" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" digest "github.com/opencontainers/go-digest" "github.com/vbatts/tar-split/archive/tar" ) @@ -164,7 +163,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { allErr = append(allErr, err) } if !found { - return nil, errorutil.Aggregate(allErr) + return nil, errors.Join(allErr...) } if err := r.initFields(); err != nil { return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) @@ -192,7 +191,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e } allErr = append(allErr, err) } - return 0, 0, errorutil.Aggregate(allErr) + return 0, 0, errors.Join(allErr...) } // initFields populates the Reader from r.toc after decoding it from diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/externaltoc/externaltoc.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/externaltoc/externaltoc.go index be8de8dfa68d..37dba017291e 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/externaltoc/externaltoc.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/externaltoc/externaltoc.go @@ -126,23 +126,20 @@ const FooterSize = 46 // gzipFooterBytes returns the 104 bytes footer. func gzipFooterBytes() ([]byte, error) { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - - // Extra header indicating the offset of TOCJSON + // Extra header indicating the external toc // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 header := make([]byte, 4) header[0], header[1] = 'S', 'G' subfield := "STARGZEXTERNALTOC" // len("STARGZEXTERNALTOC") = 17 binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - if err := gz.Close(); err != nil { - return nil, err - } - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) + extra := append(header, []byte(subfield)...) + + buf := estargz.CreateGzipFooter(extra) + + if len(buf) != FooterSize { + panic(fmt.Sprintf("footer buffer = %d, not %d", len(buf), FooterSize)) } - return buf.Bytes(), nil + return buf, nil } func NewGzipDecompressor(provideTOCFunc func() ([]byte, error)) *GzipDecompressor { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go index 88fa13b19139..dbf1458d329d 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go @@ -100,21 +100,52 @@ func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, d // gzipFooterBytes returns the 51 bytes footer. func gzipFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - // Extra header indicating the offset of TOCJSON // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 header := make([]byte, 4) header[0], header[1] = 'S', 'G' subfield := fmt.Sprintf("%016xSTARGZ", tocOff) binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) + extra := append(header, []byte(subfield)...) + + buf := CreateGzipFooter(extra) + if len(buf) != FooterSize { + panic(fmt.Sprintf("footer buffer = %d, not %d", len(buf), FooterSize)) + } + return buf +} + +// CreateGzipFooter creates eStargz's footer with the specified extra field. +func CreateGzipFooter(extra []byte) []byte { + // Gzip header (10 bytes) + // https://datatracker.ietf.org/doc/html/rfc1952 + buf := make([]byte, 10) + buf[0] = 0x1f // ID1 + buf[1] = 0x8b // ID2 + buf[2] = 8 // "deflate" + buf[3] = 0x4 // FEXTRA + // MTIME = 0 + // XFL = 0 + buf[9] = 255 // unknown + + // Extra field + xlen := make([]byte, 2) + binary.LittleEndian.PutUint16(xlen, uint16(len(extra))) + buf = append(buf, append(xlen, extra...)...) + + // Flate header (5 bytes) + // https://datatracker.ietf.org/doc/html/rfc1951 + flateHeader := []byte{ + 1, // BFINAL = 1 (last block), BFTYPE = 0 (no compression), remaining bits are ignored + 0, 0, // LEN = 0 + 0xff, 0xff, // NLEN (the one's complement of LEN) } - return buf.Bytes() + buf = append(buf, flateHeader...) + + // Gzip footer (8 bytes): CRC32 = 0, ISIZE = 0 + buf = append(buf, make([]byte, 8)...) + + return buf } type GzipDecompressor struct{} @@ -137,6 +168,9 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t } defer zr.Close() extra := zr.Extra + if len(extra) < 4 { + return 0, 0, 0, fmt.Errorf("invalid extra field length %d; expected >= 4", len(extra)) + } si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] if si1 != 'S' || si2 != 'G' { return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go index ff165e090e38..038e2d2314b0 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go @@ -40,7 +40,6 @@ import ( "strings" "time" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" "github.com/klauspost/compress/zstd" digest "github.com/opencontainers/go-digest" ) @@ -187,15 +186,10 @@ func testBuild(t *TestRunner, controllers ...TestingControllerFactory) { tt.minChunkSize = []int{0} } for _, srcCompression := range srcCompressions { - srcCompression := srcCompression for _, newCL := range controllers { - newCL := newCL for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, prefix := range allowedPrefix { - prefix := prefix for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) { tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) // Test divideEntries() @@ -675,15 +669,10 @@ func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory) tt.minChunkSize = []int{0} } for _, srcCompression := range srcCompressions { - srcCompression := srcCompression for _, newCL := range controllers { - newCL := newCL for _, prefix := range allowedPrefix { - prefix := prefix for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) { // Get original tar file and chunk digests dgstMap := make(map[string]digest.Digest) @@ -1488,11 +1477,8 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { for _, tt := range tests { for _, newCL := range controllers { - newCL := newCL for _, prefix := range allowedPrefix { - prefix := prefix for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, lossless := range []bool{true, false} { t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) { var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) @@ -1650,7 +1636,7 @@ func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) er if !deserializeTocLatencyCalled { allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) } - return errorutil.Aggregate(allErr) + return errors.Join(allErr...) } } @@ -2072,7 +2058,7 @@ func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format return f(tw, prefix, format) } -func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { +func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...any) *io.SectionReader { format := tar.FormatUnknown for _, opt := range opts { switch v := opt.(type) { @@ -2096,7 +2082,7 @@ func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) * return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) } -func dir(name string, opts ...interface{}) tarEntry { +func dir(name string, opts ...any) tarEntry { return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var o owner mode := os.FileMode(0755) @@ -2137,7 +2123,7 @@ type owner struct { gid int } -func file(name, contents string, opts ...interface{}) tarEntry { +func file(name, contents string, opts ...any) tarEntry { return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var xattrs xAttr var o owner @@ -2349,7 +2335,7 @@ func (f fileInfoOnlyMode) Size() int64 { return 0 } func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } +func (f fileInfoOnlyMode) Sys() any { return nil } func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) { if len(streams) == 0 { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go index 57e0aa614e4a..41eb6571fbb2 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go @@ -246,7 +246,7 @@ func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } func (fi fileInfo) Size() int64 { return fi.e.Size } func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() } -func (fi fileInfo) Sys() interface{} { return fi.e } +func (fi fileInfo) Sys() any { return fi.e } func (fi fileInfo) Mode() (m os.FileMode) { // TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg. m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() & diff --git a/vendor/modules.txt b/vendor/modules.txt index 199b09c2959f..aba515dfe2b5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -500,10 +500,9 @@ github.com/containerd/stargz-snapshotter/task github.com/containerd/stargz-snapshotter/util/cacheutil github.com/containerd/stargz-snapshotter/util/namedmutex github.com/containerd/stargz-snapshotter/util/testutil -# github.com/containerd/stargz-snapshotter/estargz v0.18.2 -## explicit; go 1.24.0 +# github.com/containerd/stargz-snapshotter/estargz v0.18.3-0.20260607065519-4daea594a1da +## explicit; go 1.25.0 github.com/containerd/stargz-snapshotter/estargz -github.com/containerd/stargz-snapshotter/estargz/errorutil github.com/containerd/stargz-snapshotter/estargz/externaltoc github.com/containerd/stargz-snapshotter/estargz/zstdchunked # github.com/containerd/ttrpc v1.2.9