diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go index 19a7c84733a3..c8e95b447a9e 100644 --- a/frontend/dockerfile/builder/caps.go +++ b/frontend/dockerfile/builder/caps.go @@ -14,6 +14,7 @@ var enabledCaps = map[string]struct{}{ "moby.buildkit.frontend.subrequests": {}, "moby.buildkit.frontend.contexts": {}, "moby.buildkit.frontend.gitquerystring": {}, + "moby.buildkit.frontend.contexts.zstd": {}, } func validateCaps(req string) (forward bool, err error) { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index 9435f3e8ccd9..0ef913101886 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -72,7 +72,7 @@ EOT FROM scratch AS release LABEL moby.buildkit.frontend.network.none="true" -LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts,moby.buildkit.frontend.gitquerystring" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts,moby.buildkit.frontend.gitquerystring,moby.buildkit.frontend.contexts.zstd" COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] diff --git a/frontend/dockerfile/dockerfile_add_test.go b/frontend/dockerfile/dockerfile_add_test.go index ef18a9ddf74f..a345a3e7ace6 100644 --- a/frontend/dockerfile/dockerfile_add_test.go +++ b/frontend/dockerfile/dockerfile_add_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/containerd/continuity/fs/fstest" + "github.com/klauspost/compress/zstd" "github.com/moby/buildkit/client" "github.com/moby/buildkit/frontend/dockerui" "github.com/moby/buildkit/identity" @@ -191,6 +192,37 @@ ADD t.tar.gz / require.NoError(t, err) require.Equal(t, expectedContent, dt) + // add zstd tar + buf3 := bytes.NewBuffer(nil) + zw, err := zstd.NewWriter(buf3) + require.NoError(t, err) + _, err = zw.Write(buf.Bytes()) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + dockerfile = fmt.Appendf(nil, ` +FROM %s +ADD t.tar.zst / +`, baseImage) + + dir = integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("t.tar.zst", buf3.Bytes(), 0600), + ) + + args, trace = f.DFCmdArgs(dir.Name, dir.Name) + defer os.RemoveAll(trace) + + destDir = t.TempDir() + + cmd = sb.Cmd(args + fmt.Sprintf(" --output type=local,dest=%s", destDir)) + require.NoError(t, cmd.Run()) + + dt, err = os.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, expectedContent, dt) + // add with unpack=false dockerfile = fmt.Appendf(nil, ` FROM %s diff --git a/frontend/dockerfile/docs/reference.md b/frontend/dockerfile/docs/reference.md index 5a366a1b90e9..eb3cf759cbb8 100644 --- a/frontend/dockerfile/docs/reference.md +++ b/frontend/dockerfile/docs/reference.md @@ -1459,9 +1459,9 @@ ADD arr[[]0].txt /dest/ #### Adding local tar archives When using a local tar archive as the source for `ADD`, and the archive is in a -recognized compression format (`gzip`, `bzip2` or `xz`, or uncompressed), the -archive is decompressed and extracted into the specified destination. Local tar -archives are extracted by default, see the [`ADD --unpack` flag]. +recognized compression format (`gzip`, `bzip2`, `xz` or `zstd`, or +uncompressed), the archive is decompressed and extracted into the specified +destination. Local tar archives are extracted by default, see the [`ADD --unpack` flag]. When a directory is extracted, it has the same behavior as `tar -x`. The result is the union of: @@ -1639,9 +1639,10 @@ ADD [--unpack=] ... ``` The `--unpack` flag controls whether or not to automatically unpack tar -archives (including compressed formats like `gzip` or `bzip2`) when adding them -to the image. Local tar archives are unpacked by default, whereas remote tar -archives (where `src` is a URL) are downloaded without unpacking. +archives (including compressed formats like `gzip`, `bzip2`, `xz` or `zstd`) +when adding them to the image. Local tar archives are unpacked by default, +whereas remote tar archives (where `src` is a URL) are downloaded without +unpacking. ```dockerfile # syntax=docker/dockerfile:1 diff --git a/frontend/dockerui/build_test.go b/frontend/dockerui/build_test.go index 378f209b285f..11a5608b7009 100644 --- a/frontend/dockerui/build_test.go +++ b/frontend/dockerui/build_test.go @@ -1,6 +1,8 @@ package dockerui import ( + "archive/tar" + "bytes" "testing" "github.com/containerd/platforms" @@ -9,6 +11,43 @@ import ( "github.com/stretchr/testify/require" ) +func TestIsArchive(t *testing.T) { + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + err := tw.WriteHeader(&tar.Header{ + Name: "file", + Mode: 0o644, + Size: 4, + }) + require.NoError(t, err) + _, err = tw.Write([]byte("test")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + + testCases := []struct { + name string + header []byte + expected bool + }{ + {name: "bzip2", header: []byte{0x42, 0x5A, 0x68}, expected: true}, + {name: "gzip", header: []byte{0x1F, 0x8B, 0x08}, expected: true}, + {name: "xz", header: []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, expected: true}, + {name: "zstd", header: []byte{0x28, 0xB5, 0x2F, 0xFD}, expected: true}, + {name: "zstd skippable frame", header: []byte{0x50, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00}, expected: true}, + {name: "zstd skippable frame range end", header: []byte{0x5F, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00}, expected: true}, + {name: "tar", header: tarBuf.Bytes(), expected: true}, + {name: "unknown", header: []byte("not an archive"), expected: false}, + {name: "short zstd prefix", header: []byte{0x28, 0xB5, 0x2F}, expected: false}, + {name: "short zstd skippable frame", header: []byte{0x50, 0x2A, 0x4D, 0x18}, expected: false}, + {name: "outside zstd skippable frame range", header: []byte{0x60, 0x2A, 0x4D, 0x18, 0x00, 0x00, 0x00, 0x00}, expected: false}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, isArchive(tc.header)) + }) + } +} + func TestNormalizePlatform(t *testing.T) { testCases := []struct { p, imgP ocispecs.Platform diff --git a/frontend/dockerui/context.go b/frontend/dockerui/context.go index 6fb3ebd1be60..fc75a5062d12 100644 --- a/frontend/dockerui/context.go +++ b/frontend/dockerui/context.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "context" + "encoding/binary" "io" "maps" "path/filepath" @@ -339,15 +340,16 @@ func isArchive(header []byte) bool { {0x42, 0x5A, 0x68}, // bzip2 {0x1F, 0x8B, 0x08}, // gzip {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz + {0x28, 0xB5, 0x2F, 0xFD}, // zstd } { - if len(header) < len(m) { - continue - } - if bytes.Equal(m, header[:len(m)]) { + if bytes.HasPrefix(header, m) { return true } } - + // zstd skippable frames use magic numbers from 0x184D2A50 to 0x184D2A5F + if len(header) >= 8 && binary.LittleEndian.Uint32(header[:4])&0xFFFFFFF0 == 0x184D2A50 { + return true + } r := tar.NewReader(bytes.NewBuffer(header)) _, err := r.Next() return err == nil