diff --git a/errors/errors.go b/errors/errors.go index 91e322a1422..f7cd45d8f15 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1608,6 +1608,8 @@ var ( ErrArtifactStoreInvalidArgs = errors.New("invalid artifact store arguments") ErrArtifactMetadataFailed = errors.New("failed to load artifact metadata") ErrArtifactIntegrityFailed = errors.New("artifact integrity check failed") + ErrArtifactSigningFailed = errors.New("failed to sign artifact") + ErrPathThroughNonDirectory = errors.New("path passes through a non-directory") // CI cache errors. ErrCacheUnavailable = errors.New("CI cache is not available in this environment") @@ -2117,6 +2119,22 @@ var ( // ErrArtifactTooLarge is returned when a record, document, or evidence blob exceeds its bounded-read limit. ErrArtifactTooLarge = errors.New("artifact data exceeds the size limit") + // ErrArtifactSourceChanged is returned when source content changed while it was being captured, + // so an inventory would describe different bytes than the ones staged. + ErrArtifactSourceChanged = errors.New("artifact source changed during capture") + + // ErrArtifactVerificationFailed is returned when no permitted signature satisfies a required + // trust set. It is a policy error raised before any consumption or destination publication. + ErrArtifactVerificationFailed = errors.New("artifact signature verification failed") + + // ErrArtifactPublicationAmbiguous is returned when a mutation failed in a way that could not be + // reconciled: the completed reference could not be proven to hold, or to lack, the expected content. + ErrArtifactPublicationAmbiguous = errors.New("artifact publication outcome is ambiguous") + + // ErrSymlinkLoop is returned when resolving a path follows more symlinks than the bound allows, which + // means the links form a loop. + ErrSymlinkLoop = errors.New("symlink loop") + // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The // underlying I/O error is wrapped alongside it for diagnostics. ErrArtifactStreamFailed = errors.New("artifact stream failed") diff --git a/errors/formatter_test.go b/errors/formatter_test.go index 50171afc82c..6bfbc1aceba 100644 --- a/errors/formatter_test.go +++ b/errors/formatter_test.go @@ -248,6 +248,8 @@ func TestShouldUseColorInCI(t *testing.T) { } func TestFormatInCIUsesColorUnlessDisabled(t *testing.T) { + // TERM=dumb deliberately disables automatic styling, even in CI. + t.Setenv("TERM", "xterm-256color") t.Setenv("CI", "true") t.Setenv("NO_COLOR", "") diff --git a/errors/signature.go b/errors/signature.go new file mode 100644 index 00000000000..d8868820895 --- /dev/null +++ b/errors/signature.go @@ -0,0 +1,22 @@ +package errors + +import "errors" + +// Shared signature-verification sentinels used by pkg/signature. Adapters that predate the shared +// package translate these to their legacy sentinels so existing callers keep matching. +var ( + // ErrSignatureInvalid indicates signature evidence is malformed, untrusted, or does not match the subject. + ErrSignatureInvalid = errors.New("signature does not satisfy the trust policy") + + // ErrSignatureConfigInvalid indicates the verification policy itself is unusable (for example an empty identity). + ErrSignatureConfigInvalid = errors.New("invalid signature verification configuration") + + // ErrSignatureKeyUnsupported indicates a configured trusted key cannot verify bundles by subject digest alone. + ErrSignatureKeyUnsupported = errors.New("signature key type cannot verify by digest") + + // ErrSignatureOffline indicates trust material cannot be fetched while verifying offline. + ErrSignatureOffline = errors.New("signature trust material is unavailable offline") + + // ErrSignatureSigningFailed indicates a signing backend failed to produce signature evidence. + ErrSignatureSigningFailed = errors.New("signing backend failed to produce a signature") +) diff --git a/go.mod b/go.mod index 11bee90ea65..9dac93c69cd 100644 --- a/go.mod +++ b/go.mod @@ -447,7 +447,7 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/xurls/v2 v2.6.0 // indirect - oras.land/oras-go/v2 v2.6.2 // indirect + oras.land/oras-go/v2 v2.6.2 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect diff --git a/pkg/archive/deterministic.go b/pkg/archive/deterministic.go new file mode 100644 index 00000000000..a869fc58f10 --- /dev/null +++ b/pkg/archive/deterministic.go @@ -0,0 +1,259 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "sort" + "strings" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MemberKind is the type of a [Member] written by [WriteDeterministic]. +type MemberKind int + +// Member kinds. Devices, sockets, and other special files are never representable. +const ( + // MemberFile is a regular file whose bytes come from [Member.Open]. + MemberFile MemberKind = iota + + // MemberDirectory is a directory entry. + MemberDirectory + + // MemberSymlink is a symbolic link to [Member.Target]. + MemberSymlink +) + +// deterministicPermMask holds the permission bits a deterministic member may carry. Privileged +// bits (setuid, setgid, sticky) are always cleared. +const deterministicPermMask fs.FileMode = 0o777 + +// Member is one explicitly described archive entry for [WriteDeterministic]. Unlike [PackOptions] +// sources, members never touch the filesystem on their own: the caller supplies the bytes, so the +// output depends only on the declared paths, modes, and content. +type Member struct { + // Path is the relative, slash-separated path inside the archive. + Path string + + // Kind is file, directory, or symlink. + Kind MemberKind + + // Mode holds the ordinary permission bits. Executable bits are preserved; other bits are + // written as given. Privileged bits are cleared. + Mode fs.FileMode + + // Size is the exact byte length of a file's content. + Size int64 + + // Target is a symlink's target; empty for other kinds. + Target string + + // Open returns a reader over a file's content. The writer closes it. Nil for other kinds. + Open func() (io.ReadCloser, error) +} + +// tarEpoch is the timestamp stamped into every deterministic tar entry. +var tarEpoch = time.Unix(0, 0).UTC() + +// zipEpoch is the timestamp stamped into every deterministic zip entry. 1980-01-01 is the earliest +// time the zip DOS timestamp field can represent. +var zipEpoch = time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC) + +// WriteDeterministic writes members to w as a zip, tar, or tgz archive whose bytes depend only on +// the members: entries are sorted by path, timestamps are pinned to a fixed epoch, owners are +// normalized to zero and empty names, gzip headers carry no name or time, and permission bits +// (including executable bits) are preserved. Identical members always produce identical bytes for +// one Go toolchain. It does not alter the behavior of [Run]; that path keeps its own defaults. +func WriteDeterministic(w io.Writer, format string, members []Member) error { + defer perf.Track(nil, "archive.WriteDeterministic")() + + sorted, err := sortMembers(members) + if err != nil { + return err + } + switch normalizeFormat(format) { + case FormatTar: + return writeDeterministicTar(w, sorted) + case FormatTGZ: + return writeDeterministicTGZ(w, sorted) + case FormatZip: + return writeDeterministicZip(w, sorted) + default: + return errUtils.Build(errUtils.ErrArchiveFormatNotImplemented). + WithExplanationf("deterministic writing supports zip, tar, and tgz, not %q", format). + WithContext("format", format). + Err() + } +} + +// sortMembers returns a path-sorted copy and rejects duplicate paths. +func sortMembers(members []Member) ([]Member, error) { + sorted := make([]Member, len(members)) + copy(sorted, members) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + for index := range sorted { + if sorted[index].Path == "" || strings.HasPrefix(sorted[index].Path, "/") { + return nil, memberError(sorted[index].Path, errUtils.ErrArchiveInvalidSubpath) + } + if index > 0 && sorted[index].Path == sorted[index-1].Path { + return nil, memberError(sorted[index].Path, errUtils.ErrArchiveWriteFailed) + } + } + return sorted, nil +} + +func memberError(path string, sentinel error) error { + return fmt.Errorf("%w: member %q is empty, absolute, or duplicated", sentinel, path) +} + +func writeDeterministicTGZ(w io.Writer, members []Member) error { + gz := gzip.NewWriter(w) + if err := writeDeterministicTar(gz, members); err != nil { + _ = gz.Close() + return err + } + if err := gz.Close(); err != nil { + return writeFailedError("gzip", err) + } + return nil +} + +func writeDeterministicTar(w io.Writer, members []Member) error { + tw := tar.NewWriter(w) + for index := range members { + if err := writeTarMember(tw, &members[index]); err != nil { + _ = tw.Close() + return err + } + } + if err := tw.Close(); err != nil { + return writeFailedError("tar", err) + } + return nil +} + +func writeTarMember(tw *tar.Writer, member *Member) error { + header := &tar.Header{ + Name: member.Path, + Mode: int64(member.Mode & deterministicPermMask), + ModTime: tarEpoch, + } + switch member.Kind { + case MemberDirectory: + header.Typeflag = tar.TypeDir + header.Name += "/" + case MemberSymlink: + header.Typeflag = tar.TypeSymlink + header.Linkname = member.Target + header.Mode = 0 + case MemberFile: + header.Typeflag = tar.TypeReg + header.Size = member.Size + default: + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + if err := tw.WriteHeader(header); err != nil { + return writeFailedError(member.Path, err) + } + if member.Kind != MemberFile { + return nil + } + return copyMemberContent(tw, member) +} + +func writeDeterministicZip(w io.Writer, members []Member) error { + zw := zip.NewWriter(w) + for index := range members { + if err := writeZipMember(zw, &members[index]); err != nil { + _ = zw.Close() + return err + } + } + if err := zw.Close(); err != nil { + return writeFailedError("zip", err) + } + return nil +} + +func writeZipMember(zw *zip.Writer, member *Member) error { + header := &zip.FileHeader{Name: member.Path, Method: zip.Deflate, Modified: zipEpoch} + perm := member.Mode & deterministicPermMask + switch member.Kind { + case MemberDirectory: + header.Name += "/" + header.Method = zip.Store + header.SetMode(fs.ModeDir | perm) + case MemberSymlink: + header.Method = zip.Store + header.SetMode(fs.ModeSymlink) + case MemberFile: + header.SetMode(perm) + default: + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + out, err := zw.CreateHeader(header) + if err != nil { + return writeFailedError(member.Path, err) + } + switch member.Kind { + case MemberSymlink: + if _, err := io.WriteString(out, member.Target); err != nil { + return writeFailedError(member.Path, err) + } + case MemberFile: + return copyMemberContent(out, member) + } + return nil +} + +// copyMemberContent copies exactly member.Size bytes and fails if the source is shorter or longer, +// so an archive never describes bytes other than the ones it carries. +func copyMemberContent(dst io.Writer, member *Member) (err error) { + if member.Open == nil { + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + src, err := member.Open() + if err != nil { + return writeFailedError(member.Path, err) + } + // A failed close can mean the source did not verify, so it is kept alongside any copy error. + defer func() { + if closeErr := src.Close(); closeErr != nil { + err = errors.Join(err, writeFailedError(member.Path, closeErr)) + } + }() + + written, err := io.Copy(dst, io.LimitReader(src, member.Size)) + if err != nil { + return writeFailedError(member.Path, err) + } + if written != member.Size { + return writeFailedError(member.Path, io.ErrUnexpectedEOF) + } + return rejectExtraContent(src, member) +} + +// rejectExtraContent reads until a byte or the end of the source. A reader may legally return no +// bytes and no error from a single call, so one read cannot prove the source has ended. The probe +// never touches the archive writer, so extra content is not written before the failure. +func rejectExtraContent(src io.Reader, member *Member) error { + var extra [1]byte + for { + n, readErr := src.Read(extra[:]) + switch { + case n > 0: + return writeFailedError(member.Path, fmt.Errorf("%w: content is longer than the declared %d bytes", errUtils.ErrArchiveWriteFailed, member.Size)) + case errors.Is(readErr, io.EOF): + return nil + case readErr != nil: + return writeFailedError(member.Path, readErr) + } + } +} diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go new file mode 100644 index 00000000000..618517f99a6 --- /dev/null +++ b/pkg/archive/deterministic_test.go @@ -0,0 +1,229 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "errors" + "io" + "io/fs" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func fileMember(path, content string, mode fs.FileMode) Member { + return Member{ + Path: path, Kind: MemberFile, Mode: mode, Size: int64(len(content)), + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader(content)), nil }, + } +} + +func sampleMembers() []Member { + return []Member{ + fileMember("z/last.txt", "last", 0o644), + {Path: "z", Kind: MemberDirectory, Mode: 0o755}, + fileMember("bin/run.sh", "#!/bin/sh\n", 0o755), + {Path: "bin", Kind: MemberDirectory, Mode: 0o755}, + {Path: "link", Kind: MemberSymlink, Target: "z/last.txt"}, + fileMember("unicode-é.txt", "ü", 0o640), + fileMember("empty", "", 0o644), + } +} + +func TestWriteDeterministic_IdenticalBytesRegardlessOfInputOrder(t *testing.T) { + for _, format := range []string{FormatTar, FormatTGZ, FormatZip} { + t.Run(format, func(t *testing.T) { + forward := sampleMembers() + reversed := sampleMembers() + for i, j := 0, len(reversed)-1; i < j; i, j = i+1, j-1 { + reversed[i], reversed[j] = reversed[j], reversed[i] + } + var first, second, third bytes.Buffer + require.NoError(t, WriteDeterministic(&first, format, forward)) + require.NoError(t, WriteDeterministic(&second, format, reversed)) + require.NoError(t, WriteDeterministic(&third, format, forward)) + assert.Equal(t, first.Bytes(), second.Bytes(), "member order must not change the bytes") + assert.Equal(t, first.Bytes(), third.Bytes(), "repeated writes must be identical") + }) + } +} + +func TestWriteDeterministic_TarNormalizesMetadata(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatTar, sampleMembers())) + reader := tar.NewReader(&buf) + var names []string + modes := map[string]int64{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + names = append(names, header.Name) + modes[header.Name] = header.Mode + assert.True(t, header.ModTime.Equal(tarEpoch), "%s mtime", header.Name) + assert.Zero(t, header.Uid) + assert.Zero(t, header.Gid) + assert.Empty(t, header.Uname) + assert.Empty(t, header.Gname) + } + assert.Equal(t, []string{"bin/", "bin/run.sh", "empty", "link", "unicode-é.txt", "z/", "z/last.txt"}, names) + assert.Equal(t, int64(0o755), modes["bin/run.sh"], "executable bit must survive") + assert.Equal(t, int64(0o640), modes["unicode-é.txt"]) +} + +func TestWriteDeterministic_TGZHeaderCarriesNoNameOrTime(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatTGZ, sampleMembers())) + gz, err := gzip.NewReader(&buf) + require.NoError(t, err) + assert.Empty(t, gz.Name) + assert.True(t, gz.ModTime.IsZero() || gz.ModTime.Unix() == 0) +} + +func TestWriteDeterministic_ZipEntries(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatZip, sampleMembers())) + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + byName := map[string]*zip.File{} + for _, f := range reader.File { + byName[f.Name] = f + assert.True(t, f.Modified.UTC().Equal(zipEpoch), "%s modified", f.Name) + } + require.Contains(t, byName, "bin/run.sh") + assert.Equal(t, fs.FileMode(0o755), byName["bin/run.sh"].Mode().Perm()) + assert.True(t, byName["bin/"].Mode().IsDir()) + assert.NotZero(t, byName["link"].Mode()&fs.ModeSymlink) + rc, err := byName["link"].Open() + require.NoError(t, err) + target, err := io.ReadAll(rc) + require.NoError(t, err) + require.NoError(t, rc.Close()) + assert.Equal(t, "z/last.txt", string(target)) +} + +func TestWriteDeterministic_Rejections(t *testing.T) { + tests := []struct { + name string + format string + members []Member + want error + }{ + {"duplicate path", FormatTar, []Member{fileMember("a", "x", 0o644), fileMember("a", "y", 0o644)}, errUtils.ErrArchiveWriteFailed}, + {"absolute path", FormatTar, []Member{fileMember("/a", "x", 0o644)}, errUtils.ErrArchiveInvalidSubpath}, + {"empty path", FormatZip, []Member{fileMember("", "x", 0o644)}, errUtils.ErrArchiveInvalidSubpath}, + {"unsupported format", "tar.xz", nil, errUtils.ErrArchiveFormatNotImplemented}, + {"file without opener", FormatTar, []Member{{Path: "a", Kind: MemberFile, Size: 1}}, errUtils.ErrArchiveWriteFailed}, + {"unknown kind", FormatTar, []Member{{Path: "a", Kind: MemberKind(9)}}, errUtils.ErrArchiveWriteFailed}, + {"shorter than declared", FormatTar, []Member{{ + Path: "a", Kind: MemberFile, Size: 5, + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("ab")), nil }, + }}, errUtils.ErrArchiveWriteFailed}, + {"longer than declared", FormatZip, []Member{{ + Path: "a", Kind: MemberFile, Size: 1, + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("abc")), nil }, + }}, errUtils.ErrArchiveWriteFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := WriteDeterministic(io.Discard, tt.format, tt.members) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + }) + } +} + +func TestWriteDeterministic_ClearsPrivilegedBits(t *testing.T) { + var buf bytes.Buffer + member := fileMember("a", "x", 0o4755|fs.ModeSetuid) + require.NoError(t, WriteDeterministic(&buf, FormatTar, []Member{member})) + header, err := tar.NewReader(&buf).Next() + require.NoError(t, err) + assert.Equal(t, int64(0o755), header.Mode) +} + +// stutteringReader returns one empty, error-free read before it yields its content. +type stutteringReader struct { + stuttered bool + rest io.Reader +} + +func (r *stutteringReader) Read(p []byte) (int, error) { + if !r.stuttered { + r.stuttered = true + return 0, nil + } + return r.rest.Read(p) +} + +func TestRejectExtraContent_ReadsPastEmptyReads(t *testing.T) { + tests := []struct { + name string + rest string + wantErr bool + }{ + {"byte hidden behind an empty read", "x", true}, + {"end of source behind an empty read", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := rejectExtraContent(&stutteringReader{rest: strings.NewReader(tt.rest)}, &Member{Path: "a", Size: 0}) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + return + } + require.NoError(t, err) + }) + } +} + +type failingWriter struct{ after int } + +func (f *failingWriter) Write(p []byte) (int, error) { + if f.after <= 0 { + return 0, errors.New("disk full") + } + f.after -= len(p) + return len(p), nil +} + +func TestWriteDeterministic_PropagatesWriterFailures(t *testing.T) { + for _, format := range []string{FormatTar, FormatTGZ, FormatZip} { + t.Run(format, func(t *testing.T) { + err := WriteDeterministic(&failingWriter{}, format, sampleMembers()) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + }) + } +} + +var errTestSourceClose = errors.New("source close failed") + +type closeFailReader struct { + io.Reader +} + +func (closeFailReader) Close() error { return errTestSourceClose } + +func TestWriteDeterministic_SurfacesSourceCloseFailure(t *testing.T) { + for _, format := range []string{FormatTar, FormatZip} { + t.Run(format, func(t *testing.T) { + member := Member{ + Path: "a", Kind: MemberFile, Size: 2, + Open: func() (io.ReadCloser, error) { return closeFailReader{strings.NewReader("ab")}, nil }, + } + err := WriteDeterministic(io.Discard, format, []Member{member}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + assert.ErrorIs(t, err, errTestSourceClose) + }) + } +} diff --git a/pkg/artifact/artifacttest/clone.go b/pkg/artifact/artifacttest/clone.go new file mode 100644 index 00000000000..432a5b77cad --- /dev/null +++ b/pkg/artifact/artifacttest/clone.go @@ -0,0 +1,36 @@ +package artifacttest + +import ( + "bytes" + "maps" + "slices" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// The repository never shares maps or slices with its callers: whatever crosses the boundary in +// either direction is copied, so a caller editing its own value can never change a stored record, +// a capability guarantee, or a descriptor after the fact. + +// cloneDescriptor returns a descriptor that does not share maps, slices, or pointers with d. +func cloneDescriptor(in *ocispec.Descriptor) ocispec.Descriptor { + d := *in + d.URLs = slices.Clone(d.URLs) + d.Annotations = maps.Clone(d.Annotations) + d.Data = bytes.Clone(d.Data) + if d.Platform != nil { + platform := *d.Platform + platform.OSFeatures = slices.Clone(platform.OSFeatures) + d.Platform = &platform + } + return d +} + +// cloneEvidence returns evidence whose descriptor does not share storage with e. +func cloneEvidence(in *artifact.SignatureEvidence) artifact.SignatureEvidence { + e := *in + e.Descriptor = cloneDescriptor(&in.Descriptor) + return e +} diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go new file mode 100644 index 00000000000..d0be52cad36 --- /dev/null +++ b/pkg/artifact/artifacttest/conformance.go @@ -0,0 +1,488 @@ +package artifacttest + +import ( + "context" + "errors" + "io" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Names and modes the suite uses for its fixtures. +const ( + appName = "team/app" + firstVersion = "1.0.0" + secondVersion = "2.0.0" + filePerm = 0o644 + execPerm = 0o755 + pageSafety = 100 +) + +// Harness configures [RunRepositoryConformance] for one backend. +type Harness struct { + // New returns a fresh, empty repository. It is called once per subtest; the harness never + // shares state between subtests. Register cleanup with t.Cleanup. + New func(t *testing.T) artifact.Repository + + // Packaging is the payload packaging used for publications; empty uses the first packaging + // the repository advertises. + Packaging artifact.Packaging + + // Signer produces evidence the repository can store. Nil is allowed only for repositories that + // advertise no evidence formats; the suite then asserts Sign fails with a capability error. + Signer artifact.Signer + + // TrustedPolicy is satisfied by evidence the Signer produced. Required when Signer is set. + TrustedPolicy artifact.VerificationPolicy + + // UntrustedPolicy is a policy that evidence from the Signer must NOT satisfy. Required when + // Signer is set. + UntrustedPolicy artifact.VerificationPolicy +} + +// published is one artifact stored through the repository under test. +type published struct { + prepared *artifact.PreparedArtifact + result *artifact.PublishResult + ref artifact.Ref +} + +// RunRepositoryConformance runs the shared repository contract suite against h. Every backend +// calls it from its own tests; backend-specific differences are expressed through Capabilities, +// never by omitting an operation that claims success. Operations a repository does not support +// must return a capability error, which the suite asserts. +func RunRepositoryConformance(t *testing.T, h *Harness) { + defer perf.Track(nil, "artifacttest.RunRepositoryConformance")() + + t.Helper() + suite := &conformance{h: *h} + t.Run("CapabilitiesAreStatic", suite.capabilities) + t.Run("MultipleNamesAndVersions", suite.multipleNamesAndVersions) + t.Run("PublishReadRoundTrip", suite.roundTrip) + t.Run("MetadataAndDocumentsWithoutPayload", suite.metadataAndDocuments) + t.Run("DigestReferences", suite.digestReferences) + t.Run("Enumeration", suite.enumeration) + t.Run("ForceReplacementNeverServesNewBytesUnderOldDigest", suite.replacement) + t.Run("ConditionalPublish", suite.conditionalPublish) + t.Run("Delete", suite.deletion) + t.Run("Pin", suite.pin) + t.Run("SignVerify", suite.signVerify) + t.Run("Lock", suite.lock) + t.Run("ReadOnlyOperationsReturnCapabilityErrors", suite.capabilityErrors) +} + +type conformance struct { + h Harness +} + +func (c *conformance) packaging(repo artifact.Repository) artifact.Packaging { + if c.h.Packaging != "" { + return c.h.Packaging + } + caps := repo.Capabilities() + if len(caps.Packagings) > 0 { + return caps.Packagings[0] + } + return artifact.PackagingNone +} + +// spec names and fills one artifact the suite publishes. +type spec struct { + name string + version string + content string +} + +// publish prepares a file package and publishes it. The spec's content varies the payload. +func (c *conformance) publish(t *testing.T, repo artifact.Repository, sp spec, options artifact.PublishOptions) published { + t.Helper() + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", sp.content, filePerm) + fixtures.WriteFile(t, src, "bin/run.sh", fixtures.HelperScript, execPerm) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "readme for "+sp.name+"\n", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: sp.name, Version: sp.version, Packaging: c.packaging(repo), + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "conformance"}}, + OperationID: "op-" + sp.name + "-" + sp.version, PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + result, err := repo.Publish(context.Background(), prepared, options) + require.NoError(t, err) + return published{prepared: prepared, result: result, ref: artifact.Ref{Name: sp.name, Version: sp.version}} +} + +func (c *conformance) requireWritable(t *testing.T, repo artifact.Repository) { + t.Helper() + if !repo.Capabilities().Write { + t.Skipf("the repository is read-only; capability errors are asserted by the dedicated subtest") + } +} + +func (c *conformance) requireReadable(t *testing.T, repo artifact.Repository) { + t.Helper() + if !repo.Capabilities().Read { + _, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: firstVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + t.Skipf("successful reads require the read capability; unsupported Resolve was checked") + } +} + +func (c *conformance) capabilities(t *testing.T) { + repo := c.h.New(t) + first, second := repo.Capabilities(), repo.Capabilities() + assert.Equal(t, first, second, "Capabilities must be stable and need no network access") + assert.NotEmpty(t, first.LockScopes, "every repository can coordinate at least host-scope writes") +} + +func (c *conformance) multipleNamesAndVersions(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + items := []struct{ name, version string }{{appName, "1.0.0"}, {appName, "2.0.0"}, {"other/tool", "1.0.0"}} + var all []published + for _, item := range items { + all = append(all, c.publish(t, repo, spec{name: item.name, version: item.version, content: item.name + item.version}, artifact.PublishOptions{})) + } + seen := map[digest.Digest]bool{} + for _, item := range all { + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.NotNil(t, inspection.Record) + assert.Equal(t, item.ref.Name, inspection.Record.Name) + assert.Equal(t, item.ref.Version, inspection.Record.Version) + assert.False(t, seen[pinned.ResolvedDigest], "distinct artifacts resolve to distinct digests") + seen[pinned.ResolvedDigest] = true + } +} + +func (c *conformance) roundTrip(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "round trip"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NotNil(t, content.Record) + + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "pulled content matches the inventory, modes included") +} + +func (c *conformance) metadataAndDocuments(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "docs"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + data, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + require.NoError(t, stream.Reader.Close()) + assert.Equal(t, "readme for team/app\n", string(data)) + assert.Equal(t, digest.FromBytes(data), stream.Document.Digest) + + _, err = repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a missing optional document is a typed error") +} + +func (c *conformance) digestReferences(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "digest"}, artifact.PublishOptions{}) + byVersion, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + + byDigest, err := repo.Resolve(context.Background(), artifact.Ref{Name: item.ref.Name, Digest: byVersion.ResolvedDigest}) + require.NoError(t, err) + assert.Equal(t, byVersion.ResolvedDigest, byDigest.ResolvedDigest) + + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: "other/name", Digest: byVersion.ResolvedDigest}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a digest read checks the Record's embedded name") + + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: item.ref.Name, Version: "9.9.9"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func (c *conformance) enumeration(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + caps := repo.Capabilities() + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}} { + c.publish(t, repo, spec{name: item.name, version: item.version, content: item.name + item.version}, artifact.PublishOptions{}) + } + if !caps.EnumerateNamed { + _, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + } + if !caps.EnumerateCollection { + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + var names []string + cursor := "" + for pages := 0; pages < pageSafety; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{Cursor: cursor}) + require.NoError(t, err) + for index := range page.Entries { + names = append(names, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + require.NotEqual(t, cursor, page.NextCursor, "the cursor must advance") + cursor = page.NextCursor + } + assert.Equal(t, []string{"a/app:1", "a/app:2", "b/app:1"}, names, "lexical order, every entry exactly once") + + if caps.EnumerateNamed { + page, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app", PageSize: pageSafety}) + require.NoError(t, err) + require.Len(t, page.Entries, 2) + assert.Equal(t, "1", page.Entries[0].Version) + assert.Equal(t, "2", page.Entries[1].Version) + } +} + +func (c *conformance) replacement(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + first := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "first"}, artifact.PublishOptions{}) + oldPinned, err := repo.Resolve(context.Background(), first.ref) + require.NoError(t, err) + + second := c.publishForce(t, repo, first.ref, "second", &oldPinned) + newPinned, err := repo.Resolve(context.Background(), first.ref) + require.NoError(t, err) + assert.NotEqual(t, oldPinned.ResolvedDigest, newPinned.ResolvedDigest) + assert.Equal(t, artifact.OutcomeReplaced, second.result.Outcome) + + content, err := repo.Read(context.Background(), oldPinned) + if err != nil { + assert.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "retired bytes report not found") + return + } + defer content.Close() + require.NotNil(t, content.Record) + recordDigest, err := content.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, oldPinned.RecordDigest, recordDigest, "an old digest never serves the replacement's record") +} + +func (c *conformance) publishForce(t *testing.T, repo artifact.Repository, ref artifact.Ref, content string, prior *artifact.PinnedReference) published { + return c.publish(t, repo, spec{name: ref.Name, version: ref.Version, content: content}, artifact.PublishOptions{Force: true, ExpectedPrior: prior}) +} + +func (c *conformance) conditionalPublish(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + if !repo.Capabilities().ConditionalUpdate { + t.Skipf("the repository does not advertise conditional updates") + } + ref := artifact.Ref{Name: appName, Version: "1.0.0"} + c.publish(t, repo, spec{name: ref.Name, version: ref.Version, content: "base"}, artifact.PublishOptions{}) + + stale := artifact.PinnedReference{Ref: ref, ResolvedDigest: digest.FromString("stale")} + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "changed", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: ref.Name, Version: ref.Version, Packaging: c.packaging(repo), + OperationID: "op-stale", PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{Force: true, ExpectedPrior: &stale}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected-prior reference is a conflict, not a silent overwrite") + + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "publishing over an existing reference expecting none is a conflict") +} + +func (c *conformance) deletion(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + keep := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "keep"}, artifact.PublishOptions{}) + drop := c.publish(t, repo, spec{name: appName, version: secondVersion, content: "drop"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), drop.ref) + require.NoError(t, err) + + caps := repo.Capabilities() + if caps.Delete == artifact.DeleteNone || caps.Delete == "" { + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: caps.Delete}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: caps.Delete}) + require.NoError(t, err) + assert.NotEmpty(t, result.Deleted) + _, err = repo.Resolve(context.Background(), drop.ref) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Resolve(context.Background(), keep.ref) + require.NoError(t, err, "deletion is scoped to the selected artifact") +} + +func (c *conformance) pin(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "pin"}, artifact.PublishOptions{}) + resolved, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + pinned, err := repo.Pin(context.Background(), item.ref) + require.NoError(t, err) + assert.Equal(t, resolved.ResolvedDigest, pinned.ResolvedDigest) + assert.Equal(t, resolved.RecordDigest, pinned.RecordDigest) + + _, err = repo.Pin(context.Background(), artifact.Ref{Name: "missing/artifact", Version: "1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func (c *conformance) signVerify(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + c.requireReadable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "sign"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + + caps := repo.Capabilities() + if c.h.Signer == nil { + require.Empty(t, caps.EvidenceFormats, "a repository that stores evidence needs a Signer in the harness") + _, err := repo.Sign(context.Background(), subject, unsupportedSigner(t), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + require.NotEmpty(t, c.h.TrustedPolicy.TrustSets, "TrustedPolicy is required with a Signer") + + unsigned, err := repo.Verify(context.Background(), subject, c.h.TrustedPolicy) + require.Error(t, err, "no evidence never verifies") + if unsigned != nil { + assert.NotEqual(t, artifact.VerificationVerified, unsigned.Status) + } + + evidence, err := repo.Sign(context.Background(), subject, c.h.Signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) + require.NotEmpty(t, evidence) + assert.Equal(t, subject.Digest, evidence[0].Subject.Digest) + + verified, err := repo.Verify(context.Background(), subject, c.h.TrustedPolicy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + assert.Equal(t, subject.Digest, verified.Subject.Digest) + assert.NotEmpty(t, verified.EvidenceDigests) + + rejected, err := repo.Verify(context.Background(), subject, c.h.UntrustedPolicy) + require.Error(t, err, "evidence from an untrusted signer never satisfies the policy") + if rejected != nil { + assert.NotEqual(t, artifact.VerificationVerified, rejected.Status) + } + + other := subject + other.Digest = digest.FromString("another subject") + if mismatched, err := repo.Verify(context.Background(), other, c.h.TrustedPolicy); err == nil && mismatched != nil { + assert.NotEqual(t, artifact.VerificationVerified, mismatched.Status, "evidence is bound to its subject digest") + } +} + +// unsupportedSigner returns a generated signer for a format no repository under test advertises. +// Sign has no expectation, so the mock fails the test if a repository ever invokes it. +func unsupportedSigner(t *testing.T) artifact.Signer { + t.Helper() + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return([]artifact.EvidenceFormat{"unsupported"}).AnyTimes() + return signer +} + +// leakCheckTimeout bounds the second acquisition so a leaked claim fails the test instead of hanging it. +const leakCheckTimeout = 5 * time.Second + +func (c *conformance) lock(t *testing.T) { + repo := c.h.New(t) + ctx := context.Background() + request := artifact.LockRequest{Resource: "team/app:1.0.0", Owner: "owner-a", OperationID: "op-a", Required: artifact.LockScopeHost} + first, err := repo.Lock(ctx, request) + require.NoError(t, err) + assert.Equal(t, request.Resource, first.Resource()) + assert.Equal(t, "owner-a", first.Owner()) + assert.NotEmpty(t, first.Token()) + assert.True(t, first.Scope().Satisfies(artifact.LockScopeHost)) + + caps := repo.Capabilities() + if !caps.HasLockScope(artifact.LockScopeRepository) { + stronger := request + stronger.Required = artifact.LockScopeRepository + _, err := repo.Lock(ctx, stronger) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an insufficient guarantee is an error, never a silent downgrade") + } + + blocked, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + second := request + second.Owner = "owner-b" + handle, err := repo.Lock(blocked, second) + require.Error(t, err, "a competing owner cannot take a held lock") + assert.Nil(t, handle) + assert.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errUtils.ErrArtifactConflict), "waiting is bounded by the context or rejected as a conflict: %v", err) + + require.NoError(t, first.Release(ctx)) + if errors.Is(err, errUtils.ErrArtifactPublicationAmbiguous) { + // The deadline cut a claim create short and the backend reported that it may still commit. + // Such a claim is left for deliberate operator recovery, so leak-freedom is not promised. + return + } + retry, retryCancel := context.WithTimeout(ctx, leakCheckTimeout) + defer retryCancel() + again, err := repo.Lock(retry, second) + require.NoError(t, err, "a canceled acquisition leaked no ownership") + require.NoError(t, again.Release(ctx)) +} + +func (c *conformance) capabilityErrors(t *testing.T) { + repo := c.h.New(t) + caps := repo.Capabilities() + if caps.Write { + t.Skipf("the repository is writable; read-only behavior is not applicable") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: "x/y", Version: "1", Packaging: c.packaging(repo), OperationID: "op", PreparedAt: time.Now(), + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/artifacttest/conformance_test.go b/pkg/artifact/artifacttest/conformance_test.go new file mode 100644 index 00000000000..bb7c42bd083 --- /dev/null +++ b/pkg/artifact/artifacttest/conformance_test.go @@ -0,0 +1,116 @@ +package artifacttest_test + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +func TestInMemoryRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { return artifacttest.New(artifacttest.Options{}) }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} + +func TestInMemoryRepositoryConformance_RestrictedCapabilities(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Write = false + caps.EnumerateCollection = false + caps.Delete = artifact.DeleteNone + caps.EvidenceFormats = nil + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} + +func TestInMemoryRepositoryConformance_NoEnumerationNoDelete(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.EnumerateCollection = false + caps.EnumerateNamed = false + caps.Delete = artifact.DeleteNone + caps.EvidenceFormats = nil + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} + +func TestRepository_LockRejectsLeasesAndWrongTokens(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + ctx := context.Background() + _, err := repo.Lock(ctx, artifact.LockRequest{Resource: "r", Owner: "o", Lease: 1}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: "r", Owner: "o"}) + require.NoError(t, err) + assert.Equal(t, []string{"r"}, repo.HeldLocks()) + require.ErrorIs(t, handle.Renew(ctx, 1), errUtils.ErrArtifactUnsupportedCapability) + _, expires := handle.ExpiresAt() + assert.False(t, expires) + require.NoError(t, handle.Release(ctx)) + require.ErrorIs(t, handle.Release(ctx), errUtils.ErrArtifactConflict, "a stale handle cannot release a lock it no longer owns") + assert.Empty(t, repo.HeldLocks()) +} + +// deadlineRecorder notes every competing-owner Lock call made without a deadline. +type deadlineRecorder struct { + artifact.Repository + unbounded atomic.Int32 +} + +func (r *deadlineRecorder) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + if request.Owner == "owner-b" { + if _, ok := ctx.Deadline(); !ok { + r.unbounded.Add(1) + } + } + return r.Repository.Lock(ctx, request) +} + +func TestConformance_LockAcquisitionsAreBounded(t *testing.T) { + var recorders []*deadlineRecorder + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + New: func(*testing.T) artifact.Repository { + recorder := &deadlineRecorder{Repository: artifacttest.New(artifacttest.Options{})} + recorders = append(recorders, recorder) + return recorder + }, + }) + require.NotEmpty(t, recorders) + for _, recorder := range recorders { + assert.Zero(t, recorder.unbounded.Load(), "a competing acquisition must carry a deadline so a leaked claim cannot hang the suite") + } +} + +func TestInMemoryRepositoryConformance_WriteWithoutRead(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} diff --git a/pkg/artifact/artifacttest/doc.go b/pkg/artifact/artifacttest/doc.go new file mode 100644 index 00000000000..d70c45ad13c --- /dev/null +++ b/pkg/artifact/artifacttest/doc.go @@ -0,0 +1,11 @@ +// Package artifacttest provides test support for artifact repositories and the artifact service: +// a minimal in-memory [Repository] that implements the full [artifact.Repository] contract with +// failure injection, a fake signer and evidence format, and [RunRepositoryConformance], the shared +// conformance suite every backend (local directory, local archive, S3, OCI layout, OCI registry) +// runs against its own implementation. +// +// The in-memory repository is a harness, not a backend: it models the publication lifecycle +// (pending, completed, unavailable after interruption), conditional updates, digest-addressed +// reads that never serve replacement bytes, pagination, and ownership-checked locks, so service +// behavior such as reconciliation can be tested without a real store. +package artifacttest diff --git a/pkg/artifact/artifacttest/lock.go b/pkg/artifact/artifacttest/lock.go new file mode 100644 index 00000000000..7e37bfa3b6a --- /dev/null +++ b/pkg/artifact/artifacttest/lock.go @@ -0,0 +1,150 @@ +package artifacttest + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// lockState is one claimed resource. +type lockState struct { + owner string + operationID string + token string + released chan struct{} +} + +// handle is an ownership-checked lock handle. +type handle struct { + repo *Repository + resource string + owner string + token string + scope artifact.LockScope +} + +var _ artifact.LockHandle = (*handle)(nil) + +// Lock implements [artifact.LockProvider]. Acquisition waits for a competing owner and returns the +// context error, leaking nothing, when canceled. Locks never expire and are never stolen. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "artifacttest.Repository.Lock")() + + r.mu.Lock() + r.count("Lock") + r.mu.Unlock() + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "lock", "lease", "claims never expire, so a lease cannot be fenced") + } + required := request.Required + if required == "" { + required = artifact.LockScopeHost + } + if !r.caps.HasLockScope(required) { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "lock", "lock/"+string(required), "the repository cannot provide this guarantee") + } + for { + r.mu.Lock() + held, busy := r.locks[request.Resource] + if !busy { + token := uuid.NewString() + r.locks[request.Resource] = &lockState{owner: request.Owner, operationID: request.OperationID, token: token, released: make(chan struct{})} + r.mu.Unlock() + return &handle{repo: r, resource: request.Resource, owner: request.Owner, token: token, scope: r.bestScope()}, nil + } + wait := held.released + r.mu.Unlock() + select { + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for lock %s held by %s: %w", request.Resource, held.owner, ctx.Err()) + case <-wait: + } + } +} + +// bestScope is the strongest scope the repository advertises. +func (r *Repository) bestScope() artifact.LockScope { + if r.caps.HasLockScope(artifact.LockScopeRepository) { + return artifact.LockScopeRepository + } + return artifact.LockScopeHost +} + +// HeldLocks lists the currently claimed resources, sorted. +func (r *Repository) HeldLocks() []string { + defer perf.Track(nil, "artifacttest.Repository.HeldLocks")() + + r.mu.Lock() + defer r.mu.Unlock() + resources := make([]string, 0, len(r.locks)) + for resource := range r.locks { + resources = append(resources, resource) + } + sort.Strings(resources) + return resources +} + +// Resource implements [artifact.LockHandle]. +func (h *handle) Resource() string { + defer perf.Track(nil, "artifacttest.handle.Resource")() + + return h.resource +} + +// Owner implements [artifact.LockHandle]. +func (h *handle) Owner() string { + defer perf.Track(nil, "artifacttest.handle.Owner")() + + return h.owner +} + +// Scope implements [artifact.LockHandle]. +func (h *handle) Scope() artifact.LockScope { + defer perf.Track(nil, "artifacttest.handle.Scope")() + + return h.scope +} + +// Token implements [artifact.LockHandle]. +func (h *handle) Token() string { + defer perf.Track(nil, "artifacttest.handle.Token")() + + return h.token +} + +// ExpiresAt implements [artifact.LockHandle]; claims never expire. +func (h *handle) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "artifacttest.handle.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; there are no leases. +func (h *handle) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "artifacttest.handle.Renew")() + + return artifact.NewUnsupportedCapability(h.repo.opts.Alias, "lock", "lease", "claims never expire") +} + +// Release implements [artifact.LockHandle]. Releasing checks the ownership token. +func (h *handle) Release(context.Context) error { + defer perf.Track(nil, "artifacttest.handle.Release")() + + h.repo.mu.Lock() + defer h.repo.mu.Unlock() + h.repo.count("Release") + held, ok := h.repo.locks[h.resource] + if !ok || held.token != h.token { + return fmt.Errorf("%w: lock %s is not owned by token %s", errUtils.ErrArtifactConflict, h.resource, h.token) + } + delete(h.repo.locks, h.resource) + close(held.released) + return nil +} diff --git a/pkg/artifact/artifacttest/read.go b/pkg/artifact/artifacttest/read.go new file mode 100644 index 00000000000..24d5aa9409c --- /dev/null +++ b/pkg/artifact/artifacttest/read.go @@ -0,0 +1,277 @@ +package artifacttest + +import ( + "bytes" + "context" + "fmt" + "io" + "maps" + "slices" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Resolve implements [artifact.Repository]. A pending reference is unavailable to readers; a +// digest reference checks the Record's embedded name. +func (r *Repository) Resolve(_ context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "artifacttest.Repository.Resolve")() + + if err := r.requireRead("resolve"); err != nil { + return artifact.PinnedReference{}, err + } + pinned, err := r.resolve(ref) + r.mu.Lock() + hook := r.opts.OnResolved + r.mu.Unlock() + if err == nil && ref.Version != "" && hook != nil { + hook(ref) + } + return pinned, err +} + +func (r *Repository) resolve(ref artifact.Ref) (artifact.PinnedReference, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.count("Resolve") + if err := ref.Validate(); err != nil { + return artifact.PinnedReference{}, err + } + if ref.IsDigest() { + s, ok := r.records[ref.Digest] + if !ok || s.record.Name != ref.Name { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + return r.pinnedFor(ref, s), nil + } + key := refKey(ref.Name, ref.Version) + if p, isPending := r.pendings[key]; isPending { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s has a pending publication by operation %s", errUtils.ErrArtifactPublicationUnavailable, ref, p.operationID) + } + current, ok := r.refs[key] + if !ok { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + return r.pinnedFor(ref, r.records[current]), nil +} + +// Inspect implements [artifact.Repository]. It reads only the retained record and descriptors. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Inspect(_ context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "artifacttest.Repository.Inspect")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Inspect") + if err := r.requireRead("inspect"); err != nil { + return nil, err + } + s, ok := r.records[pinned.RecordDigest] + if !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + record := s.record.Clone() + pinnedOut := r.pinnedFor(pinned.Ref, s) + inspection := &artifact.Inspection{Pinned: pinnedOut, Record: &record, Representation: r.representation(s)} + held := r.evidence[s.recordDigest] + for index := range held { + inspection.Evidence = append(inspection.Evidence, cloneEvidence(&held[index].Evidence)) + } + return inspection, nil +} + +// OpenDocument implements [artifact.Repository] without touching the payload. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) OpenDocument(_ context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "artifacttest.Repository.OpenDocument")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("OpenDocument") + if err := r.requireRead("open-document"); err != nil { + return nil, err + } + s, ok := r.records[pinned.RecordDigest] + if !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + for index := range s.record.Documents { + if s.record.Documents[index].Role != role { + continue + } + for blobIndex := range s.blobs { + held := &s.blobs[blobIndex] + if held.role == artifact.BlobDocument && held.documentRole == role { + return &artifact.DocumentStream{Document: s.record.Documents[index], Reader: newVerifyingReader(held.data, held.descriptor.Digest)}, nil + } + } + } + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDocumentMissing, role) +} + +// Read implements [artifact.Repository]. It serves exactly the pinned digest; once the payload of +// a replaced or deleted record is gone it reports not found rather than the current bytes. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Read(_ context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "artifacttest.Repository.Read")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Read") + if err := r.requireRead("read"); err != nil { + return nil, err + } + s, ok := r.records[pinned.RecordDigest] + if !ok || !s.payloadAvailable { + return nil, fmt.Errorf("%w: the payload of %s is no longer available", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + record := s.record.Clone() + content := &artifact.Content{Record: &record, Representation: r.representation(s)} + for index := range s.blobs { + held := &s.blobs[index] + content.Streams = append(content.Streams, artifact.ContentStream{ + Descriptor: cloneDescriptor(&held.descriptor), Role: held.role, Path: held.path, Reader: newVerifyingReader(held.data, held.descriptor.Digest), + }) + } + return content, nil +} + +// Pin implements [artifact.Repository]. It acquires no lock and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "artifacttest.Repository.Pin")() + + pinned, err := r.Resolve(ctx, ref) + if err != nil { + return pinned, err + } + r.mu.Lock() + defer r.mu.Unlock() + r.count("Pin") + held := r.evidence[pinned.RecordDigest] + for index := range held { + pinned.Evidence = append(pinned.Evidence, cloneDescriptor(&held[index].Evidence.Descriptor)) + } + return pinned, nil +} + +// List implements [artifact.Repository], paginating over completed references only. +func (r *Repository) List(_ context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "artifacttest.Repository.List")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("List") + if query.Name != "" && !r.caps.EnumerateNamed || query.Name == "" && !r.caps.EnumerateCollection { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "list", "enumerate", "enumeration is not supported") + } + var entries []artifact.ListEntry + for key, recordDigest := range r.refs { + if _, isPending := r.pendings[key]; isPending { + continue + } + s := r.records[recordDigest] + if query.Name != "" && s.record.Name != query.Name { + continue + } + if !labelsMatch(s.record.Metadata.Labels, query.Labels) { + continue + } + entries = append(entries, r.listEntry(s)) + } + sort.Slice(entries, func(i, j int) bool { return listKey(&entries[i]) < listKey(&entries[j]) }) + return r.page(entries, query), nil +} + +func (r *Repository) listEntry(s *stored) artifact.ListEntry { + ref := artifact.Ref{Name: s.record.Name, Version: s.record.Version} + return artifact.ListEntry{ + Pinned: r.pinnedFor(ref, s), + Name: s.record.Name, + Version: s.record.Version, + Kind: s.record.Kind, + Description: s.record.Metadata.Description, + Labels: maps.Clone(s.record.Metadata.Labels), + Documents: slices.Clone(s.record.Documents), + } +} + +func listKey(entry *artifact.ListEntry) string { + return entry.Name + "\x00" + entry.Version +} + +func (r *Repository) page(entries []artifact.ListEntry, query artifact.ListQuery) *artifact.ListPage { + size := query.PageSize + if size <= 0 { + size = r.opts.PageSize + } + start := 0 + if query.Cursor != "" { + start = sort.Search(len(entries), func(i int) bool { return listKey(&entries[i]) > query.Cursor }) + } + end := len(entries) + if size < len(entries)-start { + end = start + size + } + page := &artifact.ListPage{} + if end < len(entries) { + page.NextCursor = listKey(&entries[end-1]) + } + page.Entries = append(page.Entries, entries[start:end]...) + return page +} + +func labelsMatch(have map[string]string, want []artifact.Label) bool { + for _, label := range want { + if value, ok := have[label.Key]; !ok || value != label.Value { + return false + } + } + return true +} + +// verifyingReader verifies the digest of the bytes it yields at EOF. +type verifyingReader struct { + reader *bytes.Reader + data []byte + expected digest.Digest + done bool +} + +func newVerifyingReader(data []byte, expected digest.Digest) io.ReadCloser { + return &verifyingReader{reader: bytes.NewReader(data), data: data, expected: expected} +} + +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "artifacttest.verifyingReader.Read")() + + n, err := v.reader.Read(p) + if err == io.EOF && !v.done { + v.done = true + if digest.FromBytes(v.data) != v.expected { + return n, fmt.Errorf("%w: blob does not match descriptor %s", errUtils.ErrArtifactIntegrityFailed, v.expected) + } + } + return n, err +} + +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "artifacttest.verifyingReader.Close")() + + return nil +} + +// requireRead rejects a read operation when the repository does not advertise the read capability, even +// if writes are enabled: a write-only repository must not serve reads. +func (r *Repository) requireRead(operation string) error { + if !r.caps.Read { + return artifact.NewUnsupportedCapability(r.opts.Alias, operation, "read", "the repository is not readable") + } + return nil +} diff --git a/pkg/artifact/artifacttest/repository.go b/pkg/artifact/artifacttest/repository.go new file mode 100644 index 00000000000..c6270ca22f2 --- /dev/null +++ b/pkg/artifact/artifacttest/repository.go @@ -0,0 +1,303 @@ +package artifacttest + +import ( + "fmt" + "strings" + "sync" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultPageSize is the fake's list page size; it is small so tests exercise pagination. +const DefaultPageSize = 2 + +// FailPoint names an injectable interruption in the publication lifecycle. +type FailPoint string + +// Injectable interruptions, in lifecycle order. +const ( + // FailBeforePayload fails before anything changed. + FailBeforePayload FailPoint = "before-payload" + + // FailDuringPayload fails while payload blobs are being read; the publication stays pending. + FailDuringPayload FailPoint = "during-payload" + + // FailAfterMetadata fails after the immutable record was written but before the reference moved. + FailAfterMetadata FailPoint = "after-metadata" + + // FailAfterCommit fails after the reference was published: the server-side commit succeeded + // but the response was lost. + FailAfterCommit FailPoint = "after-commit" +) + +// Options configure a [Repository]. +type Options struct { + // Alias names the repository in locators and errors; default "memory". + Alias string + + // Capabilities overrides the default full-featured capabilities. + Capabilities *artifact.Capabilities + + // PageSize is the list page size; default [DefaultPageSize]. + PageSize int + + // OnResolved runs after Resolve has pinned a version reference and released its internal + // lock, so a test can move the reference to simulate a tag changing mid-operation. + OnResolved func(ref artifact.Ref) +} + +// blob is one stored payload or document blob. +type blob struct { + descriptor ocispec.Descriptor + role artifact.BlobRole + documentRole artifact.DocumentRole + path string + data []byte +} + +// stored is one immutable published record with its blobs. +type stored struct { + record artifact.Record + canonical []byte + recordDigest digest.Digest + blobs []blob + packaging artifact.Packaging + payloadAvailable bool +} + +// pending is an in-flight or interrupted publication. +type pending struct { + operationID string + intent digest.Digest +} + +// failure is an armed injected failure. +type failure struct { + err error + times int +} + +// Repository is an in-memory [artifact.Repository]. It is safe for concurrent use. +type Repository struct { + mu sync.Mutex + opts Options + caps artifact.Capabilities + refs map[string]digest.Digest + records map[digest.Digest]*stored + pendings map[string]*pending + evidence map[digest.Digest][]artifact.EvidenceBlob + locks map[string]*lockState + failures map[FailPoint]*failure + calls map[string]int +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// New returns an empty in-memory repository. +func New(opts Options) *Repository { + defer perf.Track(nil, "artifacttest.New")() + + if opts.Alias == "" { + opts.Alias = "memory" + } + if opts.PageSize <= 0 { + opts.PageSize = DefaultPageSize + } + caps := DefaultCapabilities() + if opts.Capabilities != nil { + caps = opts.Capabilities.Clone() + } + // The repository keeps its own copy of the capabilities, never the caller's pointer. + opts.Capabilities = nil + return &Repository{ + opts: opts, + caps: caps, + refs: map[string]digest.Digest{}, + records: map[digest.Digest]*stored{}, + pendings: map[string]*pending{}, + evidence: map[digest.Digest][]artifact.EvidenceBlob{}, + locks: map[string]*lockState{}, + failures: map[FailPoint]*failure{}, + calls: map[string]int{}, + } +} + +// DefaultCapabilities is the full-featured capability set of the in-memory repository. +func DefaultCapabilities() artifact.Capabilities { + defer perf.Track(nil, "artifacttest.DefaultCapabilities")() + + return artifact.Capabilities{ + Read: true, + Write: true, + EnumerateNamed: true, + EnumerateCollection: true, + Delete: artifact.DeleteObject, + ConditionalUpdate: true, + Packagings: []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip}, + PreservesModes: true, + SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}, + EvidenceFormats: []artifact.EvidenceFormat{FakeEvidenceFormat}, + LockScopes: []artifact.LockScope{artifact.LockScopeHost}, + } +} + +// Capabilities implements [artifact.Repository]. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "artifacttest.Repository.Capabilities")() + + return r.caps.Clone() +} + +// InjectFailure arms err to be returned from Publish at point, times times (0 means once). +func (r *Repository) InjectFailure(point FailPoint, err error, times int) { + defer perf.Track(nil, "artifacttest.Repository.InjectFailure")() + + r.mu.Lock() + defer r.mu.Unlock() + if times <= 0 { + times = 1 + } + r.failures[point] = &failure{err: err, times: times} +} + +// Calls returns how many times the named repository method was called. +func (r *Repository) Calls(method string) int { + defer perf.Track(nil, "artifacttest.Repository.Calls")() + + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[method] +} + +// TotalCalls returns the total number of repository method calls (including Capabilities-free +// methods only), which dry-run tests assert is zero. +func (r *Repository) TotalCalls() int { + defer perf.Track(nil, "artifacttest.Repository.TotalCalls")() + + r.mu.Lock() + defer r.mu.Unlock() + total := 0 + for _, n := range r.calls { + total += n + } + return total +} + +// MoveRef repoints the version reference at another stored record digest, simulating a mutable +// tag being moved. It reports false when the digest is unknown. +func (r *Repository) MoveRef(ref artifact.Ref, to digest.Digest) bool { + defer perf.Track(nil, "artifacttest.Repository.MoveRef")() + + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.records[to]; !ok { + return false + } + r.refs[refKey(ref.Name, ref.Version)] = to + return true +} + +// SetOnResolved installs (or clears) the hook that runs after a version reference was pinned. +func (r *Repository) SetOnResolved(hook func(ref artifact.Ref)) { + defer perf.Track(nil, "artifacttest.Repository.SetOnResolved")() + + r.mu.Lock() + defer r.mu.Unlock() + r.opts.OnResolved = hook +} + +// IsPending reports whether the reference has an in-flight or interrupted publication. +func (r *Repository) IsPending(ref artifact.Ref) bool { + defer perf.Track(nil, "artifacttest.Repository.IsPending")() + + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.pendings[refKey(ref.Name, ref.Version)] + return ok +} + +// RecordDigestOf returns the record digest a version reference currently points at. +func (r *Repository) RecordDigestOf(ref artifact.Ref) (digest.Digest, bool) { + defer perf.Track(nil, "artifacttest.Repository.RecordDigestOf")() + + r.mu.Lock() + defer r.mu.Unlock() + d, ok := r.refs[refKey(ref.Name, ref.Version)] + return d, ok +} + +func refKey(name, version string) string { + return name + ":" + version +} + +// refFromKey is the inverse of [refKey]. Names never contain a colon, so the last one separates +// the version. +func refFromKey(key string) artifact.Ref { + cut := strings.LastIndexByte(key, ':') + if cut < 0 { + return artifact.Ref{Name: key} + } + return artifact.Ref{Name: key[:cut], Version: key[cut+1:]} +} + +// locatorForKey is the location of the reference stored under key. +func (r *Repository) locatorForKey(key string) string { + ref := refFromKey(key) + return r.locator(ref.Name, ref.Version) +} + +// count records a call; callers hold r.mu. +func (r *Repository) count(method string) { + r.calls[method]++ +} + +// trip consumes an armed failure at point; callers hold r.mu. +func (r *Repository) trip(point FailPoint) error { + armed, ok := r.failures[point] + if !ok || armed.times <= 0 { + return nil + } + armed.times-- + return armed.err +} + +func (r *Repository) locator(name, version string) string { + return fmt.Sprintf("mem://%s/%s/%s", r.opts.Alias, name, version) +} + +func (r *Repository) pinnedFor(ref artifact.Ref, s *stored) artifact.PinnedReference { + return artifact.PinnedReference{ + Ref: ref, + Repository: "mem://" + r.opts.Alias, + ResolvedDigest: s.recordDigest, + RecordDigest: s.recordDigest, + ContentDigest: s.record.ContentDigest, + } +} + +func (r *Repository) representation(s *stored) artifact.Representation { + rep := artifact.Representation{ + Packaging: s.packaging, + Locator: r.locator(s.record.Name, s.record.Version), + RecordDigest: s.recordDigest, + } + payloadDigests := "" + for index := range s.blobs { + if s.blobs[index].role == artifact.BlobPayload { + rep.Payload = append(rep.Payload, cloneDescriptor(&s.blobs[index].descriptor)) + payloadDigests += s.blobs[index].descriptor.Digest.String() + "\n" + } else { + rep.Documents = append(rep.Documents, cloneDescriptor(&s.blobs[index].descriptor)) + } + } + rep.RepresentationDigest = digest.FromString(payloadDigests) + return rep +} diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go new file mode 100644 index 00000000000..4ef08502940 --- /dev/null +++ b/pkg/artifact/artifacttest/repository_test.go @@ -0,0 +1,429 @@ +package artifacttest_test + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +const ( + testName = "team/app" + testPerm = 0o644 + aliasName = "team/alias" +) + +// publishOne publishes a single-file artifact and returns its pinned reference. +func publishOne(t *testing.T, repo *artifacttest.Repository) artifact.PinnedReference { + t.Helper() + name, version := testName, "1.0.0" + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "content of "+name+"@"+version, testPerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: name, Version: version, Packaging: artifact.PackagingTar, + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "isolation"}}, + OperationID: "op-" + name + "-" + version, PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: name, Version: version}) + require.NoError(t, err) + return pinned +} + +// TestRepository_CapabilitySlicesAreIsolated proves the repository's guarantees are static: the +// slices a caller passes in or receives back never share storage with the repository. +func TestRepository_CapabilitySlicesAreIsolated(t *testing.T) { + tests := []struct { + name string + mutate func(c *artifact.Capabilities) + read func(c artifact.Capabilities) any + want any + }{ + { + name: "packagings", + mutate: func(c *artifact.Capabilities) { c.Packagings[0] = artifact.PackagingZip }, + read: func(c artifact.Capabilities) any { return c.Packagings }, + want: artifacttest.DefaultCapabilities().Packagings, + }, + { + name: "subject kinds", + mutate: func(c *artifact.Capabilities) { c.SubjectKinds[0] = "other" }, + read: func(c artifact.Capabilities) any { return c.SubjectKinds }, + want: artifacttest.DefaultCapabilities().SubjectKinds, + }, + { + name: "evidence formats", + mutate: func(c *artifact.Capabilities) { c.EvidenceFormats[0] = "other" }, + read: func(c artifact.Capabilities) any { return c.EvidenceFormats }, + want: artifacttest.DefaultCapabilities().EvidenceFormats, + }, + { + name: "lock scopes", + mutate: func(c *artifact.Capabilities) { c.LockScopes[0] = artifact.LockScopeRepository }, + read: func(c artifact.Capabilities) any { return c.LockScopes }, + want: artifacttest.DefaultCapabilities().LockScopes, + }, + } + for _, tt := range tests { + t.Run(tt.name+"/returned value", func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + returned := repo.Capabilities() + tt.mutate(&returned) + assert.Equal(t, tt.want, tt.read(repo.Capabilities()), "mutating the returned slices must not change the repository") + }) + t.Run(tt.name+"/accepted option", func(t *testing.T) { + supplied := artifacttest.DefaultCapabilities() + repo := artifacttest.New(artifacttest.Options{Capabilities: &supplied}) + tt.mutate(&supplied) + assert.Equal(t, tt.want, tt.read(repo.Capabilities()), "mutating the supplied option must not change the repository") + }) + } +} + +func TestRepository_MutatedCapabilitiesDoNotChangeLockGuarantees(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + returned := repo.Capabilities() + returned.LockScopes[0] = artifact.LockScopeRepository + + _, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "r", Owner: "o", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a host-only repository never grants repository scope because a caller edited its copy") +} + +func TestRepository_DeleteScope(t *testing.T) { + aliasRef := artifact.Ref{Name: aliasName, Version: "latest"} + primary := artifact.Ref{Name: testName, Version: "1.0.0"} + + tests := []struct { + name string + caps artifact.DeleteSemantics + scope artifact.DeleteSemantics + pin artifact.Ref + + wantErr error + wantDeleted []string + wantAffected []artifact.Ref + wantRecordAlive bool + wantPrimary bool + wantAlias bool + wantEvidence bool + }{ + { + name: "object scope removes every alias and reports each location once", caps: artifact.DeleteObject, scope: artifact.DeleteObject, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{aliasRef}, + }, + { + name: "object scope through the alias reports the other reference as affected", caps: artifact.DeleteObject, scope: artifact.DeleteObject, pin: aliasRef, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{primary}, + }, + { + name: "manifest scope affects every reference", caps: artifact.DeleteManifest, scope: artifact.DeleteManifest, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{aliasRef}, + }, + { + name: "reference scope removes only the pinned reference and keeps shared data", caps: artifact.DeleteReference, scope: artifact.DeleteReference, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0"}, wantRecordAlive: true, wantAlias: true, wantEvidence: true, + }, + { + name: "reference scope through the alias leaves the primary reference", caps: artifact.DeleteReference, scope: artifact.DeleteReference, pin: aliasRef, + wantDeleted: []string{"mem://memory/team/alias/latest"}, wantRecordAlive: true, wantPrimary: true, wantEvidence: true, + }, + {name: "an empty scope is rejected", caps: artifact.DeleteObject, scope: "", pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + {name: "an unknown scope is rejected", caps: artifact.DeleteObject, scope: "everything", pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + {name: "a scope the repository does not advertise is rejected", caps: artifact.DeleteReference, scope: artifact.DeleteObject, pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = tt.caps + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + require.True(t, repo.MoveRef(aliasRef, pinned.RecordDigest)) + signEvidence(t, repo, &pinned) + + target, err := repo.Resolve(context.Background(), tt.pin) + require.NoError(t, err) + result, err := repo.Delete(context.Background(), target, artifact.DeleteOptions{ExpectedCurrent: &target, Scope: tt.scope}) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + assert.ElementsMatch(t, tt.wantDeleted, result.Deleted, "each removed reference is reported once, by its own location") + assert.Len(t, result.Deleted, len(tt.wantDeleted)) + assert.ElementsMatch(t, tt.wantAffected, result.Affected) + assert.Equal(t, pinned.RecordDigest, result.Digest) + } + + _, primaryAlive := repo.RecordDigestOf(primary) + _, aliasAlive := repo.RecordDigestOf(aliasRef) + assert.Equal(t, tt.wantPrimary, primaryAlive, "primary reference") + assert.Equal(t, tt.wantAlias, aliasAlive, "alias reference") + assert.Equal(t, tt.wantRecordAlive, recordAlive(repo, pinned.RecordDigest), "shared record") + assert.Equal(t, tt.wantEvidence, evidenceAlive(t, repo, &pinned), "shared evidence") + }) + } +} + +func TestRepository_DeleteReferenceScopeRemovesRecordWithItsLastReference(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteReference + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + signEvidence(t, repo, &pinned) + + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteReference}) + require.NoError(t, err) + assert.Equal(t, []string{"mem://memory/team/app/1.0.0"}, result.Deleted) + assert.Empty(t, result.Affected) + assert.False(t, recordAlive(repo, pinned.RecordDigest), "an unreferenced record is no longer retained") + assert.False(t, evidenceAlive(t, repo, &pinned)) +} + +func TestRepository_DeleteReferenceScopeNeedsAMutableReference(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteReference + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + byDigest := pinned + byDigest.Ref = artifact.Ref{Name: testName, Digest: pinned.RecordDigest} + + _, err := repo.Delete(context.Background(), byDigest, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + _, alive := repo.RecordDigestOf(artifact.Ref{Name: testName, Version: "1.0.0"}) + assert.True(t, alive, "a rejected delete changes nothing") +} + +func recordAlive(repo *artifacttest.Repository, recordDigest digest.Digest) bool { + _, err := repo.Inspect(context.Background(), artifact.PinnedReference{RecordDigest: recordDigest, ResolvedDigest: recordDigest}) + return err == nil +} + +func evidenceAlive(t *testing.T, repo *artifacttest.Repository, pinned *artifact.PinnedReference) bool { + t.Helper() + blobs, err := repo.ReadEvidence(context.Background(), pinned) + require.NoError(t, err) + return len(blobs) > 0 +} + +func signEvidence(t *testing.T, repo *artifacttest.Repository, pinned *artifact.PinnedReference) { + t.Helper() + inspection, err := repo.Inspect(context.Background(), *pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) +} + +// TestRepository_StoredStateIsIsolatedFromCallers proves records, labels, and descriptors handed +// to or returned from the repository never alias its stored state, in either direction. +func TestRepository_StoredStateIsIsolatedFromCallers(t *testing.T) { + ctx := context.Background() + ref := artifact.Ref{Name: testName, Version: "1.0.0"} + tests := []struct { + name string + tamper func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) + }{ + {"inspect record labels", func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) { + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + inspection.Record.Metadata.Labels["suite"] = "tampered" + inspection.Record.Inventory[0].Path = "tampered" + inspection.Record.Documents = append(inspection.Record.Documents[:0], artifact.Document{Role: "tampered"}) + }}, + {"read record labels", func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) { + content, err := repo.Read(ctx, pinned) + require.NoError(t, err) + content.Record.Metadata.Labels["suite"] = "tampered" + content.Representation.Payload[0].Digest = digest.FromString("tampered") + }}, + {"list entry labels", func(t *testing.T, repo *artifacttest.Repository, _ artifact.PinnedReference) { + page, err := repo.List(ctx, artifact.ListQuery{}) + require.NoError(t, err) + require.NotEmpty(t, page.Entries) + page.Entries[0].Labels["suite"] = "tampered" + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, repo) + before, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + wantPath := before.Record.Inventory[0].Path + wantDocs := len(before.Record.Documents) + wantPayload := slices.Clone(before.Representation.Payload) + + tt.tamper(t, repo, pinned) + + after, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, map[string]string{"suite": "isolation"}, after.Record.Metadata.Labels) + assert.Equal(t, wantPath, after.Record.Inventory[0].Path) + assert.Len(t, after.Record.Documents, wantDocs) + assert.Equal(t, wantPayload, after.Representation.Payload) + _, err = repo.Resolve(ctx, ref) + require.NoError(t, err) + }) + } +} + +func TestRepository_PublishCopiesThePreparedRecord(t *testing.T) { + ctx := context.Background() + repo := artifacttest.New(artifacttest.Options{}) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload", testPerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(ctx, artifact.PrepareRequest{ + Source: src, Name: testName, Version: "1.0.0", Packaging: artifact.PackagingTar, + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "isolation"}}, + OperationID: "op", PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + _, err = repo.Publish(ctx, prepared, artifact.PublishOptions{}) + require.NoError(t, err) + + prepared.Record.Metadata.Labels["suite"] = "tampered" + prepared.Record.Inventory[0].Path = "tampered" + + pinned, err := repo.Resolve(ctx, artifact.Ref{Name: testName, Version: "1.0.0"}) + require.NoError(t, err) + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, map[string]string{"suite": "isolation"}, inspection.Record.Metadata.Labels, "the stored record is independent of the prepared one") + assert.Equal(t, "data.txt", inspection.Record.Inventory[0].Path) +} + +// TestRepository_SignDoesNotStoreEvidenceForARecordDeletedWhileSigning proves the subject is checked +// again once the signer returned: Sign releases the lock while the signer runs, so a concurrent +// Delete can remove the record in between. +func TestRepository_SignDoesNotStoreEvidenceForARecordDeletedWhileSigning(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteObject + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + + fake := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return(fake.Formats()).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), subject).DoAndReturn(func(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + // The record disappears while the signer is still working. + _, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteObject}) + require.NoError(t, err) + return fake.Sign(ctx, subject) + }) + + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Empty(t, evidence) + assert.False(t, recordAlive(repo, pinned.RecordDigest)) + assert.False(t, evidenceAlive(t, repo, &pinned), "no evidence may be left for a deleted record") +} + +// A repository that does not advertise the read capability rejects every read entrypoint, even when +// writes are enabled. +func TestRepository_ReadsAreRejectedWithoutTheReadCapability(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + ctx := context.Background() + + checks := map[string]func() error{ + "Resolve": func() error { _, err := repo.Resolve(ctx, artifact.Ref{Name: testName, Version: "1.0.0"}); return err }, + "Inspect": func() error { _, err := repo.Inspect(ctx, artifact.PinnedReference{}); return err }, + "OpenDocument": func() error { + _, err := repo.OpenDocument(ctx, artifact.PinnedReference{}, artifact.DocumentRole("readme")) + return err + }, + "Read": func() error { _, err := repo.Read(ctx, artifact.PinnedReference{}); return err }, + } + for name, call := range checks { + t.Run(name, func(t *testing.T) { + var capability *artifact.CapabilityError + require.ErrorAs(t, call(), &capability) + assert.Equal(t, "read", capability.Capability) + }) + } +} + +// A failure from the signer carries the signing sentinel and keeps its cause. +func TestRepository_SignClassifiesSignerFailures(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, repo) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + cause := errors.New("the signing service refused the request") + fake := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return(fake.Formats()).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), subject).Return(nil, cause) + + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + + require.ErrorIs(t, err, errUtils.ErrArtifactSigningFailed) + assert.ErrorIs(t, err, cause) +} + +func TestRepository_VerifyRejectsEvidenceIntegrityMismatch(t *testing.T) { + source := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, source) + signEvidence(t, source, &pinned) + valid, err := source.ReadEvidence(t.Context(), &pinned) + require.NoError(t, err) + require.Len(t, valid, 1) + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + for _, mismatch := range []string{"size", "digest"} { + t.Run(mismatch, func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + target := publishOne(t, repo) + corrupt := valid[0] + if mismatch == "size" { + corrupt.Evidence.Descriptor.Size++ + } else { + corrupt.Evidence.Descriptor.Digest = digest.FromString("different bytes") + } + _, err := repo.WriteEvidence(t.Context(), &target, []artifact.EvidenceBlob{corrupt}, nil) + require.NoError(t, err) + result, err := repo.Verify(t.Context(), valid[0].Evidence.Subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Empty(t, result.Claims) + assert.Empty(t, result.EvidenceDigests) + require.NotEmpty(t, result.Diagnostics) + assert.Equal(t, "evidence-integrity", result.Diagnostics[0].Code) + _, err = repo.WriteEvidence(t.Context(), &target, valid, nil) + require.NoError(t, err) + result, err = repo.Verify(t.Context(), valid[0].Evidence.Subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Len(t, result.Claims, 1) + assert.Equal(t, []digest.Digest{valid[0].Evidence.Descriptor.Digest}, result.EvidenceDigests) + }) + } +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go new file mode 100644 index 00000000000..fa946717fb8 --- /dev/null +++ b/pkg/artifact/artifacttest/sign.go @@ -0,0 +1,238 @@ +package artifacttest + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// FakeEvidenceFormat is the evidence format of [FakeSigner]; the in-memory repository stores and +// verifies it. It stands in for a real format only inside tests. +const FakeEvidenceFormat = artifact.EvidenceSigstoreBundle + +// fakeEvidenceMediaType is the media type of fake evidence. +const fakeEvidenceMediaType = "application/vnd.atmos.artifacttest.evidence.v1+json" + +// fakeEvidence is the payload [FakeSigner] produces. +type fakeEvidence struct { + Subject digest.Digest `json:"subject"` + Identity string `json:"identity"` + Issuer string `json:"issuer"` +} + +// FakeSigner signs with a fixed identity and issuer. Evidence is bound to the subject digest. +type FakeSigner struct { + // Identity is the signer identity claim. + Identity string + + // Issuer is the issuer claim. + Issuer string +} + +var _ artifact.Signer = (*FakeSigner)(nil) + +// Formats implements [artifact.Signer]. +func (f *FakeSigner) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "artifacttest.FakeSigner.Formats")() + + return []artifact.EvidenceFormat{FakeEvidenceFormat} +} + +// Sign implements [artifact.Signer]. +func (f *FakeSigner) Sign(_ context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + defer perf.Track(nil, "artifacttest.FakeSigner.Sign")() + + data, err := json.Marshal(fakeEvidence{Subject: subject.Digest, Identity: f.Identity, Issuer: f.Issuer}) + if err != nil { + return nil, err + } + return &artifact.EvidencePayload{Format: FakeEvidenceFormat, MediaType: fakeEvidenceMediaType, Data: data}, nil +} + +// TrustedPolicy returns a policy that only [FakeSigner] with the same identity and issuer satisfies. +func (f *FakeSigner) TrustedPolicy() artifact.VerificationPolicy { + defer perf.Track(nil, "artifacttest.FakeSigner.TrustedPolicy")() + + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: f.Identity, Issuer: f.Issuer}}}}, + } +} + +// UntrustedPolicy returns a policy that no evidence from [FakeSigner] satisfies. +func (f *FakeSigner) UntrustedPolicy() artifact.VerificationPolicy { + defer perf.Track(nil, "artifacttest.FakeSigner.UntrustedPolicy")() + + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: f.Identity + "-other", Issuer: f.Issuer}}}}, + } +} + +// Sign implements [artifact.Repository]: the evidence is stored keyed by the subject digest. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "artifacttest.Repository.Sign")() + + r.mu.Lock() + r.count("Sign") + if err := r.checkSubject(subject); err != nil { + r.mu.Unlock() + return nil, err + } + r.mu.Unlock() + if err := r.caps.CheckSignerFormats(r.opts.Alias, signer); err != nil { + return nil, err + } + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactSigningFailed, err) + } + if !r.caps.HasEvidenceFormat(payload.Format) { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "evidence/"+string(payload.Format), "the repository cannot store this evidence format") + } + evidence := artifact.SignatureEvidence{ + Subject: subject, + Format: payload.Format, + Descriptor: ocispec.Descriptor{ + MediaType: payload.MediaType, Digest: digest.FromBytes(payload.Data), Size: int64(len(payload.Data)), + }, + Locator: fmt.Sprintf("mem://%s/signatures/%s/%s", r.opts.Alias, subject.Digest.Encoded(), digest.FromBytes(payload.Data).Encoded()), + } + r.mu.Lock() + defer r.mu.Unlock() + // The lock was released while the signer ran, so a Delete may have removed the subject since. + if err := r.checkSubject(subject); err != nil { + return nil, err + } + r.evidence[subject.Digest] = append(r.evidence[subject.Digest], artifact.EvidenceBlob{Evidence: evidence, Data: bytes.Clone(payload.Data)}) + return []artifact.SignatureEvidence{evidence}, nil +} + +// checkSubject requires the subject to be a stored record; callers hold r.mu. +func (r *Repository) checkSubject(subject artifact.SignatureSubject) error { + if !r.caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "subject/"+string(subject.Kind), "unsupported subject kind") + } + if _, ok := r.records[subject.Digest]; !ok { + return fmt.Errorf("%w: subject %s", errUtils.ErrArtifactNotFound, subject.Digest) + } + return nil +} + +// Verify implements [artifact.Repository]. It authenticates stored evidence against the policy: +// every trust set must be satisfied by at least one evidence item bound to the subject. Unrelated +// or unsupported evidence is reported and ignored; it can never override matching evidence. +func (r *Repository) Verify(_ context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Verify")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Verify") + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, err := policy.Fingerprint(); err == nil { + result.PolicyFingerprint = fingerprint + } + var claims []artifact.SignerClaim + held := r.evidence[subject.Digest] + for index := range held { + claim, ok := r.authenticate(subject, &held[index], result) + if ok { + claims = append(claims, claim) + result.EvidenceDigests = append(result.EvidenceDigests, held[index].Evidence.Descriptor.Digest) + } + } + if !satisfies(policy, claims) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-permitted-signature", Message: "no valid permitted signature satisfies the trust set"}) + return result, fmt.Errorf("%w: no valid permitted signature for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + result.Claims = claims + result.Status = artifact.VerificationVerified + return result, nil +} + +// authenticate checks one evidence item; callers hold r.mu. +func (r *Repository) authenticate(subject artifact.SignatureSubject, held *artifact.EvidenceBlob, result *artifact.VerificationResult) (artifact.SignerClaim, bool) { + skip := func(code, message string) (artifact.SignerClaim, bool) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) + return artifact.SignerClaim{}, false + } + if !r.caps.HasEvidenceFormat(held.Evidence.Format) { + return skip("unsupported-evidence", "evidence in an unsupported format was ignored") + } + if held.Evidence.Descriptor.Size != int64(len(held.Data)) || held.Evidence.Descriptor.Digest != digest.FromBytes(held.Data) { + return skip("evidence-integrity", "evidence whose bytes do not match its descriptor was ignored") + } + var parsed fakeEvidence + if err := json.Unmarshal(held.Data, &parsed); err != nil { + return skip("malformed-evidence", "malformed evidence was ignored") + } + if parsed.Subject != subject.Digest { + return skip("subject-mismatch", "evidence bound to a different subject was ignored") + } + return artifact.SignerClaim{Identity: parsed.Identity, Issuer: parsed.Issuer}, true +} + +// satisfies reports whether the claims meet every trust set; without sets any valid claim suffices. +func satisfies(policy artifact.VerificationPolicy, claims []artifact.SignerClaim) bool { + if len(claims) == 0 { + return false + } + for _, set := range policy.TrustSets { + if !setSatisfied(set, claims) { + return false + } + } + return true +} + +func setSatisfied(set artifact.TrustSet, claims []artifact.SignerClaim) bool { + for _, claim := range claims { + for _, trusted := range set.Identities { + if trusted.Identity == claim.Identity && trusted.Issuer == claim.Issuer { + return true + } + } + } + return false +} + +// ReadEvidence implements [artifact.EvidenceReader]. +func (r *Repository) ReadEvidence(_ context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "artifacttest.Repository.ReadEvidence")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("ReadEvidence") + held := r.evidence[pinned.RecordDigest] + blobs := make([]artifact.EvidenceBlob, len(held)) + for index := range held { + blobs[index] = artifact.EvidenceBlob{Evidence: cloneEvidence(&held[index].Evidence), Data: bytes.Clone(held[index].Data)} + } + return blobs, nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]. +func (r *Repository) WriteEvidence(_ context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "artifacttest.Repository.WriteEvidence")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("WriteEvidence") + if _, ok := r.records[pinned.RecordDigest]; !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], artifact.EvidenceBlob{Evidence: cloneEvidence(&evidence[index].Evidence), Data: bytes.Clone(evidence[index].Data)}) + stored = append(stored, cloneEvidence(&evidence[index].Evidence)) + } + return stored, nil +} diff --git a/pkg/artifact/artifacttest/write.go b/pkg/artifact/artifacttest/write.go new file mode 100644 index 00000000000..972fc3b285d --- /dev/null +++ b/pkg/artifact/artifacttest/write.go @@ -0,0 +1,272 @@ +package artifacttest + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Publish implements [artifact.Repository] with the publication lifecycle of the contracts: the +// pending operation is recorded before payload changes, the immutable record is written, and the +// completed reference is published last. Injected failures leave the state a real interruption +// would. A reference that is pending for a different intent stays unavailable, with the owning +// operation ID in the error, instead of being stolen. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Publish")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Publish") + if !r.caps.Write { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "publish", "write", "the repository is read-only") + } + if err := r.trip(FailBeforePayload); err != nil { + return nil, err + } + record := prepared.Record.Clone() + key := refKey(record.Name, record.Version) + recordDigest, err := record.RecordDigest() + if err != nil { + return nil, err + } + outcome, existing, err := r.admit(key, prepared, options, recordDigest) + if err != nil { + return nil, err + } + if outcome == artifact.OutcomeUnchanged { + return r.publishResult(&existing.record, existing, artifact.OutcomeUnchanged), nil + } + r.pendings[key] = &pending{operationID: prepared.OperationID, intent: prepared.IntentDigest} + blobs, err := r.readBlobs(ctx, prepared) + if err != nil { + return nil, err + } + s := &stored{record: record, recordDigest: recordDigest, packaging: prepared.Packaging, blobs: blobs, payloadAvailable: true} + if s.canonical, err = record.Canonical(); err != nil { + return nil, err + } + r.records[recordDigest] = s + if err := r.trip(FailAfterMetadata); err != nil { + return nil, err + } + r.commit(key, recordDigest) + if err := r.trip(FailAfterCommit); err != nil { + return nil, err + } + return r.publishResult(&record, s, outcome), nil +} + +// admit applies conditional-update, intent, and force rules and returns the outcome to produce. +// For an unchanged outcome it also returns the stored record that already satisfies the intent. +func (r *Repository) admit(key string, prepared *artifact.PreparedArtifact, options artifact.PublishOptions, recordDigest digest.Digest) (artifact.Outcome, *stored, error) { + record := &prepared.Record + if held, isPending := r.pendings[key]; isPending { + if held.intent != prepared.IntentDigest { + return "", nil, fmt.Errorf("%w: %s:%s is pending for operation %s with a different intent; recover it before publishing", errUtils.ErrArtifactPublicationUnavailable, record.Name, record.Version, held.operationID) + } + if _, exists := r.refs[key]; exists { + return artifact.OutcomeReplaced, nil, nil + } + return artifact.OutcomeCreated, nil, nil + } + current, exists := r.refs[key] + if err := checkExpectedPrior(key, current, exists, options.ExpectedPrior); err != nil { + return "", nil, err + } + if !exists { + return artifact.OutcomeCreated, nil, nil + } + existing := r.records[current] + if current == recordDigest { + return artifact.OutcomeUnchanged, existing, nil + } + if intent, err := existing.record.IntentDigest(); err == nil && intent == prepared.IntentDigest { + return artifact.OutcomeUnchanged, existing, nil + } + if !options.Force { + return "", nil, fmt.Errorf("%w: %s already exists with a different intent", errUtils.ErrArtifactConflict, key) + } + return artifact.OutcomeReplaced, nil, nil +} + +// checkExpectedPrior is the compare-and-swap guard. +func checkExpectedPrior(key string, current digest.Digest, exists bool, expected *artifact.PinnedReference) error { + switch { + case expected == nil && exists: + return fmt.Errorf("%w: %s now exists (%s) but the writer expected it to be absent", errUtils.ErrArtifactConflict, key, current) + case expected != nil && !exists: + return fmt.Errorf("%w: %s no longer exists but the writer expected %s", errUtils.ErrArtifactConflict, key, expected.ResolvedDigest) + case expected != nil && expected.ResolvedDigest != current: + return fmt.Errorf("%w: %s is %s but the writer expected %s", errUtils.ErrArtifactConflict, key, current, expected.ResolvedDigest) + } + return nil +} + +// readBlobs reads every staged blob, verifying its descriptor. +func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedArtifact) ([]blob, error) { + blobs := make([]blob, 0, len(prepared.Blobs)) + for index := range prepared.Blobs { + staged := &prepared.Blobs[index] + if err := ctx.Err(); err != nil { + return nil, err + } + reader, err := staged.Open(ctx) + if err != nil { + return nil, fmt.Errorf("%w: open staged blob %q: %w", errUtils.ErrArtifactPackagingInvalid, staged.Path, err) + } + if index == 0 { + if err := r.trip(FailDuringPayload); err != nil { + _ = reader.Close() + return nil, err + } + } + data, readErr := io.ReadAll(reader) + closeErr := reader.Close() + if err := errors.Join(readErr, closeErr); err != nil { + return nil, fmt.Errorf("%w: read staged blob %q: %w", errUtils.ErrArtifactPackagingInvalid, staged.Path, err) + } + if int64(len(data)) != staged.Descriptor.Size || digest.FromBytes(data) != staged.Descriptor.Digest { + return nil, fmt.Errorf("%w: staged blob %q does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, staged.Path) + } + blobs = append(blobs, blob{descriptor: cloneDescriptor(&staged.Descriptor), role: staged.Role, documentRole: staged.DocumentRole, path: staged.Path, data: data}) + } + return blobs, nil +} + +// commit publishes the completed reference last and retires the replaced payload. +func (r *Repository) commit(key string, recordDigest digest.Digest) { + previous, hadPrevious := r.refs[key] + r.refs[key] = recordDigest + delete(r.pendings, key) + if hadPrevious && previous != recordDigest && !r.referenced(previous) { + r.records[previous].payloadAvailable = false + } +} + +// referenced reports whether any reference still points at the record digest. +func (r *Repository) referenced(recordDigest digest.Digest) bool { + for _, d := range r.refs { + if d == recordDigest { + return true + } + } + return false +} + +func (r *Repository) publishResult(record *artifact.Record, s *stored, outcome artifact.Outcome) *artifact.PublishResult { + ref := artifact.Ref{Name: record.Name, Version: record.Version} + return &artifact.PublishResult{ + Outcome: outcome, + Pinned: r.pinnedFor(ref, s), + Representation: r.representation(s), + Location: r.locator(record.Name, record.Version), + LockScope: artifact.LockScopeHost, + } +} + +// checkExpectedCurrent is the delete staleness guard. It compares the digest the caller observed +// with the digest currently stored under the pinned reference's own key, so a reference that moved +// after pinning is a conflict. The key comes from the pinned reference rather than the Record's +// name and version, because an alias can point at a Record named differently. A digest-addressed +// pin has no mutable reference, so it is compared with the Record digest. Callers hold r.mu. +func (r *Repository) checkExpectedCurrent(pinned *artifact.PinnedReference, expected *artifact.PinnedReference) error { + if expected == nil { + return nil + } + current := pinned.RecordDigest + if pinned.Ref.Digest == "" && pinned.Ref.Version != "" { + stored, ok := r.refs[refKey(pinned.Ref.Name, pinned.Ref.Version)] + if !ok { + return fmt.Errorf("%w: %s no longer exists", errUtils.ErrArtifactConflict, pinned.Ref) + } + current = stored + } + if expected.ResolvedDigest != current { + return fmt.Errorf("%w: the delete target changed (%s is now %s, the caller observed %s)", errUtils.ErrArtifactConflict, pinned.Ref, current, expected.ResolvedDigest) + } + return nil +} + +// Delete implements [artifact.Repository]. The requested scope decides what goes: a reference +// scope removes only the pinned reference and keeps the record and its evidence while another +// reference still uses them, while a manifest or object scope removes the record, its evidence, +// and every reference to it. A scope the repository does not advertise is rejected, never widened. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Delete(_ context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Delete")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Delete") + if r.caps.Delete == artifact.DeleteNone || r.caps.Delete == "" { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "delete", "delete", "the repository cannot delete") + } + if options.Scope != r.caps.Delete { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "delete", "delete/"+string(options.Scope), "the repository only deletes with the "+string(r.caps.Delete)+" scope") + } + if _, ok := r.records[pinned.RecordDigest]; !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + if err := r.checkExpectedCurrent(&pinned, options.ExpectedCurrent); err != nil { + return nil, err + } + if options.Scope == artifact.DeleteReference { + return r.deleteReference(&pinned) + } + return r.deleteRecord(&pinned), nil +} + +// deleteReference removes only the pinned reference. The record and its evidence survive while any +// other reference still points at them; callers hold r.mu. +func (r *Repository) deleteReference(pinned *artifact.PinnedReference) (*artifact.DeleteResult, error) { + if pinned.Ref.Version == "" || pinned.Ref.Digest != "" { + return nil, fmt.Errorf("%w: a reference-scoped delete needs a version reference, got %s", errUtils.ErrArtifactRefInvalid, pinned.Ref) + } + key := refKey(pinned.Ref.Name, pinned.Ref.Version) + if current, ok := r.refs[key]; !ok || current != pinned.RecordDigest { + return nil, fmt.Errorf("%w: %s no longer points at %s", errUtils.ErrArtifactConflict, pinned.Ref, pinned.RecordDigest) + } + delete(r.refs, key) + if !r.referenced(pinned.RecordDigest) { + delete(r.records, pinned.RecordDigest) + delete(r.evidence, pinned.RecordDigest) + } + return &artifact.DeleteResult{Digest: pinned.RecordDigest, Deleted: []string{r.locatorForKey(key)}}, nil +} + +// deleteRecord removes the record, its evidence, and every reference to it. Each removed reference +// is reported once by its own location, and every removed reference other than the pinned one is +// reported as affected; callers hold r.mu. +func (r *Repository) deleteRecord(pinned *artifact.PinnedReference) *artifact.DeleteResult { + pinnedKey := "" + if pinned.Ref.Version != "" && pinned.Ref.Digest == "" { + pinnedKey = refKey(pinned.Ref.Name, pinned.Ref.Version) + } + var removed []string + for key, d := range r.refs { + if d == pinned.RecordDigest { + removed = append(removed, key) + } + } + sort.Strings(removed) + result := &artifact.DeleteResult{Digest: pinned.RecordDigest} + for _, key := range removed { + delete(r.refs, key) + result.Deleted = append(result.Deleted, r.locatorForKey(key)) + if key != pinnedKey { + result.Affected = append(result.Affected, refFromKey(key)) + } + } + delete(r.records, pinned.RecordDigest) + delete(r.evidence, pinned.RecordDigest) + return result +} diff --git a/pkg/artifact/capabilities.go b/pkg/artifact/capabilities.go index ddd00f13639..98260ff6b30 100644 --- a/pkg/artifact/capabilities.go +++ b/pkg/artifact/capabilities.go @@ -2,6 +2,7 @@ package artifact import ( "slices" + "strings" "github.com/cloudposse/atmos/pkg/perf" ) @@ -74,6 +75,23 @@ func (c *Capabilities) HasEvidenceFormat(f EvidenceFormat) bool { return slices.Contains(c.EvidenceFormats, f) } +// CheckSignerFormats refuses a signer before it is invoked when the repository can store none of +// its evidence formats, so an unusable signer never signs anything. The repository argument +// names the repository in the capability error. +func (c *Capabilities) CheckSignerFormats(repository string, signer Signer) error { + defer perf.Track(nil, "artifact.Capabilities.CheckSignerFormats")() + + formats := signer.Formats() + names := make([]string, 0, len(formats)) + for _, format := range formats { + if c.HasEvidenceFormat(format) { + return nil + } + names = append(names, string(format)) + } + return NewUnsupportedCapability(repository, "sign", "evidence/"+strings.Join(names, ","), "the repository cannot store this evidence format") +} + // HasSubjectKind reports whether the repository supports the signature subject kind. func (c *Capabilities) HasSubjectKind(k SubjectKind) bool { defer perf.Track(nil, "artifact.Capabilities.HasSubjectKind")() diff --git a/pkg/artifact/clone.go b/pkg/artifact/clone.go new file mode 100644 index 00000000000..93c24f228f1 --- /dev/null +++ b/pkg/artifact/clone.go @@ -0,0 +1,70 @@ +package artifact + +import ( + "bytes" + "maps" + "slices" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Clone returns a Record that shares no maps, slices, or pointers with r, so a caller can keep +// either copy and edit the other without changing the first. +func (r *Record) Clone() Record { + defer perf.Track(nil, "artifact.Record.Clone")() + + out := *r + out.Metadata = r.Metadata.Clone() + out.Inventory = slices.Clone(r.Inventory) + out.Documents = slices.Clone(r.Documents) + out.KindData = bytes.Clone(r.KindData) + return out +} + +// Clone returns Capabilities that share no slices with c, so the guarantees a repository +// advertises stay static whatever a caller does with the returned value. +func (c *Capabilities) Clone() Capabilities { + defer perf.Track(nil, "artifact.Capabilities.Clone")() + + out := *c + out.Packagings = slices.Clone(c.Packagings) + out.SubjectKinds = slices.Clone(c.SubjectKinds) + out.EvidenceFormats = slices.Clone(c.EvidenceFormats) + out.LockScopes = slices.Clone(c.LockScopes) + return out +} + +// Clone returns Metadata that shares no map or pointer with m. +func (m *Metadata) Clone() Metadata { + defer perf.Track(nil, "artifact.Metadata.Clone")() + + out := *m + out.Labels = maps.Clone(m.Labels) + if m.Provenance != nil { + provenance := *m.Provenance + if provenance.CI != nil { + ci := *provenance.CI + provenance.CI = &ci + } + if provenance.Source != nil { + source := *provenance.Source + provenance.Source = &source + } + out.Provenance = &provenance + } + return out +} + +// Clone returns a VerificationPolicy that shares no slice with p, so editing the trust sets of a +// description handed to a caller never reaches the configured policy. +func (p *VerificationPolicy) Clone() VerificationPolicy { + defer perf.Track(nil, "artifact.VerificationPolicy.Clone")() + + out := *p + out.TrustSets = slices.Clone(p.TrustSets) + for index := range out.TrustSets { + out.TrustSets[index].Identities = slices.Clone(out.TrustSets[index].Identities) + out.TrustSets[index].Keys = slices.Clone(out.TrustSets[index].Keys) + } + return out +} diff --git a/pkg/artifact/clone_test.go b/pkg/artifact/clone_test.go new file mode 100644 index 00000000000..884b016ff6e --- /dev/null +++ b/pkg/artifact/clone_test.go @@ -0,0 +1,158 @@ +package artifact_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +func sampleRecord() artifact.Record { + return artifact.Record{ + SchemaVersion: artifact.SchemaVersion, + Name: "team/app", + Version: "1.0.0", + Kind: artifact.KindFiles, + Metadata: artifact.Metadata{ + Description: "sample", + Labels: map[string]string{"a": "1"}, + Provenance: &artifact.Provenance{ + CI: &artifact.CIProvenance{Provider: "github", RunID: "1"}, + Source: &artifact.SourceProvenance{Repository: "repo", Commit: "abc"}, + }, + }, + Inventory: []artifact.Entry{{Path: "a.txt", Type: artifact.EntryFile, Size: 1}, {Path: "b.txt", Type: artifact.EntryFile, Size: 2}}, + Documents: []artifact.Document{{Role: artifact.DocumentReadme, SourcePath: "README.md"}}, + KindData: artifact.KindData(`{"k":1}`), + } +} + +// TestRecord_CloneIsIndependentInBothDirections mutates the clone and then the original, and +// requires that neither edit shows up in the other. +func TestRecord_CloneIsIndependentInBothDirections(t *testing.T) { + tests := []struct { + name string + mutate func(r *artifact.Record) + read func(r *artifact.Record) any + want any + }{ + {"labels", func(r *artifact.Record) { r.Metadata.Labels["a"] = "changed" }, func(r *artifact.Record) any { return r.Metadata.Labels }, map[string]string{"a": "1"}}, + {"inventory", func(r *artifact.Record) { r.Inventory[0].Path = "changed" }, func(r *artifact.Record) any { return r.Inventory[0].Path }, "a.txt"}, + {"inventory last", func(r *artifact.Record) { r.Inventory[1].Size = 99 }, func(r *artifact.Record) any { return r.Inventory[1].Size }, int64(2)}, + {"documents", func(r *artifact.Record) { r.Documents[0].SourcePath = "changed" }, func(r *artifact.Record) any { return r.Documents[0].SourcePath }, "README.md"}, + {"kind data", func(r *artifact.Record) { r.KindData[0] = 'X' }, func(r *artifact.Record) any { return string(r.KindData) }, `{"k":1}`}, + {"CI provenance", func(r *artifact.Record) { r.Metadata.Provenance.CI.RunID = "changed" }, func(r *artifact.Record) any { return r.Metadata.Provenance.CI.RunID }, "1"}, + {"source provenance", func(r *artifact.Record) { r.Metadata.Provenance.Source.Commit = "changed" }, func(r *artifact.Record) any { return r.Metadata.Provenance.Source.Commit }, "abc"}, + {"provenance", func(r *artifact.Record) { r.Metadata.Provenance.CI = nil }, func(r *artifact.Record) any { return r.Metadata.Provenance.CI != nil }, true}, + } + for _, tt := range tests { + t.Run(tt.name+"/clone to original", func(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + tt.mutate(&clone) + assert.Equal(t, tt.want, tt.read(&original)) + }) + t.Run(tt.name+"/original to clone", func(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + tt.mutate(&original) + assert.Equal(t, tt.want, tt.read(&clone)) + }) + } +} + +func TestRecord_CloneKeepsContent(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + assert.Equal(t, original, clone) + + empty := artifact.Record{} + assert.Equal(t, empty, empty.Clone(), "nil maps and slices stay nil") +} + +func TestCapabilities_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.Capabilities { + return artifact.Capabilities{ + Read: true, + Packagings: []artifact.Packaging{artifact.PackagingTar, artifact.PackagingZip}, + SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}, + EvidenceFormats: []artifact.EvidenceFormat{"one", "two"}, + LockScopes: []artifact.LockScope{artifact.LockScopeHost}, + } + } + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + + clone.Packagings[0], clone.SubjectKinds[0], clone.EvidenceFormats[1], clone.LockScopes[0] = artifact.PackagingTgz, "x", "x", artifact.LockScopeRepository + assert.Equal(t, sample(), original, "editing the clone leaves the original alone") + + original = sample() + clone = original.Clone() + original.Packagings[1], original.SubjectKinds[0], original.EvidenceFormats[0], original.LockScopes[0] = artifact.PackagingNone, "x", "x", artifact.LockScopeRepository + assert.Equal(t, sample(), clone, "editing the original leaves the clone alone") +} + +func TestMetadata_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.Metadata { return sampleRecord().Metadata } + mutations := map[string]func(m *artifact.Metadata){ + "labels": func(m *artifact.Metadata) { m.Labels["a"] = "changed" }, + "CI provenance": func(m *artifact.Metadata) { m.Provenance.CI.RunID = "changed" }, + "source provenance": func(m *artifact.Metadata) { m.Provenance.Source.Commit = "changed" }, + "provenance": func(m *artifact.Metadata) { m.Provenance.CI = nil }, + } + for name, mutate := range mutations { + t.Run(name+"/clone to original", func(t *testing.T) { + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + mutate(&clone) + assert.Equal(t, sample(), original) + }) + t.Run(name+"/original to clone", func(t *testing.T) { + original := sample() + clone := original.Clone() + mutate(&original) + assert.Equal(t, sample(), clone) + }) + } + assert.Equal(t, artifact.Metadata{}, (&artifact.Metadata{}).Clone(), "nil maps and pointers stay nil") +} + +func TestVerificationPolicy_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.VerificationPolicy { + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: "first", Issuer: "issuer-1"}, {Identity: "second", Issuer: "issuer-2"}}, Keys: []string{"a.pub"}}, + {Keys: []string{"b.pub", "c.pub"}}, + }, + Offline: true, + } + } + mutations := map[string]func(p *artifact.VerificationPolicy){ + "trust set order": func(p *artifact.VerificationPolicy) { p.TrustSets[0], p.TrustSets[1] = p.TrustSets[1], p.TrustSets[0] }, + "first identity": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Identities[0].Identity = "changed" }, + "last identity issuer": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Identities[1].Issuer = "changed" }, + "first key": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Keys[0] = "changed" }, + "last key": func(p *artifact.VerificationPolicy) { p.TrustSets[1].Keys[1] = "changed" }, + } + for name, mutate := range mutations { + t.Run(name+"/clone to original", func(t *testing.T) { + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + mutate(&clone) + assert.Equal(t, sample(), original) + }) + t.Run(name+"/original to clone", func(t *testing.T) { + original := sample() + clone := original.Clone() + mutate(&original) + assert.Equal(t, sample(), clone) + }) + } + assert.Equal(t, artifact.VerificationPolicy{}, (&artifact.VerificationPolicy{}).Clone(), "an empty policy stays empty") +} diff --git a/pkg/artifact/evidence.go b/pkg/artifact/evidence.go new file mode 100644 index 00000000000..78e9400ec88 --- /dev/null +++ b/pkg/artifact/evidence.go @@ -0,0 +1,20 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_evidence.go github.com/cloudposse/atmos/pkg/artifact EvidenceReader,EvidenceWriter + +// EvidenceReader is an optional Repository extension that exports the signature evidence stored +// for a pinned artifact, so a copy through the shared staging fallback can carry it along. +type EvidenceReader interface { + // ReadEvidence returns the evidence blobs bound to the pinned artifact. The evidence is + // untrusted until verified. + ReadEvidence(ctx context.Context, pinned *PinnedReference) ([]EvidenceBlob, error) +} + +// EvidenceWriter is an optional Repository extension that imports signature evidence for an +// artifact the repository already holds. It never changes the signed Record or native graph root. +type EvidenceWriter interface { + // WriteEvidence stores evidence for the pinned artifact and returns its stored descriptors. + WriteEvidence(ctx context.Context, pinned *PinnedReference, evidence []EvidenceBlob, lock LockHandle) ([]SignatureEvidence, error) +} diff --git a/pkg/artifact/fixtures/fixtures.go b/pkg/artifact/fixtures/fixtures.go new file mode 100644 index 00000000000..9b960e4973c --- /dev/null +++ b/pkg/artifact/fixtures/fixtures.go @@ -0,0 +1,153 @@ +// Package fixtures builds the artifact test fixtures described by the artifact validation plan +// ("File package" and "Invalid package") with Go code inside a caller-supplied temporary +// directory. It never shells out, so it behaves identically on every platform. +package fixtures + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Permission bits used by the fixtures. +const ( + dirPerm = 0o755 + tempPerm = 0o600 + execPerm = 0o755 + filePerm = 0o644 + privPerm = 0o640 +) + +// Well-known fixture content. +const ( + // RootReadme is the content of the root README.md, which stays ordinary payload. + RootReadme = "# Root readme\n" + + // PackageReadme is the content of the distribution README in the reserved directory. + PackageReadme = "# Package readme\n" + + // PackageChangelog is the content of the distribution CHANGELOG in the reserved directory. + PackageChangelog = "# Changelog\n\n- first\n" + + // HelperScript is the content of the executable helper. + HelperScript = "#!/bin/sh\necho helper\n" + + // UnicodeName is the Unicode filename in the file package. + UnicodeName = "unicode-é-日本.txt" + + // ReservedDir is the reserved distribution directory below the source root. + ReservedDir = ".atmos/artifact" +) + +// POSIXModes reports whether the platform honors Unix permission bits. Windows does not, so tests +// that assert executable bits or mode-only changes skip there. +func POSIXModes() bool { + defer perf.Track(nil, "fixtures.POSIXModes")() + + return runtime.GOOS != "windows" +} + +// RequirePOSIXModes skips the test on platforms that ignore Unix permission bits. +func RequirePOSIXModes(t testing.TB) { + defer perf.Track(nil, "fixtures.RequirePOSIXModes")() + + t.Helper() + if !POSIXModes() { + t.Skipf("this platform does not honor Unix permission bits") + } +} + +// WriteFile writes content at the slash-separated rel path below dir and sets mode explicitly so +// the result does not depend on the process umask. +func WriteFile(t testing.TB, dir, rel string, content string, mode fs.FileMode) { + defer perf.Track(nil, "fixtures.WriteFile")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), dirPerm)) + require.NoError(t, os.WriteFile(path, []byte(content), tempPerm)) + require.NoError(t, os.Chmod(path, mode)) +} + +// MkdirAll creates the slash-separated directory below dir with mode 0755. +func MkdirAll(t testing.TB, dir, rel string) { + defer perf.Track(nil, "fixtures.MkdirAll")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(path, dirPerm)) + require.NoError(t, os.Chmod(path, dirPerm)) +} + +// Symlink creates a symlink at rel pointing at target and reports false (leaving nothing behind) +// when the platform or account cannot create symlinks. +func Symlink(t testing.TB, dir, rel, target string) bool { + defer perf.Track(nil, "fixtures.Symlink")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), dirPerm)) + if err := os.Symlink(filepath.FromSlash(target), path); err != nil { + t.Logf("symlinks are unavailable here: %v", err) + return false + } + return true +} + +// FilePackage describes the files written by [WriteFilePackage]. +type FilePackage struct { + // Dir is the source directory. + Dir string + + // Symlinks reports whether the confined symlink could be created. + Symlinks bool +} + +// WriteFilePackage writes the valid "File package" fixture into dir: nested files, an executable +// helper, an empty file, an empty directory, a confined symlink, a Unicode filename, a root README +// (ordinary payload), and a distinct package README and CHANGELOG in the reserved directory. +func WriteFilePackage(t testing.TB, dir string) FilePackage { + defer perf.Track(nil, "fixtures.WriteFilePackage")() + + t.Helper() + WriteFile(t, dir, "README.md", RootReadme, filePerm) + WriteFile(t, dir, "bin/helper.sh", HelperScript, execPerm) + WriteFile(t, dir, "nested/deep/file.txt", "deep content\n", filePerm) + WriteFile(t, dir, "empty.txt", "", filePerm) + MkdirAll(t, dir, "emptydir") + WriteFile(t, dir, UnicodeName, "unicode content\n", privPerm) + WriteFile(t, dir, ReservedDir+"/README.md", PackageReadme, filePerm) + WriteFile(t, dir, ReservedDir+"/CHANGELOG.md", PackageChangelog, filePerm) + symlinks := Symlink(t, dir, "link-to-file", "nested/deep/file.txt") + return FilePackage{Dir: dir, Symlinks: symlinks} +} + +// FilePackagePaths lists the inventory paths of the file package in sorted order, excluding the +// reserved distribution directory and, when symlinks are unavailable, the symlink. +func FilePackagePaths(symlinks bool) []string { + defer perf.Track(nil, "fixtures.FilePackagePaths")() + + paths := []string{ + "README.md", + "bin", + "bin/helper.sh", + "empty.txt", + "emptydir", + "nested", + "nested/deep", + "nested/deep/file.txt", + UnicodeName, + } + if symlinks { + paths = append(paths, "link-to-file") + } + sort.Strings(paths) + return paths +} diff --git a/pkg/artifact/fixtures/fixtures_other.go b/pkg/artifact/fixtures/fixtures_other.go new file mode 100644 index 00000000000..806e24665a7 --- /dev/null +++ b/pkg/artifact/fixtures/fixtures_other.go @@ -0,0 +1,11 @@ +//go:build !unix + +package fixtures + +import "testing" + +// SpecialFile reports false: special files cannot be created portably on this platform. +func SpecialFile(t testing.TB, _, _ string) bool { + t.Helper() + return false +} diff --git a/pkg/artifact/fixtures/fixtures_unix.go b/pkg/artifact/fixtures/fixtures_unix.go new file mode 100644 index 00000000000..f31e6af1dfd --- /dev/null +++ b/pkg/artifact/fixtures/fixtures_unix.go @@ -0,0 +1,27 @@ +//go:build unix + +package fixtures + +import ( + "path/filepath" + "syscall" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// pipePerm is the mode of the named pipe fixture. +const pipePerm = 0o600 + +// SpecialFile creates a named pipe (a special file) at rel below dir and reports whether it could. +func SpecialFile(t testing.TB, dir, rel string) bool { + defer perf.Track(nil, "fixtures.SpecialFile")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + if err := syscall.Mkfifo(path, pipePerm); err != nil { + t.Logf("named pipes are unavailable here: %v", err) + return false + } + return true +} diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go new file mode 100644 index 00000000000..e20aac6b8f3 --- /dev/null +++ b/pkg/artifact/linkpath/linkpath.go @@ -0,0 +1,209 @@ +// Package linkpath resolves symlink chains physically so containment checks can see where a +// link really points, including links whose final components do not exist yet. +package linkpath + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxLinkHops bounds the symlinks followed while resolving one path, so a loop terminates. +const ( + maxLinkHops = 255 + parentComponent = ".." +) + +// Resolve returns the physical absolute path that target refers to. It follows every symlink it +// meets, like filepath.EvalSymlinks. Unlike EvalSymlinks, it also resolves a path whose tail does +// not exist: the longest existing prefix is resolved physically, and the missing remainder is +// joined lexically. A dangling link therefore still reports where it would land once the missing +// path is created. Parent components are applied only after the symlinks before them are resolved, +// so "link/../x" is not collapsed to a sibling of "link". +// +// Resolve fails with [errUtils.ErrSymlinkLoop] when the links form a loop, with +// [errUtils.ErrPathThroughNonDirectory] when the path passes through a non-directory, and with another +// error when it cannot be inspected. Callers decide separately +// what each failure means. +func Resolve(target string) (string, error) { + defer perf.Track(nil, "linkpath.Resolve")() + + absolute, err := absolutePath(target) + if err != nil { + return "", fmt.Errorf("%w: make %q absolute: %w", errUtils.ErrArtifactUnsafePath, target, err) + } + volume := filepath.VolumeName(absolute) + w := &walker{base: volume + string(filepath.Separator), pending: splitComponents(absolute[len(volume):])} + w.resolved = w.base + for len(w.pending) > 0 { + if err := w.step(); err != nil { + return "", err + } + } + return filepath.Join(append([]string{w.resolved}, w.missing...)...), nil +} + +// absolutePath makes target absolute without cleaning it: filepath.Abs would collapse parent +// components lexically, before any symlink in front of them has been resolved. +func absolutePath(target string) (string, error) { + if filepath.IsAbs(target) { + return target, nil + } + working, err := os.Getwd() + if err != nil { + return "", err + } + return working + string(filepath.Separator) + target, nil +} + +// Escapes reports whether the symlink at from leaves the tree that links describes. The tree is +// recorded, not read from disk: links maps the slash-separated path of every symlink, relative to +// the tree root, to its target. Resolution follows the recorded links exactly like Resolve follows +// physical ones (a dangling tail is resolved lexically), so the answer cannot change when the +// files the links were read from change afterwards. A link chain that loops does not escape. +func Escapes(links map[string]string, from string) bool { + defer perf.Track(nil, "linkpath.Escapes")() + + pending := splitComponents(from) + var resolved []string + for hops := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case ".": + case parentComponent: + if len(resolved) == 0 { + return true + } + resolved = resolved[:len(resolved)-1] + default: + current := strings.Join(append(resolved, component), "/") + target, isLink := links[current] + if !isLink { + resolved = append(resolved, component) + continue + } + hops++ + if hops > maxLinkHops { + return false + } + pending = append(splitComponents(target), pending...) + } + } + return false +} + +// walker resolves a path one component at a time. +type walker struct { + // base is the filesystem root, with its volume on Windows. + base string + + // resolved is the physical directory reached so far. + resolved string + + // pending holds the components still to resolve. + pending []string + + // missing holds the lexical tail below the first missing component. + missing []string + + // hops counts the symlinks followed. + hops int +} + +// step consumes one component. It returns nil to continue and an error when resolution failed. +func (w *walker) step() error { + component := w.pending[0] + w.pending = w.pending[1:] + if len(w.missing) > 0 { + w.stepMissing(component) + return nil + } + if component == "." || component == parentComponent { + w.resolved = stepDot(w.resolved, component) + return nil + } + next := filepath.Join(w.resolved, component) + info, err := os.Lstat(next) + if errors.Is(err, fs.ErrNotExist) { + w.missing = append(w.missing, component) + return nil + } + if err != nil { + return fmt.Errorf("%w: inspect %q: %w", errUtils.ErrArtifactUnsafePath, next, err) + } + if info.Mode()&fs.ModeSymlink == 0 { + if !info.IsDir() && len(w.pending) > 0 { + return fmt.Errorf("%w: %w: %q is not a directory", errUtils.ErrArtifactUnsafePath, errUtils.ErrPathThroughNonDirectory, next) + } + w.resolved = next + return nil + } + return w.follow(next) +} + +// stepMissing resolves the missing tail lexically until it returns to an existing directory. +func (w *walker) stepMissing(component string) { + switch component { + case ".": + case parentComponent: + w.missing = w.missing[:len(w.missing)-1] + default: + w.missing = append(w.missing, component) + } +} + +// follow splices the target of the symlink at link in front of the pending components. +func (w *walker) follow(link string) error { + w.hops++ + if w.hops > maxLinkHops { + return fmt.Errorf("%w: more than %d links followed at %q", errUtils.ErrSymlinkLoop, maxLinkHops, link) + } + components, absolute, err := linkComponents(link) + if err != nil { + return fmt.Errorf("%w: read link %q: %w", errUtils.ErrArtifactUnsafePath, link, err) + } + if absolute { + w.resolved = w.base + } + w.pending = append(components, w.pending...) + return nil +} + +// stepDot applies a current or parent directory component to a fully resolved directory. +func stepDot(resolved, component string) string { + if component == parentComponent { + return filepath.Dir(resolved) + } + return resolved +} + +// linkComponents reads the symlink at link and returns the components of its target and whether the +// target is absolute, in which case resolution restarts from the root. +func linkComponents(link string) ([]string, bool, error) { + target, err := os.Readlink(link) + if err != nil { + return nil, false, err + } + absolute := filepath.IsAbs(target) || strings.HasPrefix(target, string(filepath.Separator)) + return splitComponents(target[len(filepath.VolumeName(target)):]), absolute, nil +} + +// Within reports whether target is root or lies beneath it. +func Within(root, target string) bool { + defer perf.Track(nil, "linkpath.Within")() + + relative, err := filepath.Rel(root, target) + return err == nil && relative != parentComponent && !strings.HasPrefix(relative, parentComponent+string(filepath.Separator)) +} + +// splitComponents splits a path into its non-empty components. +func splitComponents(path string) []string { + return strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == filepath.Separator }) +} diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go new file mode 100644 index 00000000000..cc39e4ff87f --- /dev/null +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -0,0 +1,137 @@ +package linkpath_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" +) + +// realTemp returns a temporary directory with symlinks in its own path already resolved. +func realTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + return dir +} + +func TestResolve(t *testing.T) { + root := realTemp(t) + fixtures.WriteFile(t, root, "tree/real/file.txt", "x", 0o644) + if !fixtures.Symlink(t, root, "tree/up", "..") { + t.Skip("this platform cannot create symlinks") + } + require.True(t, fixtures.Symlink(t, root, "tree/to-real", "real")) + require.True(t, fixtures.Symlink(t, root, "tree/dangling", "real/missing/deeper")) + require.True(t, fixtures.Symlink(t, root, "tree/escapes", "up/../outside/missing")) + require.True(t, fixtures.Symlink(t, root, "tree/missing-return", "absent/../up/../outside")) + require.True(t, fixtures.Symlink(t, root, "tree/loop-a", "loop-b")) + require.True(t, fixtures.Symlink(t, root, "tree/loop-b", "loop-a")) + + tests := []struct { + name string + path string + want string + wantErr error + }{ + {"plain file", filepath.Join(root, "tree", "real", "file.txt"), filepath.Join(root, "tree", "real", "file.txt"), nil}, + {"link to directory", filepath.Join(root, "tree", "to-real"), filepath.Join(root, "tree", "real"), nil}, + {"dangling tail stays lexical under the link target", filepath.Join(root, "tree", "dangling"), filepath.Join(root, "tree", "real", "missing", "deeper"), nil}, + {"dangling tail walking out through a link", filepath.Join(root, "tree", "escapes"), filepath.Join(filepath.Dir(root), "outside", "missing"), nil}, + {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), nil}, + {"missing tail returns to physical resolution", filepath.Join(root, "tree", "missing-return"), filepath.Join(filepath.Dir(root), "outside"), nil}, + {"loop", filepath.Join(root, "tree", "loop-a"), "", errUtils.ErrSymlinkLoop}, + {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", errUtils.ErrPathThroughNonDirectory}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := linkpath.Resolve(tt.path) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// A parent component after a symlink applies to the link's physical target, not to the link's +// directory: "jump/../secret" is a sibling of the directory jump points into. +func TestResolve_ParentComponentFollowsTheLinkPhysically(t *testing.T) { + root := realTemp(t) + fixtures.WriteFile(t, root, "elsewhere/deep/keep.txt", "x", 0o644) + if !fixtures.Symlink(t, root, "tree/jump", "../elsewhere/deep") { + t.Skip("this platform cannot create symlinks") + } + // Written as a joined string so the path keeps its ".." component. + input := filepath.Join(root, "tree", "jump") + string(filepath.Separator) + ".." + string(filepath.Separator) + "secret" + + got, err := linkpath.Resolve(input) + require.NoError(t, err) + assert.Equal(t, filepath.Join(root, "elsewhere", "secret"), got) +} + +func TestResolve_AbsoluteLinkTarget(t *testing.T) { + root := realTemp(t) + target := filepath.Join(root, "target") + require.NoError(t, os.MkdirAll(target, 0o755)) + link := filepath.Join(root, "abs") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks are unavailable here: %v", err) + } + got, err := linkpath.Resolve(filepath.Join(link, "missing")) + require.NoError(t, err) + assert.Equal(t, filepath.Join(target, "missing"), got) +} + +func TestWithin(t *testing.T) { + root := filepath.Join(string(filepath.Separator), "base", "root") + tests := []struct { + name string + target string + want bool + }{ + {"root itself", root, true}, + {"descendant", filepath.Join(root, "a", "b"), true}, + {"sibling with shared prefix", root + "-other", false}, + {"parent", filepath.Dir(root), false}, + {"unrelated", filepath.Join(string(filepath.Separator), "elsewhere"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, linkpath.Within(root, tt.target)) + }) + } +} + +func TestEscapes(t *testing.T) { + tests := []struct { + name string + links map[string]string + from string + want bool + }{ + {"link to a sibling stays inside", map[string]string{"a": "b"}, "a", false}, + {"link above the root escapes", map[string]string{"a": ".."}, "a", true}, + {"link to a missing path below a link to the root escapes through it", map[string]string{"d": ".", "L": "d/../outside/missing"}, "L", true}, + {"the same link inside the root when the directory link goes deeper", map[string]string{"d": "sub", "L": "d/../outside/missing"}, "L", false}, + {"chained links that climb out escape", map[string]string{"a/b": "..", "c": "a/b/../.."}, "c", true}, + {"chained links that stay inside", map[string]string{"a/b": "..", "c": "a/b/x"}, "c", false}, + {"a link inside a linked directory is read through the link", map[string]string{"d": "sub", "sub/L": "../.."}, "d/L", true}, + {"a dangling link beyond the root escapes", map[string]string{"a": "x/../../y"}, "a", true}, + {"a loop does not escape and terminates", map[string]string{"p": "q", "q": "p"}, "p", false}, + {"a plain name is not a link", map[string]string{}, "plain", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, linkpath.Escapes(tc.links, tc.from)) + }) + } +} diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go new file mode 100644 index 00000000000..ca6dbcbd5a2 --- /dev/null +++ b/pkg/artifact/materialize/archive.go @@ -0,0 +1,419 @@ +package materialize + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "io/fs" + "math" + "os" + "path" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Names of the distribution members generated inside a packaged representation. +const ( + // The distribution directory. + managedDir = ".atmos/artifact" + + // The parent of the distribution directory. + managedParent = ".atmos" + + // The generated manifest. + managedManifest = "manifest.json" + + // The longest symlink target read from a zip member. + maxLinkTargetBytes = 4096 + + // The chunk size of context-aware copies. + copyChunkSize = 32 * 1024 + + // The most decompressed bytes one call to the tar reader's Next may consume before it returns a + // visible member. Successive local PAX headers are consumed inside a single call, so this bounds + // the metadata a hostile archive can make the extractor read without producing a member. + maxTarMetadataBytes = 1 << 20 +) + +// fillArchive extracts the single payload stream of a tar, tgz, or zip representation. +func fillArchive(ctx context.Context, t *tree, content *artifact.Content, scratch string) error { + var payload *artifact.ContentStream + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if payload != nil { + return fmt.Errorf("%w: an archive representation must have exactly one payload", errUtils.ErrArtifactIntegrityFailed) + } + payload = stream + } + if payload == nil || payload.Reader == nil { + return fmt.Errorf("%w: the archive payload is missing", errUtils.ErrArtifactIntegrityFailed) + } + checker, err := newManagedChecker(content.Record) + if err != nil { + return err + } + if content.Representation.Packaging == artifact.PackagingZip { + return fillZip(ctx, t, payload, scratch, checker) + } + return fillTar(ctx, t, payload.Reader, content.Representation.Packaging == artifact.PackagingTgz, checker) +} + +func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, checker *managedChecker) error { + // Every read goes through the context, metadata included: Next consumes a run of local PAX + // headers before it returns a visible member, so the loop's own check cannot interrupt it. + reader = contextReader{ctx: ctx, reader: reader} + if gzipped { + gz, err := gzip.NewReader(reader) + if err != nil { + return archiveReadError(ctx, "gzip", err) + } + defer gz.Close() + reader = contextReader{ctx: ctx, reader: gz} + } + budget := &metadataBudget{reader: reader} + tr := tar.NewReader(budget) + for { + if err := ctx.Err(); err != nil { + return err + } + budget.arm(maxTarMetadataBytes) + header, err := tr.Next() + budget.disarm() + if errors.Is(err, io.EOF) { + // Drain so the verifying payload reader checks its full digest at end of stream. + if _, err = copyContext(ctx, io.Discard, reader); err != nil { + return archiveReadError(ctx, "tar trailer", err) + } + return nil + } + if err != nil { + return archiveReadError(ctx, "tar", err) + } + if err := applyTarHeader(ctx, t, tr, header, checker); err != nil { + return err + } + } +} + +// metadataBudget bounds the bytes read while it is armed. The tar reader is armed only inside Next, so +// member bodies, which are bounded by their own declared sizes, do not count against it. +type metadataBudget struct { + reader io.Reader + remaining int64 + armed bool +} + +func (b *metadataBudget) arm(limit int64) { + b.remaining = limit + b.armed = true +} + +func (b *metadataBudget) disarm() { + b.armed = false +} + +func (b *metadataBudget) Read(p []byte) (int, error) { + defer perf.Track(nil, "materialize.metadataBudget.Read")() + + if !b.armed { + return b.reader.Read(p) + } + if b.remaining <= 0 { + return 0, fmt.Errorf("%w: tar metadata exceeds %d bytes before a member", errUtils.ErrArtifactTooLarge, int64(maxTarMetadataBytes)) + } + if int64(len(p)) > b.remaining { + p = p[:b.remaining] + } + n, err := b.reader.Read(p) + b.remaining -= int64(n) + return n, err +} + +// archiveReadError reports a failed archive read. A canceled context is returned as such, because +// the cancellation is what interrupted the read and not damage to the archive. +func archiveReadError(ctx context.Context, stage string, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactIntegrityFailed, stage, err) +} + +func applyTarHeader(ctx context.Context, t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { + name := strings.TrimSuffix(path.Clean(header.Name), "/") + switch header.Typeflag { + case tar.TypeDir: + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberDir, size: header.Size}) + case tar.TypeSymlink: + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberLink, target: header.Linkname}) + case tar.TypeReg: + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberFile, size: header.Size, body: tr}) + default: + return fmt.Errorf("%w: archive member %q has unsupported type %q (hard links, devices, and special files are rejected)", errUtils.ErrArtifactPackagingInvalid, header.Name, header.Typeflag) + } +} + +func fillZip(ctx context.Context, t *tree, payload *artifact.ContentStream, scratch string, checker *managedChecker) error { + spool, err := os.CreateTemp(scratch, "archive-*.zip") + if err != nil { + return fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer os.Remove(spool.Name()) + defer spool.Close() + size, err := spoolBounded(ctx, spool, payload) + if err != nil { + return err + } + zr, err := zip.NewReader(spool, size) + if err != nil { + return fmt.Errorf("%w: zip: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + for _, file := range zr.File { + if err := ctx.Err(); err != nil { + return err + } + if err := applyZipFile(ctx, t, file, checker); err != nil { + return err + } + } + return nil +} + +// spoolBounded copies the payload into the spool file without exceeding the size declared by its +// descriptor, so a padded or oversized stream cannot fill the scratch disk. +func spoolBounded(ctx context.Context, spool io.Writer, payload *artifact.ContentStream) (int64, error) { + limit := max(payload.Descriptor.Size, 0) + // Read one byte past the declared size so an oversized stream is detected without spooling it all. + probe := limit + if probe < math.MaxInt64 { + probe++ + } + size, err := copyContext(ctx, spool, io.LimitReader(payload.Reader, probe)) + if err != nil { + return size, fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + if size > limit { + return size, fmt.Errorf("%w: spool archive: the payload is longer than the declared %d bytes", errUtils.ErrArtifactIntegrityFailed, limit) + } + return size, nil +} + +func applyZipFile(ctx context.Context, t *tree, file *zip.File, checker *managedChecker) error { + name := strings.TrimSuffix(path.Clean(file.Name), "/") + mode := file.Mode() + switch { + case mode.IsDir(): + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberDir}) + case mode&fs.ModeSymlink != 0: + target, err := readZipText(file) + if err != nil { + return err + } + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberLink, target: target}) + case mode.IsRegular(): + rc, err := file.Open() + if err != nil { + return fmt.Errorf("%w: zip member %q: %w", errUtils.ErrArtifactIntegrityFailed, file.Name, err) + } + defer rc.Close() + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberFile, size: file.FileInfo().Size(), body: rc}) + default: + return fmt.Errorf("%w: archive member %q has an unsupported type", errUtils.ErrArtifactPackagingInvalid, file.Name) + } +} + +func readZipText(file *zip.File) (string, error) { + rc, err := file.Open() + if err != nil { + return "", fmt.Errorf("%w: zip member %q: %w", errUtils.ErrArtifactIntegrityFailed, file.Name, err) + } + defer rc.Close() + data, err := artifact.ReadBounded(rc, maxLinkTargetBytes) + if err != nil { + return "", err + } + return string(data), nil +} + +type memberKind int + +const ( + memberFile memberKind = iota + memberDir + memberLink +) + +// archiveMember is one decoded archive member. +type archiveMember struct { + name string + kind memberKind + target string + size int64 + body io.Reader +} + +// applyMember validates one archive member against the inventory (or the managed directory) and +// writes it through the tree. The member body is read through the context, so a cancellation stops +// a large member mid-copy instead of after it. +func applyMember(ctx context.Context, t *tree, checker *managedChecker, m *archiveMember) error { + if err := validateMemberName(m.name); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if m.body != nil { + m.body = contextReader{ctx: ctx, reader: m.body} + } + if checker.owns(m.name, t) { + return checker.check(m) + } + entry, err := t.entryFor(m.name) + if err != nil { + return err + } + if !m.matches(entry) { + return fmt.Errorf("%w: archive member %q does not match its inventory entry (%s)", errUtils.ErrArtifactIntegrityFailed, m.name, entry.Type) + } + switch entry.Type { + case artifact.EntryDirectory: + return t.mkdir(entry) + case artifact.EntrySymlink: + return t.symlink(entry) + default: + return t.writeFile(entry, m.body) + } +} + +// matches reports whether the member has the kind, link target, and size its inventory entry declares. +func (m *archiveMember) matches(entry *artifact.Entry) bool { + switch entry.Type { + case artifact.EntryDirectory: + return m.kind == memberDir + case artifact.EntrySymlink: + return m.kind == memberLink && m.target == entry.Target + case artifact.EntryFile: + return m.kind == memberFile && m.size == entry.Size + default: + return false + } +} + +// validateMemberName rejects traversal, absolute, backslash, and empty names. Managed names pass +// through the same check, so even the reserved directory cannot be used to escape. +func validateMemberName(name string) error { + if name == "." || name == "" || name == ".." || strings.HasPrefix(name, "../") || filepath.IsAbs(name) { + return fmt.Errorf("%w: archive member %q escapes the destination", errUtils.ErrArtifactUnsafePath, name) + } + return artifact.ValidatePayloadPath(name) +} + +// managedChecker verifies the generated distribution members of a packaged representation +// (manifest.json plus embedded documents) without ever writing them to the user's tree. +type managedChecker struct { + manifest digest.Digest + documents map[string]artifact.Document + seen map[string]bool +} + +func newManagedChecker(record *artifact.Record) (*managedChecker, error) { + canonical, err := record.Canonical() + if err != nil { + return nil, err + } + checker := &managedChecker{manifest: digest.FromBytes(canonical), documents: map[string]artifact.Document{}, seen: map[string]bool{}} + for _, doc := range record.Documents { + checker.documents[managedDir+"/"+strings.ToUpper(string(doc.Role))+".md"] = doc + } + return checker, nil +} + +// owns reports whether name belongs to the managed directory and the inventory does not claim it. +func (m *managedChecker) owns(name string, t *tree) bool { + if _, claimed := t.inventory[name]; claimed { + return false + } + return name == managedParent || name == managedDir || strings.HasPrefix(name, managedDir+"/") +} + +// check verifies one managed member without ever writing it into the user's tree. +func (m *managedChecker) check(member *archiveMember) error { + name := member.name + if name == managedParent || name == managedDir { + if member.kind != memberDir || m.seen[name] { + return fmt.Errorf("%w: %q must be a unique directory", errUtils.ErrArtifactPackagingInvalid, name) + } + m.seen[name] = true + return nil + } + want, ok := m.expected(name) + if member.kind != memberFile || m.seen[name] || !ok { + return fmt.Errorf("%w: unexpected managed member %q", errUtils.ErrArtifactPackagingInvalid, name) + } + m.seen[name] = true + return verifyManagedFile(member, want) +} + +// verifyManagedFile reads a managed file within its size bound and compares it to the Record. +func verifyManagedFile(member *archiveMember, want digest.Digest) error { + limit := artifact.MaxDocumentSize + if member.name == managedDir+"/"+managedManifest { + limit = artifact.MaxRecordSize + } + if member.size > limit { + return fmt.Errorf("%w: managed member %q is %d bytes", errUtils.ErrArtifactTooLarge, member.name, member.size) + } + data, err := artifact.ReadBounded(member.body, limit) + if err != nil { + return err + } + if digest.FromBytes(data) != want { + return fmt.Errorf("%w: embedded %q does not match the Record", errUtils.ErrArtifactIntegrityFailed, member.name) + } + return nil +} + +func (m *managedChecker) expected(name string) (digest.Digest, bool) { + if name == managedDir+"/"+managedManifest { + return m.manifest, true + } + doc, ok := m.documents[name] + return doc.Digest, ok +} + +// copyContext copies src to dst in chunks and stops as soon as ctx is canceled. +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, copyChunkSize) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + n, readErr := src.Read(buffer) + if n > 0 { + written, writeErr := dst.Write(buffer[:n]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} diff --git a/pkg/artifact/materialize/archive_internal_test.go b/pkg/artifact/materialize/archive_internal_test.go new file mode 100644 index 00000000000..d510b98800d --- /dev/null +++ b/pkg/artifact/materialize/archive_internal_test.go @@ -0,0 +1,264 @@ +package materialize + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "math/rand/v2" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +const ( + internalBigBytes = 2 << 20 + internalMaxAfter = 256 << 10 +) + +// cancelOnBytesReader cancels once it served cancelAfter bytes and counts the bytes served after. +type cancelOnBytesReader struct { + data []byte + cancel context.CancelFunc + cancelAfter int + + served int + afterCanceled int + canceled bool +} + +func (c *cancelOnBytesReader) Read(p []byte) (int, error) { + if len(c.data) == 0 { + return 0, io.EOF + } + n := copy(p, c.data) + c.data = c.data[n:] + c.served += n + if c.canceled { + c.afterCanceled += n + } + if !c.canceled && c.served >= c.cancelAfter { + c.canceled = true + c.cancel() + } + return n, nil +} + +func noise(size int) []byte { + data := make([]byte, size) + random := rand.New(rand.NewPCG(3, 4)) + for index := range data { + data[index] = byte(random.Uint32()) + } + return data +} + +// newTestTree returns a tree over a temporary root whose inventory holds one file entry. +func newTestTree(t *testing.T, name string, body []byte) (*tree, string) { + t.Helper() + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + record := &artifact.Record{Inventory: []artifact.Entry{{Path: name, Type: artifact.EntryFile, Size: int64(len(body)), Mode: 0o644, Digest: digest.FromBytes(body)}}} + return newTree(root, record), dir +} + +func TestApplyMember_StopsInProgressCopyWhenCanceled(t *testing.T) { + body := noise(internalBigBytes) + tests := []struct { + name string + preCanceled bool + }{ + {"pre-canceled", true}, + {"canceled mid-copy", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr, _ := newTestTree(t, "big.bin", body) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tt.preCanceled { + cancel() + } + reader := &cancelOnBytesReader{data: body, cancel: cancel, cancelAfter: 64 << 10} + err := applyMember(ctx, tr, &managedChecker{}, &archiveMember{name: "big.bin", kind: memberFile, size: int64(len(body)), body: reader}) + require.ErrorIs(t, err, context.Canceled) + assert.Less(t, reader.afterCanceled, internalMaxAfter, "the copy stops soon after the cancellation") + assert.Less(t, reader.served, len(body), "the member is not read to the end") + }) + } +} + +func TestApplyZipFile_StopsInflatingWhenCanceled(t *testing.T) { + body := noise(internalBigBytes) + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("big.bin") + require.NoError(t, err) + _, err = w.Write(body) + require.NoError(t, err) + require.NoError(t, zw.Close()) + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, zr.File, 1) + + tr, dir := newTestTree(t, "big.bin", body) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = applyZipFile(ctx, tr, zr.File[0], &managedChecker{}) + require.ErrorIs(t, err, context.Canceled) + info, statErr := os.Stat(filepath.Join(dir, "big.bin")) + if statErr == nil { + assert.Zero(t, info.Size(), "a canceled member writes no bytes") + } +} + +// paxRecord encodes one PAX extended header record, whose leading decimal counts its own digits. +func paxRecord(key, value string) string { + body := " " + key + "=" + value + "\n" + length := len(body) + 1 + for len(fmt.Sprint(length))+len(body) != length { + length = len(fmt.Sprint(length)) + len(body) + } + return fmt.Sprintf("%d%s", length, body) +} + +// rawPaxHeader hand-encodes a local PAX extended header, which archive/tar refuses to write itself. +func rawPaxHeader(records string) []byte { + const block = 512 + header := make([]byte, block) + copy(header[0:], "pax") + copy(header[100:], "0000644\x00") + copy(header[108:], "0000000\x00") + copy(header[116:], "0000000\x00") + copy(header[124:], fmt.Sprintf("%011o\x00", len(records))) + copy(header[136:], "00000000000\x00") + copy(header[148:], " ") + header[156] = tar.TypeXHeader + copy(header[257:], "ustar\x0000") + sum := 0 + for _, b := range header { + sum += int(b) + } + copy(header[148:], fmt.Sprintf("%06o\x00 ", sum)) + padded := make([]byte, (len(records)+block-1)/block*block) + copy(padded, records) + return append(header, padded...) +} + +// successivePaxTar returns a tar stream of count local PAX headers in a row that are followed by one +// regular file. Reading the headers is metadata work that precedes any visible member. +func successivePaxTar(t *testing.T, count int, name string, body []byte) []byte { + t.Helper() + var buf bytes.Buffer + for index := range count { + buf.Write(rawPaxHeader(paxRecord("comment", fmt.Sprintf("header-%d", index)))) + } + writer := tar.NewWriter(&buf) + require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: name, Size: int64(len(body)), Mode: 0o644})) + _, err := writer.Write(body) + require.NoError(t, err) + require.NoError(t, writer.Close()) + return buf.Bytes() +} + +func TestFillTar_CancellationInterruptsMetadataReads(t *testing.T) { + body := []byte("payload") + const ( + headers = 2000 + cancelAfter = 8 << 10 + // One block of read-ahead is the most a reader may serve after the cancellation. + maxAfterCanceled = 4 << 10 + ) + tests := []struct { + name string + preCanceled bool + }{ + {"canceled before the first header", true}, + {"canceled between successive headers", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data := successivePaxTar(t, headers, "file.txt", body) + require.Greater(t, len(data), 100*cancelAfter, "the headers must outlast the cancellation point") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader := &cancelOnBytesReader{data: data, cancel: cancel, cancelAfter: cancelAfter} + if tc.preCanceled { + cancel() + } + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + + err := fillTar(ctx, target, reader, false, checker) + + require.ErrorIs(t, err, context.Canceled) + assert.False(t, errors.Is(err, errUtils.ErrArtifactIntegrityFailed), "cancellation is not an integrity failure") + assert.LessOrEqual(t, reader.afterCanceled, maxAfterCanceled, "metadata reads must stop once the context is canceled") + }) + } +} + +// A gzip trailer that fails its checksum is only noticed while the tail of the stream is drained after +// the last tar member; it must still be reported as an integrity failure, not as a bare gzip error. +func TestFillTar_ClassifiesACorruptGzipTrailer(t *testing.T) { + body := []byte("payload") + var archive bytes.Buffer + gz := gzip.NewWriter(&archive) + tw := tar.NewWriter(gz) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file.txt", Typeflag: tar.TypeReg, Size: int64(len(body)), Mode: 0o644})) + _, err := tw.Write(body) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + data := archive.Bytes() + data[len(data)-8] ^= 0xff // The CRC32 in the gzip trailer. + + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + err = fillTar(t.Context(), target, bytes.NewReader(data), true, checker) + + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +// The context never cancels here: the metadata budget alone must stop a run of PAX headers that +// outgrows it, before Next returns a visible member. +func TestFillTar_BoundsSuccessiveMetadataHeaders(t *testing.T) { + body := []byte("payload") + tests := []struct { + name string + headers int + wantErr bool + }{ + {"headers within the budget", 100, false}, + {"headers beyond the budget", 2000, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data := successivePaxTar(t, tc.headers, "file.txt", body) + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + + err := fillTar(t.Context(), target, bytes.NewReader(data), false, checker) + + if !tc.wantErr { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + assert.NotErrorIs(t, err, context.Canceled) + }) + } +} diff --git a/pkg/artifact/materialize/cancel_test.go b/pkg/artifact/materialize/cancel_test.go new file mode 100644 index 00000000000..b12a550a137 --- /dev/null +++ b/pkg/artifact/materialize/cancel_test.go @@ -0,0 +1,139 @@ +package materialize_test + +import ( + "context" + "io" + "math/rand/v2" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" +) + +// The sizes below make the tests meaningful: the member is far larger than any read buffer, and +// the cancellation arrives long before the end of it. +const ( + bigMemberBytes = 4 << 20 + cancelAfterBytes = 64 << 10 + maxReadsAfterCancel = 256 << 10 +) + +// cancelingReader cancels a context once it served cancelAfter bytes and counts what it serves +// afterwards. When atEOF is set it cancels at the end of the stream instead. +type cancelingReader struct { + inner io.ReadCloser + cancel context.CancelFunc + cancelAfter int + atEOF bool + + served int + servedCancel int + afterCanceled int + canceled bool +} + +func (c *cancelingReader) Read(p []byte) (int, error) { + n, err := c.inner.Read(p) + c.served += n + if c.canceled { + c.afterCanceled += n + } + if !c.canceled && ((!c.atEOF && c.served >= c.cancelAfter) || (c.atEOF && err == io.EOF)) { + c.canceled = true + c.servedCancel = c.served + c.cancel() + } + return n, err +} + +func (c *cancelingReader) Close() error { return c.inner.Close() } + +// noisyFile writes an incompressible file so the packaged payload stays large for every packaging. +func noisyFile(t *testing.T, src string) { + t.Helper() + data := make([]byte, bigMemberBytes) + random := rand.New(rand.NewPCG(1, 2)) + for index := range data { + data[index] = byte(random.Uint32()) + } + fixtures.WriteFile(t, src, "big.bin", string(data), 0o644) +} + +func payloadStream(t *testing.T, content *artifact.Content) *artifact.ContentStream { + t.Helper() + for index := range content.Streams { + if content.Streams[index].Role == artifact.BlobPayload { + return &content.Streams[index] + } + } + require.FailNow(t, "the fixture must have a payload stream") + return nil +} + +func TestMaterialize_CancellationStopsInProgressArchiveMemberCopy(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + noisyFile(t, src) + content := contentOf(t, prepared(t, src, packaging)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := payloadStream(t, content) + reader := &cancelingReader{inner: stream.Reader, cancel: cancel, cancelAfter: cancelAfterBytes} + stream.Reader = reader + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + require.True(t, reader.canceled, "the fixture must reach the cancellation point") + assert.Less(t, reader.afterCanceled, maxReadsAfterCancel, "reading must stop soon after the cancellation, not drain the member") + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr), "a canceled extraction installs nothing") + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_CancellationAtEndOfPayloadNeverInstalls(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + content := contentOf(t, prepared(t, src, packaging)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := payloadStream(t, content) + reader := &cancelingReader{inner: stream.Reader, cancel: cancel, atEOF: true} + stream.Reader = reader + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.True(t, reader.canceled, "the fixture must reach the end of the payload") + require.ErrorIs(t, err, context.Canceled, "a context canceled while the last member finished must not install") + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_PreCanceledZipWritesNothing(t *testing.T) { + // A zip member is inflated from the local spool; the in-package test covers the cancellation + // arriving mid-member, this one the whole pipeline with a context that is already canceled. + src := t.TempDir() + noisyFile(t, src) + content := contentOf(t, prepared(t, src, artifact.PackagingZip)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + _, statErr := os.Lstat(filepath.Join(out, "big.bin")) + assert.True(t, os.IsNotExist(statErr)) +} diff --git a/pkg/artifact/materialize/doc.go b/pkg/artifact/materialize/doc.go new file mode 100644 index 00000000000..e59c22b41d2 --- /dev/null +++ b/pkg/artifact/materialize/doc.go @@ -0,0 +1,10 @@ +// Package materialize extracts verified artifact content into a local directory. +// +// Extraction always happens in a private staging directory next to the destination, written through +// an os.Root so no entry can land outside it. Every path is validated against the Record +// inventory: traversal, entries absent from the inventory, escaping or chained symlinks, special +// members, size or digest mismatches, and oversize content are explicit errors. A newly pulled tree +// is never merged over old files: replacing an existing destination deletes the old children +// inside the destination root only and then moves the staged tree in. Ordinary permission bits +// (including executable bits) are restored exactly; owner, group, and ACLs never are. +package materialize diff --git a/pkg/artifact/materialize/exchange_darwin.go b/pkg/artifact/materialize/exchange_darwin.go new file mode 100644 index 00000000000..2aa8dbe5f3e --- /dev/null +++ b/pkg/artifact/materialize/exchange_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package materialize + +import ( + "os" + "runtime" + + "golang.org/x/sys/unix" +) + +func exchangeDirectories(stageParent *os.File, stage string, destParent *os.File, dest string) error { + err := unix.RenameatxNp(int(stageParent.Fd()), stage, int(destParent.Fd()), dest, unix.RENAME_SWAP) + runtime.KeepAlive(stageParent) + runtime.KeepAlive(destParent) + return err +} + +func renameExclusive(from, to string) error { + return unix.RenameatxNp(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_EXCL) +} diff --git a/pkg/artifact/materialize/exchange_linux.go b/pkg/artifact/materialize/exchange_linux.go new file mode 100644 index 00000000000..deb64c6bc26 --- /dev/null +++ b/pkg/artifact/materialize/exchange_linux.go @@ -0,0 +1,21 @@ +//go:build linux + +package materialize + +import ( + "os" + "runtime" + + "golang.org/x/sys/unix" +) + +func exchangeDirectories(stageParent *os.File, stage string, destParent *os.File, dest string) error { + err := unix.Renameat2(int(stageParent.Fd()), stage, int(destParent.Fd()), dest, unix.RENAME_EXCHANGE) + runtime.KeepAlive(stageParent) + runtime.KeepAlive(destParent) + return err +} + +func renameExclusive(from, to string) error { + return unix.Renameat2(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_NOREPLACE) +} diff --git a/pkg/artifact/materialize/exchange_other.go b/pkg/artifact/materialize/exchange_other.go new file mode 100644 index 00000000000..59a29432eb7 --- /dev/null +++ b/pkg/artifact/materialize/exchange_other.go @@ -0,0 +1,18 @@ +//go:build !linux && !darwin && !windows + +package materialize + +import ( + "fmt" + "os" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func exchangeDirectories(_ *os.File, _ string, _ *os.File, _ string) error { + return fmt.Errorf("%w: atomic directory replacement is unavailable on this platform", errUtils.ErrArtifactUnsupportedCapability) +} + +func renameExclusive(_, _ string) error { + return fmt.Errorf("%w: exclusive directory install is unavailable on this platform", errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/materialize/exchange_windows.go b/pkg/artifact/materialize/exchange_windows.go new file mode 100644 index 00000000000..9bfcac6df81 --- /dev/null +++ b/pkg/artifact/materialize/exchange_windows.go @@ -0,0 +1,39 @@ +//go:build windows + +package materialize + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/windows" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func exchangeDirectories(_ *os.File, _ string, _ *os.File, _ string) error { + return fmt.Errorf("%w: atomic directory replacement is unavailable on Windows", errUtils.ErrArtifactUnsupportedCapability) +} + +func renameExclusive(from, to string) error { + source, err := moveFilePath(from) + if err != nil { + return err + } + target, err := moveFilePath(to) + if err != nil { + return err + } + return windows.MoveFile(source, target) +} + +// moveFilePath returns the UTF-16 form of path as MoveFile accepts it: an absolute path beyond MAX_PATH +// gets the extended-length prefix, and a short path keeps its value. +func moveFilePath(path string) (*uint16, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return nil, err + } + return windows.UTF16PtrFromString(extendedLengthPath(absolute)) +} diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go new file mode 100644 index 00000000000..ee72be405d2 --- /dev/null +++ b/pkg/artifact/materialize/extract.go @@ -0,0 +1,216 @@ +package materialize + +import ( + "context" + "fmt" + "io" + "os" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultMaxBytes bounds the total declared size of extracted file content. +const DefaultMaxBytes int64 = 64 << 30 + +// Option customizes extraction. +type Option func(*options) + +type options struct { + maxBytes int64 + + // replaceGuard runs immediately before an existing destination is replaced. + replaceGuard func(dest *os.Root) error + + // ownerAccess keeps owner read and traversal access on the extracted tree. + ownerAccess bool +} + +// WithOwnerAccess keeps owner read access on files and owner read and traversal access on +// directories, whatever modes the Record declares. It is for a temporary tree that is read back +// before it is discarded: a valid Record can declare mode 0000, and an ordinary user could not read +// such a tree. The Record itself is never changed. +func WithOwnerAccess() Option { + defer perf.Track(nil, "materialize.WithOwnerAccess")() + + return func(o *options) { o.ownerAccess = true } +} + +// WithReplaceGuard registers a check that runs right before an existing destination directory is +// replaced, after extraction finished and the destination was bound. A caller uses it to re-check +// the destination state it decided on earlier, because the destination may have changed during the +// download. The guard must inspect through the supplied root, never reopen its Name. +// The caller must not close the root. A non-nil error aborts the replacement and leaves the destination untouched. +func WithReplaceGuard(guard func(dest *os.Root) error) Option { + defer perf.Track(nil, "materialize.WithReplaceGuard")() + + return func(o *options) { o.replaceGuard = guard } +} + +// WithMaxBytes overrides the limit on total extracted file content. +func WithMaxBytes(limit int64) Option { + defer perf.Track(nil, "materialize.WithMaxBytes")() + + return func(o *options) { o.maxBytes = limit } +} + +func applyOptions(opts []Option) options { + applied := options{maxBytes: DefaultMaxBytes} + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.maxBytes <= 0 { + applied.maxBytes = DefaultMaxBytes + } + return applied +} + +// Extract writes content into the empty directory stage, verifying every member against the +// Record inventory, checking that no symlink resolves outside stage, and applying final modes +// last. The scratch directory is private and holds temporary spool files. Extract does not close +// content. +func Extract(ctx context.Context, content *artifact.Content, stage, scratch string, opts ...Option) error { + defer perf.Track(nil, "materialize.Extract")() + + applied := applyOptions(opts) + if content == nil || content.Record == nil { + return artifact.NewUnsupportedCapability("", "pull", "materialize/graph", "a native OCI graph without a portable Record cannot be materialized as a file tree") + } + record := content.Record + if err := record.Validate(); err != nil { + return err + } + if err := checkTotalSize(record, applied.maxBytes); err != nil { + return err + } + root, err := os.OpenRoot(stage) + if err != nil { + return fmt.Errorf("%w: open staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer root.Close() + + t := newTree(root, record) + t.ownerAccess = applied.ownerAccess + if err := fill(ctx, t, content, scratch); err != nil { + return err + } + // The last member may have finished after a cancellation; never report that as success. + if err := ctx.Err(); err != nil { + return err + } + if err := t.createLinks(ctx); err != nil { + return err + } + if err := checkLinkContainment(stage, record); err != nil { + return err + } + return t.complete() +} + +func checkTotalSize(record *artifact.Record, limit int64) error { + var total int64 + for index := range record.Inventory { + total += record.Inventory[index].Size + if total > limit || total < 0 { + return fmt.Errorf("%w: content exceeds the %d byte extraction limit", errUtils.ErrArtifactTooLarge, limit) + } + } + return nil +} + +// fill dispatches on the packaging mode. +func fill(ctx context.Context, t *tree, content *artifact.Content, scratch string) error { + switch content.Representation.Packaging { + case artifact.PackagingNone: + return fillFiles(ctx, t, content) + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return fillArchive(ctx, t, content, scratch) + default: + return fmt.Errorf("%w: cannot materialize packaging %q", errUtils.ErrArtifactPackagingInvalid, content.Representation.Packaging) + } +} + +// fillFiles writes a per-file payload: file entries come from their path-addressed streams; +// directories and symlinks come from the inventory. +func fillFiles(ctx context.Context, t *tree, content *artifact.Content) error { + streams, err := payloadStreams(t, content) + if err != nil { + return err + } + paths := make([]string, 0, len(t.inventory)) + for entryPath := range t.inventory { + paths = append(paths, entryPath) + } + sort.Strings(paths) + for _, entryPath := range paths { + if err := ctx.Err(); err != nil { + return err + } + if err := fillInventoryEntry(ctx, t, t.inventory[entryPath], streams); err != nil { + return err + } + delete(streams, entryPath) + } + for extra := range streams { + return fmt.Errorf("%w: payload %q is not in the artifact inventory", errUtils.ErrArtifactIntegrityFailed, extra) + } + return nil +} + +// payloadStreams indexes the payload streams by inventory path. A stream may address only a file +// entry, because directories and symlinks come from the inventory and a stream for them would +// never be read or verified. +func payloadStreams(t *tree, content *artifact.Content) (map[string]*artifact.ContentStream, error) { + streams := make(map[string]*artifact.ContentStream, len(content.Streams)) + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if _, duplicate := streams[stream.Path]; duplicate { + return nil, fmt.Errorf("%w: payload %q appears more than once", errUtils.ErrArtifactIntegrityFailed, stream.Path) + } + if entry, ok := t.inventory[stream.Path]; ok && entry.Type != artifact.EntryFile { + return nil, fmt.Errorf("%w: payload %q does not address a file entry", errUtils.ErrArtifactIntegrityFailed, stream.Path) + } + streams[stream.Path] = stream + } + return streams, nil +} + +// contextReader stops an in-progress copy as soon as the context is canceled. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (c contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "materialize.contextReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.reader.Read(p) +} + +func fillInventoryEntry(ctx context.Context, t *tree, entry *artifact.Entry, streams map[string]*artifact.ContentStream) error { + if _, err := t.entryFor(entry.Path); err != nil { + return err + } + switch entry.Type { + case artifact.EntryDirectory: + return t.mkdir(entry) + case artifact.EntrySymlink: + return t.symlink(entry) + default: + stream, ok := streams[entry.Path] + if !ok || stream.Reader == nil { + return fmt.Errorf("%w: payload for %q is missing", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + return t.writeFile(entry, contextReader{ctx: ctx, reader: stream.Reader}) + } +} diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go new file mode 100644 index 00000000000..57357e7b2bb --- /dev/null +++ b/pkg/artifact/materialize/links.go @@ -0,0 +1,68 @@ +package materialize + +import ( + "errors" + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" +) + +// checkLinkContainment resolves every symlink physically and rejects any that leaves root. A +// lexical check of each target is not enough, because a link chain can clean to a contained path +// yet resolve outside it. A dangling link is checked through its longest existing ancestor plus the +// missing remainder, so it cannot escape once the missing path is created. +func checkLinkContainment(stage string, record *artifact.Record) error { + realRoot, err := filepath.EvalSymlinks(stage) + if err != nil { + return fmt.Errorf("%w: resolve staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + recorded := recordedLinks(record) + for index := range record.Inventory { + entry := &record.Inventory[index] + if entry.Type != artifact.EntrySymlink { + continue + } + if err := checkLinkEntry(stage, realRoot, recorded, entry); err != nil { + return err + } + } + return nil +} + +// recordedLinks maps each recorded symlink path to its recorded target. +func recordedLinks(record *artifact.Record) map[string]string { + recorded := make(map[string]string) + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type == artifact.EntrySymlink { + recorded[entry.Path] = entry.Target + } + } + return recorded +} + +// checkLinkEntry decides whether one staged symlink stays inside the package root. +func checkLinkEntry(stage, realRoot string, recorded map[string]string, entry *artifact.Entry) error { + resolved, err := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) + switch { + case errors.Is(err, errUtils.ErrSymlinkLoop): + // A loop never reaches a path outside the root, so it is not an escape. + return nil + case errors.Is(err, errUtils.ErrPathThroughNonDirectory): + // The target runs through a regular file, so it can never resolve to anything. Capture accepts + // such a link when its recorded path stays inside the tree, and so does materialization: the + // same lexical check, over the recorded links, decides. + if linkpath.Escapes(recorded, entry.Path) { + return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) + } + return nil + case err != nil: + // Any other failure leaves the target unknown, and an unknown target cannot be shown to stay inside. + return fmt.Errorf("%w: symlink %q cannot be resolved: %w", errUtils.ErrArtifactUnsafePath, entry.Path, err) + case !linkpath.Within(realRoot, resolved): + return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) + } + return nil +} diff --git a/pkg/artifact/materialize/longpath.go b/pkg/artifact/materialize/longpath.go new file mode 100644 index 00000000000..876eb9beaab --- /dev/null +++ b/pkg/artifact/materialize/longpath.go @@ -0,0 +1,21 @@ +package materialize + +import "strings" + +// maxPathChars is the Windows MAX_PATH limit, which counts the terminating NUL. +const maxPathChars = 260 + +// extendedLengthPath converts a long absolute Windows path to the extended-length form that APIs such +// as MoveFile accept beyond MAX_PATH. A path shorter than the limit, and one that already has an +// extended-length prefix, is returned unchanged. A UNC path (\\server\share) becomes \\?\UNC\server\share. +// The path must already be absolute and use backslashes: the prefix switches off the normalization +// that would otherwise convert slashes and resolve dot segments. +func extendedLengthPath(absolute string) string { + if len(absolute) < maxPathChars || strings.HasPrefix(absolute, `\\?\`) { + return absolute + } + if strings.HasPrefix(absolute, `\\`) { + return `\\?\UNC\` + absolute[2:] + } + return `\\?\` + absolute +} diff --git a/pkg/artifact/materialize/longpath_internal_test.go b/pkg/artifact/materialize/longpath_internal_test.go new file mode 100644 index 00000000000..f293ef3dd7b --- /dev/null +++ b/pkg/artifact/materialize/longpath_internal_test.go @@ -0,0 +1,31 @@ +package materialize + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtendedLengthPath(t *testing.T) { + long := `C:\` + strings.Repeat(`a\`, 130) + "dir" + longUNC := `\\server\share\` + strings.Repeat(`a\`, 130) + "dir" + tests := []struct { + name string + in string + want string + }{ + {"short drive path is unchanged", `C:\tmp\out`, `C:\tmp\out`}, + {"short UNC path is unchanged", `\\server\share\out`, `\\server\share\out`}, + {"long drive path gets the prefix", long, `\\?\` + long}, + {"long UNC path gets the UNC prefix", longUNC, `\\?\UNC\` + longUNC[2:]}, + {"an extended-length path is unchanged", `\\?\` + long, `\\?\` + long}, + {"the boundary length is converted", `C:\` + strings.Repeat("b", maxPathChars-3), `\\?\C:\` + strings.Repeat("b", maxPathChars-3)}, + {"one below the boundary is unchanged", `C:\` + strings.Repeat("b", maxPathChars-4), `C:\` + strings.Repeat("b", maxPathChars-4)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, extendedLengthPath(tt.in)) + }) + } +} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go new file mode 100644 index 00000000000..6364383b0d5 --- /dev/null +++ b/pkg/artifact/materialize/materialize_test.go @@ -0,0 +1,861 @@ +package materialize_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +var fixedTime = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +func prepared(t *testing.T, src string, packaging artifact.Packaging) *artifact.PreparedArtifact { + t.Helper() + p, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: "pkg/example", Version: "1.0.0", Packaging: packaging, OperationID: "op", PreparedAt: fixedTime, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = p.Close() }) + return p +} + +// contentOf converts staged blobs into the streams a repository would return. +func contentOf(t *testing.T, p *artifact.PreparedArtifact) *artifact.Content { + t.Helper() + record := p.Record + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: p.Packaging}} + for index := range p.Blobs { + blob := &p.Blobs[index] + reader, err := blob.Open(context.Background()) + require.NoError(t, err) + content.Streams = append(content.Streams, artifact.ContentStream{Descriptor: blob.Descriptor, Role: blob.Role, Path: blob.Path, Reader: reader}) + } + t.Cleanup(func() { _ = content.Close() }) + return content +} + +// replaceStream swaps a stream's bytes after closing the file the repository opened for it, so the +// replaced handle never outlives the test (an open handle blocks temp-dir removal on Windows). +func replaceStream(c *artifact.Content, index int, data []byte) { + _ = c.Streams[index].Reader.Close() + c.Streams[index].Reader = io.NopCloser(bytes.NewReader(data)) +} + +func dest(t *testing.T) string { + return filepath.Join(t.TempDir(), "out", "pulled") +} + +func assertNoScratch(t *testing.T, destination string) { + t.Helper() + items, err := os.ReadDir(filepath.Dir(destination)) + require.NoError(t, err) + for _, item := range items { + assert.NotContains(t, item.Name(), ".atmos-pull-", "staging must be removed") + } +} + +func TestMaterialize_RoundTripEveryPackaging(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + p := prepared(t, src, packaging) + out := dest(t) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + assertNoScratch(t, out) + + state, err := materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + + data, err := os.ReadFile(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, fixtures.HelperScript, string(data)) + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "executable bit survives") + } + _, err = os.Stat(filepath.Join(out, "emptydir")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(out, fixtures.UnicodeName)) + require.NoError(t, err) + _, err = os.Lstat(filepath.Join(out, ".atmos")) + assert.True(t, os.IsNotExist(err), "managed distribution members are not materialized") + if pkg.Symlinks { + target, err := os.Readlink(filepath.Join(out, "link-to-file")) + require.NoError(t, err) + assert.Equal(t, "nested/deep/file.txt", filepath.ToSlash(target)) + } + }) + } +} + +func TestMaterialize_ReplacesWithoutMergingAndStaysInsideDestination(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + + root := t.TempDir() + out := filepath.Join(root, "pulled") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(outside, 0o755)) + fixtures.WriteFile(t, outside, "keep.txt", "keep", 0o644) + fixtures.WriteFile(t, root, "sibling.txt", "sibling", 0o644) + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + fixtures.WriteFile(t, out, "nested/stale2.txt", "stale", 0o644) + linkOK := fixtures.Symlink(t, out, "link-out", "../outside") + + state, err := materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state) + + err = materialize.Materialize(context.Background(), contentOf(t, p), out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, statErr := os.Stat(filepath.Join(out, "stale.txt")) + require.NoError(t, statErr, "unsupported replacement preserves the existing destination") + assertNoScratch(t, out) + return + } + require.NoError(t, err) + items, err := os.ReadDir(out) + require.NoError(t, err) + require.Len(t, items, 1, "no stale file is merged with the new tree") + assert.Equal(t, "new.txt", items[0].Name()) + assertNoScratch(t, out) + + keep, err := os.ReadFile(filepath.Join(outside, "keep.txt")) + require.NoError(t, err, "a stale link must not let deletion escape the destination") + assert.Equal(t, "keep", string(keep)) + _, err = os.Stat(filepath.Join(root, "sibling.txt")) + require.NoError(t, err) + _ = linkOK +} + +// A destination of "." names the working directory, whose parent is not ".": the destination is made +// absolute before its parent is derived, so an empty working directory is replaced without force. +func TestMaterialize_ReplacesAnEmptyWorkingDirectoryNamedByDot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the working directory cannot be replaced while it is in use on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + work := filepath.Join(t.TempDir(), "work") + require.NoError(t, os.Mkdir(work, 0o755)) + t.Chdir(work) + + state, err := materialize.InspectDestination(".", &p.Record) + require.NoError(t, err) + require.Equal(t, materialize.DestinationEmpty, state) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), ".")) + + data, err := os.ReadFile(filepath.Join(work, "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, work) +} + +func TestMaterialize_ReplacesRestrictiveDestination(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("atomic directory replacement is unavailable on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + out := dest(t) + fixtures.WriteFile(t, out, "blocked/old.txt", "old", 0o600) + restricted := filepath.Join(out, "blocked") + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + data, err := os.ReadFile(filepath.Join(out, "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, out) +} + +// A trailing separator makes the filesystem resolve a symlink as a directory, so the destination is +// normalized before it is inspected: a symlink destination is rejected whether or not it ends in one. +func TestInspectDestination_RejectsASymlinkWithATrailingSeparator(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + root := t.TempDir() + // The target holds exactly the inventory, so following the link would report a match. + fixtures.WriteFile(t, root, "target/a.txt", "a", 0o644) + if !fixtures.Symlink(t, root, "linked", "target") { + t.Skip("this platform cannot create symlinks") + } + link := filepath.Join(root, "linked") + + for name, dest := range map[string]string{ + "plain": link, + "trailing separator": link + string(filepath.Separator), + } { + t.Run(name, func(t *testing.T) { + _, err := materialize.InspectDestination(dest, &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + }) + } +} + +func TestInspectDestination_States(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + root := t.TempDir() + + state, err := materialize.InspectDestination(filepath.Join(root, "absent"), &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationAbsent, state) + + empty := filepath.Join(root, "empty") + require.NoError(t, os.Mkdir(empty, 0o755)) + state, err = materialize.InspectDestination(empty, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationEmpty, state) + + differs := filepath.Join(root, "differs") + fixtures.WriteFile(t, differs, "a.txt", "different", 0o644) + state, err = materialize.InspectDestination(differs, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state) + + extra := filepath.Join(root, "extra") + fixtures.WriteFile(t, extra, "a.txt", "a", 0o644) + fixtures.WriteFile(t, extra, "b.txt", "b", 0o644) + state, err = materialize.InspectDestination(extra, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "extra files are a difference") + + file := filepath.Join(root, "file") + fixtures.WriteFile(t, root, "file", "x", 0o644) + _, err = materialize.InspectDestination(file, &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + + if fixtures.Symlink(t, root, "linked", "empty") { + _, err = materialize.InspectDestination(filepath.Join(root, "linked"), &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + err = materialize.Materialize(context.Background(), contentOf(t, p), filepath.Join(root, "linked")) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + } +} + +func TestMaterialize_PerFileIntegrityFailures(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bbbb", 0o644) + tests := []struct { + name string + mutate func(content *artifact.Content) + want error + }{ + {"tampered bytes", func(c *artifact.Content) { + replaceStream(c, 0, []byte("xxxx")) + }, errUtils.ErrArtifactIntegrityFailed}, + {"truncated", func(c *artifact.Content) { + replaceStream(c, 0, []byte("aa")) + }, errUtils.ErrArtifactIntegrityFailed}, + {"oversize", func(c *artifact.Content) { + replaceStream(c, 0, []byte("aaaaaaaa")) + }, errUtils.ErrArtifactTooLarge}, + {"missing payload", func(c *artifact.Content) { + _ = c.Streams[0].Reader.Close() + c.Streams = c.Streams[1:] + }, errUtils.ErrArtifactIntegrityFailed}, + {"extra payload", func(c *artifact.Content) { + c.Streams = append(c.Streams, artifact.ContentStream{Role: artifact.BlobPayload, Path: "extra.txt", Reader: io.NopCloser(bytes.NewReader(nil))}) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate payload", func(c *artifact.Content) { + c.Streams = append(c.Streams, c.Streams[0]) + }, errUtils.ErrArtifactIntegrityFailed}, + {"traversal path", func(c *artifact.Content) { + c.Streams[0].Path = "../a.txt" + }, errUtils.ErrArtifactIntegrityFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + tt.mutate(content) + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr), "a failed extraction never creates the destination") + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_GraphWithoutRecordIsUnsupported(t *testing.T) { + err := materialize.Materialize(context.Background(), &artifact.Content{}, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + err = materialize.Materialize(context.Background(), nil, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + require.ErrorIs(t, materialize.Materialize(context.Background(), &artifact.Content{}, ""), errUtils.ErrArtifactConfigInvalid) +} + +func TestMaterialize_SizeLimit(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingTgz)) + err := materialize.Materialize(context.Background(), content, dest(t), materialize.WithMaxBytes(5)) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestMaterialize_CanceledContextCleansUp(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz, artifact.PackagingZip} { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := dest(t) + err := materialize.Materialize(ctx, contentOf(t, prepared(t, src, packaging)), out) + require.ErrorIs(t, err, context.Canceled, packaging) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + } +} + +func TestMaterialize_RestrictiveDirectoryModeAppliedLast(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not enforced on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "locked/inner.txt", "x", 0o644) + require.NoError(t, os.Chmod(filepath.Join(src, "locked"), 0o555)) + t.Cleanup(func() { _ = os.Chmod(filepath.Join(src, "locked"), 0o755) }) + p := prepared(t, src, artifact.PackagingTar) + out := dest(t) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + info, err := os.Stat(filepath.Join(out, "locked")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o555), info.Mode().Perm()) + require.NoError(t, os.Chmod(filepath.Join(out, "locked"), 0o755)) +} + +// sealedRecord builds a valid Record around an inventory for crafted-content tests. +func sealedRecord(t *testing.T, inventory []artifact.Entry) artifact.Record { + t.Helper() + record := artifact.Record{ + Name: "pkg/crafted", Version: "1", Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(fixedTime), Inventory: inventory, + } + require.NoError(t, record.Seal()) + return record +} + +// unsealedRecord builds a Record around an inventory without sealing it. The record layer rejects +// inventories whose symlink targets escape the package, so such a record can never be sealed; +// this helper lets a test prove Materialize still refuses one that arrives unsealed. +func unsealedRecord(inventory []artifact.Entry) artifact.Record { + return artifact.Record{ + SchemaVersion: artifact.SchemaVersion, + Name: "pkg/crafted", Version: "1", Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(fixedTime), Inventory: inventory, + } +} + +func fileEntry(path, content string, mode uint32) artifact.Entry { + return artifact.Entry{Path: path, Type: artifact.EntryFile, Size: int64(len(content)), Mode: mode, Digest: digest.FromString(content)} +} + +type tarMember struct { + header tar.Header + body string +} + +func tarOf(t *testing.T, members []tarMember) io.ReadCloser { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for index := range members { + m := &members[index] + h := m.header + h.Size = int64(len(m.body)) + require.NoError(t, tw.WriteHeader(&h)) + _, err := tw.Write([]byte(m.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + return io.NopCloser(&buf) +} + +func tgzOf(t *testing.T, members []tarMember) io.ReadCloser { + t.Helper() + raw, err := io.ReadAll(tarOf(t, members)) + require.NoError(t, err) + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err = gz.Write(raw) + require.NoError(t, err) + require.NoError(t, gz.Close()) + return io.NopCloser(&buf) +} + +func archiveContent(record *artifact.Record, packaging artifact.Packaging, reader io.ReadCloser) *artifact.Content { + return &artifact.Content{ + Record: record, + Representation: artifact.Representation{Packaging: packaging}, + Streams: []artifact.ContentStream{{Role: artifact.BlobPayload, Reader: reader}}, + } +} + +func TestMaterialize_HostileArchives(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + good := tarMember{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "good"} + tests := []struct { + name string + members []tarMember + want error + }{ + {"traversal", []tarMember{{header: tar.Header{Name: "../evil.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"nested traversal", []tarMember{{header: tar.Header{Name: "a/../../evil.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"absolute", []tarMember{{header: tar.Header{Name: "/etc/passwd", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"not in inventory", []tarMember{good, {header: tar.Header{Name: "extra.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}}, errUtils.ErrArtifactIntegrityFailed}, + {"hard link", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeLink, Linkname: "other"}}}, errUtils.ErrArtifactPackagingInvalid}, + {"device", []tarMember{{header: tar.Header{Name: "dev", Typeflag: tar.TypeChar}}}, errUtils.ErrArtifactPackagingInvalid}, + {"wrong size", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "goodgood"}}, errUtils.ErrArtifactIntegrityFailed}, + {"wrong digest", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "evil"}}, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate", []tarMember{good, good}, errUtils.ErrArtifactIntegrityFailed}, + {"missing member", nil, errUtils.ErrArtifactIntegrityFailed}, + {"unexpected managed member", []tarMember{good, {header: tar.Header{Name: ".atmos/artifact/evil.sh", Typeflag: tar.TypeReg, Mode: 0o755}, body: "x"}}, errUtils.ErrArtifactPackagingInvalid}, + {"tampered manifest", []tarMember{good, {header: tar.Header{Name: ".atmos/artifact/manifest.json", Typeflag: tar.TypeReg, Mode: 0o644}, body: "{}"}}, errUtils.ErrArtifactIntegrityFailed}, + } + for _, tt := range tests { + for _, wrap := range []struct { + name string + packaging artifact.Packaging + build func(*testing.T, []tarMember) io.ReadCloser + }{{"tar", artifact.PackagingTar, tarOf}, {"tgz", artifact.PackagingTgz, tgzOf}} { + t.Run(tt.name+"/"+wrap.name, func(t *testing.T) { + out := dest(t) + err := materialize.Materialize(context.Background(), archiveContent(&record, wrap.packaging, wrap.build(t, tt.members)), out) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + }) + } + } +} + +func TestMaterialize_ArchiveRejectsCorruptStreams(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + for _, packaging := range []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingZip, artifact.PackagingTar} { + err := materialize.Materialize(context.Background(), archiveContent(&record, packaging, io.NopCloser(bytes.NewReader([]byte("not an archive")))), dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, packaging) + } + noPayload := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingTar}} + require.ErrorIs(t, materialize.Materialize(context.Background(), noPayload, dest(t)), errUtils.ErrArtifactIntegrityFailed) + badPackaging := archiveContent(&record, "rar", io.NopCloser(bytes.NewReader(nil))) + require.ErrorIs(t, materialize.Materialize(context.Background(), badPackaging, dest(t)), errUtils.ErrArtifactPackagingInvalid) +} + +// requireSymlinks skips the test only when the platform cannot create symlinks at all. +func requireSymlinks(t *testing.T) { + t.Helper() + probe := t.TempDir() + fixtures.WriteFile(t, probe, "target.txt", "x", 0o644) + if !fixtures.Symlink(t, probe, "link", "target.txt") { + t.Skip("this platform cannot create symlinks") + } +} + +func TestMaterialize_ChainedSymlinkEscapeIsRejected(t *testing.T) { + requireSymlinks(t) + record := unsealedRecord([]artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/b/../.."}, + }) + // The record layer rejects the escape before any content is written, so it cannot be sealed. + require.ErrorIs(t, record.Seal(), errUtils.ErrArtifactUnsafePath) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + require.Error(t, err, "a chained symlink escape must be rejected") + assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestMaterialize_DanglingChainedSymlinkEscapeIsRejected(t *testing.T) { + requireSymlinks(t) + tests := []struct { + name string + inventory []artifact.Entry + wantErr bool + }{ + { + name: "dangling tail beyond the root", + inventory: []artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/b/../../missing"}, + }, + wantErr: true, + }, + { + name: "dangling tail inside the root", + inventory: []artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/missing/deeper"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr { + // The record layer rejects the escape, so the record cannot be sealed, and + // Materialize must still refuse it when it arrives unsealed. + record := unsealedRecord(tt.inventory) + require.ErrorIs(t, record.Seal(), errUtils.ErrArtifactUnsafePath) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + require.ErrorIs(t, materialize.Materialize(context.Background(), content, dest(t)), errUtils.ErrArtifactUnsafePath) + return + } + record := sealedRecord(t, tt.inventory) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + require.NoError(t, materialize.Materialize(context.Background(), content, dest(t)), "a dangling link that stays inside the tree is allowed") + }) + } +} + +func TestMaterialize_PayloadForNonFileEntryIsRejected(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "sub/a.txt", "aaaa", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + content.Streams = append(content.Streams, artifact.ContentStream{Role: artifact.BlobPayload, Path: "sub", Reader: io.NopCloser(bytes.NewReader([]byte("smuggled")))}) + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) +} + +// endlessReader yields zero bytes forever and counts how many it produced. +type endlessReader struct{ read int64 } + +func (r *endlessReader) Read(p []byte) (int, error) { + r.read += int64(len(p)) + return len(p), nil +} + +func (r *endlessReader) Close() error { return nil } + +func TestMaterialize_ZipSpoolIsBoundedByTheDeclaredSize(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + const declared = 64 + endless := &endlessReader{} + content := archiveContent(&record, artifact.PackagingZip, endless) + content.Streams[0].Descriptor.Size = declared + err := materialize.Materialize(context.Background(), content, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.LessOrEqual(t, endless.read, int64(copyChunkBound), "the spool must stop shortly after the declared size") +} + +func TestMaterialize_ZipLongerThanDeclaredSizeIsRejected(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + p := prepared(t, src, artifact.PackagingZip) + tests := []struct { + name string + adjust int64 + wantErr bool + }{ + {"declared size matches", 0, false}, + {"stream longer than declared", -1, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content := contentOf(t, p) + for index := range content.Streams { + content.Streams[index].Descriptor.Size += tt.adjust + } + err := materialize.Materialize(context.Background(), content, dest(t)) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + return + } + require.NoError(t, err) + }) + } +} + +// copyChunkBound is the largest read the bounded spool may issue: one copy chunk. +const copyChunkBound = 32 * 1024 + +func TestMaterialize_ReplaceGuard(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + errRefused := errors.New("guard refused") + tests := []struct { + name string + guard func(dest *os.Root) error + wantErr error + wantStale bool + }{ + {"guard accepts", func(*os.Root) error { return nil }, nil, false}, + {"guard refuses", func(*os.Root) error { return errRefused }, errRefused, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := dest(t) + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(tt.guard)) + switch { + case tt.wantErr != nil: + require.ErrorIs(t, err, tt.wantErr) + case runtime.GOOS == "windows": + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + tt.wantStale = true + default: + require.NoError(t, err) + } + _, staleErr := os.Stat(filepath.Join(out, "stale.txt")) + assert.Equal(t, tt.wantStale, staleErr == nil) + assertNoScratch(t, out) + }) + } + + t.Run("guard is not consulted for an absent destination", func(t *testing.T) { + called := false + out := dest(t) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(func(*os.Root) error { + called = true + return errRefused + })) + require.NoError(t, err) + assert.False(t, called) + }) +} + +func TestMaterialize_DestinationSwappedForASymlinkNeverRedirectsRemoval(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("a directory with an open handle cannot be renamed on Windows") + } + requireSymlinks(t) + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + + root := t.TempDir() + out := filepath.Join(root, "pulled") + outside := filepath.Join(root, "outside") + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + fixtures.WriteFile(t, outside, "keep.txt", "keep", 0o644) + + swap := materialize.WithReplaceGuard(func(dest *os.Root) error { + // Simulate an attacker replacing the checked destination right after it was inspected. + require.NoError(t, os.Rename(dest.Name(), filepath.Join(root, "moved-away"))) + require.NoError(t, os.Symlink(outside, dest.Name())) + return nil + }) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + keep, readErr := os.ReadFile(filepath.Join(outside, "keep.txt")) + require.NoError(t, readErr, "the swapped-in link target must not lose files") + assert.Equal(t, "keep", string(keep)) + _, newErr := os.Stat(filepath.Join(outside, "new.txt")) + assert.True(t, os.IsNotExist(newErr), "staged content must not land in the link target") +} + +// cancelOnReadReader cancels the context on its first read and then yields one byte per read. +type cancelOnReadReader struct { + cancel context.CancelFunc + data []byte +} + +func (c *cancelOnReadReader) Read(p []byte) (int, error) { + c.cancel() + if len(c.data) == 0 { + return 0, io.EOF + } + p[0] = c.data[0] + c.data = c.data[1:] + return 1, nil +} + +func (c *cancelOnReadReader) Close() error { return nil } + +func TestMaterialize_CancellationStopsInProgressFileCopy(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "big.txt", "0123456789", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + replaced := false + for index := range content.Streams { + if content.Streams[index].Role != artifact.BlobPayload { + continue + } + _ = content.Streams[index].Reader.Close() + content.Streams[index].Reader = &cancelOnReadReader{cancel: cancel, data: []byte("0123456789")} + replaced = true + } + require.True(t, replaced, "the fixture must have a payload stream") + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) +} + +func TestMaterialize_TrailingSeparatorDestination(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + + t.Run("absent destination", func(t *testing.T) { + root := t.TempDir() + out := filepath.Join(root, "out") + string(filepath.Separator) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, prepared(t, src, artifact.PackagingTgz)), out)) + data, err := os.ReadFile(filepath.Join(root, "out", "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, filepath.Join(root, "out")) + items, err := os.ReadDir(root) + require.NoError(t, err) + require.Len(t, items, 1, "staging must be created beside the destination, not inside it") + }) + + t.Run("existing destination", func(t *testing.T) { + root := t.TempDir() + existing := filepath.Join(root, "out") + fixtures.WriteFile(t, existing, "stale.txt", "stale", 0o644) + out := existing + string(filepath.Separator) + err := materialize.Materialize(context.Background(), contentOf(t, prepared(t, src, artifact.PackagingTgz)), out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + require.NoError(t, err) + items, err := os.ReadDir(existing) + require.NoError(t, err) + require.Len(t, items, 1) + assert.Equal(t, "new.txt", items[0].Name()) + assertNoScratch(t, existing) + }) +} + +func TestInspectDestination_EmptyRecordMatchesAnEmptyDestination(t *testing.T) { + p := prepared(t, t.TempDir(), artifact.PackagingNone) + require.Empty(t, p.Record.Inventory, "the sealed Record is empty") + out := dest(t) + + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + state, err := materialize.InspectDestination(out, &p.Record) + + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "a repeat pull of an empty Record is recognized as a match") + + fixtures.WriteFile(t, out, "stray.txt", "x", 0o644) + state, err = materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "content under an empty Record is a difference") +} + +func TestInspectDestination_AllowsAnUnrecordedAncestorDirectory(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "sub/a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + record := p.Record + record.Inventory = nil + for _, entry := range p.Record.Inventory { + if entry.Type != artifact.EntryDirectory { + record.Inventory = append(record.Inventory, entry) + } + } + require.Len(t, record.Inventory, 1, "the inventory names the file but not its directory") + + root := t.TempDir() + matching := filepath.Join(root, "matching") + fixtures.WriteFile(t, matching, "sub/a.txt", "a", 0o644) + state, err := materialize.InspectDestination(matching, &record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "a directory that only holds recorded entries is not a difference") + + unrelated := filepath.Join(root, "unrelated") + fixtures.WriteFile(t, unrelated, "sub/a.txt", "a", 0o644) + require.NoError(t, os.Mkdir(filepath.Join(unrelated, "other"), 0o755)) + state, err = materialize.InspectDestination(unrelated, &record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "an unrelated extra directory is still a difference") +} + +func TestMaterialize_RepeatedManagedDirectories(t *testing.T) { + for _, name := range []string{".atmos/", ".atmos/artifact/"} { + for _, populated := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/populated=%t", name, populated), func(t *testing.T) { + var inventory []artifact.Entry + var members []tarMember + if populated { + inventory = []artifact.Entry{fileEntry("payload.txt", "payload", 0o600)} + members = append(members, tarMember{header: tar.Header{Name: "payload.txt", Typeflag: tar.TypeReg, Mode: 0o600}, body: "payload"}) + } + directory := tarMember{header: tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0o755}} + members = append(members, directory, directory) + record := sealedRecord(t, inventory) + content := archiveContent(&record, artifact.PackagingTgz, tgzOf(t, members)) + t.Cleanup(func() { _ = content.Close() }) + require.ErrorIs(t, materialize.Materialize(t.Context(), content, dest(t)), errUtils.ErrArtifactPackagingInvalid) + }) + } + } +} + +func TestMaterialize_ReplaceGuardInspectsBoundDirectoryDuringPathSwap(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("a directory with an open handle cannot be renamed on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + out := dest(t) + fixtures.WriteFile(t, out, "keep.txt", "unapproved", 0o644) + guard := materialize.WithReplaceGuard(func(root *os.Root) error { + moved := out + "-moved" + require.NoError(t, os.Rename(out, moved)) + require.NoError(t, os.Mkdir(out, 0o755)) + state, err := materialize.InspectBoundDestination(root, &p.Record) + require.NoError(t, err) + require.NoError(t, os.Remove(out)) + require.NoError(t, os.Rename(moved, out)) + assert.Equal(t, materialize.DestinationDiffers, state) + if state == materialize.DestinationDiffers { + return errUtils.ErrArtifactConflict + } + return nil + }) + require.ErrorIs(t, materialize.Materialize(t.Context(), contentOf(t, p), out, guard), errUtils.ErrArtifactConflict) + data, err := os.ReadFile(filepath.Join(out, "keep.txt")) + require.NoError(t, err) + assert.Equal(t, "unapproved", string(data)) + assertNoScratch(t, out) +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go new file mode 100644 index 00000000000..b2d3bf0eefb --- /dev/null +++ b/pkg/artifact/materialize/replace.go @@ -0,0 +1,452 @@ +package materialize + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + scratchPattern = ".atmos-pull-*" + parentDirMode = 0o755 + treeDirName = "tree" + rootDirName = "." +) + +// errMismatch stops a destination walk at the first difference. It never leaves this package. +var errMismatch = errors.New("destination differs from the artifact inventory") + +// DestinationState classifies an output directory relative to the artifact being pulled. +type DestinationState int + +// Destination states. +const ( + // DestinationAbsent means the destination does not exist. + DestinationAbsent DestinationState = iota + + // DestinationEmpty means the destination is an existing empty directory. + DestinationEmpty + + // DestinationMatches means the destination already holds exactly the artifact's inventory. + DestinationMatches + + // DestinationDiffers means the destination holds other content; replacing it requires force. + DestinationDiffers +) + +// InspectDestination classifies dest without modifying it. A symlink or non-directory +// destination is an error: a replacement must never follow a link out of the requested root. +func InspectDestination(dest string, record *artifact.Record) (DestinationState, error) { + defer perf.Track(nil, "materialize.InspectDestination")() + + // A trailing separator makes the filesystem resolve a symlink as a directory, which would let a + // matching link target pass the symlink check below. Inspect the cleaned path instead. + if dest != "" { + dest = filepath.Clean(dest) + } + info, err := os.Lstat(dest) + switch { + case os.IsNotExist(err): + return DestinationAbsent, nil + case err != nil: + return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + case info.Mode()&os.ModeSymlink != 0: + return DestinationDiffers, fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) + case !info.IsDir(): + return DestinationDiffers, fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) + } + bound, err := bindDestination(dest, info) + if err != nil { + return DestinationDiffers, err + } + defer bound.root.Close() + return InspectBoundDestination(bound.root, record) +} + +// InspectBoundDestination classifies the directory held by root without resolving its pathname. +// The caller retains ownership of root and must keep it open throughout inspection. +func InspectBoundDestination(root *os.Root, record *artifact.Record) (DestinationState, error) { + defer perf.Track(nil, "materialize.InspectBoundDestination")() + + children, err := fs.ReadDir(root.FS(), rootDirName) + if err != nil { + return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, root.Name(), err) + } + if len(children) == 0 { + if len(record.Inventory) == 0 { + return DestinationMatches, nil + } + return DestinationEmpty, nil + } + if treeMatches(root, record) { + return DestinationMatches, nil + } + return DestinationDiffers, nil +} + +// treeMatches reports whether dir holds exactly the Record inventory: the same paths, types, link +// targets, file sizes and digests, and (off Windows) permission bits. Any unreadable entry counts +// as a difference. +func treeMatches(root *os.Root, record *artifact.Record) bool { + want := make(map[string]artifact.Entry, len(record.Inventory)) + for _, entry := range record.Inventory { + want[entry.Path] = entry + } + ancestors := requiredAncestors(want) + matched := 0 + walkErr := fs.WalkDir(root.FS(), rootDirName, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == rootDirName { + return nil + } + key := p + entry, ok := want[key] + if !ok { + // A directory the inventory does not record is still required when an entry lives under it. + if d.IsDir() && ancestors[key] { + return nil + } + return errMismatch + } + if !entryMatches(root, p, d, &entry) { + return errMismatch + } + matched++ + return nil + }) + return walkErr == nil && matched == len(want) +} + +// requiredAncestors returns the directories that hold at least one inventory entry. +func requiredAncestors(want map[string]artifact.Entry) map[string]bool { + ancestors := make(map[string]bool) + for name := range want { + for dir := path.Dir(name); dir != "." && dir != "/" && !ancestors[dir]; dir = path.Dir(dir) { + ancestors[dir] = true + } + } + return ancestors +} + +func entryMatches(root *os.Root, p string, d fs.DirEntry, entry *artifact.Entry) bool { + info, err := d.Info() + if err != nil { + return false + } + switch entry.Type { + case artifact.EntryDirectory: + return info.IsDir() && modeMatches(info, entry) + case artifact.EntrySymlink: + return linkMatches(root, p, info, entry) + default: + return fileMatches(root, p, info, entry) + } +} + +func linkMatches(root *os.Root, p string, info os.FileInfo, entry *artifact.Entry) bool { + target, err := root.Readlink(p) + return err == nil && info.Mode()&os.ModeSymlink != 0 && filepath.ToSlash(target) == entry.Target +} + +func fileMatches(root *os.Root, p string, info os.FileInfo, entry *artifact.Entry) bool { + return info.Mode().IsRegular() && info.Size() == entry.Size && modeMatches(info, entry) && fileDigest(root, p) == entry.Digest +} + +func modeMatches(info os.FileInfo, entry *artifact.Entry) bool { + if runtime.GOOS == "windows" { + return true + } + return uint32(info.Mode().Perm()) == entry.Mode +} + +func fileDigest(root *os.Root, p string) digest.Digest { + file, err := root.Open(p) + if err != nil { + return "" + } + defer file.Close() + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "" + } + return digest.NewDigest(digest.SHA256, hasher) +} + +// Materialize extracts content into a private staging directory beside dest and then replaces dest +// with it. An existing destination is exchanged atomically with the staged tree. If atomic +// exchange is unavailable (including on Windows), replacing an existing directory fails without +// modifying it. The caller decides beforehand (see [InspectDestination]) whether replacement is permitted. +func Materialize(ctx context.Context, content *artifact.Content, dest string, opts ...Option) error { + defer perf.Track(nil, "materialize.Materialize")() + + if dest == "" { + return fmt.Errorf("%w: the destination is empty", errUtils.ErrArtifactConfigInvalid) + } + // Normalize first so a trailing separator ("out/") cannot make the parent the destination itself, + // and make the path absolute so a relative destination such as "." has a real parent directory. + dest, err := filepath.Abs(dest) + if err != nil { + return fmt.Errorf("%w: resolve destination: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + parent := filepath.Dir(dest) + if err := os.MkdirAll(parent, parentDirMode); err != nil { + return fmt.Errorf("%w: create %q: %w", errUtils.ErrArtifactPackagingInvalid, parent, err) + } + scratch, err := os.MkdirTemp(parent, scratchPattern) + if err != nil { + return fmt.Errorf("%w: create staging beside %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + scratchRoot, err := os.OpenRoot(scratch) + if err != nil { + _ = os.Remove(scratch) + return fmt.Errorf("%w: open staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + preserveScratch := false + defer func() { + if !preserveScratch { + if cleanupErr := removeScratch(scratchRoot, scratch); cleanupErr != nil { + log.Warn("Failed to remove artifact staging directory", "path", scratch, "error", cleanupErr) + } + } + _ = scratchRoot.Close() + }() + + stage := filepath.Join(scratch, treeDirName) + if err := os.Mkdir(stage, stagingDirMode); err != nil { + return fmt.Errorf("%w: create staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + if err := Extract(ctx, content, stage, scratch, opts...); err != nil { + return err + } + preserveScratch, err = installStaged(ctx, stage, dest, applyOptions(opts).replaceGuard) + return err +} + +// installStaged moves the extracted tree into place. Cancellation is final: a canceled pull never +// installs, even when extraction had just finished. +func installStaged(ctx context.Context, stage, dest string, guard func(dest *os.Root) error) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + if err := os.Chmod(stage, parentDirMode); err != nil { + return false, fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return replaceDestination(ctx, stage, dest, guard) +} + +// replaceDestination atomically exchanges the staged and existing trees. No children are +// removed or installed through the destination path, and a failed exchange leaves both intact. +// The boolean preserves scratch when a concurrent replacement cannot be safely rolled back. +func replaceDestination(ctx context.Context, stage, dest string, guard func(dest *os.Root) error) (bool, error) { + info, err := os.Lstat(dest) + switch { + case os.IsNotExist(err): + return false, rename(stage, dest) + case err != nil: + return false, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + case info.Mode()&os.ModeSymlink != 0: + return false, fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) + case !info.IsDir(): + return false, fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) + } + bound, err := bindDestination(dest, info) + if err != nil { + return false, err + } + defer bound.root.Close() + if guard != nil { + if err := guard(bound.root); err != nil { + return false, err + } + } + if err := bound.verify(); err != nil { + return false, err + } + // The guard can run long on a large destination, and cancellation is final: it must not exchange. + if err := ctx.Err(); err != nil { + return false, err + } + return exchangeStagedTree(stage, dest, bound.info) +} + +func exchangeStagedTree(stage, dest string, destInfo os.FileInfo) (bool, error) { + stageInfo, err := os.Lstat(stage) + if err != nil { + return false, fmt.Errorf("%w: inspect staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + stageParent, err := os.Open(filepath.Dir(stage)) + if err != nil { + return false, fmt.Errorf("%w: open staging parent: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer stageParent.Close() + destParent, err := os.Open(filepath.Dir(dest)) + if err != nil { + return false, fmt.Errorf("%w: open destination parent: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer destParent.Close() + pair := exchangePair{stageParent: stageParent, stage: filepath.Base(stage), destParent: destParent, dest: filepath.Base(dest), stageInfo: stageInfo, destInfo: destInfo} + return exchangeDestination(&pair, exchangeDirectories) +} + +type exchangePair struct { + stageParent *os.File + stage string + destParent *os.File + dest string + stageInfo os.FileInfo + destInfo os.FileInfo +} + +type exchangeFunc func(stageParent *os.File, stage string, destParent *os.File, dest string) error + +// exchangeDestination checks the identity displaced by the atomic exchange. If an attacker +// swapped the entry after verification, reverse the exchange; never remove an unrecognized tree. +func exchangeDestination(pair *exchangePair, exchange exchangeFunc) (bool, error) { + stageParent, stage, destParent, dest := pair.stageParent, pair.stage, pair.destParent, pair.dest + stageInfo, destInfo := pair.stageInfo, pair.destInfo + stagingRoot, err := checkedParent(stageParent) + if err != nil { + return false, err + } + defer stagingRoot.Close() + destinationRoot, err := checkedParent(destParent) + if err != nil { + return false, err + } + defer destinationRoot.Close() + if !sameRootEntry(destinationRoot, dest, destInfo) { + return false, changedDestination(filepath.Join(destParent.Name(), dest)) + } + if err := exchange(stageParent, stage, destParent, dest); err != nil { + return false, fmt.Errorf("%w: atomically replace destination: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + if pair.matchesExchange(stagingRoot, destinationRoot) { + return false, nil + } + rollbackErr := exchange(stageParent, stage, destParent, dest) + if rollbackErr == nil && sameRootEntry(stagingRoot, stage, stageInfo) { + return false, changedDestination(filepath.Join(destParent.Name(), dest)) + } + return true, errors.Join(fmt.Errorf("%w: destination changed during replacement; recovery data retained in %q", errUtils.ErrArtifactUnsafePath, stageParent.Name()), rollbackErr) +} + +func (pair *exchangePair) matchesExchange(stageRoot, destRoot *os.Root) bool { + return sameRootEntry(stageRoot, pair.stage, pair.destInfo) && sameRootEntry(destRoot, pair.dest, pair.stageInfo) +} + +func sameRootEntry(root *os.Root, name string, expected os.FileInfo) bool { + info, err := root.Lstat(name) + return err == nil && os.SameFile(info, expected) +} + +// checkedParent binds path-based inspection to the same directory as the syscall descriptor. +func checkedParent(parent *os.File) (*os.Root, error) { + root, err := os.OpenRoot(parent.Name()) + if err != nil { + return nil, changedDestination(parent.Name()) + } + opened, err := parent.Stat() + if err == nil { + var current os.FileInfo + current, err = root.Stat(rootDirName) + if err == nil && os.SameFile(opened, current) { + return root, nil + } + } + _ = root.Close() + return nil, changedDestination(parent.Name()) +} + +type boundDestination struct { + path string + root *os.Root + info os.FileInfo +} + +func bindDestination(dest string, checked os.FileInfo) (*boundDestination, error) { + root, err := os.OpenRoot(dest) + if err != nil { + return nil, fmt.Errorf("%w: open destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + opened, err := root.Stat(rootDirName) + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("%w: inspect destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + if !os.SameFile(checked, opened) { + _ = root.Close() + return nil, changedDestination(dest) + } + return &boundDestination{path: dest, root: root, info: opened}, nil +} + +func (b *boundDestination) verify() error { + now, err := os.Lstat(b.path) + if err != nil || now.Mode()&os.ModeSymlink != 0 || !os.SameFile(now, b.info) { + return changedDestination(b.path) + } + return nil +} + +func changedDestination(dest string) error { + return fmt.Errorf("%w: destination %q changed while it was being replaced", errUtils.ErrArtifactUnsafePath, dest) +} + +// removeScratch restores owner access before removing a displaced restrictive tree. WalkDir +// never follows symlinks and Root bounds permission changes to the private staging tree. +func removeScratch(root *os.Root, scratch string) error { + walkErr := fs.WalkDir(root.FS(), rootDirName, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + return root.Chmod(path, info.Mode().Perm()|0o700) + } + return nil + }) + children, readErr := fs.ReadDir(root.FS(), rootDirName) + var removeErr error + for _, child := range children { + removeErr = errors.Join(removeErr, root.RemoveAll(child.Name())) + } + opened, statErr := root.Stat(rootDirName) + if statErr == nil { + var current os.FileInfo + current, statErr = os.Lstat(scratch) + if statErr == nil && !os.SameFile(opened, current) { + statErr = changedDestination(scratch) + } + } + closeErr := root.Close() + if statErr == nil && closeErr == nil { + statErr = os.Remove(scratch) + } + return errors.Join(walkErr, readErr, removeErr, statErr, closeErr) +} + +func rename(from, to string) error { + if err := renameExclusive(from, to); err != nil { + return fmt.Errorf("%w: move %q into place: %w", errUtils.ErrArtifactPackagingInvalid, to, err) + } + return nil +} diff --git a/pkg/artifact/materialize/replace_cancel_internal_test.go b/pkg/artifact/materialize/replace_cancel_internal_test.go new file mode 100644 index 00000000000..d0241135df5 --- /dev/null +++ b/pkg/artifact/materialize/replace_cancel_internal_test.go @@ -0,0 +1,67 @@ +package materialize + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stagedAndDestination lays out a staged tree and an existing destination that hold different content. +func stagedAndDestination(t *testing.T) (stage, dest string) { + t.Helper() + root := t.TempDir() + stage = filepath.Join(root, "scratch", "tree") + dest = filepath.Join(root, "dest") + require.NoError(t, os.MkdirAll(stage, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(stage, "new"), []byte("new"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + return stage, dest +} + +func TestInstallStaged_CancellationDuringTheReplaceGuardIsFinal(t *testing.T) { + tests := []struct { + name string + cancelInGuard bool + }{ + {"canceled while the guard runs", true}, + {"not canceled", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if !tc.cancelInGuard && runtime.GOOS == "windows" { + t.Skipf("atomic exchange of an existing destination is unavailable on %s", runtime.GOOS) + } + stage, dest := stagedAndDestination(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + guard := func(*os.Root) error { + if tc.cancelInGuard { + cancel() + } + return nil + } + + _, err := installStaged(ctx, stage, dest, guard) + + if !tc.cancelInGuard { + require.NoError(t, err) + data, readErr := os.ReadFile(filepath.Join(dest, "new")) + require.NoError(t, readErr) + assert.Equal(t, "new", string(data)) + return + } + require.ErrorIs(t, err, context.Canceled) + data, readErr := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, readErr) + assert.Equal(t, "original", string(data), "the previous content stays in place") + assert.NoFileExists(t, filepath.Join(dest, "new")) + assert.FileExists(t, filepath.Join(stage, "new"), "the staged tree is not installed") + }) + } +} diff --git a/pkg/artifact/materialize/replace_internal_test.go b/pkg/artifact/materialize/replace_internal_test.go new file mode 100644 index 00000000000..d0f35087770 --- /dev/null +++ b/pkg/artifact/materialize/replace_internal_test.go @@ -0,0 +1,213 @@ +//go:build linux || darwin + +package materialize + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestExchangeDestinationFailurePreservesOriginal(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + restricted := filepath.Join(dest, "restricted") + require.NoError(t, os.Mkdir(restricted, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(restricted, "keep"), []byte("keep"), 0o600)) + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + sentinel := errors.New("exchange failed") + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, func(*os.File, string, *os.File, string) error { return sentinel }) + require.ErrorIs(t, err, sentinel) + require.False(t, preserve) + data, err := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, err) + require.Equal(t, "original", string(data)) + require.NoError(t, os.Chmod(restricted, 0o700)) + data, err = os.ReadFile(filepath.Join(restricted, "keep")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestExchangeDestinationRejectsLastMomentSwap(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(dest, filepath.Join(root, "original"))) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "intruder"), []byte("intact"), 0o600)) + } + return exchangeDirectories(a, b, c, d) + } + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.False(t, preserve) + require.Equal(t, 2, calls) + data, err := os.ReadFile(filepath.Join(dest, "intruder")) + require.NoError(t, err) + require.Equal(t, "intact", string(data)) + data, err = os.ReadFile(filepath.Join(root, "original", "original")) + require.NoError(t, err) + require.Equal(t, "original", string(data)) +} + +func TestRemoveScratchWithRestrictiveDirectories(t *testing.T) { + scratch := filepath.Join(t.TempDir(), "scratch") + restricted := filepath.Join(scratch, "tree", "restricted") + require.NoError(t, os.MkdirAll(restricted, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(restricted, "data"), []byte("x"), 0o600)) + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + root, err := os.OpenRoot(scratch) + require.NoError(t, err) + defer root.Close() + require.NoError(t, removeScratch(root, scratch)) + _, err = os.Lstat(scratch) + require.True(t, os.IsNotExist(err)) +} + +func TestExchangeDestinationRetainsDisplacedTreeWhenRollbackFails(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(dest, filepath.Join(root, "original"))) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "intruder"), []byte("keep"), 0o600)) + return exchangeDirectories(a, b, c, d) + } + return errors.New("rollback failed") + } + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.True(t, preserve) + require.ErrorContains(t, err, scratch) + data, err := os.ReadFile(filepath.Join(scratch, "tree", "intruder")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestRenameExclusiveNeverOverwritesExistingDestination(t *testing.T) { + root := t.TempDir() + stage := filepath.Join(root, "stage") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(stage, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.Error(t, renameExclusive(stage, dest)) + _, err := os.Stat(stage) + require.NoError(t, err) + _, err = os.Stat(dest) + require.NoError(t, err) +} + +func TestRemoveScratchDoesNotFollowSwappedPath(t *testing.T) { + parent := t.TempDir() + scratch := filepath.Join(parent, "scratch") + outside := filepath.Join(parent, "outside") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(outside, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "keep"), []byte("keep"), 0o600)) + root, err := os.OpenRoot(scratch) + require.NoError(t, err) + defer root.Close() + require.NoError(t, os.Rename(scratch, filepath.Join(parent, "moved"))) + require.NoError(t, os.Symlink(outside, scratch)) + require.ErrorIs(t, removeScratch(root, scratch), errUtils.ErrArtifactUnsafePath) + data, err := os.ReadFile(filepath.Join(outside, "keep")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestExchangeDestinationRejectsSwappedStage(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("keep"), 0o600)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(filepath.Join(scratch, "tree"), filepath.Join(scratch, "real"))) + require.NoError(t, os.Symlink(filepath.Join(root, "elsewhere"), filepath.Join(scratch, "tree"))) + } + return exchangeDirectories(a, b, c, d) + } + pair := &exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo} + preserve, err := exchangeDestination(pair, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.True(t, preserve) + require.Equal(t, 2, calls) + data, err := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) + info, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeSymlink) +} diff --git a/pkg/artifact/materialize/tree.go b/pkg/artifact/materialize/tree.go new file mode 100644 index 00000000000..bccdb97f00f --- /dev/null +++ b/pkg/artifact/materialize/tree.go @@ -0,0 +1,278 @@ +package materialize + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +const ( + stagingDirMode fs.FileMode = 0o700 + stagingFileMode fs.FileMode = 0o600 + + // Owner bits that a temporary tree keeps so it can be read back. + ownerFileAccess fs.FileMode = 0o400 + ownerDirAccess fs.FileMode = 0o700 +) + +// tree writes verified entries below a root and remembers directory modes to apply last. +type tree struct { + root *os.Root + inventory map[string]*artifact.Entry + written map[string]bool + + // links holds the symlinks validated so far. They are created after every other member + // exists (see createLinks), so a platform that types a link by its target sees the target. + links []*artifact.Entry + + // ownerAccess keeps owner read and traversal access on the final modes. The Record keeps the + // original modes; this only protects a temporary tree that is read back before it is discarded. + ownerAccess bool +} + +func newTree(root *os.Root, record *artifact.Record) *tree { + inventory := make(map[string]*artifact.Entry, len(record.Inventory)) + for index := range record.Inventory { + inventory[record.Inventory[index].Path] = &record.Inventory[index] + } + return &tree{root: root, inventory: inventory, written: make(map[string]bool, len(inventory))} +} + +// entryFor returns the inventory entry a member must match, rejecting unsafe or unknown paths. +func (t *tree) entryFor(name string) (*artifact.Entry, error) { + if err := artifact.ValidatePayloadPath(name); err != nil { + return nil, err + } + entry, ok := t.inventory[name] + if !ok { + return nil, fmt.Errorf("%w: member %q is not in the artifact inventory", errUtils.ErrArtifactIntegrityFailed, name) + } + if t.written[name] { + return nil, fmt.Errorf("%w: member %q appears more than once", errUtils.ErrArtifactIntegrityFailed, name) + } + t.written[name] = true + return entry, nil +} + +func (t *tree) mkdir(entry *artifact.Entry) error { + if err := t.root.MkdirAll(filepath.FromSlash(entry.Path), stagingDirMode); err != nil { + return fmt.Errorf("%w: create directory %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + return nil +} + +// writeFile writes a regular file, enforcing the declared size and digest. +func (t *tree) writeFile(entry *artifact.Entry, src io.Reader) error { + if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { + return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + file, err := t.root.OpenFile(filepath.FromSlash(entry.Path), os.O_WRONLY|os.O_CREATE|os.O_EXCL, stagingFileMode) + if err != nil { + return fmt.Errorf("%w: create file %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + hasher := sha256.New() + written, copyErr := io.Copy(io.MultiWriter(file, hasher), io.LimitReader(src, entry.Size+1)) + if closeErr := file.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("%w: write %q: %w", errUtils.ErrArtifactIntegrityFailed, entry.Path, copyErr) + } + if written > entry.Size { + return fmt.Errorf("%w: %q is larger than its declared %d bytes", errUtils.ErrArtifactTooLarge, entry.Path, entry.Size) + } + if written != entry.Size || digest.NewDigest(digest.SHA256, hasher) != entry.Digest { + return fmt.Errorf("%w: %q does not match its inventory size and digest", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + return nil +} + +// symlink records a symlink entry. The link itself is created by createLinks once every file and +// directory exists: Windows chooses between a file link and a directory link when the link is +// created, and a directory link to a target that does not exist yet becomes a file link that can +// never be traversed. +func (t *tree) symlink(entry *artifact.Entry) error { + t.links = append(t.links, entry) + return nil +} + +// createLinks creates the recorded symlinks. A link whose target runs through another link is +// created after that link, so the target resolves to what it will be once the tree is complete. +func (t *tree) createLinks(ctx context.Context) error { + for _, entry := range orderLinks(t.links) { + if err := ctx.Err(); err != nil { + return err + } + if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { + return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + if err := t.root.Symlink(filepath.FromSlash(entry.Target), filepath.FromSlash(entry.Path)); err != nil { + return fmt.Errorf("%w: this platform cannot create symlink %q -> %q (it is not silently replaced): %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, entry.Target, err) + } + } + return nil +} + +// orderLinks sorts links by path and then moves each link after the link its target passes through. +// Links that depend on each other in a cycle keep their path order; they can never resolve anyway. +func orderLinks(links []*artifact.Entry) []*artifact.Entry { + sorted := slices.Clone(links) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + pending, dependents := linkDependencies(sorted) + ordered := make([]*artifact.Entry, 0, len(sorted)) + done := make([]bool, len(sorted)) + var ready []int + for index := range sorted { + if pending[index] == 0 { + ready = append(ready, index) + } + } + for len(ready) > 0 { + index := ready[0] + ready = ready[1:] + done[index] = true + ordered = append(ordered, sorted[index]) + ready = append(ready, releaseDependents(dependents[index], pending)...) + } + for index, link := range sorted { + if !done[index] { + ordered = append(ordered, link) + } + } + return ordered +} + +// linkDependencies counts, for each link, the links that must exist before it, and lists the links +// that wait on each link. +func linkDependencies(sorted []*artifact.Entry) (pending []int, dependents [][]int) { + byPath := make(map[string]int, len(sorted)) + for index, link := range sorted { + byPath[link.Path] = index + } + pending = make([]int, len(sorted)) + dependents = make([][]int, len(sorted)) + for index, link := range sorted { + for _, dep := range linksThrough(link, index, sorted, byPath) { + pending[index]++ + dependents[dep] = append(dependents[dep], index) + } + } + return pending, dependents +} + +// releaseDependents counts one satisfied dependency for each waiting link and returns the links +// that are now ready. +func releaseDependents(waiting []int, pending []int) []int { + var ready []int + for _, index := range waiting { + pending[index]-- + if pending[index] == 0 { + ready = append(ready, index) + } + } + return ready +} + +// maxLinkExpansion bounds the links followed while expanding one target, so a cycle terminates. +const maxLinkExpansion = 255 + +// linksThrough lists every other link that the target of link passes through. The target is expanded +// the way the filesystem resolves it: a link met on the way is replaced by its own target, and a +// parent component applies to the directory reached after that replacement, never to the name before +// it. Cleaning the path lexically first would drop a link that a parent component follows. +func linksThrough(link *artifact.Entry, self int, links []*artifact.Entry, byPath map[string]int) []int { + pending := splitSlash(path.Dir(link.Path) + "/" + link.Target) + var deps []int + var directory []string + seen := make(map[int]bool) + for hops := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case "", ".": + continue + case "..": + if len(directory) > 0 { + directory = directory[:len(directory)-1] + } + continue + } + current := strings.Join(append(slices.Clone(directory), component), "/") + index, isLink := byPath[current] + if !isLink { + directory = append(directory, component) + continue + } + if index != self && !seen[index] { + seen[index] = true + deps = append(deps, index) + } + hops++ + if hops > maxLinkExpansion { + break + } + target := links[index].Target + if path.IsAbs(target) { + directory = nil + } + pending = append(splitSlash(target), pending...) + } + return deps +} + +// splitSlash splits a slash-separated path into its components, keeping empty and dot components so +// the caller decides what they mean. +func splitSlash(value string) []string { + return strings.Split(value, "/") +} + +// complete checks that every inventory entry was produced and applies final modes: files first, +// then directories deepest-first so a restrictive directory mode never blocks earlier chmods. +func (t *tree) complete() error { + paths := make([]string, 0, len(t.inventory)) + for entryPath := range t.inventory { + if !t.written[entryPath] { + return fmt.Errorf("%w: inventory entry %q is missing from the content", errUtils.ErrArtifactIntegrityFailed, entryPath) + } + paths = append(paths, entryPath) + } + sort.Slice(paths, func(i, j int) bool { + return len(paths[i]) > len(paths[j]) || (len(paths[i]) == len(paths[j]) && paths[i] > paths[j]) + }) + for _, entryPath := range paths { + entry := t.inventory[entryPath] + if entry.Type == artifact.EntrySymlink { + continue + } + if err := t.root.Chmod(filepath.FromSlash(entryPath), t.finalMode(entry)); err != nil { + return fmt.Errorf("%w: set mode %#o on %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Mode, entryPath, err) + } + } + return nil +} + +// finalMode is the mode applied to an entry: its recorded mode, plus owner access when the tree is a +// temporary one that is read back (read for files, read and traversal for directories). +func (t *tree) finalMode(entry *artifact.Entry) fs.FileMode { + mode := fs.FileMode(entry.Mode) + if !t.ownerAccess { + return mode + } + if entry.Type == artifact.EntryDirectory { + return mode | ownerDirAccess + } + return mode | ownerFileAccess +} diff --git a/pkg/artifact/materialize/tree_links_internal_test.go b/pkg/artifact/materialize/tree_links_internal_test.go new file mode 100644 index 00000000000..00e27153c97 --- /dev/null +++ b/pkg/artifact/materialize/tree_links_internal_test.go @@ -0,0 +1,189 @@ +package materialize + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func linkEntry(name, target string) *artifact.Entry { + return &artifact.Entry{Path: name, Type: artifact.EntrySymlink, Target: target} +} + +func linkPaths(links []*artifact.Entry) []string { + paths := make([]string, 0, len(links)) + for _, link := range links { + paths = append(paths, link.Path) + } + return paths +} + +func TestOrderLinks_CreatesALinkAfterTheLinkItsTargetPassesThrough(t *testing.T) { + tests := []struct { + name string + links []*artifact.Entry + want []string + }{ + {"independent links keep path order", []*artifact.Entry{linkEntry("b", "x"), linkEntry("a", "y")}, []string{"a", "b"}}, + {"a link to a directory that sorts after it", []*artifact.Entry{linkEntry("alias", "z-dir")}, []string{"alias"}}, + {"a chain is created from its end", []*artifact.Entry{linkEntry("a", "b"), linkEntry("b", "c"), linkEntry("c", "dir")}, []string{"c", "b", "a"}}, + {"a target below a link", []*artifact.Entry{linkEntry("x", "y/sub"), linkEntry("y", "dir")}, []string{"y", "x"}}, + {"a relative target in a subdirectory", []*artifact.Entry{linkEntry("d/a", "../e"), linkEntry("e", "dir")}, []string{"e", "d/a"}}, + // "m" reaches deep/c only after "z" is expanded, because the parent component applies to z's target. + {"a parent component after a link", []*artifact.Entry{linkEntry("m", "z/../c"), linkEntry("z", "deep/b"), linkEntry("deep/c", "dir")}, []string{"deep/c", "z", "m"}}, + // "t" passes through "r" (a link to "s"), and "s" is a link that no prefix of t's target names. + {"every link met while expanding the target", []*artifact.Entry{linkEntry("t", "r/f"), linkEntry("r", "s/sub"), linkEntry("s", "dir")}, []string{"s", "r", "t"}}, + {"a cycle keeps path order and terminates", []*artifact.Entry{linkEntry("q", "p"), linkEntry("p", "q")}, []string{"p", "q"}}, + {"a link to itself terminates", []*artifact.Entry{linkEntry("s", "s")}, []string{"s"}}, + {"no links", nil, []string{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + original := linkPaths(tc.links) + got := linkPaths(orderLinks(tc.links)) + assert.Equal(t, tc.want, got) + assert.Equal(t, original, linkPaths(tc.links), "the caller's slice is not reordered") + }) + } +} + +func TestTree_CreatesSymlinksOnlyAfterEveryOtherMemberExists(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + alias := linkEntry("alias", "z-dir") + zdir := &artifact.Entry{Path: "z-dir", Type: artifact.EntryDirectory, Mode: 0o755} + target := newTree(root, &artifact.Record{Inventory: []artifact.Entry{*alias, *zdir}}) + + require.NoError(t, target.symlink(alias)) + _, err = os.Lstat(filepath.Join(dir, "alias")) + require.ErrorIs(t, err, os.ErrNotExist, "recording a link must not create it yet") + + require.NoError(t, target.mkdir(zdir)) + requireSymlinkSupport(t) + require.NoError(t, target.createLinks(t.Context())) + info, err := os.Lstat(filepath.Join(dir, "alias")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&os.ModeSymlink) + entries, err := os.ReadDir(filepath.Join(dir, "alias")) + require.NoError(t, err, "the link must be traversable as a directory") + assert.Empty(t, entries) +} + +func TestTree_OwnerAccessKeepsATemporaryTreeReadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + body := []byte("sealed") + tests := []struct { + name string + ownerAccess bool + wantFile os.FileMode + wantDir os.FileMode + }{ + {"recorded modes are applied as declared", false, 0o000, 0o000}, + {"owner access is kept on top of the recorded modes", true, 0o400, 0o700}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + record := &artifact.Record{Inventory: []artifact.Entry{ + {Path: "d", Type: artifact.EntryDirectory, Mode: 0}, + {Path: "d/f", Type: artifact.EntryFile, Size: int64(len(body)), Mode: 0, Digest: digest.FromBytes(body)}, + }} + target := newTree(root, record) + target.ownerAccess = tc.ownerAccess + for index := range record.Inventory { + entry := &record.Inventory[index] + _, err := target.entryFor(entry.Path) + require.NoError(t, err) + if entry.Type == artifact.EntryDirectory { + require.NoError(t, target.mkdir(entry)) + continue + } + require.NoError(t, target.writeFile(entry, bytes.NewReader(body))) + } + + require.NoError(t, target.complete()) + assert.Zero(t, record.Inventory[0].Mode, "the Record keeps the modes it declared") + assert.Zero(t, record.Inventory[1].Mode, "the Record keeps the modes it declared") + t.Cleanup(func() { + _ = os.Chmod(filepath.Join(dir, "d"), 0o700) + _ = os.Chmod(filepath.Join(dir, "d", "f"), 0o600) + }) + + dirInfo, err := os.Stat(filepath.Join(dir, "d")) + require.NoError(t, err) + assert.Equal(t, tc.wantDir, dirInfo.Mode().Perm()) + if tc.ownerAccess { + fileInfo, err := os.Stat(filepath.Join(dir, "d", "f")) + require.NoError(t, err) + assert.Equal(t, tc.wantFile, fileInfo.Mode().Perm()) + data, err := os.ReadFile(filepath.Join(dir, "d", "f")) + require.NoError(t, err) + assert.Equal(t, body, data) + } + }) + } +} + +// requireSymlinkSupport skips the test only when the platform cannot create symlinks at all. It runs +// its own probe so a failure of the operation under test fails the test instead of skipping it. +func requireSymlinkSupport(t *testing.T) { + t.Helper() + probe := t.TempDir() + if err := os.WriteFile(filepath.Join(probe, "target.txt"), []byte("x"), 0o600); err != nil { + t.Fatalf("write the symlink probe target: %v", err) + } + if err := os.Symlink("target.txt", filepath.Join(probe, "link")); err != nil { + t.Skipf("this platform cannot create symlinks: %v", err) + } +} + +// A link whose target cannot be resolved for a reason other than a loop is rejected, because an unknown +// target cannot be shown to stay inside the package; a loop never leaves the root, so it is accepted. +func TestCheckLinkContainment_ResolutionFailures(t *testing.T) { + requireSymlinkSupport(t) + tests := []struct { + name string + links map[string]string + wantErr bool + }{ + // Capture accepts a link whose target runs through a regular file while its recorded path stays inside + // the tree, so materialization accepts it too; one that leaves the tree is rejected. + {name: "through a regular file, inside the tree", links: map[string]string{"dangling": "file.txt/child"}}, + {name: "through a regular file, leaving the tree", links: map[string]string{"bad": "file.txt/../../outside"}, wantErr: true}, + {name: "loop", links: map[string]string{"a": "b", "b": "a"}}, + {name: "inside the root", links: map[string]string{"ok": "file.txt"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stage := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(stage, "file.txt"), []byte("x"), 0o600)) + record := &artifact.Record{} + for name, target := range tt.links { + require.NoError(t, os.Symlink(target, filepath.Join(stage, name))) + record.Inventory = append(record.Inventory, artifact.Entry{Path: name, Type: artifact.EntrySymlink, Target: target}) + } + err := checkLinkContainment(stage, record) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + return + } + require.NoError(t, err) + }) + } +} diff --git a/pkg/artifact/mock_evidence.go b/pkg/artifact/mock_evidence.go new file mode 100644 index 00000000000..18e95df58cd --- /dev/null +++ b/pkg/artifact/mock_evidence.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: EvidenceReader,EvidenceWriter) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_evidence.go github.com/cloudposse/atmos/pkg/artifact EvidenceReader,EvidenceWriter +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockEvidenceReader is a mock of EvidenceReader interface. +type MockEvidenceReader struct { + ctrl *gomock.Controller + recorder *MockEvidenceReaderMockRecorder + isgomock struct{} +} + +// MockEvidenceReaderMockRecorder is the mock recorder for MockEvidenceReader. +type MockEvidenceReaderMockRecorder struct { + mock *MockEvidenceReader +} + +// NewMockEvidenceReader creates a new mock instance. +func NewMockEvidenceReader(ctrl *gomock.Controller) *MockEvidenceReader { + mock := &MockEvidenceReader{ctrl: ctrl} + mock.recorder = &MockEvidenceReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEvidenceReader) EXPECT() *MockEvidenceReaderMockRecorder { + return m.recorder +} + +// ReadEvidence mocks base method. +func (m *MockEvidenceReader) ReadEvidence(ctx context.Context, pinned *PinnedReference) ([]EvidenceBlob, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadEvidence", ctx, pinned) + ret0, _ := ret[0].([]EvidenceBlob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadEvidence indicates an expected call of ReadEvidence. +func (mr *MockEvidenceReaderMockRecorder) ReadEvidence(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadEvidence", reflect.TypeOf((*MockEvidenceReader)(nil).ReadEvidence), ctx, pinned) +} + +// MockEvidenceWriter is a mock of EvidenceWriter interface. +type MockEvidenceWriter struct { + ctrl *gomock.Controller + recorder *MockEvidenceWriterMockRecorder + isgomock struct{} +} + +// MockEvidenceWriterMockRecorder is the mock recorder for MockEvidenceWriter. +type MockEvidenceWriterMockRecorder struct { + mock *MockEvidenceWriter +} + +// NewMockEvidenceWriter creates a new mock instance. +func NewMockEvidenceWriter(ctrl *gomock.Controller) *MockEvidenceWriter { + mock := &MockEvidenceWriter{ctrl: ctrl} + mock.recorder = &MockEvidenceWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEvidenceWriter) EXPECT() *MockEvidenceWriterMockRecorder { + return m.recorder +} + +// WriteEvidence mocks base method. +func (m *MockEvidenceWriter) WriteEvidence(ctx context.Context, pinned *PinnedReference, evidence []EvidenceBlob, lock LockHandle) ([]SignatureEvidence, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteEvidence", ctx, pinned, evidence, lock) + ret0, _ := ret[0].([]SignatureEvidence) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteEvidence indicates an expected call of WriteEvidence. +func (mr *MockEvidenceWriterMockRecorder) WriteEvidence(ctx, pinned, evidence, lock any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteEvidence", reflect.TypeOf((*MockEvidenceWriter)(nil).WriteEvidence), ctx, pinned, evidence, lock) +} diff --git a/pkg/artifact/produce/assemble.go b/pkg/artifact/produce/assemble.go new file mode 100644 index 00000000000..bdd514abb38 --- /dev/null +++ b/pkg/artifact/produce/assemble.go @@ -0,0 +1,224 @@ +package produce + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/archive" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Constants for staged payloads. +const ( + // The media type of a per-file payload blob. + payloadMediaType = "application/octet-stream" + + // The mode of generated archive files. + generatedFilePerm = 0o644 + + // The mode of generated archive directories. + generatedDirPerm = 0o755 + + // The mode of the staged archive file itself. + archiveFilePerm = 0o600 +) + +// opener opens the bytes of one inventory file. +type opener func(entryPath string) (io.ReadCloser, error) + +// assembly turns a sealed Record, its documents, and a file opener into staged blobs. +type assembly struct { + record *artifact.Record + recordBytes []byte + docs []stagedDocument + open opener + stagingDir string +} + +// blobs stages the payload for packaging and then the separate document blobs. +func (a *assembly) blobs(ctx context.Context, packaging artifact.Packaging) ([]artifact.StagedBlob, error) { + var ( + payload []artifact.StagedBlob + err error + ) + switch packaging { + case artifact.PackagingNone: + payload = a.fileBlobs() + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + var blob artifact.StagedBlob + blob, err = a.archiveBlob(ctx, packaging) + payload = []artifact.StagedBlob{blob} + default: + err = fmt.Errorf("%w: cannot assemble packaging %q", errUtils.ErrArtifactPackagingInvalid, packaging) + } + if err != nil { + return nil, err + } + return append(payload, a.documentBlobs()...), nil +} + +func (a *assembly) fileBlobs() []artifact.StagedBlob { + var blobs []artifact.StagedBlob + for index := range a.record.Inventory { + entry := a.record.Inventory[index] + if entry.Type != artifact.EntryFile { + continue + } + blobs = append(blobs, artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: payloadMediaType, Digest: entry.Digest, Size: entry.Size}, + Role: artifact.BlobPayload, + Path: entry.Path, + Open: func(context.Context) (io.ReadCloser, error) { return a.open(entry.Path) }, + }) + } + return blobs +} + +func (a *assembly) documentBlobs() []artifact.StagedBlob { + blobs := make([]artifact.StagedBlob, 0, len(a.docs)) + for index := range a.docs { + doc := &a.docs[index] + blobs = append(blobs, artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: doc.descriptor.MediaType, Digest: doc.descriptor.Digest, Size: doc.descriptor.Size}, + Role: artifact.BlobDocument, + DocumentRole: doc.descriptor.Role, + Open: func(context.Context) (io.ReadCloser, error) { return doc.open(), nil }, + }) + } + return blobs +} + +// archiveBlob writes the deterministic archive into staging, hashing it as it is written. The +// context is checked while members are opened and read, so cancellation stops a large assembly. +func (a *assembly) archiveBlob(ctx context.Context, packaging artifact.Packaging) (artifact.StagedBlob, error) { + format, mediaType := archiveFormat(packaging) + path := filepath.Join(a.stagingDir, "payload."+format) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, archiveFilePerm) + if err != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: create archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + hasher := sha256.New() + writeErr := archive.WriteDeterministic(io.MultiWriter(file, hasher), format, withContext(ctx, a.members())) + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: %w", errUtils.ErrArtifactPackagingInvalid, writeErr) + } + info, err := os.Stat(path) + if err != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: stat archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: mediaType, Digest: digest.NewDigest(digest.SHA256, hasher), Size: info.Size()}, + Role: artifact.BlobPayload, + Open: func(context.Context) (io.ReadCloser, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%w: open staged archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return file, nil + }, + }, nil +} + +// withContext checks cancellation when opening and reading every file member. +func withContext(ctx context.Context, members []archive.Member) []archive.Member { + for index := range members { + open := members[index].Open + if open == nil { + continue + } + members[index].Open = func() (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + reader, err := open() + if err != nil { + return nil, err + } + return &contextReader{ctx: ctx, ReadCloser: reader}, nil + } + } + return members +} + +type contextReader struct { + ctx context.Context + io.ReadCloser +} + +func (r *contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "produce.contextReader.Read")() + + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.ReadCloser.Read(p) +} + +func archiveFormat(packaging artifact.Packaging) (string, string) { + switch packaging { + case artifact.PackagingTar: + return archive.FormatTar, artifact.MediaTypeTar + case artifact.PackagingZip: + return archive.FormatZip, artifact.MediaTypeZip + default: + return archive.FormatTGZ, artifact.MediaTypeTarGzip + } +} + +// members lists the archive members: the inventory plus the generated reserved directory holding +// manifest.json and the embedded documents. The generated members never enter the Record. +func (a *assembly) members() []archive.Member { + members := make([]archive.Member, 0, len(a.record.Inventory)+len(a.docs)+3) + present := make(map[string]bool, len(a.record.Inventory)) + for index := range a.record.Inventory { + entry := a.record.Inventory[index] + present[entry.Path] = true + members = append(members, a.inventoryMember(&entry)) + } + if !present[reservedParent] { + members = append(members, archive.Member{Path: reservedParent, Kind: archive.MemberDirectory, Mode: generatedDirPerm}) + } + members = append(members, archive.Member{Path: reservedDir, Kind: archive.MemberDirectory, Mode: generatedDirPerm}) + members = append(members, bytesMember(reservedDir+"/"+manifestName, a.recordBytes)) + for index := range a.docs { + members = append(members, bytesMember(reservedDir+"/"+reservedDocumentNames[a.docs[index].descriptor.Role], a.docs[index].data)) + } + return members +} + +func (a *assembly) inventoryMember(entry *artifact.Entry) archive.Member { + member := archive.Member{Path: entry.Path, Mode: os.FileMode(entry.Mode)} + switch entry.Type { + case artifact.EntryDirectory: + member.Kind = archive.MemberDirectory + case artifact.EntrySymlink: + member.Kind = archive.MemberSymlink + member.Target = entry.Target + default: + member.Kind = archive.MemberFile + member.Size = entry.Size + entryPath := entry.Path + member.Open = func() (io.ReadCloser, error) { return a.open(entryPath) } + } + return member +} + +func bytesMember(path string, data []byte) archive.Member { + doc := stagedDocument{data: data} + return archive.Member{ + Path: path, Kind: archive.MemberFile, Mode: generatedFilePerm, Size: int64(len(data)), + Open: func() (io.ReadCloser, error) { return doc.open(), nil }, + } +} diff --git a/pkg/artifact/produce/context_internal_test.go b/pkg/artifact/produce/context_internal_test.go new file mode 100644 index 00000000000..9f904a46b3b --- /dev/null +++ b/pkg/artifact/produce/context_internal_test.go @@ -0,0 +1,69 @@ +package produce + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/archive" +) + +func TestArchiveMemberStopsReadingAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + members := withContext(ctx, []archive.Member{{Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("payload")), nil }}}) + reader, err := members[0].Open() + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1) + _, err = reader.Read(buf) + require.NoError(t, err) + cancel() + _, err = reader.Read(buf) + require.True(t, errors.Is(err, context.Canceled)) +} + +func TestVerifyingReader_ClassifiesStreamFailures(t *testing.T) { + body := []byte("payload") + hash := digest.FromBytes(body) + closedFile := func(t *testing.T) *os.File { + t.Helper() + file, err := os.CreateTemp(t.TempDir(), "closed-*") + require.NoError(t, err) + require.NoError(t, file.Close()) + return file + } + + t.Run("a read failure keeps its cause and gains the packaging sentinel", func(t *testing.T) { + reader := newVerifyingReader(closedFile(t), hash, int64(len(body))) + _, err := reader.Read(make([]byte, 8)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrClosed) + }) + t.Run("a close failure keeps its cause and gains the packaging sentinel", func(t *testing.T) { + err := newVerifyingReader(closedFile(t), hash, int64(len(body))).Close() + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrClosed) + }) + t.Run("end of stream stays unchanged when the content matches", func(t *testing.T) { + reader := newVerifyingReader(io.NopCloser(bytes.NewReader(body)), hash, int64(len(body))) + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, body, got) + require.NoError(t, reader.Close()) + }) + t.Run("a digest mismatch at the end keeps the integrity classification", func(t *testing.T) { + reader := newVerifyingReader(io.NopCloser(bytes.NewReader([]byte("tampered"))), hash, int64(len(body))) + _, err := io.ReadAll(reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.NotErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) +} diff --git a/pkg/artifact/produce/doc.go b/pkg/artifact/produce/doc.go new file mode 100644 index 00000000000..b53fb20a97b --- /dev/null +++ b/pkg/artifact/produce/doc.go @@ -0,0 +1,10 @@ +// Package produce implements the generic "files" artifact producer and the seams that +// kind-specific producers (scaffold, component, and so on) reuse. +// +// Preparation snapshots the source into bounded disk staging (see package snapshot), resolves the +// artifact name and version, merges labels, recognizes the reserved .atmos/artifact distribution +// directory and explicit README/CHANGELOG overrides, seals a canonical Record, and stages the +// payload either as per-file blobs or as a deterministic tar, tgz, or zip archive. The user's +// source tree is never modified; the generated manifest.json exists only inside the packaged +// archive and never appears in the Record's inventory. +package produce diff --git a/pkg/artifact/produce/documents.go b/pkg/artifact/produce/documents.go new file mode 100644 index 00000000000..a1dc200d877 --- /dev/null +++ b/pkg/artifact/produce/documents.go @@ -0,0 +1,270 @@ +package produce + +import ( + "bytes" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +// Names of the reserved distribution paths. +const ( + // The hidden directory that holds the reserved distribution directory. + reservedParent = ".atmos" + + // The source directory reserved for README and CHANGELOG distribution documents. + reservedDir = ".atmos/artifact" + + // The generated manifest, written only inside the packaged representation. + manifestName = "manifest.json" +) + +var reservedDocumentNames = map[artifact.DocumentRole]string{ + artifact.DocumentReadme: "README.md", + artifact.DocumentChangelog: "CHANGELOG.md", +} + +// documentRoles lists the roles in canonical (sorted) order. +var documentRoles = []artifact.DocumentRole{artifact.DocumentChangelog, artifact.DocumentReadme} + +// stagedDocument is a document descriptor plus its bytes (at most [artifact.MaxDocumentSize]). +type stagedDocument struct { + descriptor artifact.Document + data []byte +} + +// checkReserved rejects reserved-directory contents other than the two distribution documents, +// including a conflicting source manifest.json, subdirectories, and links. +func checkReserved(entries []artifact.Entry) error { + for index := range entries { + entry := &entries[index] + switch { + case entry.Path == reservedParent: + // A file or symlink named .atmos would be dropped from the inventory by payloadInventory + // and then be missing after a pull, so it cannot be sealed. + if entry.Type != artifact.EntryDirectory { + return reservedError(entry.Path, "must be a directory") + } + case entry.Path == reservedDir: + if entry.Type != artifact.EntryDirectory { + return reservedError(entry.Path, "must be a directory") + } + case strings.HasPrefix(entry.Path, reservedDir+"/"): + if !isReservedDocument(entry) { + return reservedError(entry.Path, "is not README.md or CHANGELOG.md; the generated "+manifestName+" is written only inside the packaged representation") + } + } + } + return nil +} + +func isReservedDocument(entry *artifact.Entry) bool { + if entry.Type != artifact.EntryFile { + return false + } + name := strings.TrimPrefix(entry.Path, reservedDir+"/") + for _, reserved := range reservedDocumentNames { + if name == reserved { + return true + } + } + return false +} + +func reservedError(entryPath, reason string) error { + return fmt.Errorf("%w: %q in the reserved %s directory %s", errUtils.ErrArtifactPackagingInvalid, entryPath, reservedDir, reason) +} + +// payloadInventory removes the reserved distribution subtree from the snapshot inventory and drops +// the .atmos parent when nothing else lives under it. +func payloadInventory(entries []artifact.Entry) []artifact.Entry { + hasOtherAtmos := false + for index := range entries { + p := entries[index].Path + if strings.HasPrefix(p, reservedParent+"/") && p != reservedDir && !strings.HasPrefix(p, reservedDir+"/") { + hasOtherAtmos = true + break + } + } + inventory := make([]artifact.Entry, 0, len(entries)) + for index := range entries { + p := entries[index].Path + if p == reservedDir || strings.HasPrefix(p, reservedDir+"/") { + continue + } + if p == reservedParent && !hasOtherAtmos { + continue + } + inventory = append(inventory, entries[index]) + } + return inventory +} + +// documentOverrides carries the explicit --readme and --changelog paths. +type documentOverrides map[artifact.DocumentRole]string + +// collectDocuments resolves each document role: an explicit override wins over the reserved +// distribution file. Overrides resolve relative to the source directory or, for a single-file +// source, the file's parent. +func collectDocuments(snap *snapshot.Snapshot, source string, overrides documentOverrides) ([]stagedDocument, error) { + var docs []stagedDocument + for _, role := range documentRoles { + doc, found, err := collectDocument(snap, source, role, overrides[role]) + if err != nil { + return nil, err + } + if found { + docs = append(docs, doc) + } + } + return docs, nil +} + +func collectDocument(snap *snapshot.Snapshot, source string, role artifact.DocumentRole, override string) (stagedDocument, bool, error) { + if override != "" { + rel, err := cleanDocumentPath(role, override) + if err != nil { + return stagedDocument{}, false, err + } + doc, err := loadOverride(snap, source, role, rel) + return doc, err == nil, err + } + rel := reservedDir + "/" + reservedDocumentNames[role] + if snap.SourceIsFile { + return stagedDocument{}, false, nil + } + entry, ok := snap.Lookup(rel) + if !ok { + return stagedDocument{}, false, nil + } + data, err := readStagedDocument(snap, &entry) + if err != nil { + return stagedDocument{}, false, err + } + return newStagedDocument(role, rel, data), true, nil +} + +// cleanDocumentPath validates an explicit document path: relative, slash-normalized, contained. +func cleanDocumentPath(role artifact.DocumentRole, override string) (string, error) { + slashed := filepath.ToSlash(override) + cleaned := path.Clean(slashed) + if filepath.IsAbs(override) || strings.HasPrefix(slashed, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") || cleaned == "." { + return "", fmt.Errorf("%w: %s path %q must be relative to the source and stay inside it", errUtils.ErrArtifactUnsafePath, role, override) + } + if err := artifact.ValidatePayloadPath(cleaned); err != nil { + return "", fmt.Errorf("%s path %q: %w", role, override, err) + } + return cleaned, nil +} + +func loadOverride(snap *snapshot.Snapshot, source string, role artifact.DocumentRole, rel string) (stagedDocument, error) { + if snap.SourceIsFile { + data, err := readSiblingDocument(source, role, rel) + if err != nil { + return stagedDocument{}, err + } + return newStagedDocument(role, rel, data), nil + } + entry, ok := snap.Lookup(rel) + if !ok || entry.Type != artifact.EntryFile { + return stagedDocument{}, fmt.Errorf("%w: %s file %q is not a regular file inside the source", errUtils.ErrArtifactPackagingInvalid, role, rel) + } + data, err := readStagedDocument(snap, &entry) + if err != nil { + return stagedDocument{}, err + } + return newStagedDocument(role, rel, data), nil +} + +func readStagedDocument(snap *snapshot.Snapshot, entry *artifact.Entry) ([]byte, error) { + if entry.Size > artifact.MaxDocumentSize { + return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, entry.Path, entry.Size, artifact.MaxDocumentSize) + } + reader, err := snap.Open(entry.Path) + if err != nil { + return nil, err + } + defer reader.Close() + return readBounded(reader, entry.Path) +} + +// readSiblingDocument reads an explicit document next to a single-file source, rejecting anything +// that is not a regular file contained in the source's parent directory. +func readSiblingDocument(source string, role artifact.DocumentRole, rel string) ([]byte, error) { + parent, err := checkedSiblingParent(source, role, rel) + if err != nil { + return nil, err + } + // Open the document through a handle on the checked parent directory so a parent that is + // replaced after the containment check cannot redirect the read outside it. + root, err := os.OpenRoot(parent) + if err != nil { + return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + defer root.Close() + file, err := root.Open(filepath.FromSlash(rel)) + if err != nil { + return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%w: %s file %q is not a regular file", errUtils.ErrArtifactPackagingInvalid, role, rel) + } + if info.Size() > artifact.MaxDocumentSize { + return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, rel, info.Size(), artifact.MaxDocumentSize) + } + return readBounded(file, rel) +} + +// checkedSiblingParent resolves the source's parent directory and verifies that the document +// rel resolves inside it. +func checkedSiblingParent(source string, role artifact.DocumentRole, rel string) (string, error) { + resolvedSource, err := filepath.EvalSymlinks(source) + if err != nil { + return "", fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + parent := filepath.Dir(resolvedSource) + full, err := filepath.EvalSymlinks(filepath.Join(parent, filepath.FromSlash(rel))) + if err != nil { + return "", fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + if relative, err := filepath.Rel(parent, full); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%w: %s file %q resolves outside the source directory", errUtils.ErrArtifactUnsafePath, role, rel) + } + return parent, nil +} + +func readBounded(reader io.Reader, name string) ([]byte, error) { + data, err := artifact.ReadBounded(reader, artifact.MaxDocumentSize) + if err != nil { + return nil, fmt.Errorf("document %q: %w", name, err) + } + return data, nil +} + +func newStagedDocument(role artifact.DocumentRole, sourcePath string, data []byte) stagedDocument { + return stagedDocument{ + descriptor: artifact.Document{ + Role: role, + SourcePath: sourcePath, + MediaType: artifact.DocumentMediaType, + Size: int64(len(data)), + Digest: digest.FromBytes(data), + }, + data: data, + } +} + +func (d *stagedDocument) open() io.ReadCloser { + return io.NopCloser(bytes.NewReader(d.data)) +} diff --git a/pkg/artifact/produce/files.go b/pkg/artifact/produce/files.go new file mode 100644 index 00000000000..625670f472f --- /dev/null +++ b/pkg/artifact/produce/files.go @@ -0,0 +1,235 @@ +package produce + +import ( + "context" + "fmt" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Producer is the generic "files" producer. It satisfies [artifact.Producer]. +type Producer struct { + stagingRoot string + limits snapshot.Limits + metadata MetadataSource + snapshotOps []snapshot.Option +} + +var _ artifact.Producer = (*Producer)(nil) + +// Option customizes a [Producer]. +type Option func(*Producer) + +// WithStagingRoot sets the directory that holds bounded disk staging. +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "produce.WithStagingRoot")() + + return func(p *Producer) { p.stagingRoot = dir } +} + +// WithLimits overrides the staging limits. +func WithLimits(limits snapshot.Limits) Option { + defer perf.Track(nil, "produce.WithLimits")() + + return func(p *Producer) { p.limits = limits } +} + +// WithMetadataSource supplies producer metadata (name, version, description, labels, kind data). +func WithMetadataSource(source MetadataSource) Option { + defer perf.Track(nil, "produce.WithMetadataSource")() + + return func(p *Producer) { p.metadata = source } +} + +// WithSnapshotOptions passes extra options to the snapshot capture, for example a capture hook in tests. +func WithSnapshotOptions(opts ...snapshot.Option) Option { + defer perf.Track(nil, "produce.WithSnapshotOptions")() + + return func(p *Producer) { p.snapshotOps = append(p.snapshotOps, opts...) } +} + +// New returns a files producer. +func New(opts ...Option) *Producer { + defer perf.Track(nil, "produce.New")() + + p := &Producer{} + for _, opt := range opts { + if opt != nil { + opt(p) + } + } + return p +} + +// Register adds a files producer to registry. +func Register(registry *artifact.ProducerRegistry, opts ...Option) error { + defer perf.Track(nil, "produce.Register")() + + return registry.Register(New(opts...)) +} + +// Kind returns [artifact.KindFiles]. +func (p *Producer) Kind() string { + defer perf.Track(nil, "produce.Producer.Kind")() + + return artifact.KindFiles +} + +// Prepare snapshots the source and returns a sealed PreparedArtifact. The caller owns it and must +// Close it; on any error all owned temporary data is already removed. +// +//nolint:gocritic // The request is passed by value because artifact.Producer fixes the signature. +func (p *Producer) Prepare(ctx context.Context, request artifact.PrepareRequest) (*artifact.PreparedArtifact, error) { + defer perf.Track(nil, "produce.Producer.Prepare")() + + if err := validatePrepareRequest(&request); err != nil { + return nil, err + } + meta, err := p.producerMetadata(ctx, request.Source) + if err != nil { + return nil, err + } + name, version, err := ResolveIdentity(request.Name, request.Version, &meta, request.Source) + if err != nil { + return nil, err + } + if err := artifact.ValidateLabels(mergeLabels(meta.Labels, request.Metadata.Labels)); err != nil { + return nil, err + } + snap, err := snapshot.Capture(ctx, request.Source, p.captureOptions()...) + if err != nil { + return nil, err + } + prepared, err := p.prepareFrom(ctx, snap, &request, &identity{name: name, version: version, meta: meta}) + if err != nil { + _ = snap.Close() + return nil, err + } + return prepared, nil +} + +func (p *Producer) captureOptions() []snapshot.Option { + opts := []snapshot.Option{snapshot.WithStagingRoot(p.stagingRoot), snapshot.WithLimits(p.limits)} + return append(opts, p.snapshotOps...) +} + +func (p *Producer) producerMetadata(ctx context.Context, source string) (ProducerMetadata, error) { + if p.metadata == nil { + return ProducerMetadata{}, nil + } + meta, err := p.metadata(ctx, source) + if err != nil { + return ProducerMetadata{}, fmt.Errorf("%w: read producer metadata for %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + return meta, nil +} + +// identity is the resolved name, version, and producer metadata of one preparation. +type identity struct { + name string + version string + meta ProducerMetadata +} + +func (p *Producer) prepareFrom(ctx context.Context, snap *snapshot.Snapshot, request *artifact.PrepareRequest, id *identity) (*artifact.PreparedArtifact, error) { + if err := checkReserved(snap.Entries); err != nil { + return nil, err + } + docs, err := collectDocuments(snap, request.Source, documentOverrides{ + artifact.DocumentReadme: request.Readme, + artifact.DocumentChangelog: request.Changelog, + }) + if err != nil { + return nil, err + } + record := artifact.Record{ + Name: id.name, + Version: id.version, + Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(request.PreparedAt), + Metadata: artifact.Metadata{ + Description: firstNonEmpty(request.Metadata.Description, id.meta.Description), + Labels: mergeLabels(id.meta.Labels, request.Metadata.Labels), + Provenance: request.Metadata.Provenance, + }, + Inventory: payloadInventory(snap.Entries), + Documents: documentDescriptors(docs), + KindData: id.meta.KindData, + } + // The request's provenance and the metadata source's kind data are caller-owned; the sealed + // Record keeps its own copies. + record = record.Clone() + if err := record.Seal(); err != nil { + return nil, err + } + return finalize(ctx, &finalizeInput{ + record: &record, docs: docs, open: snap.Open, stagingDir: snap.Root, packaging: request.Packaging, + operationID: request.OperationID, preparedAt: request.PreparedAt, cleanup: snap.Close, + }) +} + +// finalizeInput is everything needed to stage blobs and build the PreparedArtifact. +type finalizeInput struct { + record *artifact.Record + docs []stagedDocument + open opener + stagingDir string + packaging artifact.Packaging + operationID string + preparedAt time.Time + cleanup func() error +} + +// finalize canonicalizes the Record, stages the blobs, and builds the PreparedArtifact. +func finalize(ctx context.Context, in *finalizeInput) (*artifact.PreparedArtifact, error) { + recordBytes, err := in.record.Canonical() + if err != nil { + return nil, err + } + intent, err := in.record.IntentDigest() + if err != nil { + return nil, err + } + assembled := assembly{record: in.record, recordBytes: recordBytes, docs: in.docs, open: in.open, stagingDir: in.stagingDir} + blobs, err := assembled.blobs(ctx, in.packaging) + if err != nil { + return nil, err + } + return &artifact.PreparedArtifact{ + Record: *in.record, + IntentDigest: intent, + OperationID: in.operationID, + PreparedAt: in.preparedAt, + Packaging: in.packaging, + Blobs: blobs, + Cleanup: in.cleanup, + }, nil +} + +func documentDescriptors(docs []stagedDocument) []artifact.Document { + descriptors := make([]artifact.Document, 0, len(docs)) + for index := range docs { + descriptors = append(descriptors, docs[index].descriptor) + } + return descriptors +} + +// validatePrepareRequest checks the fields the service is required to supply. +func validatePrepareRequest(request *artifact.PrepareRequest) error { + if request.Kind != "" && request.Kind != artifact.KindFiles { + return fmt.Errorf("%w: the files producer cannot prepare kind %q", errUtils.ErrArtifactConfigInvalid, request.Kind) + } + if request.OperationID == "" || request.PreparedAt.IsZero() { + return fmt.Errorf("%w: the operation ID and preparation time are captured once by the service before preparing", errUtils.ErrArtifactConfigInvalid) + } + switch request.Packaging { + case artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return nil + default: + return fmt.Errorf("%w: unsupported packaging %q", errUtils.ErrArtifactPackagingInvalid, request.Packaging) + } +} diff --git a/pkg/artifact/produce/identity.go b/pkg/artifact/produce/identity.go new file mode 100644 index 00000000000..c7bc6e3687d --- /dev/null +++ b/pkg/artifact/produce/identity.go @@ -0,0 +1,97 @@ +package produce + +import ( + "context" + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ProducerMetadata is name, version, and descriptive metadata a producer derives from its source +// (for example a scaffold manifest). Generic file packages need none of it. +type ProducerMetadata struct { + // Name is the producer-supplied artifact name. + Name string + + // Version is the producer-supplied version. + Version string + + // Description is the producer-supplied description. + Description string + + // Labels are producer-supplied default labels; explicit labels override them. + Labels map[string]string + + // KindData is optional kind-specific data stored in the Record. + KindData artifact.KindData +} + +// MetadataSource reads producer metadata for a source. A source without metadata returns the zero +// value and no error. +type MetadataSource func(ctx context.Context, source string) (ProducerMetadata, error) + +// ResolveIdentity applies the name and version precedence rules. Name precedence is the explicit +// value, then producer metadata, then the source basename. Version precedence is the explicit +// value, then producer metadata; otherwise it fails because there is no generated version and no +// implicit latest. Backend-invalid values are rejected rather than normalized. +func ResolveIdentity(explicitName, explicitVersion string, meta *ProducerMetadata, source string) (string, string, error) { + defer perf.Track(nil, "produce.ResolveIdentity")() + + name := firstNonEmpty(explicitName, meta.Name) + if name == "" { + base, err := sourceBasename(source) + if err != nil { + return "", "", err + } + name = base + if err := artifact.ValidateName(name); err != nil { + return "", "", fmt.Errorf("%w: the source basename %q is not a valid artifact name; pass an explicit name", err, name) + } + } + if err := artifact.ValidateName(name); err != nil { + return "", "", err + } + version := firstNonEmpty(explicitVersion, meta.Version) + if version == "" { + return "", "", fmt.Errorf("%w: a version is required (there is no generated version and no implicit latest)", errUtils.ErrArtifactRefInvalid) + } + if err := artifact.ValidateVersion(version); err != nil { + return "", "", err + } + return name, version, nil +} + +func sourceBasename(source string) (string, error) { + abs, err := filepath.Abs(source) + if err != nil { + return "", fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + return filepath.Base(abs), nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +// mergeLabels returns producer labels overlaid by explicit labels, or nil when there are none. +func mergeLabels(producer, explicit map[string]string) map[string]string { + if len(producer) == 0 && len(explicit) == 0 { + return nil + } + merged := make(map[string]string, len(producer)+len(explicit)) + for key, value := range producer { + merged[key] = value + } + for key, value := range explicit { + merged[key] = value + } + return merged +} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go new file mode 100644 index 00000000000..709701757fd --- /dev/null +++ b/pkg/artifact/produce/produce_test.go @@ -0,0 +1,698 @@ +package produce_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +var fixedTime = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +func request(src string, packaging artifact.Packaging) *artifact.PrepareRequest { + return &artifact.PrepareRequest{ + Source: src, + Name: "tools/example", + Version: "1.0.0", + Packaging: packaging, + OperationID: "op-1", + PreparedAt: fixedTime, + } +} + +func prepare(t *testing.T, req *artifact.PrepareRequest, opts ...produce.Option) (*artifact.PreparedArtifact, string) { + t.Helper() + staging := t.TempDir() + opts = append([]produce.Option{produce.WithStagingRoot(staging)}, opts...) + prepared, err := produce.New(opts...).Prepare(context.Background(), *req) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared, staging +} + +func readBlob(t *testing.T, blob *artifact.StagedBlob) []byte { + t.Helper() + reader, err := blob.Open(context.Background()) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + return data +} + +func payloadBlobs(prepared *artifact.PreparedArtifact) []artifact.StagedBlob { + var blobs []artifact.StagedBlob + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobPayload { + blobs = append(blobs, prepared.Blobs[index]) + } + } + return blobs +} + +func inventoryPaths(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for _, entry := range record.Inventory { + paths = append(paths, entry.Path) + } + return paths +} + +func TestFilesProducer_FilePackageRecord(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + record := prepared.Record + + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), inventoryPaths(&record), "reserved directory is not payload; root README is") + assert.Equal(t, artifact.KindFiles, record.Kind) + assert.Equal(t, "tools/example", record.Name) + assert.Equal(t, artifact.NewTimestamp(fixedTime), record.CreatedAt) + require.Len(t, record.Documents, 2) + assert.Equal(t, artifact.DocumentChangelog, record.Documents[0].Role) + assert.Equal(t, ".atmos/artifact/CHANGELOG.md", record.Documents[0].SourcePath) + assert.Equal(t, artifact.DocumentReadme, record.Documents[1].Role) + assert.Equal(t, int64(len(fixtures.PackageReadme)), record.Documents[1].Size) + assert.Equal(t, "op-1", prepared.OperationID) + assert.True(t, fixedTime.Equal(prepared.PreparedAt)) + require.NoError(t, record.Validate()) + + for _, entry := range record.Inventory { + switch entry.Path { + case "bin/helper.sh": + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), entry.Mode) + } + case "empty.txt": + assert.Zero(t, entry.Size) + } + } +} + +func TestFilesProducer_PerFileBlobs(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + + var files []string + blobs := payloadBlobs(prepared) + for index := range blobs { + files = append(files, blobs[index].Path) + assert.NotEmpty(t, blobs[index].Path) + data := readBlob(t, &blobs[index]) + assert.Equal(t, int64(len(data)), blobs[index].Descriptor.Size) + assert.Equal(t, digest.FromBytes(data), blobs[index].Descriptor.Digest) + } + want := []string{"README.md", "bin/helper.sh", "empty.txt", "nested/deep/file.txt", fixtures.UnicodeName} + assert.ElementsMatch(t, want, files, "directories and symlinks have no blob (symlinks=%v)", pkg.Symlinks) + + documents := 0 + for _, blob := range prepared.Blobs { + if blob.Role == artifact.BlobDocument { + documents++ + assert.Equal(t, artifact.DocumentMediaType, blob.Descriptor.MediaType) + } + } + assert.Equal(t, 2, documents) +} + +func TestFilesProducer_RepeatedPreparationIsByteIdentical(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + first, _ := prepare(t, request(src, packaging)) + // Touch mtimes so only content, paths, and modes can influence the output. + future := time.Now().Add(48 * time.Hour) + require.NoError(t, os.Chtimes(filepath.Join(src, "README.md"), future, future)) + second, _ := prepare(t, request(src, packaging)) + + firstRecord, err := first.Record.Canonical() + require.NoError(t, err) + secondRecord, err := second.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstRecord, secondRecord) + assert.Equal(t, first.IntentDigest, second.IntentDigest) + require.Len(t, second.Blobs, len(first.Blobs)) + for index := range first.Blobs { + assert.Equal(t, first.Blobs[index].Descriptor, second.Blobs[index].Descriptor) + assert.Equal(t, readBlob(t, &first.Blobs[index]), readBlob(t, &second.Blobs[index])) + } + }) + } +} + +func TestFilesProducer_IntentIgnoresTimeButRecordDoesNot(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + first, _ := prepare(t, request(src, artifact.PackagingTgz)) + later := request(src, artifact.PackagingTgz) + later.PreparedAt = fixedTime.Add(26 * time.Hour) + later.OperationID = "op-2" + second, _ := prepare(t, later) + + assert.Equal(t, first.IntentDigest, second.IntentDigest) + assert.Equal(t, first.Record.ContentDigest, second.Record.ContentDigest) + firstDigest, err := first.Record.RecordDigest() + require.NoError(t, err) + secondDigest, err := second.Record.RecordDigest() + require.NoError(t, err) + assert.NotEqual(t, firstDigest, secondDigest) +} + +func TestFilesProducer_ChangedModeLabelOrDocumentChangesIntent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + base, _ := prepare(t, request(src, artifact.PackagingNone)) + + labeled := request(src, artifact.PackagingNone) + labeled.Metadata.Labels = map[string]string{"team": "platform"} + withLabel, _ := prepare(t, labeled) + assert.NotEqual(t, base.IntentDigest, withLabel.IntentDigest) + assert.Equal(t, base.Record.ContentDigest, withLabel.Record.ContentDigest, "labels do not change content identity") + + withMode := base + if fixtures.POSIXModes() { + require.NoError(t, os.Chmod(filepath.Join(src, "nested", "deep", "file.txt"), 0o755)) + withMode, _ = prepare(t, request(src, artifact.PackagingNone)) + assert.NotEqual(t, base.Record.ContentDigest, withMode.Record.ContentDigest) + } + + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "changed readme\n", 0o644) + withDoc, _ := prepare(t, request(src, artifact.PackagingNone)) + assert.NotEqual(t, withMode.Record.ContentDigest, withDoc.Record.ContentDigest) +} + +func TestFilesProducer_ArchiveCarriesManifestAndLeavesSourceUntouched(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + before := listTree(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingTgz)) + assert.Equal(t, before, listTree(t, src), "the source tree must not be modified") + + blobs := payloadBlobs(prepared) + require.Len(t, blobs, 1) + assert.Equal(t, artifact.MediaTypeTarGzip, blobs[0].Descriptor.MediaType) + data := readBlob(t, &blobs[0]) + assert.Equal(t, int64(len(data)), blobs[0].Descriptor.Size) + + gz, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + reader := tar.NewReader(gz) + contents := map[string][]byte{} + modes := map[string]int64{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + modes[header.Name] = header.Mode + body, err := io.ReadAll(reader) + require.NoError(t, err) + contents[header.Name] = body + } + recordBytes, err := prepared.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, recordBytes, contents[".atmos/artifact/manifest.json"]) + assert.Equal(t, fixtures.PackageReadme, string(contents[".atmos/artifact/README.md"])) + assert.Equal(t, fixtures.PackageChangelog, string(contents[".atmos/artifact/CHANGELOG.md"])) + assert.Equal(t, fixtures.RootReadme, string(contents["README.md"]), "root README stays ordinary payload") + if fixtures.POSIXModes() { + assert.Equal(t, int64(0o755), modes["bin/helper.sh"]) + } + assert.Contains(t, contents, "emptydir/") + if pkg.Symlinks { + assert.Contains(t, modes, "link-to-file") + } + + for _, entry := range prepared.Record.Inventory { + assert.NotContains(t, entry.Path, "manifest.json", "generated members never enter the Record") + assert.NotContains(t, entry.Path, ".atmos/artifact") + } +} + +func listTree(t *testing.T, root string) []string { + t.Helper() + var out []string + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + info, err := d.Info() + require.NoError(t, err) + entry := path + "|" + info.Mode().String() + // NTFS refreshes a directory's listed modification time lazily when its files are only read, + // so directory times drift on Windows without any change. Entries added or removed are still + // caught by the path list, and file times are compared exactly. + if !info.IsDir() { + entry += "|" + info.ModTime().String() + } + out = append(out, entry) + return nil + })) + return out +} + +func TestFilesProducer_IdentityPrecedence(t *testing.T) { + src := filepath.Join(t.TempDir(), "my-source") + require.NoError(t, os.Mkdir(src, 0o755)) + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + metadata := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{Name: "from/producer", Version: "9.9.9", Description: "producer desc", Labels: map[string]string{"a": "producer", "b": "producer"}}, nil + } + tests := []struct { + name string + explicitName string + explicitVer string + opts []produce.Option + wantName string + wantVersion string + wantErr error + }{ + {"explicit beats producer", "explicit/name", "2.0.0", []produce.Option{produce.WithMetadataSource(metadata)}, "explicit/name", "2.0.0", nil}, + {"producer beats basename", "", "", []produce.Option{produce.WithMetadataSource(metadata)}, "from/producer", "9.9.9", nil}, + {"basename fallback", "", "1.0.0", nil, "my-source", "1.0.0", nil}, + {"version never generated", "x", "", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"invalid explicit name", "Bad Name", "1", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"invalid version", "x", "has space", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"traversal name", "../x", "1", nil, "", "", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Name, req.Version = tt.explicitName, tt.explicitVer + staging := t.TempDir() + prepared, err := produce.New(append([]produce.Option{produce.WithStagingRoot(staging)}, tt.opts...)...).Prepare(context.Background(), *req) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items, "nothing is staged for an invalid identity") + return + } + require.NoError(t, err) + defer prepared.Close() + assert.Equal(t, tt.wantName, prepared.Record.Name) + assert.Equal(t, tt.wantVersion, prepared.Record.Version) + }) + } +} + +func TestFilesProducer_InvalidBasenameNeedsExplicitName(t *testing.T) { + src := filepath.Join(t.TempDir(), "My Source") + require.NoError(t, os.Mkdir(src, 0o755)) + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + req := request(src, artifact.PackagingNone) + req.Name = "" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Contains(t, err.Error(), "explicit name") +} + +func TestFilesProducer_LabelMergeAndDescription(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + metadata := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{Description: "producer desc", Labels: map[string]string{"a": "producer", "b": "producer"}}, nil + } + req := request(src, artifact.PackagingNone) + req.Metadata = artifact.Metadata{Labels: map[string]string{"b": "explicit", "c": "explicit"}} + prepared, _ := prepare(t, req, produce.WithMetadataSource(metadata)) + assert.Equal(t, map[string]string{"a": "producer", "b": "explicit", "c": "explicit"}, prepared.Record.Metadata.Labels) + assert.Equal(t, "producer desc", prepared.Record.Metadata.Description) + + req.Metadata.Description = "explicit desc" + prepared, _ = prepare(t, req, produce.WithMetadataSource(metadata)) + assert.Equal(t, "explicit desc", prepared.Record.Metadata.Description) +} + +func TestFilesProducer_InvalidLabelsRejectedBeforeStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + req := request(src, artifact.PackagingNone) + req.Metadata.Labels = map[string]string{"bad=key": "v"} + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_RequestValidation(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + base := request(src, artifact.PackagingNone) + tests := []struct { + name string + mutate func(*artifact.PrepareRequest) + want error + }{ + {"other kind", func(r *artifact.PrepareRequest) { r.Kind = "scaffold" }, errUtils.ErrArtifactConfigInvalid}, + {"no operation id", func(r *artifact.PrepareRequest) { r.OperationID = "" }, errUtils.ErrArtifactConfigInvalid}, + {"no prepared time", func(r *artifact.PrepareRequest) { r.PreparedAt = time.Time{} }, errUtils.ErrArtifactConfigInvalid}, + {"bad packaging", func(r *artifact.PrepareRequest) { r.Packaging = "rar" }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := *base + tt.mutate(&req) + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), req) + require.ErrorIs(t, err, tt.want) + }) + } + assert.Equal(t, artifact.KindFiles, produce.New().Kind()) +} + +func TestFilesProducer_ReservedDirectoryConflicts(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + }{ + {"source manifest.json", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/manifest.json", "{}", 0o644) + return true + }}, + {"unknown reserved file", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/notes.txt", "x", 0o644) + return true + }}, + {"nested reserved directory", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/sub/README.md", "x", 0o644) + return true + }}, + {"regular file named .atmos", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, ".atmos", "not a directory", 0o644) + return true + }}, + {"symlink named .atmos", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, ".atmos", "a.txt") + }}, + {"reserved symlink document", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, "real.md", "x", 0o644) + return fixtures.Symlink(t, dir, fixtures.ReservedDir+"/README.md", "../../real.md") + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + if !tt.build(t, src) { + t.Skip("fixture unavailable") + } + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} + +func TestFilesProducer_OtherAtmosContentStaysPayload(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, ".atmos/other/config.yaml", "x: 1\n", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "doc", 0o644) + prepared, _ := prepare(t, request(src, artifact.PackagingTar)) + assert.Equal(t, []string{".atmos", ".atmos/other", ".atmos/other/config.yaml"}, inventoryPaths(&prepared.Record)) +} + +func TestFilesProducer_ReservedParentDroppedWhenOnlyDistribution(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "doc", 0o644) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + assert.Equal(t, []string{"a.txt"}, inventoryPaths(&prepared.Record)) +} + +func TestFilesProducer_ExplicitDocuments(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "docs/GUIDE.md", "explicit guide", 0o644) + fixtures.WriteFile(t, src, "docs/HISTORY.md", "explicit history", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "reserved readme", 0o644) + + req := request(src, artifact.PackagingNone) + req.Readme = "docs/GUIDE.md" + req.Changelog = "docs/HISTORY.md" + prepared, _ := prepare(t, req) + byRole := map[artifact.DocumentRole]artifact.Document{} + for _, doc := range prepared.Record.Documents { + byRole[doc.Role] = doc + } + assert.Equal(t, "docs/GUIDE.md", byRole[artifact.DocumentReadme].SourcePath, "explicit override wins over the reserved file") + assert.Equal(t, "docs/HISTORY.md", byRole[artifact.DocumentChangelog].SourcePath) + assert.Contains(t, inventoryPaths(&prepared.Record), "docs/GUIDE.md", "an explicit document inside the source stays payload") +} + +func TestFilesProducer_ExplicitDocumentRejections(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "big.md", string(bytes.Repeat([]byte("x"), int(artifact.MaxDocumentSize)+1)), 0o644) + fixtures.MkdirAll(t, src, "adir") + tests := []struct { + name string + override string + want error + }{ + {"traversal", "../outside.md", errUtils.ErrArtifactUnsafePath}, + {"absolute", filepath.Join(src, "a.txt"), errUtils.ErrArtifactUnsafePath}, + {"dot", ".", errUtils.ErrArtifactUnsafePath}, + {"missing", "missing.md", errUtils.ErrArtifactPackagingInvalid}, + {"directory", "adir", errUtils.ErrArtifactPackagingInvalid}, + {"oversize", "big.md", errUtils.ErrArtifactTooLarge}, + {"reserved namespace", artifact.ReservedNamespace + "/x.md", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Readme = tt.override + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, tt.want) + }) + } +} + +func TestFilesProducer_SingleFileSource(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "binary", 0o755) + fixtures.WriteFile(t, dir, "NOTES.md", "sibling notes", 0o644) + fixtures.MkdirAll(t, dir, "sub") + req := request(filepath.Join(dir, "tool.bin"), artifact.PackagingZip) + req.Name = "" + req.Readme = "NOTES.md" + prepared, _ := prepare(t, req) + assert.Equal(t, "tool.bin", prepared.Record.Name) + assert.Equal(t, []string{"tool.bin"}, inventoryPaths(&prepared.Record)) + require.Len(t, prepared.Record.Documents, 1) + assert.Equal(t, "NOTES.md", prepared.Record.Documents[0].SourcePath) + + req.Readme = "../outside.md" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + + req.Readme = "sub" + _, err = produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + + req.Readme = "missing.md" + _, err = produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestFilesProducer_SingleFileDocumentEscapingThroughSymlink(t *testing.T) { + root := t.TempDir() + parent := filepath.Join(root, "parent") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(parent, 0o755)) + require.NoError(t, os.MkdirAll(outside, 0o755)) + fixtures.WriteFile(t, parent, "tool.bin", "x", 0o644) + fixtures.WriteFile(t, outside, "secret.md", "secret", 0o644) + if err := os.Symlink(outside, filepath.Join(parent, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + req := request(filepath.Join(parent, "tool.bin"), artifact.PackagingNone) + req.Readme = "link/secret.md" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestFilesProducer_SingleFileDocumentThroughSymlinkInsideTheSourceDirectory(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "x", 0o644) + fixtures.MkdirAll(t, dir, "docs") + fixtures.WriteFile(t, dir, "docs/NOTES.md", "inside notes", 0o644) + if err := os.Symlink("docs", filepath.Join(dir, "alias")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + req := request(filepath.Join(dir, "tool.bin"), artifact.PackagingNone) + req.Readme = "alias/NOTES.md" + prepared, _ := prepare(t, req) + require.Len(t, prepared.Record.Documents, 1) + assert.Equal(t, "alias/NOTES.md", prepared.Record.Documents[0].SourcePath) + assert.Equal(t, int64(len("inside notes")), prepared.Record.Documents[0].Size) +} + +func TestFilesProducer_InvalidPackagesFailBeforeAnythingIsKept(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + want error + }{ + {"symlink escape", func(t *testing.T, dir string) bool { return fixtures.Symlink(t, dir, "escape", "../../etc/passwd") }, errUtils.ErrArtifactUnsafePath}, + {"absolute symlink", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "abs", filepath.ToSlash(os.TempDir())) + }, errUtils.ErrArtifactUnsafePath}, + {"special file", func(t *testing.T, dir string) bool { return fixtures.SpecialFile(t, dir, "pipe") }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + if !tt.build(t, src) { + t.Skip("fixture unavailable") + } + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, tt.want) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} + +func TestFilesProducer_ChangeDuringCaptureFails(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bbbb", 0o644) + staging := t.TempDir() + hook := snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + fixtures.WriteFile(t, src, "a.txt", "AAAAA", 0o644) + } + }) + _, err := produce.New(produce.WithStagingRoot(staging), produce.WithSnapshotOptions(hook)).Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_CloseRemovesStagingAndIsIdempotent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + prepared, staging := prepare(t, request(src, artifact.PackagingTgz)) + items, err := os.ReadDir(staging) + require.NoError(t, err) + require.NotEmpty(t, items) + require.NoError(t, prepared.Close()) + require.NoError(t, prepared.Close()) + items, err = os.ReadDir(staging) + require.NoError(t, err) + assert.Empty(t, items) +} + +func TestFilesProducer_CancellationRemovesStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + ctx, cancel := context.WithCancel(context.Background()) + staging := t.TempDir() + hook := snapshot.WithCaptureHook(func(string) { cancel() }) + _, err := produce.New(produce.WithStagingRoot(staging), produce.WithSnapshotOptions(hook)).Prepare(ctx, *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, context.Canceled) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_LimitsAreExplicitErrors(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + _, err := produce.New(produce.WithStagingRoot(t.TempDir()), produce.WithLimits(snapshot.Limits{MaxBytes: 5})).Prepare(context.Background(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestRegisterAddsFilesProducer(t *testing.T) { + registry := artifact.NewProducerRegistry() + require.NoError(t, produce.Register(registry)) + producer, ok := registry.Lookup("") + require.True(t, ok) + assert.Equal(t, artifact.KindFiles, producer.Kind()) + require.ErrorIs(t, produce.Register(registry), errUtils.ErrArtifactConflict) + assert.Equal(t, []string{artifact.KindFiles}, registry.Kinds()) +} + +func TestFilesProducer_MetadataSourceFailureIsClassifiedAndKeepsItsCause(t *testing.T) { + cause := errors.New("provider unavailable") + tests := []struct { + name string + cause error + }{ + {"a provider failure", cause}, + {"a canceled lookup", context.Canceled}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + source := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{}, tc.cause + } + _, err := produce.New(produce.WithStagingRoot(t.TempDir()), produce.WithMetadataSource(source)).Prepare(t.Context(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, tc.cause) + // The path is quoted with %q, which doubles backslashes on Windows. + assert.Contains(t, err.Error(), fmt.Sprintf("%q", src), "the source path is part of the error") + }) + } +} + +func TestFilesProducer_ArchivePayloadOpenFailureIsClassified(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + prepared, staging := prepare(t, request(src, artifact.PackagingTgz)) + + var removed int + require.NoError(t, filepath.WalkDir(staging, func(path string, entry os.DirEntry, err error) error { + if err == nil && !entry.IsDir() && filepath.Base(path) == "payload.tgz" { + removed++ + return os.Remove(path) + } + return err + })) + require.Equal(t, 1, removed, "exactly one staged archive must exist") + + var checked int + for index := range prepared.Blobs { + blob := &prepared.Blobs[index] + if blob.Role != artifact.BlobPayload { + continue + } + checked++ + _, err := blob.Open(t.Context()) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrNotExist) + } + require.Equal(t, 1, checked) +} diff --git a/pkg/artifact/produce/repack.go b/pkg/artifact/produce/repack.go new file mode 100644 index 00000000000..e9413cc780a --- /dev/null +++ b/pkg/artifact/produce/repack.go @@ -0,0 +1,150 @@ +package produce + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "time" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// RepackRequest rebuilds an existing, sealed Record for a different packaging. The Record bytes +// and content identity are preserved exactly; only the physical representation changes. +type RepackRequest struct { + // Record is the existing Record. It is not modified. + Record artifact.Record + + // Tree is a directory holding the materialized inventory (without the reserved directory). + Tree string + + // Documents holds the document bytes by role; each must match its Record descriptor. + Documents map[artifact.DocumentRole][]byte + + // Packaging is the destination packaging. + Packaging artifact.Packaging + + // OperationID is the logical operation identifier. + OperationID string + + // PreparedAt is the preparation time of this copy operation (the Record keeps its own created_at). + PreparedAt time.Time +} + +// Repack stages an existing Record from a materialized tree for the requested packaging. Every +// file read from the tree is verified against its inventory digest. +func (p *Producer) Repack(ctx context.Context, request *RepackRequest) (*artifact.PreparedArtifact, error) { + defer perf.Track(nil, "produce.Producer.Repack")() + + if request.OperationID == "" || request.PreparedAt.IsZero() { + return nil, fmt.Errorf("%w: the operation ID and preparation time are required", errUtils.ErrArtifactConfigInvalid) + } + // The Record is copied so later edits to the request cannot reach the prepared artifact. + record := request.Record.Clone() + if err := record.Validate(); err != nil { + return nil, err + } + docs, err := repackDocuments(&record, request.Documents) + if err != nil { + return nil, err + } + stagingDir, err := os.MkdirTemp(p.stagingRoot, "atmos-artifact-repack-*") + if err != nil { + return nil, fmt.Errorf("%w: create staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + prepared, err := finalize(ctx, &finalizeInput{ + record: &record, docs: docs, open: treeOpener(&record, request.Tree), stagingDir: stagingDir, packaging: request.Packaging, + operationID: request.OperationID, preparedAt: request.PreparedAt, cleanup: func() error { return os.RemoveAll(stagingDir) }, + }) + if err != nil { + _ = os.RemoveAll(stagingDir) + return nil, err + } + return prepared, nil +} + +// treeOpener opens inventory files from a materialized tree, verifying each against its digest. +func treeOpener(record *artifact.Record, tree string) opener { + inventory := make(map[string]*artifact.Entry, len(record.Inventory)) + for index := range record.Inventory { + inventory[record.Inventory[index].Path] = &record.Inventory[index] + } + return func(entryPath string) (io.ReadCloser, error) { + entry, ok := inventory[entryPath] + if !ok || entry.Type != artifact.EntryFile { + return nil, fmt.Errorf("%w: %q is not an inventory file", errUtils.ErrArtifactNotFound, entryPath) + } + file, err := os.Open(filepath.Join(tree, filepath.FromSlash(entryPath))) + if err != nil { + return nil, fmt.Errorf("%w: open %q: %w", errUtils.ErrArtifactPackagingInvalid, entryPath, err) + } + return newVerifyingReader(file, entry.Digest, entry.Size), nil + } +} + +// repackDocuments pairs each Record document descriptor with its bytes and verifies the digest. +// The bytes are copied after the check: document blobs are read lazily when the artifact is +// published, so they must not depend on a buffer the caller still owns. +func repackDocuments(record *artifact.Record, provided map[artifact.DocumentRole][]byte) ([]stagedDocument, error) { + docs := make([]stagedDocument, 0, len(record.Documents)) + for _, descriptor := range record.Documents { + data, ok := provided[descriptor.Role] + if !ok { + return nil, fmt.Errorf("%w: %s bytes are required to repack", errUtils.ErrArtifactDocumentMissing, descriptor.Role) + } + if int64(len(data)) != descriptor.Size || digest.FromBytes(data) != descriptor.Digest { + return nil, fmt.Errorf("%w: %s does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, descriptor.Role) + } + docs = append(docs, stagedDocument{descriptor: descriptor, data: bytes.Clone(data)}) + } + return docs, nil +} + +// verifyingReader fails at EOF unless the bytes read match the expected digest and size. +type verifyingReader struct { + rc io.ReadCloser + hasher hash.Hash + expected digest.Digest + size int64 + read int64 +} + +func newVerifyingReader(rc io.ReadCloser, expected digest.Digest, size int64) *verifyingReader { + return &verifyingReader{rc: rc, hasher: sha256.New(), expected: expected, size: size} +} + +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "produce.verifyingReader.Read")() + + n, err := v.rc.Read(p) + v.read += int64(n) + _, _ = v.hasher.Write(p[:n]) + if err == io.EOF { + if v.read != v.size || digest.NewDigest(digest.SHA256, v.hasher) != v.expected { + return n, fmt.Errorf("%w: file content does not match its inventory digest", errUtils.ErrArtifactIntegrityFailed) + } + return n, err + } + if err != nil { + return n, fmt.Errorf("%w: read staged file: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return n, nil +} + +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "produce.verifyingReader.Close")() + + if err := v.rc.Close(); err != nil { + return fmt.Errorf("%w: close staged file: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return nil +} diff --git a/pkg/artifact/produce/repack_test.go b/pkg/artifact/produce/repack_test.go new file mode 100644 index 00000000000..620893562f7 --- /dev/null +++ b/pkg/artifact/produce/repack_test.go @@ -0,0 +1,216 @@ +package produce_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +// treeOf materializes a prepared artifact's per-file payload into a directory. +func treeOf(t *testing.T, prepared *artifact.PreparedArtifact) string { + t.Helper() + tree := t.TempDir() + blobs := payloadBlobs(prepared) + for index := range blobs { + fixtures.WriteFile(t, tree, blobs[index].Path, string(readBlob(t, &blobs[index])), 0o644) + } + return tree +} + +func documentsOf(t *testing.T, prepared *artifact.PreparedArtifact) map[artifact.DocumentRole][]byte { + t.Helper() + docs := map[artifact.DocumentRole][]byte{} + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobDocument { + docs[prepared.Blobs[index].DocumentRole] = readBlob(t, &prepared.Blobs[index]) + } + } + return docs +} + +func TestRepack_PreservesRecordAndMatchesFreshArchive(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + fresh, _ := prepare(t, request(src, artifact.PackagingTgz)) + + req := &produce.RepackRequest{ + Record: original.Record, + Tree: treeOf(t, original), + Documents: documentsOf(t, original), + Packaging: artifact.PackagingTgz, + OperationID: "copy-op", + PreparedAt: fixedTime.AddDate(0, 0, 3), + } + repacked, err := produce.New(produce.WithStagingRoot(t.TempDir())).Repack(context.Background(), req) + require.NoError(t, err) + defer repacked.Close() + + originalBytes, err := original.Record.Canonical() + require.NoError(t, err) + repackedBytes, err := repacked.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, originalBytes, repackedBytes, "a copy preserves Record bytes") + assert.Equal(t, original.IntentDigest, repacked.IntentDigest) + assert.Equal(t, "copy-op", repacked.OperationID) + + repackedPayload := payloadBlobs(repacked) + freshPayload := payloadBlobs(fresh) + require.Len(t, repackedPayload, 1) + assert.Equal(t, readBlob(t, &freshPayload[0]), readBlob(t, &repackedPayload[0]), "deterministic assembly yields identical archive bytes") +} + +func TestRepack_Failures(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + good := func() *produce.RepackRequest { + return &produce.RepackRequest{ + Record: original.Record, Tree: treeOf(t, original), Documents: documentsOf(t, original), + Packaging: artifact.PackagingTar, OperationID: "op", PreparedAt: fixedTime, + } + } + tests := []struct { + name string + mutate func(*produce.RepackRequest) + want error + }{ + {"no operation", func(r *produce.RepackRequest) { r.OperationID = "" }, errUtils.ErrArtifactConfigInvalid}, + {"missing document", func(r *produce.RepackRequest) { delete(r.Documents, artifact.DocumentReadme) }, errUtils.ErrArtifactDocumentMissing}, + {"tampered document", func(r *produce.RepackRequest) { r.Documents[artifact.DocumentReadme] = []byte("tampered") }, errUtils.ErrArtifactIntegrityFailed}, + {"tampered tree file", func(r *produce.RepackRequest) { + require.NoError(t, os.WriteFile(filepath.Join(r.Tree, "nested", "deep", "file.txt"), []byte("tampered content"), 0o644)) + }, errUtils.ErrArtifactPackagingInvalid}, + {"missing tree file", func(r *produce.RepackRequest) { + require.NoError(t, os.Remove(filepath.Join(r.Tree, "empty.txt"))) + }, errUtils.ErrArtifactPackagingInvalid}, + {"invalid record", func(r *produce.RepackRequest) { r.Record.ContentDigest = "" }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := good() + tt.mutate(req) + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Repack(context.Background(), req) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} + +// TestRepack_PreparedArtifactIsIndependentOfTheRequest proves that what the caller passes in is +// copied at the boundary: editing the request after Repack returns changes nothing the prepared +// artifact serves, and editing the prepared artifact changes nothing in the request. +func TestRepack_PreparedArtifactIsIndependentOfTheRequest(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + wantDocs := documentsOf(t, original) + require.NotEmpty(t, wantDocs[artifact.DocumentReadme]) + + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + documents := documentsOf(t, original) + req := &produce.RepackRequest{ + Record: original.Record.Clone(), Tree: treeOf(t, original), Documents: documents, + Packaging: packaging, OperationID: "op", PreparedAt: fixedTime, + } + repacked, err := produce.New(produce.WithStagingRoot(t.TempDir())).Repack(context.Background(), req) + require.NoError(t, err) + defer repacked.Close() + wantRecord, err := repacked.Record.Canonical() + require.NoError(t, err) + + // Request to prepared artifact: scribble over everything the caller still owns. + for role := range req.Documents { + for index := range req.Documents[role] { + req.Documents[role][index] = 'X' + } + } + req.Record.Metadata.Labels = map[string]string{"scribble": "x"} + req.Record.Inventory[0].Path = "scribble" + req.Record.Inventory[len(req.Record.Inventory)-1].Digest = "sha256:scribble" + req.Record.Documents[0].SourcePath = "scribble" + + for role, want := range wantDocs { + blob := documentBlob(t, repacked, role) + got := readBlob(t, blob) + assert.Equal(t, want, got, "the %s document served later is the one sealed in its descriptor", role) + assert.Equal(t, blob.Descriptor.Digest, digest.FromBytes(got)) + } + gotRecord, err := repacked.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, wantRecord, gotRecord, "the prepared Record does not follow the request's Record") + if packaging == artifact.PackagingNone { + wantFiles := map[string][]byte{} + for index, blob := range payloadBlobs(original) { + wantFiles[blob.Path] = readBlob(t, &payloadBlobs(original)[index]) + } + gotBlobs := payloadBlobs(repacked) + require.Len(t, gotBlobs, len(wantFiles)) + for index := range gotBlobs { + assert.Equal(t, wantFiles[gotBlobs[index].Path], readBlob(t, &gotBlobs[index]), "payload files still verify against the original inventory: %s", gotBlobs[index].Path) + } + } + + // Prepared artifact to request: editing the result must not reach the request. + req.Record.Inventory[0].Path = "kept" + repacked.Record.Inventory[0].Path = "scribble-too" + assert.Equal(t, "kept", req.Record.Inventory[0].Path) + }) + } +} + +func documentBlob(t *testing.T, prepared *artifact.PreparedArtifact, role artifact.DocumentRole) *artifact.StagedBlob { + t.Helper() + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobDocument && prepared.Blobs[index].DocumentRole == role { + return &prepared.Blobs[index] + } + } + require.FailNow(t, "the prepared artifact has no such document blob", string(role)) + return nil +} + +// TestPrepare_RecordIsIndependentOfTheRequestAndMetadata proves the pointer and byte-slice fields +// the producer is handed (provenance and kind data) are copied into the sealed Record. +func TestPrepare_RecordIsIndependentOfTheRequestAndMetadata(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + provenance := &artifact.Provenance{ + CI: &artifact.CIProvenance{Provider: "github", RunID: "1"}, + Source: &artifact.SourceProvenance{Commit: "abc"}, + } + kindData := artifact.KindData(`{"k":1}`) + req := request(src, artifact.PackagingTar) + req.Metadata.Provenance = provenance + source := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{KindData: kindData}, nil + } + prepared, _ := prepare(t, req, produce.WithMetadataSource(source)) + want, err := prepared.Record.Canonical() + require.NoError(t, err) + + provenance.CI.RunID = "scribble" + provenance.Source.Commit = "scribble" + kindData[0] = 'X' + + got, err := prepared.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, want, got, "later edits to the request or the metadata do not reach the Record") + assert.Equal(t, "1", prepared.Record.Metadata.Provenance.CI.RunID) + assert.JSONEq(t, `{"k":1}`, string(prepared.Record.KindData)) +} diff --git a/pkg/artifact/producers.go b/pkg/artifact/producers.go new file mode 100644 index 00000000000..2b0cad4f40b --- /dev/null +++ b/pkg/artifact/producers.go @@ -0,0 +1,72 @@ +package artifact + +import ( + "fmt" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ProducerRegistry maps artifact kinds to the [Producer] that prepares them. The generic files +// producer is registered by a composition root; kind-specific producers (scaffold, component, and +// so on) register the same way and add their own validation and exclusions. +type ProducerRegistry struct { + mu sync.RWMutex + producers map[string]Producer +} + +// NewProducerRegistry returns an empty registry. +func NewProducerRegistry() *ProducerRegistry { + defer perf.Track(nil, "artifact.NewProducerRegistry")() + + return &ProducerRegistry{producers: make(map[string]Producer)} +} + +// Register adds a producer under its Kind. A duplicate kind or an invalid kind is an error. +func (r *ProducerRegistry) Register(producer Producer) error { + defer perf.Track(nil, "artifact.ProducerRegistry.Register")() + + if producer == nil { + return fmt.Errorf("%w: producer must not be nil", errUtils.ErrArtifactConfigInvalid) + } + kind := producer.Kind() + if err := ValidateKind(kind); err != nil { + return fmt.Errorf("%w: producer kind %q: %w", errUtils.ErrArtifactConfigInvalid, kind, err) + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.producers[kind]; exists { + return fmt.Errorf("%w: producer for kind %q is already registered", errUtils.ErrArtifactConflict, kind) + } + r.producers[kind] = producer + return nil +} + +// Lookup returns the producer for kind; an empty kind means [KindFiles]. +func (r *ProducerRegistry) Lookup(kind string) (Producer, bool) { + defer perf.Track(nil, "artifact.ProducerRegistry.Lookup")() + + if kind == "" { + kind = KindFiles + } + r.mu.RLock() + defer r.mu.RUnlock() + producer, ok := r.producers[kind] + return producer, ok +} + +// Kinds returns the registered kinds, sorted. +func (r *ProducerRegistry) Kinds() []string { + defer perf.Track(nil, "artifact.ProducerRegistry.Kinds")() + + r.mu.RLock() + defer r.mu.RUnlock() + kinds := make([]string, 0, len(r.producers)) + for kind := range r.producers { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + return kinds +} diff --git a/pkg/artifact/producers_test.go b/pkg/artifact/producers_test.go new file mode 100644 index 00000000000..fef643c18cc --- /dev/null +++ b/pkg/artifact/producers_test.go @@ -0,0 +1,53 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestProducerRegistry(t *testing.T) { + ctrl := gomock.NewController(t) + files := NewMockProducer(ctrl) + files.EXPECT().Kind().Return(KindFiles).AnyTimes() + scaffold := NewMockProducer(ctrl) + scaffold.EXPECT().Kind().Return("scaffold").AnyTimes() + invalid := NewMockProducer(ctrl) + invalid.EXPECT().Kind().Return("Bad Kind").AnyTimes() + + registry := NewProducerRegistry() + require.NoError(t, registry.Register(scaffold)) + require.NoError(t, registry.Register(files)) + require.ErrorIs(t, registry.Register(files), errUtils.ErrArtifactConflict) + require.ErrorIs(t, registry.Register(invalid), errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, registry.Register(nil), errUtils.ErrArtifactConfigInvalid) + + found, ok := registry.Lookup("") + require.True(t, ok) + assert.Equal(t, KindFiles, found.Kind(), "an empty kind selects the generic files producer") + found, ok = registry.Lookup("scaffold") + require.True(t, ok) + assert.Equal(t, "scaffold", found.Kind()) + _, ok = registry.Lookup("component/terraform") + assert.False(t, ok) + assert.Equal(t, []string{KindFiles, "scaffold"}, registry.Kinds()) +} + +// A signer is refused before it runs when the repository can store none of its formats. +func TestCapabilitiesCheckSignerFormats(t *testing.T) { + caps := &Capabilities{EvidenceFormats: []EvidenceFormat{EvidenceSigstoreBundle}} + + usable := NewMockSigner(gomock.NewController(t)) + usable.EXPECT().Formats().Return([]EvidenceFormat{"other", EvidenceSigstoreBundle}) + require.NoError(t, caps.CheckSignerFormats("repo", usable), "one storable format is enough") + + unusable := NewMockSigner(gomock.NewController(t)) + unusable.EXPECT().Formats().Return([]EvidenceFormat{"a", "b"}) + err := caps.CheckSignerFormats("repo", unusable) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Contains(t, err.Error(), "evidence/a,b") +} diff --git a/pkg/artifact/service/boundary_test.go b/pkg/artifact/service/boundary_test.go new file mode 100644 index 00000000000..61942f6a8bd --- /dev/null +++ b/pkg/artifact/service/boundary_test.go @@ -0,0 +1,45 @@ +package service_test + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestArtifactPackagesImportNoCLIRunnerHookPluginOrCI keeps the artifact service and the packages +// it composes (preparation, staging, materialization) free of CLI, runner, hook, plugin, and CI +// dependencies, as the package boundary contract requires. Test-support packages are checked too. +func TestArtifactPackagesImportNoCLIRunnerHookPluginOrCI(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + } + files, err := filepath.Glob(filepath.Join("..", "*", "*.go")) + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go new file mode 100644 index 00000000000..6483fde9fe9 --- /dev/null +++ b/pkg/artifact/service/commit.go @@ -0,0 +1,243 @@ +package service + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// target is the repository a mutation writes to. +type target struct { + alias string + repo artifact.Repository + info *RepositoryInfo +} + +// incoming describes the artifact being written, for comparison and reconciliation. +type incoming struct { + ref artifact.Ref + record *artifact.Record + intent digest.Digest + recordDigest digest.Digest + payload []ocispec.Descriptor + size int64 + prepared *artifact.PreparedArtifact + inspection *artifact.Inspection + + // rootDigest is the pinned source root of a native graph copy; reconciliation compares it when + // there is no Record digest to compare. + rootDigest digest.Digest +} + +func newIncoming(prepared *artifact.PreparedArtifact) (*incoming, error) { + recordDigest, err := prepared.Record.RecordDigest() + if err != nil { + return nil, err + } + in := &incoming{ref: artifact.Ref{Name: prepared.Record.Name, Version: prepared.Record.Version}, record: &prepared.Record, intent: prepared.IntentDigest, recordDigest: recordDigest, prepared: prepared} + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobPayload { + in.payload = append(in.payload, prepared.Blobs[index].Descriptor) + in.size += prepared.Blobs[index].Descriptor.Size + } + } + in.inspection = &artifact.Inspection{ + Record: in.record, + Representation: artifact.Representation{Packaging: prepared.Packaging, Payload: in.payload}, + } + return in, nil +} + +// current is what a reference holds right now. +type current struct { + pinned *artifact.PinnedReference + inspection *artifact.Inspection + unavailable bool +} + +type verdict int + +const ( + verdictCreate verdict = iota + verdictReplace + verdictUnchanged +) + +// writer performs one repository write attempt. +type writer func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) + +// publishState is filled by commit for the caller's outcome. +type publishState struct { + record artifact.Record + existing bool +} + +// publication bundles everything one write needs: the operation, the target repository, the +// incoming artifact, the force flag, the write closure, and the state commit fills in. +type publication struct { + op *operation + tgt *target + in *incoming + force bool + write writer + state *publishState +} + +// commit runs the shared publication sequence under the writer lock: read the current reference, +// compare intent, apply conflict and immutability rules, then write with reconciliation. +func (s *Service) commit(ctx context.Context, pub *publication) (err error) { + ref := pub.in.ref + pub.op.result.Ref = ref.String() + locks, err := s.acquireLocks(ctx, pub.op, opPublish, []lockTarget{{alias: pub.tgt.alias, repo: pub.tgt.repo, info: pub.tgt.info, resource: ref.String()}}) + if err != nil { + return err + } + s.event(ctx, pub, PhaseLocked, 0, nil) + defer func() { + // A failed release already raised its recovery warning with the owner and resource; the + // journal must not claim the lock was released, and the committed outcome is unchanged. + if releaseErr := locks.release(ctx, pub.op); releaseErr == nil { + s.event(ctx, pub, PhaseReleased, 0, nil) + } + if err != nil { + s.event(ctx, pub, PhaseFailed, 0, err) + } + }() + + cur, err := s.readCurrent(ctx, pub.tgt, ref) + if err != nil { + return err + } + decision, err := s.decide(pub.tgt, cur, pub.in, pub.force) + if err != nil { + return err + } + if decision == verdictUnchanged { + s.reportUnchanged(pub, cur) + return nil + } + result, err := s.execute(ctx, pub, cur, decision, locks.handleFor(pub.tgt.alias)) + if err != nil { + return err + } + s.reportWritten(pub, result, locks, decision, cur) + return nil +} + +func (s *Service) event(ctx context.Context, pub *publication, phase JournalPhase, attempt int, err error) { + s.record(ctx, &JournalEntry{ + OperationID: pub.op.id, Phase: phase, Repository: pub.tgt.alias, Ref: pub.in.ref.String(), + IntentDigest: pub.in.intent.String(), Attempt: attempt, Err: err, + }) +} + +// readCurrent resolves and inspects the reference once. A missing reference is not an error; a +// reference that exists but is unavailable (pending or interrupted) is reported so the same-intent +// retry can repair it. +func (s *Service) readCurrent(ctx context.Context, tgt *target, ref artifact.Ref) (*current, error) { + pinned, err := tgt.repo.Resolve(ctx, ref) + switch { + case errors.Is(err, errUtils.ErrArtifactNotFound): + return ¤t{}, nil + case errors.Is(err, errUtils.ErrArtifactPublicationUnavailable): + return ¤t{unavailable: true}, nil + case err != nil: + return nil, fmt.Errorf("resolve %s in repository %q: %w", ref, tgt.alias, err) + } + inspection, err := tgt.repo.Inspect(ctx, pinned) + if err != nil { + return nil, fmt.Errorf("inspect %s in repository %q: %w", ref, tgt.alias, err) + } + return ¤t{pinned: &pinned, inspection: inspection}, nil +} + +// decide applies the intent comparison, conflict, force, and immutability rules. +func (s *Service) decide(tgt *target, cur *current, in *incoming, force bool) (verdict, error) { + if cur.pinned == nil { + return verdictCreate, nil + } + existing := cur.inspection.Record + if sameIntent(existing, in) || sameGraphRoot(existing, cur, in) { + return verdictUnchanged, nil + } + if immutable, reason := s.immutableCandidate(tgt, cur, in); immutable { + return verdictCreate, fmt.Errorf("%w: %s stays immutable even with force", errUtils.ErrArtifactConflict, reason) + } + if !force { + return verdictCreate, fmt.Errorf("%w: %s already exists in repository %q with different %s; use force to replace it", + errUtils.ErrArtifactConflict, in.ref, tgt.alias, describeDifferences(existing, in.record)) + } + return verdictReplace, nil +} + +// sameIntent reports whether the stored Record carries the incoming intent. +func sameIntent(existing *artifact.Record, in *incoming) bool { + if existing == nil { + return false + } + intent, err := existing.IntentDigest() + return err == nil && intent == in.intent +} + +// sameGraphRoot reports whether a Record-free destination already points at the pinned source root +// of a native graph copy. It uses the same root identity as reconcileGraph. +func sameGraphRoot(existing *artifact.Record, cur *current, in *incoming) bool { + return existing == nil && in.rootDigest != "" && cur.pinned.ResolvedDigest == in.rootDigest +} + +// immutableCandidate asks the admission policy about both the stored and the incoming artifact. +func (s *Service) immutableCandidate(tgt *target, cur *current, in *incoming) (bool, string) { + if s.immutable == nil { + return false, "" + } + if immutable, why := s.immutable.Immutable(Candidate{Existing: true, Repository: tgt.alias, Inspection: cur.inspection}); immutable { + return true, "the existing release (" + why + ")" + } + if immutable, why := s.immutable.Immutable(Candidate{Repository: tgt.alias, Inspection: in.inspection, Prepared: in.prepared}); immutable { + return true, "the incoming release (" + why + ")" + } + return false, "" +} + +// describeDifferences names which parts of identity changed. +func describeDifferences(existing, incoming *artifact.Record) string { + if existing == nil || incoming == nil { + return "content (the existing artifact carries no portable record)" + } + var differences []string + if existing.ContentDigest != incoming.ContentDigest { + differences = append(differences, "content (files, modes, or documents)") + } + if existing.Metadata.Description != incoming.Metadata.Description { + differences = append(differences, "description") + } + if !labelsEqual(existing.Metadata.Labels, incoming.Metadata.Labels) { + differences = append(differences, "labels") + } + if existing.Kind != incoming.Kind || string(existing.KindData) != string(incoming.KindData) { + differences = append(differences, "kind data") + } + if len(differences) == 0 { + return "intent" + } + return strings.Join(differences, ", ") +} + +func labelsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for key, value := range a { + if other, ok := b[key]; !ok || other != value { + return false + } + } + return true +} diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go new file mode 100644 index 00000000000..70c6986f558 --- /dev/null +++ b/pkg/artifact/service/config_repositories.go @@ -0,0 +1,187 @@ +package service + +import ( + "context" + "errors" + "fmt" + "maps" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// ConfigRepositories implements [Repositories] from the validated artifacts configuration and a +// repository kind registry. Describe reads only configuration and kind descriptors, so it never +// authenticates; Open constructs each repository once through the registry, which is where +// identities are resolved. +type ConfigRepositories struct { + cfg *schema.ArtifactConfig + registry *artifact.RepositoryRegistry + options []artifact.RepositoryOption + mu sync.Mutex + opened map[string]*openedRepository +} + +// An openedRepository is the construction state of one alias. Its done channel closes when +// construction ends, and the repository and error are valid after that. The shared mutex guards only +// the map of these entries, never a construction: building a repository may resolve credentials and +// wait on a user. +type openedRepository struct { + done chan struct{} + repo artifact.Repository + err error +} + +var _ Repositories = (*ConfigRepositories)(nil) + +// FromConfig returns a [Repositories] backed by configuration. A nil registry uses the default +// repository registry. +func FromConfig(cfg *schema.ArtifactConfig, registry *artifact.RepositoryRegistry, opts ...artifact.RepositoryOption) *ConfigRepositories { + defer perf.Track(nil, "service.FromConfig")() + + if registry == nil { + registry = artifact.DefaultRepositoryRegistry() + } + return &ConfigRepositories{cfg: cfg, registry: registry, options: opts, opened: map[string]*openedRepository{}} +} + +// Describe implements [Repositories]. For a repository that delegates locking, the lock scopes of +// the delegate are added to its capabilities, so the lock guarantee check sees what the delegate +// can provide. +func (c *ConfigRepositories) Describe(alias string) (RepositoryInfo, error) { + defer perf.Track(nil, "service.ConfigRepositories.Describe")() + + repoCfg, ok := c.cfg.Repositories[alias] + if !ok { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, known := c.registry.Lookup(repoCfg.Kind) + if !known { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q has unknown kind %q", errUtils.ErrArtifactConfigInvalid, alias, repoCfg.Kind) + } + plan, err := artifactconfig.ResolveLockPlan(c.cfg, alias) + if err != nil { + return RepositoryInfo{}, err + } + policy, err := artifactconfig.EffectivePolicy(c.cfg, alias) + if err != nil { + return RepositoryInfo{}, err + } + info := RepositoryInfo{ + Alias: alias, + Capabilities: desc.Capabilities.Clone(), + Packaging: desc.EffectivePackaging(repoCfg.Packaging), + RequiredLockScope: plan.Required, + Policy: policy, + } + if repoCfg.Defaults != nil && repoCfg.Defaults.Metadata != nil { + info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: maps.Clone(repoCfg.Defaults.Metadata.Labels)} + } + if plan.Provider != alias { + info.LockProvider = plan.Provider + info.Capabilities.LockScopes = c.delegatedScopes(&info.Capabilities, plan.Provider) + } + return info, nil +} + +// delegatedScopes adds the delegate repository's lock scopes to the repository's own. +func (c *ConfigRepositories) delegatedScopes(caps *artifact.Capabilities, provider string) []artifact.LockScope { + scopes := append([]artifact.LockScope(nil), caps.LockScopes...) + delegate, ok := c.cfg.Repositories[provider] + if !ok { + return scopes + } + desc, known := c.registry.Lookup(delegate.Kind) + if !known { + return scopes + } + for _, scope := range desc.Capabilities.LockScopes { + if !containsScope(scopes, scope) { + scopes = append(scopes, scope) + } + } + return scopes +} + +func containsScope(scopes []artifact.LockScope, scope artifact.LockScope) bool { + for _, have := range scopes { + if have == scope { + return true + } + } + return false +} + +// Open implements [Repositories]. Each repository is constructed once, and constructing one does not +// block opening another: callers waiting for the same alias share its construction and stop waiting +// when their own context ends. A failed construction is not cached, so a later Open retries it. +func (c *ConfigRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { + defer perf.Track(nil, "service.ConfigRepositories.Open")() + + for { + c.mu.Lock() + if entry, ok := c.opened[alias]; ok { + c.mu.Unlock() + repo, retry, err := c.await(ctx, entry) + if retry { + continue + } + return repo, err + } + repoCfg, ok := c.cfg.Repositories[alias] + if !ok { + c.mu.Unlock() + return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + entry := &openedRepository{done: make(chan struct{})} + c.opened[alias] = entry + c.mu.Unlock() + return c.construct(ctx, alias, &repoCfg, entry) + } +} + +// construct builds the repository outside the shared mutex and publishes the outcome to every caller +// waiting on entry. A failure removes the entry so a later Open can try again. +func (c *ConfigRepositories) construct(ctx context.Context, alias string, repoCfg *schema.ArtifactRepositoryConfig, entry *openedRepository) (artifact.Repository, error) { + entry.repo, entry.err = c.registry.New(ctx, alias, repoCfg, c.options...) + if entry.err != nil { + c.mu.Lock() + if c.opened[alias] == entry { + delete(c.opened, alias) + } + c.mu.Unlock() + } + close(entry.done) + return entry.repo, entry.err +} + +// await waits for another caller's construction of entry, or for ctx to end. It reports retry when that +// construction ended because the other caller's own context did, which says nothing about this caller. +func (c *ConfigRepositories) await(ctx context.Context, entry *openedRepository) (artifact.Repository, bool, error) { + select { + case <-entry.done: + case <-ctx.Done(): + return nil, false, ctx.Err() + } + if entry.err != nil && ctx.Err() == nil && (errors.Is(entry.err, context.Canceled) || errors.Is(entry.err, context.DeadlineExceeded)) { + return nil, true, nil + } + return entry.repo, false, entry.err +} + +// Aliases implements [Repositories]. +func (c *ConfigRepositories) Aliases() []string { + defer perf.Track(nil, "service.ConfigRepositories.Aliases")() + + aliases := make([]string, 0, len(c.cfg.Repositories)) + for alias := range c.cfg.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/service/config_repositories_test.go b/pkg/artifact/service/config_repositories_test.go new file mode 100644 index 00000000000..d4445b39f37 --- /dev/null +++ b/pkg/artifact/service/config_repositories_test.go @@ -0,0 +1,275 @@ +package service_test + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +func configRegistry(t *testing.T, constructed *atomic.Int64) *artifact.RepositoryRegistry { + t.Helper() + registry := artifact.NewRepositoryRegistry() + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindS3} { + desc, ok := artifact.BuiltinKind(kind) + require.True(t, ok) + require.NoError(t, registry.Register(desc, func(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + constructed.Add(1) + return artifacttest.New(artifacttest.Options{Alias: alias}), nil + })) + } + return registry +} + +func boolPtr(b bool) *bool { return &b } + +func TestConfigRepositories_DescribeIsOfflineAndCombinesPolicyDefaultsAndLocking(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{ + Verification: &schema.ArtifactVerificationConfig{ + RequireSignature: boolPtr(true), + TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "global@example.com", Issuer: "https://global.example.com"}}, + }, + Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": { + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": "/tmp/shared"}, + Defaults: &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{ + Description: "default description", Labels: map[string]string{"team": "platform"}, + }}, + Verification: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "repo@example.com", Issuer: "https://repo.example.com"}}}, + Locking: &schema.ArtifactLockingConfig{RequiredScope: "repository", Repository: "coordination"}, + }, + "coordination": {Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-coordination"}}, + }, + } + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + + assert.Equal(t, []string{"coordination", "shared"}, repos.Aliases()) + info, err := repos.Describe("shared") + require.NoError(t, err) + assert.Zero(t, constructed.Load(), "describing never constructs a repository, so it never authenticates") + assert.Equal(t, artifact.PackagingNone, info.Packaging) + assert.Equal(t, artifact.LockScopeRepository, info.RequiredLockScope) + assert.True(t, info.Capabilities.HasLockScope(artifact.LockScopeRepository), "the delegate's scope is advertised") + assert.Equal(t, "coordination", info.LockProvider, "locks are routed to the delegate") + assert.Equal(t, "default description", info.Defaults.Description) + assert.Equal(t, map[string]string{"team": "platform"}, info.Defaults.Labels) + assert.True(t, info.Policy.RequireSignature) + assert.Len(t, info.Policy.TrustSets, 2, "global and repository trust sets are separate, both required") + + coordination, err := repos.Describe("coordination") + require.NoError(t, err) + assert.Equal(t, artifact.PackagingTgz, coordination.Packaging) + assert.Equal(t, artifact.LockScopeHost, coordination.RequiredLockScope) + assert.Empty(t, coordination.LockProvider, "a repository without delegation locks for itself") + assert.Len(t, coordination.Policy.TrustSets, 1) +} + +func TestConfigRepositories_OpenConstructsOnceAndRejectsUnknowns(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/shared"}}, + "ghost": {Kind: "unregistered/kind"}, + "s3": {Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifacts"}}, + }} + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + + first, err := repos.Open(context.Background(), "shared") + require.NoError(t, err) + second, err := repos.Open(context.Background(), "shared") + require.NoError(t, err) + assert.Same(t, first, second) + assert.Equal(t, int64(1), constructed.Load()) + + _, err = repos.Open(context.Background(), "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Describe("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Describe("ghost") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Open(context.Background(), "ghost") + require.Error(t, err) +} + +func TestConfigRepositories_DrivesTheServiceEndToEnd(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/shared"}}, + }} + e := newEnv(t, envConfig{}) + svc, err := service.New(service.FromConfig(cfg, configRegistry(t, &constructed)), nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + + dry := e.publishRequest("shared", e.source("hello")) + dry.DryRun = true + _, err = svc.Publish(context.Background(), dry) + require.NoError(t, err) + assert.Zero(t, constructed.Load(), "a dry run through configuration performs no repository construction or identity resolution") + + result, err := svc.Publish(context.Background(), e.publishRequest("shared", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, int64(1), constructed.Load()) +} + +// TestConfigRepositories_DescribeReturnsIndependentCopies proves a caller editing what Describe +// returned cannot change the registry's capabilities or the configured defaults, which every later +// Describe would otherwise serve. +func TestConfigRepositories_DescribeReturnsIndependentCopies(t *testing.T) { + var constructed atomic.Int64 + labels := map[string]string{"team": "platform"} + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": { + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": "/tmp/shared"}, + Defaults: &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{Labels: labels}}, + }, + }} + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + pristine, err := repos.Describe("shared") + require.NoError(t, err) + wantPackagings := append([]artifact.Packaging(nil), pristine.Capabilities.Packagings...) + wantLockScopes := append([]artifact.LockScope(nil), pristine.Capabilities.LockScopes...) + require.NotEmpty(t, wantPackagings) + require.NotEmpty(t, wantLockScopes) + + first, err := repos.Describe("shared") + require.NoError(t, err) + first.Capabilities.Packagings[0] = artifact.PackagingZip + first.Capabilities.LockScopes[0] = artifact.LockScopeRepository + first.Defaults.Labels["team"] = "scribble" + + second, err := repos.Describe("shared") + require.NoError(t, err) + assert.Equal(t, wantPackagings, second.Capabilities.Packagings) + assert.Equal(t, wantLockScopes, second.Capabilities.LockScopes) + assert.Equal(t, map[string]string{"team": "platform"}, second.Defaults.Labels) + assert.Equal(t, map[string]string{"team": "platform"}, labels, "the configured defaults are untouched") +} + +// blockingRegistry builds repositories like configRegistry, but the construction of "slow" waits for +// release, so a test can hold one alias mid-construction. +func blockingRegistry(t *testing.T, started chan<- struct{}, release <-chan struct{}, constructed *atomic.Int64) *artifact.RepositoryRegistry { + t.Helper() + registry := artifact.NewRepositoryRegistry() + desc, ok := artifact.BuiltinKind(artifact.RepositoryKindLocalDir) + require.True(t, ok) + require.NoError(t, registry.Register(desc, func(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + constructed.Add(1) + if alias == "slow" { + started <- struct{}{} + <-release + } + return artifacttest.New(artifacttest.Options{Alias: alias}), nil + })) + return registry +} + +func blockingConfig() *schema.ArtifactConfig { + return &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "fast": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/fast"}}, + "slow": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/slow"}}, + }} +} + +// While one repository is being constructed (it may be resolving credentials), opening an already +// cached repository must not wait for it. +func TestConfigRepositories_OpenOfACachedRepositoryIsNotBlockedByAnotherConstruction(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 1), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + cached, err := repos.Open(context.Background(), "fast") + require.NoError(t, err) + + slowDone := make(chan error, 1) + go func() { + _, openErr := repos.Open(context.Background(), "slow") + slowDone <- openErr + }() + <-started + + again := make(chan artifact.Repository, 1) + go func() { + repo, _ := repos.Open(context.Background(), "fast") + again <- repo + }() + select { + case repo := <-again: + assert.Same(t, cached, repo) + case <-time.After(2 * time.Second): + close(release) + t.Fatal("opening a cached repository waited for another repository's construction") + } + close(release) + require.NoError(t, <-slowDone) +} + +// A caller waiting for a repository that another caller is constructing stops waiting when its own +// context ends, and the construction still completes for everyone else. +func TestConfigRepositories_AWaitingOpenHonorsItsContext(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 1), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + + firstDone := make(chan artifact.Repository, 1) + go func() { + repo, _ := repos.Open(context.Background(), "slow") + firstDone <- repo + }() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + waiter := make(chan error, 1) + go func() { + _, err := repos.Open(ctx, "slow") + waiter <- err + }() + cancel() + select { + case err := <-waiter: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + close(release) + t.Fatal("a canceled waiter kept waiting for the construction") + } + close(release) + require.NotNil(t, <-firstDone) + assert.Equal(t, int64(1), constructed.Load(), "the waiter did not start a second construction") +} + +// Callers that race for the same alias share one construction and one repository. +func TestConfigRepositories_ConcurrentOpensOfOneAliasConstructOnce(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 8), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + + const callers = 8 + results := make([]artifact.Repository, callers) + var wg sync.WaitGroup + for i := range results { + wg.Add(1) + go func() { + defer wg.Done() + results[i], _ = repos.Open(context.Background(), "slow") + }() + } + <-started + close(release) + wg.Wait() + for _, repo := range results { + require.NotNil(t, repo) + assert.Same(t, results[0], repo) + } + assert.Equal(t, int64(1), constructed.Load()) +} diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go new file mode 100644 index 00000000000..fa77e9db59c --- /dev/null +++ b/pkg/artifact/service/consume_test.go @@ -0,0 +1,477 @@ +package service_test + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func out(t *testing.T) string { + return filepath.Join(t.TempDir(), "out", "pulled") +} + +func TestPull_RoundTripEveryPackaging(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = packaging }) + src := e.source("hello") + published := e.publish("dst", src) + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, result.Outcome) + assert.Equal(t, dest, result.Location) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + + state, err := materialize.InspectDestination(dest, &published.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + }) + } +} + +func TestPull_ExistingDestinationRules(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + dest := out(t) + request := artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest} + _, err := e.svc.Pull(context.Background(), request) + require.NoError(t, err) + reads := e.repos["dst"].Calls("Read") + + again, err := e.svc.Pull(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "an identical destination is a no-op") + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "nothing is downloaded for an identical destination") + + fixtures.WriteFile(t, dest, "stale.txt", "stale", 0o644) + require.NoError(t, os.WriteFile(filepath.Join(dest, "data.txt"), []byte("locally edited"), 0o644)) + _, err = e.svc.Pull(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a different non-empty destination needs force") + data, readErr := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, readErr) + assert.Equal(t, "locally edited", string(data), "a refused pull leaves the destination untouched") + + request.Force = true + forced, err := e.svc.Pull(context.Background(), request) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + data, readErr := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, readErr) + assert.Equal(t, "locally edited", string(data)) + empty := out(t) + require.NoError(t, os.MkdirAll(empty, 0o755)) + request.Destination, request.Force = empty, false + _, err = e.svc.Pull(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + entries, readErr := os.ReadDir(empty) + require.NoError(t, readErr) + assert.Empty(t, entries) + return + } + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, forced.Outcome) + _, statErr := os.Stat(filepath.Join(dest, "stale.txt")) + assert.True(t, os.IsNotExist(statErr), "stale files are removed, never merged") + + empty := out(t) + require.NoError(t, os.MkdirAll(empty, 0o755)) + request.Destination, request.Force = empty, false + _, err = e.svc.Pull(context.Background(), request) + require.NoError(t, err, "an empty destination needs no force") +} + +func TestPull_Failures(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + tests := []struct { + name string + request artifact.PullRequest + want error + }{ + {"missing repository", artifact.PullRequest{Ref: exampleRef, Destination: out(t)}, errUtils.ErrArtifactConfigInvalid}, + {"missing destination", artifact.PullRequest{Repository: "dst", Ref: exampleRef}, errUtils.ErrArtifactConfigInvalid}, + {"invalid ref", artifact.PullRequest{Repository: "dst", Ref: artifact.Ref{Name: "x"}, Destination: out(t)}, errUtils.ErrArtifactRefInvalid}, + {"not found", artifact.PullRequest{Repository: "dst", Ref: ref("tools/example", "9.9.9"), Destination: out(t)}, errUtils.ErrArtifactNotFound}, + {"unknown repository", artifact.PullRequest{Repository: "nope", Ref: exampleRef, Destination: out(t)}, errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := e.svc.Pull(context.Background(), tt.request) + require.ErrorIs(t, err, tt.want) + require.NotNil(t, result) + assert.NotEmpty(t, result.OperationID) + _, statErr := os.Stat(tt.request.Destination) + assert.True(t, tt.request.Destination == "" || os.IsNotExist(statErr), "a failed pull creates nothing") + }) + } +} + +func TestPull_WriteOnlyRepositoryIsACapabilityError(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Zero(t, e.repos["dst"].TotalCalls()) +} + +// moveTagOnFirstResolve repoints exampleRef at the decoy the first time it is resolved. +func moveTagOnFirstResolve(t *testing.T, e *env, decoyVersion string) { + t.Helper() + decoy, ok := e.repos["dst"].RecordDigestOf(ref("tools/example", decoyVersion)) + require.True(t, ok) + moved := false + e.repos["dst"].SetOnResolved(func(resolved artifact.Ref) { + if resolved == exampleRef && !moved { + moved = true + assert.True(t, e.repos["dst"].MoveRef(exampleRef, decoy)) + } + }) +} + +func publishDecoy(t *testing.T, e *env) { + t.Helper() + request := e.publishRequest("dst", e.source("DECOY CONTENT")) + request.Version = "2.0.0" + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) +} + +func TestPull_TagMovedMidReadStillConsumesTheOriginallyPinnedSubject(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + resolves := e.repos["dst"].Calls("Resolve") + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["dst"].Calls("Resolve")-resolves, "the mutable reference is resolved exactly once") + assert.Equal(t, original.Operation.RecordDigest, result.RecordDigest) + data, err := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, err) + assert.Equal(t, "ORIGINAL", string(data), "the moved tag never substitutes other bytes") + moved, _ := e.repos["dst"].RecordDigestOf(exampleRef) + assert.NotEqual(t, original.Operation.RecordDigest, moved, "the tag really moved") +} + +func TestInspectAndShowResolveOnce(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + before := e.repos["dst"].Calls("Resolve") + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["dst"].Calls("Resolve")-before) + assert.Equal(t, original.Operation.RecordDigest, inspected.Operation.RecordDigest) + assert.Equal(t, original.Operation.RecordDigest, inspected.Inspection.Pinned.RecordDigest) + require.NotNil(t, inspected.Inspection.Record) + assert.Equal(t, original.Record.ContentDigest, inspected.Inspection.Record.ContentDigest) + assert.Nil(t, inspected.Verification, "no trust configuration reports unchecked, never verified") +} + +func TestShowDocument(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + reads := e.repos["dst"].Calls("Read") + + result, err := e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.NoError(t, err) + assert.Equal(t, "package readme\n", string(result.Content)) + assert.Equal(t, artifact.DocumentReadme, result.Document.Role) + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "documents are read without the payload") + assert.Equal(t, int64(len(result.Content)), result.Operation.Size) + + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentChangelog}) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing) + + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: "license"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestInspectAndShow_Failures(t *testing.T) { + e := newEnv(t, envConfig{}) + _, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func seedList(t *testing.T, e *env, alias string, items map[string]string, labels map[string]string) { + t.Helper() + for nameVersion, content := range items { + name, version, _ := splitRef(nameVersion) + request := artifact.PublishRequest{Repository: alias, Source: e.source(content), Name: name, Version: version} + request.Metadata.Labels = labels + _, err := e.svc.Publish(context.Background(), request) + require.NoError(t, err) + } +} + +func splitRef(nameVersion string) (string, string, bool) { + parsed, err := artifact.ParseRef(nameVersion) + if err != nil { + return "", "", false + } + return parsed.Name, parsed.Version, true +} + +func TestList_AcrossRepositoriesWithPaginationLabelsAndOrdering(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "beta": {}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1", "app/a:2": "a2", "app/b:1": "b1"}, map[string]string{"team": "platform"}) + seedList(t, e, "beta", map[string]string{"app/a:1": "other", "app/c:1": "c1"}, map[string]string{"team": "data"}) + + all, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + var keys []string + for _, entry := range all.Artifacts { + keys = append(keys, entry.Repository+"/"+entry.Name+":"+entry.Version) + } + assert.Equal(t, []string{"alpha/app/a:1", "beta/app/a:1", "alpha/app/a:2", "alpha/app/b:1", "beta/app/c:1"}, keys, "sorted by name, version, repository; aliases stay visible; every page consumed") + require.Len(t, all.Repositories, 2) + assert.Equal(t, 3, all.Repositories[0].Count) + + named, err := e.svc.List(context.Background(), artifact.ListRequest{Name: "app/a"}) + require.NoError(t, err) + assert.Len(t, named.Artifacts, 3) + + one, err := e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"beta"}}) + require.NoError(t, err) + assert.Len(t, one.Artifacts, 2) + + platform, err := e.svc.List(context.Background(), artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}}}) + require.NoError(t, err) + assert.Len(t, platform.Artifacts, 3) + + none, err := e.svc.List(context.Background(), artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}, {Key: "team", Value: "data"}}}) + require.NoError(t, err) + assert.Empty(t, none.Artifacts, "label selectors use AND semantics") +} + +func TestSearch_MatchesNamesDescriptionsKindsAndLabels(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}}}) + request := e.publishRequest("alpha", e.source("a")) + request.Name = "net/vpc-module" + request.Metadata = artifact.Metadata{Description: "Networking Basics", Labels: map[string]string{"purpose": "backup"}} + _, err := e.svc.Publish(context.Background(), request) + require.NoError(t, err) + other := e.publishRequest("alpha", e.source("b")) + other.Name = "tools/other" + _, err = e.svc.Publish(context.Background(), other) + require.NoError(t, err) + + for query, want := range map[string]int{"vpc": 1, "networking": 1, "BACKUP": 1, "files": 2, "nomatch": 0, "": 2} { + result, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: query}) + require.NoError(t, err, query) + assert.Len(t, result.Artifacts, want, query) + } + labeled, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "files", Labels: []artifact.Label{{Key: "purpose", Value: "backup"}}}) + require.NoError(t, err) + assert.Len(t, labeled.Artifacts, 1) +} + +func TestList_CapabilityErrorsAreReportedNotHidden(t *testing.T) { + noCollection := artifacttest.DefaultCapabilities() + noCollection.EnumerateCollection = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "registry": {Capabilities: &noCollection}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1"}, nil) + + all, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err, "a default listing tolerates a repository without collection enumeration") + assert.Len(t, all.Artifacts, 1) + var diagnostic artifact.RepositoryListing + for _, listing := range all.Repositories { + if listing.Repository == "registry" { + diagnostic = listing + } + } + require.NotEmpty(t, diagnostic.Diagnostics) + assert.Contains(t, diagnostic.Diagnostics[0].Message, "direct named operations") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"registry"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an explicitly requested repository reports the capability error") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"registry"}, Name: "app/a"}) + require.NoError(t, err, "named listing works without a catalog") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"ghost"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestList_OnlyFailingRepositoriesFailTheListing(t *testing.T) { + noCollection := artifacttest.DefaultCapabilities() + noCollection.EnumerateCollection = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"registry": {Capabilities: &noCollection}}}) + _, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "when no repository can answer, the listing fails") +} + +func TestEnumeration_CancellationAfterOneDefaultRepositoryCompletesIsNotSuccess(t *testing.T) { + tests := []struct { + name string + run func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) + }{ + {"list", func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) { + return svc.List(ctx, artifact.ListRequest{}) + }}, + {"search", func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) { + return svc.Search(ctx, artifact.SearchRequest{Query: "app"}) + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "beta": {}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1"}, nil) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Alpha completes first; beta is canceled only after alpha returned its page. + alphaDone := make(chan struct{}) + var once sync.Once + e.static.Repositories["alpha"] = delegatingRepository(t, e.repos["alpha"], repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + page, err := e.repos["alpha"].List(ctx, query) + once.Do(func() { close(alphaDone) }) + return page, err + }}) + e.static.Repositories["beta"] = delegatingRepository(t, e.repos["beta"], repoHooks{list: func(ctx context.Context, _ artifact.ListQuery) (*artifact.ListPage, error) { + <-alphaDone + cancel() + return nil, ctx.Err() + }}) + + result, err := tc.run(ctx, e.svc) + + require.ErrorIs(t, err, context.Canceled, "an incomplete listing must not be reported as a success") + require.NotNil(t, result) + assert.Len(t, result.Artifacts, 1, "what alpha returned stays visible beside the error") + }) + } +} + +type listGate struct { + inflight atomic.Int32 + peak atomic.Int32 + entered chan struct{} + release chan struct{} +} + +// gatedList returns a generated repository that blocks every List on the gate before delegating. +func gatedList(t *testing.T, base *artifacttest.Repository, gate *listGate) artifact.Repository { + return delegatingRepository(t, base, repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + current := gate.inflight.Add(1) + defer gate.inflight.Add(-1) + for old := gate.peak.Load(); current > old && !gate.peak.CompareAndSwap(old, current); old = gate.peak.Load() { + } + gate.entered <- struct{}{} + <-gate.release + return base.List(ctx, query) + }}) +} + +func TestList_ConcurrencyIsInjectable(t *testing.T) { + e := newEnv(t, envConfig{ + repos: map[string]artifacttest.Options{"a": {}, "b": {}, "c": {}}, + options: []service.Option{service.WithConcurrency(1)}, + }) + for _, alias := range []string{"a", "b", "c"} { + seedList(t, e, alias, map[string]string{"app/x:1": alias}, nil) + } + gate := &listGate{entered: make(chan struct{}, 3), release: make(chan struct{})} + for _, alias := range []string{"a", "b", "c"} { + e.static.Repositories[alias] = gatedList(t, e.repos[alias], gate) + } + type listOutcome struct { + count int + err error + } + done := make(chan listOutcome, 1) + go func() { + result, err := e.svc.List(context.Background(), artifact.ListRequest{}) + outcome := listOutcome{err: err} + if err == nil { + outcome.count = len(result.Artifacts) + } + done <- outcome + }() + select { + case <-gate.entered: + case <-time.After(2 * time.Second): + close(gate.release) + t.Fatal("listing did not start") + } + select { + case <-gate.entered: + case <-time.After(200 * time.Millisecond): + } + close(gate.release) + outcome := <-done + require.NoError(t, outcome.err) + assert.Equal(t, 3, outcome.count) + assert.Equal(t, int32(1), gate.peak.Load()) +} + +func TestPull_RechecksTheDestinationBeforeReplacingIt(t *testing.T) { + tests := []struct { + name string + force bool + wantErr error + wantKept bool + }{ + {"files appeared during the download without force", false, errUtils.ErrArtifactConflict, true}, + {"files appeared during the download with force", true, nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("ORIGINAL")) + dest := out(t) + require.NoError(t, os.MkdirAll(dest, 0o755)) + // Another process touches the destination right before the artifact is read. + e.static.Repositories["dst"] = delegatingRepository(t, e.repos["dst"], repoHooks{read: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + fixtures.WriteFile(t, dest, "late.txt", "written during the download", 0o644) + return e.repos["dst"].Read(ctx, pinned) + }}) + + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: tt.force}) + if runtime.GOOS == "windows" && tt.force { + tt.wantErr = errUtils.ErrArtifactUnsupportedCapability + tt.wantKept = true + } + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + _, lateErr := os.Stat(filepath.Join(dest, "late.txt")) + assert.Equal(t, tt.wantKept, lateErr == nil, "a refused replacement leaves the destination untouched; a forced one replaces it") + _, dataErr := os.Stat(filepath.Join(dest, "data.txt")) + assert.Equal(t, !tt.wantKept, dataErr == nil) + }) + } +} diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go new file mode 100644 index 00000000000..ac25d29cb44 --- /dev/null +++ b/pkg/artifact/service/copy.go @@ -0,0 +1,387 @@ +package service + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// copyPlan is everything resolved before a copy mutates the destination. +type copyPlan struct { + ref artifact.Ref + src, dst *target + pinned artifact.PinnedReference + inspection *artifact.Inspection +} + +// copyWrite is how a copy will write the destination: the incoming description, the write +// closure, and a cleanup for any staging it owns. +type copyWrite struct { + in *incoming + write writer + cleanup func() +} + +// Copy implements [artifact.Service]. The source reference is resolved once and verified against +// BOTH the source and destination policies before the destination is touched; force never +// bypasses that. A compatible native transfer is used when available, otherwise the bounded +// staging fallback repacks the unchanged Record for the destination's packaging. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Copy(ctx context.Context, request artifact.CopyRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Copy")() + + op := s.begin(artifact.ActionCopy, request.To, request.Ref.String()) + err := s.copyArtifact(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func validateCopy(request *artifact.CopyRequest) error { + if request.From == "" || request.To == "" || request.From == request.To { + return fmt.Errorf("%w: copy needs distinct source and destination repositories", errUtils.ErrArtifactConfigInvalid) + } + return request.Ref.Validate() +} + +func (s *Service) copyArtifact(ctx context.Context, op *operation, request *artifact.CopyRequest) error { + if err := validateCopy(request); err != nil { + return err + } + plan, err := s.planCopy(ctx, op, request) + if err != nil { + return err + } + if plan.inspection.Record == nil && !canNativeCopy(plan) { + return artifact.NewUnsupportedCapability(request.To, opCopy, "transfer/graph", "a native OCI graph can only be copied between repositories that support a compatible native transfer") + } + cw, err := s.copyWriter(ctx, op, plan) + if err != nil { + return err + } + defer cw.cleanup() + + pub := &publication{op: op, tgt: plan.dst, in: cw.in, force: request.Force, write: cw.write, state: &publishState{}} + if err := s.commit(ctx, pub); err != nil { + return err + } + if op.result.Outcome == "" { + op.result.Outcome = artifact.OutcomeCreated + } + op.result.Repository = request.To + // Evidence is carried even when the destination already holds the Record, because signatures + // added to the source after the first copy would otherwise never reach the destination. + s.carryEvidence(ctx, op, plan) + return nil +} + +// planCopy checks capabilities and lock guarantees, resolves the source once, and enforces both policies. +func (s *Service) planCopy(ctx context.Context, op *operation, request *artifact.CopyRequest) (*copyPlan, error) { + srcInfo, srcRepo, err := s.openForRead(ctx, request.From, opCopy) + if err != nil { + return nil, err + } + dstInfo, _, err := s.describeForWrite(request.To, opCopy) + if err != nil { + return nil, err + } + if err := checkLockGuarantee(&dstInfo, opCopy); err != nil { + return nil, err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, srcRepo, request.From, request.Ref) + if err != nil { + return nil, err + } + if _, err := gate(ctx, request.From, srcRepo, inspection, srcInfo.Policy, dstInfo.Policy); err != nil { + return nil, err + } + dstRepo, err := s.repos.Open(ctx, request.To) + if err != nil { + return nil, err + } + return ©Plan{ + ref: request.Ref, + src: &target{alias: request.From, repo: srcRepo, info: &srcInfo}, + dst: &target{alias: request.To, repo: dstRepo, info: &dstInfo}, + pinned: pinned, + inspection: inspection, + }, nil +} + +func canNativeCopy(plan *copyPlan) bool { + copier, ok := plan.dst.repo.(artifact.Copier) + return ok && copier.CanCopyFrom(plan.src.repo) +} + +// copyWriter chooses the native transfer or the staging fallback. +func (s *Service) copyWriter(ctx context.Context, op *operation, plan *copyPlan) (*copyWrite, error) { + if canNativeCopy(plan) { + return nativeCopyWriter(plan) + } + prepared, cleanup, err := s.stageCopy(ctx, op, plan) + if err != nil { + return nil, err + } + in, err := newIncoming(prepared) + if err != nil { + cleanup() + return nil, err + } + in.inspection = plan.inspection + write := func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return plan.dst.repo.Publish(ctx, prepared, options) + } + return ©Write{in: in, write: write, cleanup: cleanup}, nil +} + +// nativeRef is the destination reference of a native transfer: the Record's name and version, or +// the requested tag for a foreign graph. A digest-only graph has no tag to publish under. +func nativeRef(plan *copyPlan) (artifact.Ref, error) { + if record := plan.inspection.Record; record != nil { + return artifact.Ref{Name: record.Name, Version: record.Version}, nil + } + if plan.ref.Version == "" { + return artifact.Ref{}, fmt.Errorf("%w: copying a native graph needs a name:version reference to publish under in the destination", errUtils.ErrArtifactRefInvalid) + } + return artifact.Ref{Name: plan.ref.Name, Version: plan.ref.Version}, nil +} + +func nativeCopyWriter(plan *copyPlan) (*copyWrite, error) { + copier, _ := plan.dst.repo.(artifact.Copier) + ref, err := nativeRef(plan) + if err != nil { + return nil, err + } + in := &incoming{ref: ref, inspection: plan.inspection, rootDigest: plan.pinned.ResolvedDigest} + if record := plan.inspection.Record; record != nil { + if err := in.bindRecord(record, plan.inspection); err != nil { + return nil, err + } + } + write := func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return copier.CopyFrom(ctx, plan.src.repo, plan.pinned, artifact.CopyFromOptions(options)) + } + return ©Write{in: in, write: write, cleanup: func() {}}, nil +} + +// bindRecord fills the identity of a stored Record into the incoming description. +func (in *incoming) bindRecord(record *artifact.Record, inspection *artifact.Inspection) error { + recordDigest, err := record.RecordDigest() + if err != nil { + return err + } + intent, err := record.IntentDigest() + if err != nil { + return err + } + in.record, in.recordDigest, in.intent = record, recordDigest, intent + in.payload = inspection.Representation.Payload + in.size = payloadSize(inspection) + return nil +} + +// stageCopy reads the pinned source into a bounded temporary tree and repacks the unchanged Record +// for the destination packaging. The returned cleanup removes every owned temporary directory. +func (s *Service) stageCopy(ctx context.Context, op *operation, plan *copyPlan) (*artifact.PreparedArtifact, func(), error) { + content, err := plan.src.repo.Read(ctx, plan.pinned) + if err != nil { + return nil, nil, fmt.Errorf("read %s from repository %q: %w", plan.pinned.Ref, plan.src.alias, err) + } + defer content.Close() + if err := bindContent(content, plan.inspection); err != nil { + return nil, nil, err + } + docs, err := collectDocumentBytes(content) + if err != nil { + return nil, nil, err + } + tree, err := os.MkdirTemp(s.stagingRoot, "atmos-artifact-copy-*") + if err != nil { + return nil, nil, fmt.Errorf("%w: create copy staging: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + cleanup := func() { + if err := removeStagedTree(tree); err != nil { + op.warn("staging-cleanup-failed", fmt.Sprintf("remove copy staging %q: %v", tree, err)) + } + } + files := filepath.Join(tree, "files") + // The tree is read back by the repack and then discarded, so it keeps owner read and traversal + // access even when the Record declares a restrictive mode; the Record itself is repacked unchanged. + if err := materialize.Materialize(ctx, content, files, materialize.WithOwnerAccess()); err != nil { + cleanup() + return nil, nil, err + } + prepared, err := s.repacker.Repack(ctx, &produce.RepackRequest{ + Record: *plan.inspection.Record, + Tree: files, + Documents: docs, + Packaging: packagingFor(plan.dst.info), + OperationID: op.id, + PreparedAt: op.preparedAt, + }) + if err != nil { + cleanup() + return nil, nil, err + } + prior := prepared.Cleanup + prepared.Cleanup = func() error { + var firstErr error + if prior != nil { + firstErr = prior() + } + cleanup() + return firstErr + } + return prepared, func() { _ = prepared.Close() }, nil +} + +func packagingFor(info *RepositoryInfo) artifact.Packaging { + if info.Packaging != "" { + return info.Packaging + } + if len(info.Capabilities.Packagings) > 0 { + return info.Capabilities.Packagings[0] + } + return artifact.PackagingNone +} + +// collectDocumentBytes reads the (bounded) document streams of the content by role. +func collectDocumentBytes(content *artifact.Content) (map[artifact.DocumentRole][]byte, error) { + docs := map[artifact.DocumentRole][]byte{} + if content.Record == nil { + return docs, nil + } + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobDocument || stream.Reader == nil { + continue + } + data, err := artifact.ReadBounded(stream.Reader, artifact.MaxDocumentSize) + if err != nil { + return nil, err + } + role, ok := documentRoleFor(content.Record, stream.Descriptor.Digest.String()) + if !ok { + return nil, fmt.Errorf("%w: a document stream matches no descriptor in the Record", errUtils.ErrArtifactIntegrityFailed) + } + docs[role] = data + } + return docs, nil +} + +func documentRoleFor(record *artifact.Record, digestValue string) (artifact.DocumentRole, bool) { + for _, doc := range record.Documents { + if doc.Digest.String() == digestValue { + return doc.Role, true + } + } + return "", false +} + +// carryEvidence moves signature evidence along with a copy when both repositories support it. +// Evidence is never invented: without both optional interfaces, a diagnostic explains that the +// destination holds no signatures. Evidence is written only to the exact record the copy produced +// or found, never to a record a concurrent writer repointed the destination reference to. +func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPlan) { + reader, canRead := plan.src.repo.(artifact.EvidenceReader) + writer, canWrite := plan.dst.repo.(artifact.EvidenceWriter) + if !canRead || !canWrite { + if verificationRequired(plan.dst.info.Policy) { + op.warn("evidence-not-carried", "the destination policy requires verification but the repositories cannot transfer signature evidence; consumers of the destination copy must verify against the source evidence") + } + return + } + blobs, err := reader.ReadEvidence(ctx, &plan.pinned) + if err != nil { + op.warn("evidence-read-failed", err.Error()) + return + } + if len(blobs) == 0 { + return + } + dstPinned, err := resolveWritten(ctx, op, plan) + if err == nil { + blobs = missingEvidence(ctx, plan.dst.repo, &dstPinned, blobs) + if len(blobs) > 0 { + _, err = writer.WriteEvidence(ctx, &dstPinned, blobs, nil) + } + } + if err != nil { + op.warn("evidence-write-failed", err.Error()) + } +} + +// resolveWritten resolves the destination reference and confirms it still points at the digest the +// copy wrote or found. A mismatch means another writer repointed the reference after the +// destination lock was released. +func resolveWritten(ctx context.Context, op *operation, plan *copyPlan) (artifact.PinnedReference, error) { + dstRef, err := nativeRef(plan) + if err != nil { + return artifact.PinnedReference{}, err + } + dstPinned, err := plan.dst.repo.Resolve(ctx, dstRef) + if err != nil { + return artifact.PinnedReference{}, err + } + if want := op.result.ResolvedDigest; want != "" && dstPinned.ResolvedDigest != want { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s in repository %q now points at %s, not the copied %s; evidence was not written", + errUtils.ErrArtifactConflict, dstRef, plan.dst.alias, dstPinned.ResolvedDigest, want) + } + return dstPinned, nil +} + +// missingEvidence drops the blobs the destination already holds, so repeating a copy never +// duplicates evidence. A destination that cannot list its evidence receives every blob. +func missingEvidence(ctx context.Context, repo artifact.Repository, pinned *artifact.PinnedReference, blobs []artifact.EvidenceBlob) []artifact.EvidenceBlob { + existing, ok := repo.(artifact.EvidenceReader) + if !ok { + return blobs + } + held, err := existing.ReadEvidence(ctx, pinned) + if err != nil { + return blobs + } + have := make(map[string]bool, len(held)) + for index := range held { + have[held[index].Evidence.Descriptor.Digest.String()] = true + } + var missing []artifact.EvidenceBlob + for index := range blobs { + if !have[blobs[index].Evidence.Descriptor.Digest.String()] { + missing = append(missing, blobs[index]) + } + } + return missing +} + +// removeStagedTree restores owner access as it walks directories, without following symlinks. +func removeStagedTree(tree string) error { + root, err := os.OpenRoot(tree) + if err != nil { + return err + } + walkErr := fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + return root.Chmod(path, info.Mode().Perm()|0o700) + } + return nil + }) + closeErr := root.Close() + return errors.Join(walkErr, closeErr, os.RemoveAll(tree)) +} diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go new file mode 100644 index 00000000000..098bbb4206b --- /dev/null +++ b/pkg/artifact/service/copy_test.go @@ -0,0 +1,488 @@ +package service_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "sync/atomic" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// copyEnv has a "src" repository and a "dst" repository with chosen packagings. +func copyEnv(t *testing.T, srcPackaging, dstPackaging artifact.Packaging, options ...service.Option) *env { + t.Helper() + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"src": {}, "dst": {}}, options: options}) + e.setInfo("src", func(info *service.RepositoryInfo) { info.Packaging = srcPackaging }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = dstPackaging }) + return e +} + +func TestCopy_PreservesRecordAcrossPackagingChanges(t *testing.T) { + packagings := []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} + for _, from := range packagings { + for _, to := range packagings { + t.Run(string(from)+"->"+string(to), func(t *testing.T) { + e := copyEnv(t, from, to) + e.setInfo("dst", func(info *service.RepositoryInfo) { + info.Defaults = artifact.Metadata{Labels: map[string]string{"destination-default": "x"}} + }) + published := e.publish("src", e.source("hello")) + e.clock.Advance(72 * 3600e9) + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, "dst", result.Repository) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest, "a copy preserves Record bytes and identity") + assert.Equal(t, published.Record.ContentDigest, result.ContentDigest) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, to, inspected.Inspection.Representation.Packaging) + want, err := published.Record.Canonical() + require.NoError(t, err) + got, err := inspected.Inspection.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, want, got, "created_at and labels are not restamped; destination defaults do not apply to copies") + + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + state, err := materialize.InspectDestination(dest, &published.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + e.assertStagingClean() + }) + } + } +} + +func TestCopy_ConflictForceAndUnchanged(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz) + e.publish("src", e.source("original")) + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, first.Outcome) + + again, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "copying an identical artifact is a no-op") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) + + e.publishForce("src", e.source("different")) + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + forced, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, forced.Outcome) + e.assertStagingClean() +} + +func (e *env) publishForce(repository, src string) { + e.t.Helper() + request := e.publishRequest(repository, src) + request.Force = true + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(e.t, err) +} + +func TestCopy_PluginReleasesStayImmutableOnTheDestinationEvenWithForce(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithImmutabilityPolicy(pluginPolicy())) + pluginRequest := e.publishRequest("dst", e.source("plugin build")) + pluginRequest.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.Publish(context.Background(), pluginRequest) + require.NoError(t, err) + e.publish("src", e.source("generic")) + + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "immutable") +} + +func TestCopy_TagMovedMidCopyCopiesTheOriginallyPinnedSubject(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz) + original := e.publish("src", e.source("ORIGINAL")) + decoy := e.publishRequest("src", e.source("DECOY")) + decoy.Version = "2.0.0" + _, err := e.svc.Publish(context.Background(), decoy) + require.NoError(t, err) + decoyDigest, _ := e.repos["src"].RecordDigestOf(ref("tools/example", "2.0.0")) + moved := false + e.repos["src"].SetOnResolved(func(resolved artifact.Ref) { + if resolved == exampleRef && !moved { + moved = true + e.repos["src"].MoveRef(exampleRef, decoyDigest) + } + }) + before := e.repos["src"].Calls("Resolve") + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["src"].Calls("Resolve")-before, "the source reference is resolved once") + assert.Equal(t, original.Operation.RecordDigest, result.RecordDigest) + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + data, err := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, err) + assert.Equal(t, "ORIGINAL", string(data)) +} + +func TestCopy_ValidationAndCapabilitiesPrecedeAnyRead(t *testing.T) { + readOnly := artifacttest.DefaultCapabilities() + readOnly.Write = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"src": {}, "dst": {Capabilities: &readOnly}}}) + e.publish("src", e.source("hello")) + reads := e.repos["src"].TotalCalls() + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, reads, e.repos["src"].TotalCalls(), "the source is untouched when the destination cannot be written") + + for name, request := range map[string]artifact.CopyRequest{ + "same repository": {Ref: exampleRef, From: "src", To: "src"}, + "missing source": {Ref: exampleRef, To: "dst"}, + "missing target": {Ref: exampleRef, From: "src"}, + "invalid ref": {Ref: artifact.Ref{Name: "x"}, From: "src", To: "dst"}, + } { + _, err := e.svc.Copy(context.Background(), request) + require.Error(t, err, name) + } + writable := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + _, err = writable.svc.Copy(context.Background(), artifact.CopyRequest{Ref: ref("tools/example", "9"), From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestCopy_SourceAndDestinationPoliciesBothApplyAndForceNeverBypassesThem(t *testing.T) { + trusted := releaseSigner.TrustedPolicy() + tests := []struct { + name string + srcPolicy artifact.VerificationPolicy + dstPolicy artifact.VerificationPolicy + }{ + {"source requires", trusted, artifact.VerificationPolicy{}}, + {"destination requires", artifact.VerificationPolicy{}, trusted}, + {"both require", trusted, trusted}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + e.setInfo("src", func(info *service.RepositoryInfo) { info.Policy = tt.srcPolicy }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = tt.dstPolicy }) + dstCalls := e.repos["dst"].TotalCalls() + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "unsigned content cannot satisfy either policy, and force does not help") + assert.Equal(t, dstCalls, e.repos["dst"].TotalCalls(), "verification ran before the destination was touched") + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + if verificationConfigured(tt.dstPolicy) { + require.NoError(t, err, "evidence travels with the copy so the destination verifies independently") + assert.Equal(t, artifact.VerificationVerified, verified.Verification.Status) + } + }) + } +} + +func verificationConfigured(policy artifact.VerificationPolicy) bool { + return policy.RequireSignature || policy.HasTrustConstraints() +} + +// plainRepository hides optional extension interfaces of the wrapped repository. +type plainRepository struct{ artifact.Repository } + +func TestCopy_WarnsWhenEvidenceCannotTravel(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.static.Repositories["dst"] = plainRepository{Repository: e.repos["dst"]} + e.setInfo("src", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + var warned bool + for _, diagnostic := range result.Diagnostics { + warned = warned || diagnostic.Code == "evidence-not-carried" + } + assert.True(t, warned, "missing evidence transfer is reported, never silently dropped") +} + +func TestCopy_UsesNativeTransferWhenCompatible(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + published := e.publish("src", e.source("hello")) + native := newNativeRepository(t, e.repos["dst"]) + e.static.Repositories["dst"] = native + reads := e.repos["src"].Calls("Read") + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies) + assert.Equal(t, reads, e.repos["src"].Calls("Read"), "the staging fallback is not used") + assert.Equal(t, "native://copy", result.Location) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest) + assert.Nil(t, native.options.ExpectedPrior) + assert.NotNil(t, native.options.Lock, "the destination lock is held across a native transfer") + assert.False(t, native.options.Force, "force applies only when replacement was decided") + + native.compat = false + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies, "an incompatible pair falls back to staging") +} + +func TestCopy_RepeatedNativeGraphCopyWithoutRecordIsUnchanged(t *testing.T) { + for _, force := range []bool{false, true} { + t.Run(map[bool]string{false: "without force", true: "with force"}[force], func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + e.publish("src", e.source("hello")) + source := delegatingRepository(t, e.repos["src"], repoHooks{inspect: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := e.repos["src"].Inspect(ctx, pinned) + if err != nil { + return nil, err + } + return withoutRecord(inspection), nil + }}) + pinned, err := e.repos["src"].Resolve(context.Background(), exampleRef) + require.NoError(t, err) + var native *nativeRepository + destination := delegatingRepository(t, e.repos["dst"], repoHooks{ + resolve: func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + if native.copies == 0 { + return artifact.PinnedReference{}, errUtils.ErrArtifactNotFound + } + return pinned, nil + }, + inspect: func(ctx context.Context, p artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := e.repos["src"].Inspect(ctx, p) + if err != nil { + return nil, err + } + return withoutRecord(inspection), nil + }, + }) + native = newNativeRepository(t, destination) + e.static.Repositories["src"] = source + e.static.Repositories["dst"] = native + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, first.Outcome) + + again, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: force}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "the destination already points at the source root") + assert.Equal(t, 1, native.copies, "an unchanged destination is not copied again") + }) + } +} + +func diagnosticCodes(result *artifact.OperationResult) []string { + codes := make([]string, 0, len(result.Diagnostics)) + for _, diagnostic := range result.Diagnostics { + codes = append(codes, diagnostic.Code) + } + return codes +} + +func TestCopy_UnchangedDestinationStillReceivesLaterSignatures(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + require.Equal(t, artifact.OutcomeCreated, first.Outcome) + assert.Zero(t, e.repos["dst"].Calls("WriteEvidence"), "the source was unsigned, so nothing could travel") + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + second, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, second.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("WriteEvidence"), "evidence is carried even though the Record was already there") + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Verification.Status) + + third, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, third.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("WriteEvidence"), "repeating a copy never duplicates evidence the destination already holds") + held, err := e.repos["dst"].ReadEvidence(context.Background(), &artifact.PinnedReference{RecordDigest: published.Operation.RecordDigest}) + require.NoError(t, err) + assert.Len(t, held, 1) +} + +// repointedRepository returns a generated repository that answers Resolve with a different digest +// once a publish completed, as if another writer forced the reference elsewhere after the +// destination lock was released. It keeps the evidence writer of the base repository. +func repointedRepository(t *testing.T, base *artifacttest.Repository) artifact.Repository { + t.Helper() + var published atomic.Bool + mock := delegatingRepository(t, base, repoHooks{ + publish: func(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + result, err := base.Publish(ctx, prepared, options) + if err == nil { + published.Store(true) + } + return result, err + }, + resolve: func(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + pinned, err := base.Resolve(ctx, ref) + if err == nil && published.Load() { + pinned.ResolvedDigest = digest.FromString("another writer") + } + return pinned, err + }, + }) + return struct { + artifact.Repository + artifact.EvidenceWriter + }{mock, base} +} + +func TestCopy_EvidenceIsNotWrittenToARepointedDestination(t *testing.T) { + tests := []struct { + name string + repoint bool + wantWrite int + wantCode bool + }{ + {"reference still points at the copied record", false, 1, false}, + {"reference repointed by another writer", true, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + dst := e.repos["dst"] + if tt.repoint { + e.static.Repositories["dst"] = repointedRepository(t, dst) + } + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err, "a skipped evidence write is a diagnostic, not a failed copy") + assert.Equal(t, tt.wantWrite, dst.Calls("WriteEvidence")) + assert.Equal(t, tt.wantCode, slices.Contains(diagnosticCodes(result), "evidence-write-failed")) + }) + } +} + +// TestCopy_StagingKeepsOwnerAccessWhileRepacking proves a Record that declares mode 0000 can still +// be repacked: the temporary tree stays readable until the repack finished, and the Record the +// repack receives keeps the declared modes. +func TestCopy_StagingKeepsOwnerAccessWhileRepacking(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skipf("permission bits are not enforced for this process on %s (uid %d)", runtime.GOOS, os.Geteuid()) + } + var repacked bool + repacker := service.NewMockRepacker(gomock.NewController(t)) + repacker.EXPECT().Repack(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) { + repacked = true + data, err := os.ReadFile(filepath.Join(request.Tree, "data.txt")) + require.NoError(t, err, "a file declared with mode 0000 stays readable for the repack") + assert.Equal(t, "hello", string(data)) + entries, err := os.ReadDir(filepath.Join(request.Tree, "bin")) + require.NoError(t, err, "a directory declared with mode 0000 stays traversable for the repack") + assert.NotEmpty(t, entries) + for _, entry := range request.Record.Inventory { + if entry.Type == artifact.EntryFile || entry.Type == artifact.EntryDirectory { + assert.Zero(t, entry.Mode, "the Record keeps the declared mode for %s", entry.Path) + } + } + return nil, errTransport + }) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithRepacker(repacker)) + e.publish("src", e.source("hello")) + base := e.repos["src"] + e.static.Repositories["src"] = delegatingRepository(t, base, repoHooks{ + inspect: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := base.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + record := inspection.Record.Clone() + for index := range record.Inventory { + record.Inventory[index].Mode = 0 + } + require.NoError(t, record.Seal(), "the content digest follows the declared modes") + inspection.Record = &record + return inspection, nil + }, + read: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + content, err := base.Read(ctx, pinned) + if err != nil { + return nil, err + } + content.Record = nil + return content, nil + }, + }) + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + + require.ErrorIs(t, err, errTransport) + require.True(t, repacked, "the copy must reach the repack") + e.assertStagingClean() +} + +func TestCopy_InspectionPolicySeesIncomingReleaseOnBothPaths(t *testing.T) { + for _, native := range []bool{false, true} { + t.Run(fmt.Sprintf("native=%t", native), func(t *testing.T) { + var sawIncoming bool + policy := service.ImmutabilityFunc(func(c service.Candidate) (bool, string) { + require.NotNil(t, c.Inspection) + if c.Existing { + return false, "" + } + sawIncoming = true + assert.Equal(t, native, c.Prepared == nil) + return c.Inspection.Record.Metadata.Labels["format"] == "plugin", "incoming plugin" + }) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithImmutabilityPolicy(policy)) + e.publish("dst", e.source("generic")) + request := e.publishRequest("src", e.source("plugin")) + request.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.Publish(t.Context(), request) + require.NoError(t, err) + if native { + e.static.Repositories["dst"] = newNativeRepository(t, e.repos["dst"]) + } + _, err = e.svc.Copy(t.Context(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "incoming plugin") + assert.True(t, sawIncoming) + }) + } +} diff --git a/pkg/artifact/service/delegate_test.go b/pkg/artifact/service/delegate_test.go new file mode 100644 index 00000000000..4e32f3f7e98 --- /dev/null +++ b/pkg/artifact/service/delegate_test.go @@ -0,0 +1,96 @@ +package service_test + +import ( + "context" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// repoHooks overrides individual repository methods of a delegating mock. A nil hook delegates to +// the base repository. +type repoHooks struct { + resolve func(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) + inspect func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) + read func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) + list func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) + lock func(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) + publish func(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) +} + +// delegatingRepository returns a generated MockRepository whose methods forward to base, except +// for the ones a hook replaces. +func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHooks) *artifact.MockRepository { + t.Helper() + if hooks.resolve == nil { + hooks.resolve = base.Resolve + } + if hooks.inspect == nil { + hooks.inspect = base.Inspect + } + if hooks.read == nil { + hooks.read = base.Read + } + if hooks.list == nil { + hooks.list = base.List + } + if hooks.lock == nil { + hooks.lock = base.Lock + } + if hooks.publish == nil { + hooks.publish = base.Publish + } + mock := artifact.NewMockRepository(gomock.NewController(t)) + mock.EXPECT().Capabilities().DoAndReturn(base.Capabilities).AnyTimes() + mock.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(hooks.lock).AnyTimes() + mock.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(hooks.resolve).AnyTimes() + mock.EXPECT().Inspect(gomock.Any(), gomock.Any()).DoAndReturn(hooks.inspect).AnyTimes() + mock.EXPECT().OpenDocument(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.OpenDocument).AnyTimes() + mock.EXPECT().Read(gomock.Any(), gomock.Any()).DoAndReturn(hooks.read).AnyTimes() + mock.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(hooks.publish).AnyTimes() + mock.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(hooks.list).AnyTimes() + mock.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Delete).AnyTimes() + mock.EXPECT().Pin(gomock.Any(), gomock.Any()).DoAndReturn(base.Pin).AnyTimes() + mock.EXPECT().Sign(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Sign).AnyTimes() + mock.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Verify).AnyTimes() + return mock +} + +// nativeRepository adds a native transfer, backed by a generated Copier, to a repository. It +// records how often and with which options the transfer ran. +type nativeRepository struct { + artifact.Repository + artifact.Copier + copies int + options artifact.CopyFromOptions + compat bool +} + +func newNativeRepository(t *testing.T, base artifact.Repository) *nativeRepository { + t.Helper() + native := &nativeRepository{Repository: base, compat: true} + copier := artifact.NewMockCopier(gomock.NewController(t)) + copier.EXPECT().CanCopyFrom(gomock.Any()).DoAndReturn(func(artifact.Repository) bool { return native.compat }).AnyTimes() + copier.EXPECT().CopyFrom(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, source artifact.Repository, pinned artifact.PinnedReference, options artifact.CopyFromOptions) (*artifact.PublishResult, error) { + native.copies++ + native.options = options + inspection, err := source.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: pinned, Representation: inspection.Representation, Location: "native://copy"}, nil + }, + ).AnyTimes() + native.Copier = copier + return native +} + +// withoutRecord returns a copy of the inspection that carries no portable Record, like a foreign graph. +func withoutRecord(inspection *artifact.Inspection) *artifact.Inspection { + stripped := *inspection + stripped.Record = nil + return &stripped +} diff --git a/pkg/artifact/service/doc.go b/pkg/artifact/service/doc.go new file mode 100644 index 00000000000..4003b651c6e --- /dev/null +++ b/pkg/artifact/service/doc.go @@ -0,0 +1,18 @@ +// Package service implements [artifact.Service]: the repository-independent orchestration of +// publish, pull, copy, inspect, list, search, delete, document reads, sign, verify, and pin over +// [artifact.Repository] implementations. +// +// The service owns name and version selection, producer preparation, effective metadata, digest +// pinning, conflict policy, retries, and result aggregation; repositories own storage mechanics +// and capabilities. Three rules hold for every operation: +// +// - A mutable reference is resolved exactly once; every later read, verification, and copy uses +// the pinned subject, so a tag moved mid-operation can never substitute different bytes. +// - Capabilities and lock guarantees are checked before the first mutation, locks are acquired in +// canonical repository/resource order, and they are released on success, error, and cancellation. +// - Required verification runs before destination publication or local materialization, and +// force never bypasses it. Without any trust configuration the result is "unchecked", never +// "verified". +// +// Every method returns its typed result alongside the error so reporting can describe completed work. +package service diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go new file mode 100644 index 00000000000..91bf1d996cb --- /dev/null +++ b/pkg/artifact/service/edge_test.go @@ -0,0 +1,461 @@ +package service_test + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// mockSetup wires a service around one mock repository named "dst" (and optionally "src"). +type mockSetup struct { + ctrl *gomock.Controller + repo *artifact.MockRepository + svc *service.Service + e *env +} + +func newMockSetup(t *testing.T, caps *artifact.Capabilities, extra ...service.Option) *mockSetup { + t.Helper() + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(*caps).AnyTimes() + e := newEnv(t, envConfig{}) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"dst": repo}, + Info: map[string]service.RepositoryInfo{"dst": {Capabilities: *caps, Packaging: artifact.PackagingNone}}, + } + options := append([]service.Option{service.WithStagingRoot(e.staging), service.WithClock(e.clock.Now), service.WithRetry(2, 0), service.WithSleep(func(context.Context, time.Duration) error { return nil })}, extra...) + svc, err := service.New(repos, nil, options...) + require.NoError(t, err) + return &mockSetup{ctrl: ctrl, repo: repo, svc: svc, e: e} +} + +func sealedRecord(t *testing.T) (artifact.Record, digest.Digest) { + t.Helper() + record := artifact.Record{Name: "tools/example", Version: "1.0.0", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + return record, recordDigest +} + +// expectNoLock lets a mock repository accept lock calls with a trivial handle. +func (m *mockSetup) expectLock() { + handle := artifact.NewMockLockHandle(m.ctrl) + handle.EXPECT().Resource().Return("r").AnyTimes() + handle.EXPECT().Owner().Return("o").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + handle.EXPECT().Release(gomock.Any()).Return(nil).AnyTimes() + m.repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil).AnyTimes() +} + +func TestPublish_ReconciliationNeverClaimsSuccessWithoutProof(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + _, otherDigest := sealedRecord(t) + + tests := []struct { + name string + reconcile func(m *mockSetup) + wantErr error + }{ + {"reference cannot be read", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errTransport) + }, errUtils.ErrArtifactPublicationAmbiguous}, + {"reference now exists with unrelated content", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{ResolvedDigest: otherDigest, RecordDigest: otherDigest}, nil) + }, errUtils.ErrArtifactConflict}, + {"matching record but descriptors differ", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).DoAndReturn(func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + return artifact.PinnedReference{ResolvedDigest: "sha256:x", RecordDigest: lastPreparedDigest}, nil + }) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.Inspection{ + Representation: artifact.Representation{Payload: []ocispec.Descriptor{{Digest: digest.FromString("other-bytes"), Size: 1}}}, + }, nil) + }, errUtils.ErrArtifactIntegrityFailed}, + {"matching record but inspection fails", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).DoAndReturn(func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + return artifact.PinnedReference{ResolvedDigest: "sha256:x", RecordDigest: lastPreparedDigest}, nil + }) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(nil, errTransport) + }, errUtils.ErrArtifactPublicationAmbiguous}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + // The first Resolve is the pre-write read of the current state. + first := m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound) + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, prepared *artifact.PreparedArtifact, _ artifact.PublishOptions) (*artifact.PublishResult, error) { + d, err := prepared.Record.RecordDigest() + require.NoError(t, err) + lastPreparedDigest = d + return nil, errTransport + }, + ).After(first) + tt.reconcile(m) + result, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", src)) + require.ErrorIs(t, err, tt.wantErr) + assert.Empty(t, result.Outcome, "never reported as success") + }) + } +} + +// lastPreparedDigest is set by the publish expectation for use in later reconcile expectations. +var lastPreparedDigest digest.Digest + +func TestPublish_ExistingReferenceVanishingDuringReconcileIsAmbiguous(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + e := newEnv(t, envConfig{}) + prior := e.publish("dst", src) + changed := e.source("changed") + + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: prior.Operation.RecordDigest, RecordDigest: prior.Operation.RecordDigest} + record := prior.Record + gomock.InOrder( + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil), + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errTransport), + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound), + ) + request := m.e.publishRequest("dst", changed) + request.Force = true + _, err := m.svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) +} + +func TestPublish_NotCommittedReconciliationReturnsTheOriginalError(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + e := newEnv(t, envConfig{}) + prior := e.publish("dst", src) + + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: prior.Operation.RecordDigest, RecordDigest: prior.Operation.RecordDigest} + record := prior.Record + gomock.InOrder( + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil), + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errTransport), + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + ) + request := m.e.publishRequest("dst", m.e.source("changed")) + request.Force = true + _, err := m.svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errTransport) + require.NotErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous, "the reference provably still holds the prior state") +} + +func TestPublish_ReadFailuresOfTheCurrentStateAreReported(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + t.Run("resolve fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errTransport) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errTransport) + }) + t.Run("inspect fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{ResolvedDigest: "sha256:abc"}, nil) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(nil, errTransport) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errTransport) + }) + t.Run("open fails", func(t *testing.T) { + e := newEnv(t, envConfig{}) + ctrl := gomock.NewController(t) + repos := service.NewMockRepositories(ctrl) + repos.EXPECT().Describe("dst").Return(service.RepositoryInfo{Alias: "dst", Capabilities: caps, Packaging: artifact.PackagingNone}, nil) + repos.EXPECT().Open(gomock.Any(), "dst").Return(nil, errTransport) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + _, err = svc.Publish(context.Background(), e.publishRequest("dst", e.source("x"))) + require.ErrorIs(t, err, errTransport) + e.assertStagingClean() + }) +} + +func TestPublish_ExistingForeignGraphIsNotComparableAndNeedsForce(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: digest.FromString("graph")} + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned}, nil) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "no portable record") +} + +func TestPull_Edges(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + + t.Run("native graph cannot be materialized", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned}, nil) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("content bound to another record is rejected", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + other := record + other.Version = "9.9.9" + require.NoError(t, other.Seal()) + m.repo.EXPECT().Read(gomock.Any(), pinned).Return(&artifact.Content{Record: &other}, nil) + dest := out(t) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.True(t, missing(dest)) + }) + t.Run("read fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + m.repo.EXPECT().Read(gomock.Any(), pinned).Return(nil, errTransport) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errTransport) + }) + t.Run("inspect fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(nil, errTransport) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errTransport) + }) +} + +func TestShowDocument_TamperedDocumentFailsItsDescriptor(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + want := []byte("genuine readme") + descriptor := artifact.Document{Role: artifact.DocumentReadme, SourcePath: "README.md", MediaType: artifact.DocumentMediaType, Size: int64(len(want)), Digest: digest.FromBytes(want)} + record.Documents = []artifact.Document{descriptor} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + pinned.ResolvedDigest, pinned.RecordDigest = recordDigest, recordDigest + + for name, content := range map[string][]byte{"altered bytes": []byte("tampered readme"), "oversize": bytes.Repeat([]byte("x"), int(artifact.MaxDocumentSize)+1)} { + t.Run(name, func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(&artifact.DocumentStream{Document: descriptor, Reader: io.NopCloser(bytes.NewReader(content))}, nil) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.Error(t, err) + assert.True(t, errors.Is(err, errUtils.ErrArtifactIntegrityFailed) || errors.Is(err, errUtils.ErrArtifactTooLarge), "%v", err) + }) + } + t.Run("forged descriptor", func(t *testing.T) { + m := newMockSetup(t, &caps) + forged := []byte("forged readme") + other := descriptor + other.Size, other.Digest = int64(len(forged)), digest.FromBytes(forged) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(&artifact.DocumentStream{Document: other, Reader: io.NopCloser(bytes.NewReader(forged))}, nil) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("open fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(nil, errTransport) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errTransport) + }) +} + +func TestList_PaginationGuards(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + t.Run("non-advancing cursor", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).Return(&artifact.ListPage{NextCursor: "stuck"}, nil).Times(2) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + }) + t.Run("cursor cycle across pages", func(t *testing.T) { + m := newMockSetup(t, &caps) + next := map[string]string{"": "a", "a": "b", "b": "a"} + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + return &artifact.ListPage{NextCursor: next[query.Cursor]}, nil + }).Times(3) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + }) + t.Run("list error", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, errTransport) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errTransport) + }) + t.Run("canceled", func(t *testing.T) { + m := newMockSetup(t, &caps) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := m.svc.List(ctx, artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, context.Canceled) + }) + t.Run("write-only repository", func(t *testing.T) { + writeOnly := caps + writeOnly.Read = false + m := newMockSetup(t, &writeOnly) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) +} + +// graphSource is a source repository whose artifact is a foreign OCI graph without a Record. +func graphSource(ctrl *gomock.Controller, pinned *artifact.PinnedReference) *artifact.MockRepository { + repo := artifact.NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(artifacttest.DefaultCapabilities()).AnyTimes() + repo.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(*pinned, nil).AnyTimes() + root := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString("graph-root"), Size: 10} + repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.Inspection{ + Pinned: *pinned, + Representation: artifact.Representation{Packaging: artifact.PackagingOCIGraph, OriginalRoot: &root}, + }, nil).AnyTimes() + return repo +} + +func TestCopy_ForeignGraphNeedsANativeTransfer(t *testing.T) { + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: digest.FromString("graph-root")} + build := func(t *testing.T, dst artifact.Repository) *service.Service { + ctrl := gomock.NewController(t) + e := newEnv(t, envConfig{}) + src := graphSource(ctrl, &pinned) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"src": src, "dst": dst}, + Info: map[string]service.RepositoryInfo{ + "src": {Capabilities: artifacttest.DefaultCapabilities()}, + "dst": {Capabilities: artifacttest.DefaultCapabilities(), Packaging: artifact.PackagingNone}, + }, + } + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + return svc + } + + t.Run("no native transfer is a capability error", func(t *testing.T) { + svc := build(t, artifacttest.New(artifacttest.Options{})) + _, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("native transfer carries the graph", func(t *testing.T) { + native := newNativeRepository(t, artifacttest.New(artifacttest.Options{})) + svc := build(t, native) + result, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + }) + t.Run("a digest-only graph has no tag to publish under", func(t *testing.T) { + native := newNativeRepository(t, artifacttest.New(artifacttest.Options{})) + svc := build(t, native) + digestRef := artifact.Ref{Name: "tools/example", Digest: digest.FromString("graph-root")} + _, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: digestRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) +} + +func TestCopy_EvidenceCarryFailuresAreReported(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + reader := artifact.NewMockEvidenceReader(gomock.NewController(t)) + reader.EXPECT().ReadEvidence(gomock.Any(), gomock.Any()).Return(nil, errTransport).AnyTimes() + e.static.Repositories["src"] = struct { + artifact.Repository + artifact.EvidenceReader + }{e.repos["src"], reader} + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + var warned bool + for _, diagnostic := range result.Diagnostics { + warned = warned || diagnostic.Code == "evidence-read-failed" + } + assert.True(t, warned) +} + +func TestSign_RejectsADigestThatIsNotTheSignableSubject(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + // An OCI-style repository resolves the digest to a manifest that is not the Record digest. + manifest := digest.FromString("manifest") + pinned := artifact.PinnedReference{Ref: artifact.Ref{Name: "tools/example", Digest: manifest}, ResolvedDigest: manifest, RecordDigest: recordDigest} + m := newMockSetup(t, &caps, service.WithSigner(releaseSigner)) + m.repo.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + _, err := m.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: manifest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestCopy_RepackFailureCleansUpStaging(t *testing.T) { + repacker := service.NewMockRepacker(gomock.NewController(t)) + repacker.EXPECT().Repack(gomock.Any(), gomock.Any()).Return(nil, errTransport) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithRepacker(repacker)) + e.publish("src", e.source("hello")) + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errTransport) + e.assertStagingClean() + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) +} + +func TestPublish_DefaultBackoffSleepAndCancellationDuringBackoff(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 1) + svc, err := service.New(e.static, nil, service.WithRetry(2, time.Millisecond), service.WithStagingRoot(e.staging)) + require.NoError(t, err) + result, err := svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err, "the default context-aware sleep elapses and the retry succeeds") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + + other := newEnv(t, envConfig{}) + other.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 5) + ctx, cancel := context.WithCancel(context.Background()) + svc, err = service.New(other.static, nil, service.WithRetry(3, time.Hour), service.WithStagingRoot(other.staging), + service.WithSleep(func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() })) + require.NoError(t, err) + _, err = svc.Publish(ctx, other.publishRequest("dst", other.source("hello"))) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, other.repos["dst"].Calls("Publish")) + + real, err := service.New(other.static, nil, service.WithRetry(2, time.Hour), service.WithStagingRoot(other.staging)) + require.NoError(t, err) + other.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 5) + cancelled, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stop() + _, err = real.Publish(cancelled, other.publishRequest("dst", other.source("hello"))) + require.ErrorIs(t, err, context.DeadlineExceeded, "an hour-long default backoff is interrupted by the context") +} diff --git a/pkg/artifact/service/enumerate.go b/pkg/artifact/service/enumerate.go new file mode 100644 index 00000000000..c62260cba9a --- /dev/null +++ b/pkg/artifact/service/enumerate.go @@ -0,0 +1,131 @@ +package service + +import ( + "context" + "fmt" + "sort" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// maxListPages bounds pagination so a repository with a non-advancing cursor cannot loop forever. +const maxListPages = 100_000 + +// listOutcome is the result of enumerating one repository. +type listOutcome struct { + entries []artifact.ListEntry + listing artifact.RepositoryListing + err error +} + +// listRepository enumerates one repository, consuming pagination, and applies the label and text +// filters. A repository without the needed enumeration capability yields a capability error that +// explains direct named operations still work. +func (s *Service) listRepository(ctx context.Context, alias string, request *enumeration) listOutcome { + outcome := listOutcome{listing: artifact.RepositoryListing{Repository: alias}} + fail := func(err error) listOutcome { + outcome.err = fmt.Errorf("repository %q: %w", alias, err) + outcome.listing.Diagnostics = append(outcome.listing.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "list-failed", Message: err.Error()}) + return outcome + } + info, err := s.repos.Describe(alias) + if err != nil { + return fail(err) + } + if err := checkEnumeration(alias, &info.Capabilities, request.name); err != nil { + return fail(err) + } + repo, err := s.repos.Open(ctx, alias) + if err != nil { + return fail(err) + } + entries, err := consumePages(ctx, repo, request) + if err != nil { + return fail(err) + } + for index := range entries { + entries[index].Repository = alias + if matchesFilters(&entries[index], request) { + outcome.entries = append(outcome.entries, entries[index]) + } + } + outcome.listing.Count = len(outcome.entries) + return outcome +} + +func checkEnumeration(alias string, caps *artifact.Capabilities, name string) error { + if !caps.Read { + return artifact.NewUnsupportedCapability(alias, opList, "read", "the repository is write-only") + } + if name != "" && !caps.EnumerateNamed { + return artifact.NewUnsupportedCapability(alias, opList, "enumerate/named", "this repository cannot list versions of a name; direct named operations such as inspect still work") + } + if name == "" && !caps.EnumerateCollection { + return artifact.NewUnsupportedCapability(alias, opList, "enumerate/collection", "this repository has no native collection listing or catalog; direct named operations such as inspect still work") + } + return nil +} + +func consumePages(ctx context.Context, repo artifact.Repository, request *enumeration) ([]artifact.ListEntry, error) { + var entries []artifact.ListEntry + cursor := "" + seen := map[string]bool{} + for page := 0; page < maxListPages; page++ { + if err := ctx.Err(); err != nil { + return entries, err + } + result, err := repo.List(ctx, artifact.ListQuery{Name: request.name, Labels: request.labels, Cursor: cursor, Refresh: request.refresh}) + if err != nil { + return entries, err + } + entries = append(entries, result.Entries...) + if result.NextCursor == "" { + return entries, nil + } + // Any cursor already fetched means the listing cycles, not only one that repeats immediately. + seen[cursor] = true + if seen[result.NextCursor] { + return entries, fmt.Errorf("%w: the repository returned list cursor %q again", errUtils.ErrArtifactPublicationUnavailable, result.NextCursor) + } + cursor = result.NextCursor + } + return entries, fmt.Errorf("%w: the listing exceeded %d pages", errUtils.ErrArtifactTooLarge, maxListPages) +} + +// matchesFilters applies the exact label selectors (AND semantics) and the search text. +func matchesFilters(entry *artifact.ListEntry, request *enumeration) bool { + for _, label := range request.labels { + if value, ok := entry.Labels[label.Key]; !ok || value != label.Value { + return false + } + } + if request.text == "" { + return true + } + needle := strings.ToLower(request.text) + haystack := []string{entry.Name, entry.Kind, entry.Description} + for key, value := range entry.Labels { + haystack = append(haystack, key, value) + } + for _, text := range haystack { + if strings.Contains(strings.ToLower(text), needle) { + return true + } + } + return false +} + +// sortEntries orders entries by name, then version, then repository alias. +func sortEntries(entries []artifact.ListEntry) { + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].Name != entries[j].Name { + return entries[i].Name < entries[j].Name + } + if entries[i].Version != entries[j].Version { + return entries[i].Version < entries[j].Version + } + return entries[i].Repository < entries[j].Repository + }) +} diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go new file mode 100644 index 00000000000..171f49fb47d --- /dev/null +++ b/pkg/artifact/service/execute.go @@ -0,0 +1,195 @@ +package service + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// reconcileState is what a read of the completed reference proves. +type reconcileState int + +const ( + // The reference could not be proven to hold or lack the expected content. + reconcileUnknown reconcileState = iota + + // The completed reference holds exactly the expected content. + reconcileCommitted + + // The reference still equals the observed prior state. + reconcileNotCommitted +) + +// execute writes with reconciliation. It never blind-retries: after an ambiguous failure it reads +// the completed reference and verifies the expected digests; only a proven not-committed state +// with a temporary error is retried, reusing the prepared artifact, operation ID, and time. +func (s *Service) execute(ctx context.Context, pub *publication, cur *current, decision verdict, lock artifact.LockHandle) (*artifact.PublishResult, error) { + for attempt := 1; ; attempt++ { + s.event(ctx, pub, PhasePending, attempt, nil) + options := artifact.PublishOptions{ExpectedPrior: cur.pinned, Force: decision == verdictReplace, Lock: lock} + result, err := pub.write(ctx, options) + if err == nil { + s.event(ctx, pub, PhaseCommitted, attempt, nil) + return result, nil + } + if isDefinitive(err) { + return nil, err + } + recovered, retry, failure := s.afterFailure(ctx, pub, cur, err, attempt) + if !retry { + return recovered, failure + } + } +} + +// afterFailure decides what an ambiguous write failure means by reading the completed reference. +// It returns a recovered result when the write is proven committed, retry when it is proven +// uncommitted and worth repeating, and otherwise the error to report. +func (s *Service) afterFailure(ctx context.Context, pub *publication, cur *current, writeErr error, attempt int) (*artifact.PublishResult, bool, error) { + reconcileCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + found := reconcile(reconcileCtx, pub.tgt, pub.in, cur) + cancel() + switch found.state { + case reconcileCommitted: + s.event(ctx, pub, PhaseReconciled, attempt, nil) + pub.op.info("reconciled", fmt.Sprintf("the repository reported %q but the completed reference holds the expected digests; success was confirmed by reading it", writeErr)) + return syntheticResult(&found.pinned, found.inspection), false, nil + case reconcileNotCommitted: + if ctx.Err() != nil { + return nil, false, errors.Join(writeErr, ctx.Err()) + } + if !isTemporary(writeErr) || attempt >= s.attempts { + return nil, false, writeErr + } + if sleepErr := s.sleep(ctx, s.backoff<<(attempt-1)); sleepErr != nil { + return nil, false, errors.Join(writeErr, sleepErr) + } + return nil, true, nil + case reconcileUnknown: + fallthrough + default: + ambiguous := fmt.Errorf("%w: operation %s on %s in repository %q failed and could not be reconciled; check the reference before retrying", errUtils.ErrArtifactPublicationAmbiguous, pub.op.id, pub.in.ref, pub.tgt.alias) + return nil, false, errors.Join(ambiguous, writeErr, found.err) + } +} + +// isDefinitive reports errors that prove nothing was committed, so reconciliation is unnecessary. A +// publication-unavailable rejection (a reference pending for another operation) is definitive even +// when the error also carries the ambiguous sentinel. +func isDefinitive(err error) bool { + for _, sentinel := range []error{ + errUtils.ErrArtifactPublicationUnavailable, + errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactRecordInvalid, + errUtils.ErrArtifactTooLarge, errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, + errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactVerificationFailed, + } { + if errors.Is(err, sentinel) { + return true + } + } + return false +} + +// isTemporary reports errors that advertise themselves as retryable. +func isTemporary(err error) bool { + var temporary interface{ Temporary() bool } + return errors.As(err, &temporary) && temporary.Temporary() +} + +// reconciliation is what reading the completed reference proved. +type reconciliation struct { + state reconcileState + pinned artifact.PinnedReference + inspection *artifact.Inspection + err error +} + +// reconcile reads the completed reference and classifies it against the expected content. +func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) reconciliation { + ref := in.ref + pinned, err := tgt.repo.Resolve(ctx, ref) + switch { + case errors.Is(err, errUtils.ErrArtifactNotFound): + if cur.pinned == nil && !cur.unavailable { + return reconciliation{state: reconcileNotCommitted} + } + return reconciliation{err: fmt.Errorf("the previously present reference %s is now absent: %w", ref, err)} + case err != nil: + return reconciliation{err: fmt.Errorf("reconcile %s: %w", ref, err)} + } + if in.recordDigest == "" { + return reconcileGraph(ref, &pinned, in, cur) + } + if pinned.RecordDigest == in.recordDigest { + return reconcileRecord(ctx, tgt, &pinned, in) + } + if cur.pinned != nil && pinned.ResolvedDigest == cur.pinned.ResolvedDigest { + return reconciliation{state: reconcileNotCommitted, pinned: pinned} + } + return reconciliation{pinned: pinned, err: fmt.Errorf("%w: %s now resolves to %s, which is neither the prior state nor the expected content", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest)} +} + +// reconcileRecord inspects a reference whose Record digest matches the expected one and confirms +// that the stored content is the expected content. +func reconcileRecord(ctx context.Context, tgt *target, pinned *artifact.PinnedReference, in *incoming) reconciliation { + inspection, err := tgt.repo.Inspect(ctx, *pinned) + if err != nil { + return reconciliation{pinned: *pinned, err: fmt.Errorf("reconcile %s: %w", in.ref, err)} + } + if matchErr := matchesExpected(in, inspection); matchErr != nil { + return reconciliation{pinned: *pinned, inspection: inspection, err: matchErr} + } + return reconciliation{state: reconcileCommitted, pinned: *pinned, inspection: inspection} +} + +// reconcileGraph classifies a native OCI graph without a Record. Empty record digests prove nothing, +// so success requires the destination root to be exactly the pinned source root. +func reconcileGraph(ref artifact.Ref, pinned *artifact.PinnedReference, in *incoming, cur *current) reconciliation { + if in.rootDigest != "" && pinned.ResolvedDigest == in.rootDigest { + return reconciliation{state: reconcileCommitted, pinned: *pinned, inspection: in.inspection} + } + if cur.pinned != nil && pinned.ResolvedDigest == cur.pinned.ResolvedDigest { + return reconciliation{state: reconcileNotCommitted, pinned: *pinned} + } + return reconciliation{pinned: *pinned, err: fmt.Errorf("%w: %s now resolves to %s, not the source graph root %s", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest, in.rootDigest)} +} + +// matchesExpected verifies the stored Record and, when both sides describe them, the payload +// descriptors against what this operation prepared. +func matchesExpected(in *incoming, inspection *artifact.Inspection) error { + if inspection.Record != nil { + stored, err := inspection.Record.RecordDigest() + if err != nil || stored != in.recordDigest { + return fmt.Errorf("%w: the stored record does not match the prepared record", errUtils.ErrArtifactIntegrityFailed) + } + } + if len(in.payload) == 0 || len(inspection.Representation.Payload) == 0 { + return nil + } + want := descriptorDigests(in.payload) + got := descriptorDigests(inspection.Representation.Payload) + if !slices.Equal(want, got) { + return fmt.Errorf("%w: the stored payload descriptors do not match the prepared payload", errUtils.ErrArtifactIntegrityFailed) + } + return nil +} + +func descriptorDigests(descriptors []ocispec.Descriptor) []string { + digests := make([]string, 0, len(descriptors)) + for index := range descriptors { + digests = append(digests, descriptors[index].Digest.String()) + } + slices.Sort(digests) + return digests +} + +func syntheticResult(pinned *artifact.PinnedReference, inspection *artifact.Inspection) *artifact.PublishResult { + return &artifact.PublishResult{Pinned: *pinned, Representation: inspection.Representation, Location: inspection.Representation.Locator} +} diff --git a/pkg/artifact/service/helpers_test.go b/pkg/artifact/service/helpers_test.go new file mode 100644 index 00000000000..6274a2a51ff --- /dev/null +++ b/pkg/artifact/service/helpers_test.go @@ -0,0 +1,208 @@ +package service_test + +import ( + "context" + "fmt" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// startTime is the fixed clock origin for service tests. +var startTime = time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC) + +// fakeClock is a settable clock. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func (c *fakeClock) Set(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = t +} + +// journalLog collects the lifecycle events that the generated MockJournal receives. It is a +// synchronized collector only; it does not implement service.Journal. +type journalLog struct { + mu sync.Mutex + entries []service.JournalEntry +} + +func (j *journalLog) add(entry *service.JournalEntry) { + j.mu.Lock() + defer j.mu.Unlock() + j.entries = append(j.entries, *entry) +} + +func (j *journalLog) phases() []service.JournalPhase { + j.mu.Lock() + defer j.mu.Unlock() + phases := make([]service.JournalPhase, 0, len(j.entries)) + for _, entry := range j.entries { + phases = append(phases, entry.Phase) + } + return phases +} + +func (j *journalLog) snapshot() []service.JournalEntry { + j.mu.Lock() + defer j.mu.Unlock() + return append([]service.JournalEntry(nil), j.entries...) +} + +// newRecordingJournal returns a generated MockJournal that records every entry into the log. +func newRecordingJournal(t *testing.T) (*service.MockJournal, *journalLog) { + t.Helper() + log := &journalLog{} + journal := service.NewMockJournal(gomock.NewController(t)) + journal.EXPECT().Record(gomock.Any(), gomock.Any()).Do(func(_ context.Context, entry *service.JournalEntry) { log.add(entry) }).AnyTimes() + return journal, log +} + +// temporaryError is a retryable failure. +type temporaryError struct{ message string } + +func (e *temporaryError) Error() string { return e.message } +func (e *temporaryError) Temporary() bool { return true } + +// env is a service wired to named in-memory repositories. +type env struct { + t *testing.T + repos map[string]*artifacttest.Repository + static *service.StaticRepositories + clock *fakeClock + journal *journalLog + staging string + sleeps []time.Duration + sleepMu sync.Mutex + onSleep func(time.Duration) + svc *service.Service +} + +type envConfig struct { + repos map[string]artifacttest.Options + infos map[string]service.RepositoryInfo + options []service.Option +} + +// newEnv builds an environment with a "dst" repository by default. +func newEnv(t *testing.T, config envConfig) *env { + t.Helper() + if config.repos == nil { + config.repos = map[string]artifacttest.Options{"dst": {Alias: "dst"}} + } + journal, log := newRecordingJournal(t) + e := &env{ + t: t, + repos: map[string]*artifacttest.Repository{}, + clock: &fakeClock{now: startTime}, + journal: log, + staging: t.TempDir(), + } + e.static = &service.StaticRepositories{Repositories: map[string]artifact.Repository{}, Info: config.infos} + if e.static.Info == nil { + e.static.Info = map[string]service.RepositoryInfo{} + } + for alias, opts := range config.repos { + opts.Alias = alias + repo := artifacttest.New(opts) + e.repos[alias] = repo + e.static.Repositories[alias] = repo + } + var counter atomic.Int64 + options := []service.Option{ + service.WithClock(e.clock.Now), + service.WithIDGenerator(func() string { return fmt.Sprintf("op-%d", counter.Add(1)) }), + service.WithJournal(journal), + service.WithSleep(func(_ context.Context, d time.Duration) error { + e.sleepMu.Lock() + e.sleeps = append(e.sleeps, d) + hook := e.onSleep + e.sleepMu.Unlock() + if hook != nil { + hook(d) + } + return nil + }), + service.WithRetry(3, time.Millisecond), + service.WithStagingRoot(e.staging), + service.WithOwner("test-owner"), + } + svc, err := service.New(e.static, nil, append(options, config.options...)...) + require.NoError(t, err) + e.svc = svc + return e +} + +func (e *env) setInfo(alias string, mutate func(*service.RepositoryInfo)) { + info, err := e.static.Describe(alias) + require.NoError(e.t, err) + mutate(&info) + e.static.Info[alias] = info +} + +// source writes a small file package whose data.txt holds content and returns its directory. +func (e *env) source(content string) string { + e.t.Helper() + src := e.t.TempDir() + fixtures.WriteFile(e.t, src, "data.txt", content, 0o644) + fixtures.WriteFile(e.t, src, "bin/run.sh", fixtures.HelperScript, 0o755) + fixtures.WriteFile(e.t, src, fixtures.ReservedDir+"/README.md", "package readme\n", 0o644) + return src +} + +func (e *env) publishRequest(repository, src string) artifact.PublishRequest { + return artifact.PublishRequest{Repository: repository, Source: src, Name: "tools/example", Version: "1.0.0"} +} + +func (e *env) publish(repository, src string) *service.PublishOutcome { + e.t.Helper() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest(repository, src)) + require.NoError(e.t, err) + return outcome +} + +// assertStagingClean asserts no owned temporary data remains. +func (e *env) assertStagingClean() { + e.t.Helper() + items, err := os.ReadDir(e.staging) + require.NoError(e.t, err) + assert.Empty(e.t, items, "owned temporary staging must be removed") +} + +func ref(name, version string) artifact.Ref { + return artifact.Ref{Name: name, Version: version} +} + +var exampleRef = ref("tools/example", "1.0.0") + +// missing reports whether the path does not exist, for readable "nothing was created" assertions. +func missing(path string) bool { + _, err := os.Stat(path) + return os.IsNotExist(err) +} diff --git a/pkg/artifact/service/internal_helpers_test.go b/pkg/artifact/service/internal_helpers_test.go new file mode 100644 index 00000000000..0d628dc3a2c --- /dev/null +++ b/pkg/artifact/service/internal_helpers_test.go @@ -0,0 +1,10 @@ +package service + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func ocispecDescriptor() ocispec.Descriptor { + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageIndex, Digest: digest.FromString("root"), Size: 4} +} diff --git a/pkg/artifact/service/internal_test.go b/pkg/artifact/service/internal_test.go new file mode 100644 index 00000000000..79faaef74a7 --- /dev/null +++ b/pkg/artifact/service/internal_test.go @@ -0,0 +1,212 @@ +package service + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func TestCanonicalOrder_SortsByRepositoryThenResourceAndDropsDuplicates(t *testing.T) { + targets := []lockTarget{ + {alias: "b", resource: "z"}, + {alias: "a", resource: "y"}, + {alias: "a", resource: "x"}, + {alias: "b", resource: "a"}, + {alias: "a", resource: "x"}, + } + ordered := canonicalOrder(targets) + var got []string + for _, target := range ordered { + got = append(got, target.alias+"/"+target.resource) + } + assert.Equal(t, []string{"a/x", "a/y", "b/a", "b/z"}, got) + assert.Equal(t, "b", targets[0].alias, "the caller's slice is not reordered") +} + +func TestAcquireLocks_MultiResourceUsesCanonicalOrderAndReleasesInReverse(t *testing.T) { + ctrl := gomock.NewController(t) + var order []string + mkRepo := func(alias string) *artifact.MockRepository { + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + handle.EXPECT().Resource().Return("res").AnyTimes() + handle.EXPECT().Owner().Return("owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + order = append(order, "lock "+alias+"/"+request.Resource) + assert.Equal(t, "owner-x", request.Owner) + assert.Equal(t, "op-9", request.OperationID) + assert.Equal(t, artifact.LockScopeHost, request.Required) + return handle, nil + }) + handle.EXPECT().Release(gomock.Any()).DoAndReturn(func(context.Context) error { + order = append(order, "release "+alias) + return nil + }) + return repo + } + repoA := mkRepo("a") + repoB := mkRepo("b") + caps := artifact.Capabilities{LockScopes: []artifact.LockScope{artifact.LockScopeHost}} + infoA := &RepositoryInfo{Alias: "a", Capabilities: caps} + infoB := &RepositoryInfo{Alias: "b", Capabilities: caps} + + s := &Service{owner: "owner-x"} + op := &operation{id: "op-9"} + set, err := s.acquireLocks(context.Background(), op, "copy", []lockTarget{ + {alias: "b", repo: repoB, info: infoB, resource: "r2"}, + {alias: "a", repo: repoA, info: infoA, resource: "r1"}, + }) + require.NoError(t, err) + assert.NotNil(t, set.handleFor("a")) + assert.Nil(t, set.handleFor("c")) + assert.Equal(t, artifact.LockScopeHost, set.scope()) + require.NoError(t, set.release(context.Background(), op)) + assert.Equal(t, []string{"lock a/r1", "lock b/r2", "release b", "release a"}, order) +} + +func TestAcquireLocks_FailureReleasesWhatWasTakenAndInsufficientScopeFailsFirst(t *testing.T) { + ctrl := gomock.NewController(t) + first := artifact.NewMockRepository(ctrl) + second := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + handle.EXPECT().Resource().Return("res").AnyTimes() + handle.EXPECT().Owner().Return("owner").AnyTimes() + first.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + handle.EXPECT().Release(gomock.Any()).Return(nil) + second.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(nil, errUtils.ErrArtifactConflict) + caps := artifact.Capabilities{LockScopes: []artifact.LockScope{artifact.LockScopeHost}} + + s := &Service{owner: "o"} + op := &operation{id: "op"} + _, err := s.acquireLocks(context.Background(), op, "copy", []lockTarget{ + {alias: "a", repo: first, info: &RepositoryInfo{Capabilities: caps}, resource: "r"}, + {alias: "b", repo: second, info: &RepositoryInfo{Capabilities: caps}, resource: "r"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + + weak := &RepositoryInfo{Alias: "w", Capabilities: caps, RequiredLockScope: artifact.LockScopeRepository} + untouched := artifact.NewMockRepository(ctrl) // No Lock expectation: it must never be called. + _, err = s.acquireLocks(context.Background(), op, "copy", []lockTarget{{alias: "w", repo: untouched, info: weak, resource: "r"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestLockSet_NilAndEmptySetsAreSafe(t *testing.T) { + var set *lockSet + assert.Nil(t, set.handleFor("a")) + assert.Empty(t, set.scope()) + require.NoError(t, set.release(context.Background(), &operation{})) + assert.Empty(t, (&lockSet{}).scope()) +} + +func TestLockSet_ScopeIsTheWeakestHeld(t *testing.T) { + ctrl := gomock.NewController(t) + repoHandle := artifact.NewMockLockHandle(ctrl) + hostHandle := artifact.NewMockLockHandle(ctrl) + repoHandle.EXPECT().Scope().Return(artifact.LockScopeRepository).AnyTimes() + hostHandle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + set := &lockSet{held: []heldLock{{handle: repoHandle}, {handle: hostHandle}}} + assert.Equal(t, artifact.LockScopeHost, set.scope()) + set = &lockSet{held: []heldLock{{handle: hostHandle}, {handle: repoHandle}}} + assert.Equal(t, artifact.LockScopeHost, set.scope()) +} + +func TestSplitPolicyAndMerge(t *testing.T) { + global := artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: []string{"g.pub"}}}} + repo := artifact.VerificationPolicy{TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: "i", Issuer: "https://x"}}}}, Offline: true} + merged := mergePolicies(global, repo) + assert.True(t, merged.RequireSignature) + assert.True(t, merged.Offline) + require.Len(t, merged.TrustSets, 2, "sets are kept separate, never unioned") + calls := splitPolicy(merged) + require.Len(t, calls, 2) + assert.True(t, calls[0].Offline) + assert.Len(t, calls[1].TrustSets, 1) + + bare := splitPolicy(artifact.VerificationPolicy{}) + require.Len(t, bare, 1) + assert.True(t, bare[0].RequireSignature, "an empty policy that is consulted still demands a signature") + + assert.False(t, verificationRequired(artifact.VerificationPolicy{})) + assert.True(t, verificationRequired(artifact.VerificationPolicy{RequireSignature: true})) + assert.True(t, verificationRequired(artifact.VerificationPolicy{TrustSets: []artifact.TrustSet{{Keys: []string{"k"}}}})) +} + +func TestSubjectOf(t *testing.T) { + _, err := subjectOf(&artifact.Inspection{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + + root := artifact.Inspection{Representation: artifact.Representation{OriginalRoot: &ocispecRoot}} + subject, err := subjectOf(&root) + require.NoError(t, err) + assert.Equal(t, artifact.SubjectOCIRoot, subject.Kind) + assert.Equal(t, ocispecRoot.Digest, subject.Digest, "a native graph is signed by its original root, never an archive checksum") +} + +func TestMergeMetadataAndDifferences(t *testing.T) { + merged := mergeMetadata(artifact.Metadata{Description: "d", Labels: map[string]string{"a": "1"}}, artifact.Metadata{Labels: map[string]string{"a": "2", "b": "3"}}) + assert.Equal(t, "d", merged.Description) + assert.Equal(t, map[string]string{"a": "2", "b": "3"}, merged.Labels) + assert.Nil(t, mergeMetadata(artifact.Metadata{}, artifact.Metadata{}).Labels) + + a := &artifact.Record{Kind: "files"} + b := &artifact.Record{Kind: "other", KindData: artifact.KindData(`{"x":1}`)} + assert.Contains(t, describeDifferences(a, b), "kind data") + assert.Equal(t, "intent", describeDifferences(a, a)) + assert.Contains(t, describeDifferences(nil, a), "no portable record") +} + +func TestIsDefinitiveAndTemporary(t *testing.T) { + assert.True(t, isDefinitive(errUtils.ErrArtifactConflict)) + assert.True(t, isDefinitive(errUtils.ErrArtifactUnsupportedCapability)) + assert.False(t, isDefinitive(context.Canceled)) + assert.False(t, isTemporary(context.Canceled)) + assert.True(t, isDefinitive(errUtils.ErrArtifactPublicationUnavailable)) + assert.True(t, isDefinitive(errors.Join(errUtils.ErrArtifactPublicationAmbiguous, errUtils.ErrArtifactPublicationUnavailable)), + "an unavailable rejection stays definitive even when it also carries the ambiguous sentinel") + assert.False(t, isDefinitive(errUtils.ErrArtifactPublicationAmbiguous)) +} + +func TestReconcileGraphComparesTheSourceRoot(t *testing.T) { + ref := artifact.Ref{Name: "pkg/graph", Version: "1"} + root := digest.FromString("source root") + prior := artifact.PinnedReference{ResolvedDigest: digest.FromString("prior root")} + other := artifact.PinnedReference{ResolvedDigest: digest.FromString("someone else")} + + found := reconcileGraph(ref, &artifact.PinnedReference{ResolvedDigest: root}, &incoming{rootDigest: root}, ¤t{}) + assert.Equal(t, reconcileCommitted, found.state, "the destination root is the pinned source root") + + found = reconcileGraph(ref, &other, &incoming{rootDigest: root}, ¤t{}) + assert.Equal(t, reconcileUnknown, found.state, "empty record digests on both sides never prove success") + require.ErrorIs(t, found.err, errUtils.ErrArtifactConflict) + + found = reconcileGraph(ref, &prior, &incoming{rootDigest: root}, ¤t{pinned: &prior}) + assert.Equal(t, reconcileNotCommitted, found.state) + + found = reconcileGraph(ref, &other, &incoming{}, ¤t{}) + assert.Equal(t, reconcileUnknown, found.state, "without a source root nothing can be proven") +} + +var ocispecRoot = ocispecDescriptor() + +func TestRemoveStagedTreeRestoresDirectoryAccess(t *testing.T) { + tree := filepath.Join(t.TempDir(), "tree") + dir := filepath.Join(tree, "restricted") + require.NoError(t, os.MkdirAll(dir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "file"), []byte("x"), 0o600)) + require.NoError(t, os.Chmod(dir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + require.NoError(t, removeStagedTree(tree)) + _, err := os.Lstat(tree) + require.True(t, os.IsNotExist(err)) +} diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go new file mode 100644 index 00000000000..0e5f3f2bcf6 --- /dev/null +++ b/pkg/artifact/service/interrupt_test.go @@ -0,0 +1,325 @@ +package service_test + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +var errTransport = errors.New("connection reset") + +func TestPublish_InterruptionBeforePayloadLeavesNothingAndRetryCreates(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.ErrorIs(t, err, errTransport) + assert.Empty(t, outcome.Operation.Outcome, "a failed write never reports success") + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + assert.False(t, e.repos["dst"].IsPending(exampleRef)) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "a non-temporary failure is never blind-retried") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + retry, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retry.Operation.Outcome) +} + +func TestPublish_InterruptionDuringPayloadOrAfterMetadataIsAmbiguousUntilSameIntentRetry(t *testing.T) { + for _, point := range []artifacttest.FailPoint{artifacttest.FailDuringPayload, artifacttest.FailAfterMetadata} { + t.Run(string(point), func(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(point, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + require.ErrorIs(t, err, errTransport, "the cause is retained for diagnostics") + assert.Contains(t, err.Error(), outcome.Operation.OperationID, "recovery diagnostics name the operation") + assert.Empty(t, outcome.Operation.Outcome, "partial work is never advertised as success") + assert.True(t, e.repos["dst"].IsPending(exampleRef), "the reference stays unavailable") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "no blind retry of an ambiguous mutation") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "readers cannot see a pending publication") + + retry, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err, "the same-intent retry repairs the interrupted publication") + assert.Equal(t, artifact.OutcomeCreated, retry.Operation.Outcome) + assert.False(t, e.repos["dst"].IsPending(exampleRef)) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + }) + } +} + +func TestPublish_DifferentIntentCannotStealAPendingPublication(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailDuringPayload, errTransport, 1) + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("first"))) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + + other, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("different"))) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + assert.Contains(t, err.Error(), "op-1", "the pending operation ID is surfaced for recovery") + assert.Empty(t, other.Operation.Outcome) +} + +func TestPublish_CommitWithLostResponseIsReconciledNotRetried(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err, "success only after reconciliation read the completed reference") + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "reconciliation replaces the retry") + var reconciled bool + for _, diagnostic := range outcome.Operation.Diagnostics { + reconciled = reconciled || diagnostic.Code == "reconciled" + } + assert.True(t, reconciled, "the reconciliation is reported") + assert.Contains(t, e.journal.phases(), service.PhaseReconciled) + current, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok) + assert.Equal(t, outcome.Operation.RecordDigest, current) +} + +func TestPublish_CommitWithLostResponseOnReplacementIsReconciled(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("first")) + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, errTransport, 1) + request := e.publishRequest("dst", e.source("second")) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, outcome.Operation.Outcome) +} + +func TestPublish_TemporaryFailuresRetryOnlyAfterReconciliationProvesNothingCommitted(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 2) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 3, e.repos["dst"].Calls("Publish")) + e.sleepMu.Lock() + assert.Equal(t, []time.Duration{time.Millisecond, 2 * time.Millisecond}, e.sleeps, "exponential backoff") + e.sleepMu.Unlock() + + var attempts []int + for _, entry := range e.journal.snapshot() { + assert.Equal(t, "op-1", entry.OperationID, "every attempt reuses the operation ID") + if entry.Phase == service.PhasePending { + attempts = append(attempts, entry.Attempt) + } + } + assert.Equal(t, []int{1, 2, 3}, attempts) +} + +func TestPublish_TemporaryFailureExhaustsAttempts(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 10) + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + var temporary *temporaryError + require.ErrorAs(t, err, &temporary) + assert.Equal(t, 3, e.repos["dst"].Calls("Publish")) +} + +func TestPublish_TemporaryFailureAfterCommitIsReconciledWithoutRetry(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, &temporaryError{"timeout"}, 5) + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "a committed publication is never repeated") +} + +func TestPublish_DateRolloverAcrossRetriesReusesCapturedTime(t *testing.T) { + e := newEnv(t, envConfig{}) + e.clock.Set(time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC)) + e.onSleep = func(time.Duration) { e.clock.Advance(2 * time.Second) } // Crosses midnight between attempts. + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, "2026-03-04T23:59:59Z", outcome.Record.CreatedAt.String(), "the retry reuses the time captured before the first attempt") + assert.Equal(t, 2, e.repos["dst"].Calls("Publish")) + assert.Equal(t, "op-1", outcome.Operation.OperationID) + assert.Equal(t, []service.JournalPhase{ + service.PhasePrepared, service.PhaseLocked, service.PhasePending, service.PhasePending, service.PhaseCommitted, service.PhaseReleased, + }, e.journal.phases(), "one lifecycle, two attempts") +} + +func TestPublish_FailedForceReplacementLeavesReferenceUnavailableUntilSameIntentRetry(t *testing.T) { + e := newEnv(t, envConfig{}) + first := e.publish("dst", e.source("first")) + oldPinned, err := e.repos["dst"].Resolve(context.Background(), exampleRef) + require.NoError(t, err) + + e.repos["dst"].InjectFailure(artifacttest.FailDuringPayload, errTransport, 1) + request := e.publishRequest("dst", e.source("second")) + request.Force = true + _, err = e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + assert.True(t, e.repos["dst"].IsPending(exampleRef)) + _, err = e.repos["dst"].Resolve(context.Background(), exampleRef) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "no rollback is claimed: the reference is unavailable, not silently the old version") + + retried, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, retried.Operation.Outcome) + assert.NotEqual(t, first.Operation.RecordDigest, retried.Operation.RecordDigest) + + _, err = e.repos["dst"].Read(context.Background(), oldPinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the old digest never serves the replacement's bytes") +} + +func TestPublish_TwoWritersOneReferenceLockSerializesThem(t *testing.T) { + const writers = 8 + e := newEnv(t, envConfig{}) + sources := make([]string, writers) + for i := range sources { + sources[i] = e.source(fmt.Sprintf("content-%d", i)) + } + var wg sync.WaitGroup + var created, conflicts atomic.Int64 + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", sources[i])) + switch { + case err == nil: + created.Add(1) + case errors.Is(err, errUtils.ErrArtifactConflict): + conflicts.Add(1) + default: + t.Errorf("unexpected error: %v", err) + } + }() + } + wg.Wait() + assert.Equal(t, int64(1), created.Load(), "exactly one writer wins the reference") + assert.Equal(t, int64(writers-1), conflicts.Load()) + assert.Empty(t, e.repos["dst"].HeldLocks()) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "losers conflict before writing anything") +} + +func TestPublish_TwoWritersWithIdenticalIntentCreateOnce(t *testing.T) { + const writers = 6 + e := newEnv(t, envConfig{}) + src := e.source("same") + var wg sync.WaitGroup + var created, unchanged atomic.Int64 + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + switch outcome.Operation.Outcome { + case artifact.OutcomeCreated: + created.Add(1) + case artifact.OutcomeUnchanged: + unchanged.Add(1) + } + }() + } + wg.Wait() + assert.Equal(t, int64(1), created.Load()) + assert.Equal(t, int64(writers-1), unchanged.Load()) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) +} + +// unlockedRepository hides the writer lock so the repository's own conditional guard is exercised. +// It forwards every other operation to base through the generated repository mock. +func unlockedRepository(t *testing.T, base artifact.Repository) *artifact.MockRepository { + t.Helper() + return delegatingRepository(t, base, repoHooks{lock: func(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + handle := artifact.NewMockLockHandle(gomock.NewController(t)) + handle.EXPECT().Resource().Return(request.Resource).AnyTimes() + handle.EXPECT().Owner().Return("noop").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + handle.EXPECT().Token().Return("noop").AnyTimes() + handle.EXPECT().ExpiresAt().Return(time.Time{}, false).AnyTimes() + handle.EXPECT().Renew(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + handle.EXPECT().Release(gomock.Any()).Return(nil).AnyTimes() + return handle, nil + }}) +} + +func TestPublish_ConditionalGuardRejectsRacingWritersEvenWithoutAWriterLock(t *testing.T) { + const writers = 6 + e := newEnv(t, envConfig{}) + bare := artifacttest.New(artifacttest.Options{Alias: "bare"}) + e.static.Repositories["bare"] = unlockedRepository(t, bare) + e.static.Info["bare"] = service.RepositoryInfo{Alias: "bare", Capabilities: bare.Capabilities(), Packaging: artifact.PackagingNone} + + sources := make([]string, writers) + for i := range sources { + sources[i] = e.source(fmt.Sprintf("racing-%d", i)) + } + var wg sync.WaitGroup + var wins atomic.Int64 + var digests sync.Map + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("bare", sources[i])) + if err == nil { + wins.Add(1) + digests.Store(outcome.Operation.RecordDigest, true) + } + }() + } + wg.Wait() + assert.Equal(t, int64(1), wins.Load(), "exactly one writer wins the reference") + current, ok := bare.RecordDigestOf(exampleRef) + require.True(t, ok) + var count int + digests.Range(func(key, _ any) bool { count++; return true }) + assert.Equal(t, int64(count), wins.Load(), "each reported success corresponds to a distinct committed record") + pinned, err := bare.Resolve(context.Background(), exampleRef) + require.NoError(t, err) + assert.Equal(t, current, pinned.RecordDigest, "the reference is bound to exactly one artifact") +} + +func TestPublish_DelegatedLockIsTakenFromTheProvider(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {}, "coord": {}}}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.LockProvider = "coord" }) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["coord"].Calls("Lock"), "the delegate provides the lock") + assert.Zero(t, e.repos["dst"].Calls("Lock"), "the target repository is not asked to lock") + assert.Empty(t, e.repos["coord"].HeldLocks(), "the delegated lock is released") + + e.setInfo("dst", func(info *service.RepositoryInfo) { info.LockProvider = "" }) + _, err = e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("other"))) + require.Error(t, err, "the second publish conflicts, but only after locking") + assert.Equal(t, 1, e.repos["dst"].Calls("Lock"), "without delegation the target repository locks for itself") +} diff --git a/pkg/artifact/service/locks.go b/pkg/artifact/service/locks.go new file mode 100644 index 00000000000..7149c1f40d7 --- /dev/null +++ b/pkg/artifact/service/locks.go @@ -0,0 +1,177 @@ +package service + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// lockTarget is one resource to lock in one repository. +type lockTarget struct { + alias string + repo artifact.Repository + info *RepositoryInfo + resource string +} + +// heldLock is an acquired lock together with where it came from. +type heldLock struct { + target lockTarget + handle artifact.LockHandle +} + +// lockSet is a group of held locks released together in reverse acquisition order. +type lockSet struct { + held []heldLock +} + +// requiredScope returns the guarantee a repository requires, defaulting to host scope. +func requiredScope(info *RepositoryInfo) artifact.LockScope { + if info.RequiredLockScope == "" { + return artifact.LockScopeHost + } + return info.RequiredLockScope +} + +// checkLockGuarantee rejects an insufficient lock guarantee before anything is mutated. +func checkLockGuarantee(info *RepositoryInfo, operation string) error { + required := requiredScope(info) + if info.Capabilities.HasLockScope(required) { + return nil + } + return artifact.NewUnsupportedCapability(info.Alias, operation, "lock/"+string(required), + fmt.Sprintf("the repository cannot provide %s-scope write coordination; configure locking.repository to delegate to a repository that can, or relax locking.required_scope", required)) +} + +// acquireLocks checks every guarantee first, then acquires the locks in canonical repository alias +// then resource order. Any failure (including cancellation) releases whatever was already taken. +func (s *Service) acquireLocks(ctx context.Context, op *operation, operation string, targets []lockTarget) (*lockSet, error) { + for index := range targets { + if err := checkLockGuarantee(targets[index].info, operation); err != nil { + return nil, err + } + } + ordered := canonicalOrder(targets) + set := &lockSet{} + for index := range ordered { + target := ordered[index] + locker, resource, err := s.lockerFor(ctx, &target) + if err != nil { + releaseErr := set.release(ctx, op) + return nil, errors.Join(err, releaseErr) + } + handle, err := locker.Lock(ctx, artifact.LockRequest{ + Resource: resource, + Owner: s.owner, + OperationID: op.id, + Required: requiredScope(target.info), + }) + if err != nil { + releaseErr := set.release(ctx, op) + return nil, errors.Join(fmt.Errorf("lock %s in repository %q: %w", target.resource, target.alias, err), releaseErr) + } + set.held = append(set.held, heldLock{target: target, handle: handle}) + } + return set, nil +} + +// lockerFor returns the repository that provides the target's lock and the resource to lock there. +// A repository that delegates locking (locking.repository) is locked through its delegate, whose +// resource is qualified by the target alias because one delegate can coordinate several repositories. +func (s *Service) lockerFor(ctx context.Context, target *lockTarget) (artifact.Repository, string, error) { + provider := target.info.LockProvider + if provider == "" || provider == target.alias { + return target.repo, target.resource, nil + } + repo, err := s.repos.Open(ctx, provider) + if err != nil { + return nil, "", fmt.Errorf("open lock repository %q delegated by %q: %w", provider, target.alias, err) + } + return repo, target.alias + "/" + target.resource, nil +} + +// canonicalOrder sorts targets by repository alias then resource and drops exact duplicates. +func canonicalOrder(targets []lockTarget) []lockTarget { + ordered := append([]lockTarget(nil), targets...) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].alias != ordered[j].alias { + return ordered[i].alias < ordered[j].alias + } + return ordered[i].resource < ordered[j].resource + }) + unique := ordered[:0] + for index := range ordered { + if index > 0 && ordered[index].alias == ordered[index-1].alias && ordered[index].resource == ordered[index-1].resource { + continue + } + unique = append(unique, ordered[index]) + } + return unique +} + +// handleFor returns the lock held for the repository alias, or nil. +func (l *lockSet) handleFor(alias string) artifact.LockHandle { + if l == nil { + return nil + } + for index := range l.held { + if l.held[index].target.alias == alias { + return l.held[index].handle + } + } + return nil +} + +// handleForResource returns the handle for a specific lock target. +func (l *lockSet) handleForResource(alias, resource string) artifact.LockHandle { + if l != nil { + for i := range l.held { + if l.held[i].target.alias == alias && l.held[i].target.resource == resource { + return l.held[i].handle + } + } + } + return nil +} + +func subjectLockResource(name string) string { return "subject:" + name } + +// scope returns the weakest actual scope across held locks, which results report. +func (l *lockSet) scope() artifact.LockScope { + if l == nil || len(l.held) == 0 { + return "" + } + weakest := l.held[0].handle.Scope() + for index := range l.held { + if scope := l.held[index].handle.Scope(); !scope.Satisfies(artifact.LockScopeRepository) { + weakest = scope + } + } + return weakest +} + +// release relinquishes every held lock in reverse order. It uses a context detached from +// cancellation so a canceled operation never leaks ownership. A failed release is reported as a +// warning with the owner and resource an operator needs for recovery, and returned. +func (l *lockSet) release(ctx context.Context, op *operation) error { + if l == nil { + return nil + } + var errs []error + for index := len(l.held) - 1; index >= 0; index-- { + held := l.held[index] + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + err := held.handle.Release(releaseCtx) + cancel() + if err != nil { + op.warn("lock-release-failed", fmt.Sprintf("repository %q resource %q owner %q: operator recovery may be required: %v", held.target.alias, held.handle.Resource(), held.handle.Owner(), err)) + errs = append(errs, fmt.Errorf("release lock %s in repository %q: %w", held.handle.Resource(), held.target.alias, err)) + } + } + l.held = nil + return errors.Join(errs...) +} diff --git a/pkg/artifact/service/mock_repositories.go b/pkg/artifact/service/mock_repositories.go new file mode 100644 index 00000000000..d687b2bc5ae --- /dev/null +++ b/pkg/artifact/service/mock_repositories.go @@ -0,0 +1,201 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/service (interfaces: Repositories,Journal,ImmutabilityPolicy,Repacker) +// +// Generated by this command: +// +// mockgen -package service -destination mock_repositories.go github.com/cloudposse/atmos/pkg/artifact/service Repositories,Journal,ImmutabilityPolicy,Repacker +// + +// Package service is a generated GoMock package. +package service + +import ( + context "context" + reflect "reflect" + + artifact "github.com/cloudposse/atmos/pkg/artifact" + produce "github.com/cloudposse/atmos/pkg/artifact/produce" + gomock "go.uber.org/mock/gomock" +) + +// MockRepositories is a mock of Repositories interface. +type MockRepositories struct { + ctrl *gomock.Controller + recorder *MockRepositoriesMockRecorder + isgomock struct{} +} + +// MockRepositoriesMockRecorder is the mock recorder for MockRepositories. +type MockRepositoriesMockRecorder struct { + mock *MockRepositories +} + +// NewMockRepositories creates a new mock instance. +func NewMockRepositories(ctrl *gomock.Controller) *MockRepositories { + mock := &MockRepositories{ctrl: ctrl} + mock.recorder = &MockRepositoriesMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepositories) EXPECT() *MockRepositoriesMockRecorder { + return m.recorder +} + +// Aliases mocks base method. +func (m *MockRepositories) Aliases() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Aliases") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Aliases indicates an expected call of Aliases. +func (mr *MockRepositoriesMockRecorder) Aliases() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Aliases", reflect.TypeOf((*MockRepositories)(nil).Aliases)) +} + +// Describe mocks base method. +func (m *MockRepositories) Describe(alias string) (RepositoryInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Describe", alias) + ret0, _ := ret[0].(RepositoryInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Describe indicates an expected call of Describe. +func (mr *MockRepositoriesMockRecorder) Describe(alias any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Describe", reflect.TypeOf((*MockRepositories)(nil).Describe), alias) +} + +// Open mocks base method. +func (m *MockRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Open", ctx, alias) + ret0, _ := ret[0].(artifact.Repository) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Open indicates an expected call of Open. +func (mr *MockRepositoriesMockRecorder) Open(ctx, alias any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Open", reflect.TypeOf((*MockRepositories)(nil).Open), ctx, alias) +} + +// MockJournal is a mock of Journal interface. +type MockJournal struct { + ctrl *gomock.Controller + recorder *MockJournalMockRecorder + isgomock struct{} +} + +// MockJournalMockRecorder is the mock recorder for MockJournal. +type MockJournalMockRecorder struct { + mock *MockJournal +} + +// NewMockJournal creates a new mock instance. +func NewMockJournal(ctrl *gomock.Controller) *MockJournal { + mock := &MockJournal{ctrl: ctrl} + mock.recorder = &MockJournalMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockJournal) EXPECT() *MockJournalMockRecorder { + return m.recorder +} + +// Record mocks base method. +func (m *MockJournal) Record(ctx context.Context, entry *JournalEntry) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Record", ctx, entry) +} + +// Record indicates an expected call of Record. +func (mr *MockJournalMockRecorder) Record(ctx, entry any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Record", reflect.TypeOf((*MockJournal)(nil).Record), ctx, entry) +} + +// MockImmutabilityPolicy is a mock of ImmutabilityPolicy interface. +type MockImmutabilityPolicy struct { + ctrl *gomock.Controller + recorder *MockImmutabilityPolicyMockRecorder + isgomock struct{} +} + +// MockImmutabilityPolicyMockRecorder is the mock recorder for MockImmutabilityPolicy. +type MockImmutabilityPolicyMockRecorder struct { + mock *MockImmutabilityPolicy +} + +// NewMockImmutabilityPolicy creates a new mock instance. +func NewMockImmutabilityPolicy(ctrl *gomock.Controller) *MockImmutabilityPolicy { + mock := &MockImmutabilityPolicy{ctrl: ctrl} + mock.recorder = &MockImmutabilityPolicyMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockImmutabilityPolicy) EXPECT() *MockImmutabilityPolicyMockRecorder { + return m.recorder +} + +// Immutable mocks base method. +func (m *MockImmutabilityPolicy) Immutable(candidate Candidate) (bool, string) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Immutable", candidate) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + return ret0, ret1 +} + +// Immutable indicates an expected call of Immutable. +func (mr *MockImmutabilityPolicyMockRecorder) Immutable(candidate any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Immutable", reflect.TypeOf((*MockImmutabilityPolicy)(nil).Immutable), candidate) +} + +// MockRepacker is a mock of Repacker interface. +type MockRepacker struct { + ctrl *gomock.Controller + recorder *MockRepackerMockRecorder + isgomock struct{} +} + +// MockRepackerMockRecorder is the mock recorder for MockRepacker. +type MockRepackerMockRecorder struct { + mock *MockRepacker +} + +// NewMockRepacker creates a new mock instance. +func NewMockRepacker(ctrl *gomock.Controller) *MockRepacker { + mock := &MockRepacker{ctrl: ctrl} + mock.recorder = &MockRepackerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepacker) EXPECT() *MockRepackerMockRecorder { + return m.recorder +} + +// Repack mocks base method. +func (m *MockRepacker) Repack(ctx context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Repack", ctx, request) + ret0, _ := ret[0].(*artifact.PreparedArtifact) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Repack indicates an expected call of Repack. +func (mr *MockRepackerMockRecorder) Repack(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Repack", reflect.TypeOf((*MockRepacker)(nil).Repack), ctx, request) +} diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go new file mode 100644 index 00000000000..4189bf49dd6 --- /dev/null +++ b/pkg/artifact/service/mutate.go @@ -0,0 +1,105 @@ +package service + +import ( + "context" + "fmt" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Service]. The reference is resolved once; deletion is scoped to the +// pinned subject within the repository's declared delete capability and guarded by the writer lock. +func (s *Service) Delete(ctx context.Context, request artifact.DeleteRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Delete")() + + op := s.begin(artifact.ActionDelete, request.Repository, request.Ref.String()) + err := s.deleteArtifact(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func (s *Service) deleteArtifact(ctx context.Context, op *operation, request *artifact.DeleteRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opDelete) + if err != nil { + return err + } + if caps := info.Capabilities; caps.Delete == artifact.DeleteNone || caps.Delete == "" || !caps.Write { + return artifact.NewUnsupportedCapability(request.Repository, opDelete, opDelete, "the repository cannot delete artifacts") + } + return s.runDelete(ctx, op, request, &info, repo) +} + +// validateRepositoryRef checks the two fields every single-artifact operation needs. +func validateRepositoryRef(repository string, ref artifact.Ref) error { + if err := requireRepository(repository); err != nil { + return err + } + return ref.Validate() +} + +func (s *Service) runDelete(ctx context.Context, op *operation, request *artifact.DeleteRequest, info *RepositoryInfo, repo artifact.Repository) error { + resource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: request.Ref.String()} + subjectResource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: subjectLockResource(request.Ref.Name)} + locks, err := s.acquireLocks(ctx, op, opDelete, []lockTarget{resource, subjectResource}) + if err != nil { + return err + } + defer func() { _ = locks.release(ctx, op) }() + + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleForResource(request.Repository, resource.resource)}) + if err != nil { + return fmt.Errorf("delete %s in repository %q: %w", request.Ref, request.Repository, err) + } + op.result.Outcome = artifact.OutcomeDeleted + op.result.LockScope = locks.scope() + op.result.Size = payloadSize(inspection) + if len(result.Deleted) > 0 { + op.result.Location = result.Deleted[0] + op.result.Aliases = result.Deleted[1:] + } + for _, affected := range result.Affected { + op.info("affected-reference", fmt.Sprintf("the deletion also affected %s", affected)) + } + return nil +} + +// Pin implements [artifact.Service]. A pin is immutable reference data for consumer lockfiles; it +// acquires no write lock and is not a substitute for signature verification. +func (s *Service) Pin(ctx context.Context, request artifact.PinRequest) (*artifact.PinResult, error) { + defer perf.Track(nil, "service.Service.Pin")() + + op := s.begin(artifact.ActionPin, request.Repository, request.Ref.String()) + result := &artifact.PinResult{} + err := s.pin(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) pin(ctx context.Context, op *operation, request *artifact.PinRequest, result *artifact.PinResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + _, repo, err := s.openForRead(ctx, request.Repository, opPin) + if err != nil { + return err + } + pinned, err := repo.Pin(ctx, request.Ref) + if err != nil { + return fmt.Errorf("pin %s in repository %q: %w", request.Ref, request.Repository, err) + } + result.Pinned = pinned + op.result.Outcome = artifact.OutcomeRead + op.result.ResolvedDigest = pinned.ResolvedDigest + op.result.RecordDigest = pinned.RecordDigest + op.result.ContentDigest = pinned.ContentDigest + op.info("pin-not-trust", "a pin fixes the immutable reference; it does not verify signatures, and trust policy is reevaluated when it is used") + return nil +} diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go new file mode 100644 index 00000000000..07436eac227 --- /dev/null +++ b/pkg/artifact/service/mutate_test.go @@ -0,0 +1,249 @@ +package service_test + +import ( + "context" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func TestDelete_ScopedToTheSelectedArtifact(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("keep")) + other := e.publishRequest("dst", e.source("drop")) + other.Version = "2.0.0" + _, err := e.svc.Publish(context.Background(), other) + require.NoError(t, err) + + result, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDeleted, result.Outcome) + assert.NotEmpty(t, result.Location) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + assert.Empty(t, e.repos["dst"].HeldLocks()) + + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err, "other versions are untouched") + + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Empty(t, e.repos["dst"].HeldLocks(), "locks are released on failure too") +} + +func TestDelete_RefusesAReferenceThatMovedAfterItWasPinned(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the reference now names another record than the one the caller observed") + + moved, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok, "the moved reference is left in place") + assert.NotEqual(t, original.Operation.RecordDigest, moved) + _, ok = e.repos["dst"].RecordDigestOf(ref("tools/example", "2.0.0")) + assert.True(t, ok, "the record the reference moved to is not deleted") + assert.Empty(t, e.repos["dst"].HeldLocks()) +} + +func TestDelete_CapabilityAndRequestChecks(t *testing.T) { + noDelete := artifacttest.DefaultCapabilities() + noDelete.Delete = artifact.DeleteNone + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &noDelete}}}) + e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "an unsupported delete takes no lock") + + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: "x"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestDelete_ReportsAffectedReferences(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + caps := artifacttest.DefaultCapabilities() + handle := artifact.NewMockLockHandle(ctrl) + record := artifact.Record{Name: "x/y", Version: "1", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + pinned := artifact.PinnedReference{Ref: ref("x/y", "1"), ResolvedDigest: recordDigest, RecordDigest: recordDigest} + + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil).Times(2) + handle.EXPECT().Release(gomock.Any()).Return(nil).Times(2) + handle.EXPECT().Resource().Return("x/y:1").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Resolve(gomock.Any(), ref("x/y", "1")).Return(pinned, nil) + repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + repo.EXPECT().Delete(gomock.Any(), pinned, gomock.Any()).DoAndReturn( + func(_ context.Context, _ artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + assert.Equal(t, artifact.DeleteObject, options.Scope) + require.NotNil(t, options.ExpectedCurrent) + assert.Equal(t, recordDigest, options.ExpectedCurrent.ResolvedDigest) + assert.NotNil(t, options.Lock) + return &artifact.DeleteResult{Deleted: []string{"loc-1", "loc-2"}, Affected: []artifact.Ref{ref("x/y", "latest-ish")}}, nil + }, + ) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps}}} + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + result, err := svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("x/y", "1")}) + require.NoError(t, err) + assert.Equal(t, "loc-1", result.Location) + assert.Equal(t, []string{"loc-2"}, result.Aliases) + var affected bool + for _, diagnostic := range result.Diagnostics { + affected = affected || diagnostic.Code == "affected-reference" + } + assert.True(t, affected) +} + +func TestPin_IsImmutableDataTakesNoLockAndGrantsNoTrust(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + + result, err := e.svc.Pin(context.Background(), artifact.PinRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, published.Operation.RecordDigest, result.Pinned.ResolvedDigest) + assert.Equal(t, published.Operation.RecordDigest, result.Operation.RecordDigest) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "a pin is not a write mutex") + assert.Equal(t, 0, e.repos["dst"].Calls("Verify"), "a pin is not a substitute for verification") + var note bool + for _, diagnostic := range result.Operation.Diagnostics { + note = note || diagnostic.Code == "pin-not-trust" + } + assert.True(t, note) + + _, err = e.svc.Pin(context.Background(), artifact.PinRequest{Repository: "dst", Ref: ref("tools/example", "9")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.Pin(context.Background(), artifact.PinRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestLocks_HeldByAnotherOwnerBlocksUntilCanceledAndLeaksNothing(t *testing.T) { + e := newEnv(t, envConfig{}) + other, err := e.repos["dst"].Lock(context.Background(), artifact.LockRequest{Resource: exampleRef.String(), Owner: "someone-else"}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) + defer cancel() + outcome, err := e.svc.PublishDetailed(ctx, e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Zero(t, e.repos["dst"].Calls("Publish"), "nothing is written without the lock") + assert.Equal(t, []string{exampleRef.String()}, e.repos["dst"].HeldLocks(), "the other owner's claim is neither stolen nor leaked") + assert.Empty(t, outcome.Operation.Outcome) + + require.NoError(t, other.Release(context.Background())) + retried, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retried.Operation.Outcome) +} + +func TestLocks_ReleasedWithADetachedContextAfterCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + handle.EXPECT().Resource().Return("tools/example:1.0.0").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound).AnyTimes() + + ctx, cancel := context.WithCancel(context.Background()) + repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *artifact.PreparedArtifact, artifact.PublishOptions) (*artifact.PublishResult, error) { + cancel() + return nil, context.Canceled + }, + ) + var released bool + handle.EXPECT().Release(gomock.Any()).DoAndReturn(func(releaseCtx context.Context) error { + released = true + assert.NoError(t, releaseCtx.Err(), "release must not inherit the canceled context") + return nil + }) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone}}} + e := newEnv(t, envConfig{}) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging), service.WithClock(e.clock.Now), service.WithRetry(1, 0)) + require.NoError(t, err) + + _, err = svc.Publish(ctx, e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, released, "the lock was released after cancellation") + e.assertStagingClean() +} + +func TestLocks_ReleaseFailureIsReportedForOperatorRecovery(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + handle.EXPECT().Resource().Return("tools/example:1.0.0").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound) + repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: artifact.PinnedReference{ResolvedDigest: digest.FromString("x")}, Representation: artifact.Representation{Payload: []ocispec.Descriptor{}}}, nil, + ) + handle.EXPECT().Release(gomock.Any()).Return(errTransport) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone}}} + e := newEnv(t, envConfig{}) + journal, log := newRecordingJournal(t) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging), service.WithJournal(journal)) + require.NoError(t, err) + + result, err := svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err, "the publication itself succeeded") + assert.Contains(t, log.phases(), service.PhaseCommitted) + assert.NotContains(t, log.phases(), service.PhaseReleased, "a failed release must not be journaled as released") + var warning artifact.Diagnostic + for _, diagnostic := range result.Diagnostics { + if diagnostic.Code == "lock-release-failed" { + warning = diagnostic + } + } + assert.Contains(t, warning.Message, "test-owner") + assert.Contains(t, warning.Message, "tools/example:1.0.0") +} + +func TestStaticRepositories(t *testing.T) { + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"b": artifacttest.New(artifacttest.Options{}), "a": artifacttest.New(artifacttest.Options{})}} + assert.Equal(t, []string{"a", "b"}, repos.Aliases()) + info, err := repos.Describe("a") + require.NoError(t, err) + assert.Equal(t, "a", info.Alias) + assert.True(t, info.Capabilities.Write) + assert.Equal(t, artifact.PackagingNone, info.Packaging) + _, err = repos.Describe("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Open(context.Background(), "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + repo, err := repos.Open(context.Background(), "a") + require.NoError(t, err) + assert.NotNil(t, repo) +} diff --git a/pkg/artifact/service/publish.go b/pkg/artifact/service/publish.go new file mode 100644 index 00000000000..5140c00abd8 --- /dev/null +++ b/pkg/artifact/service/publish.go @@ -0,0 +1,246 @@ +package service + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// PublishOutcome is the detailed result of [Service.PublishDetailed]. On an unchanged no-op, +// Record is the EXISTING record, byte for byte, and Operation reports the current execution +// separately (its own operation ID and duration). +type PublishOutcome struct { + // Operation is the typed operation result. + Operation artifact.OperationResult + + // Record is the Record now published (the existing one on a no-op); it is empty on failure + // before preparation and for dry runs holds the Record that would be published. + Record artifact.Record + + // Existing reports an unchanged no-op that returned the stored Record. + Existing bool +} + +// Publish implements [artifact.Service]. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Publish(ctx context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Publish")() + + outcome, err := s.PublishDetailed(ctx, request) + return &outcome.Operation, err +} + +// PublishDetailed prepares the source and stores it in one repository, returning the Record +// alongside the typed operation result. The result is always non-nil, even on error. +// +//nolint:gocritic // The request is passed by value to mirror Publish, which artifact.Service fixes. +func (s *Service) PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*PublishOutcome, error) { + defer perf.Track(nil, "service.Service.PublishDetailed")() + + op := s.begin(artifact.ActionPublish, request.Repository, requestedName(request.Name, request.Version)) + state := &publishState{} + err := s.publish(ctx, op, &request, state) + return &PublishOutcome{Operation: op.done(err), Record: state.record, Existing: state.existing}, err +} + +func requestedName(name, version string) string { + if name == "" || version == "" { + return name + } + return artifact.Ref{Name: name, Version: version}.String() +} + +func (s *Service) publish(ctx context.Context, op *operation, request *artifact.PublishRequest, state *publishState) error { + if err := validatePublish(request); err != nil { + return err + } + info, packaging, err := s.describeForWrite(request.Repository, opPublish) + if err != nil { + return err + } + if err := checkLockGuarantee(&info, opPublish); err != nil { + return err + } + producer, ok := s.producers.Lookup(request.Kind) + if !ok { + return fmt.Errorf("%w: no producer is registered for kind %q (registered: %v)", errUtils.ErrArtifactConfigInvalid, request.Kind, s.producers.Kinds()) + } + prepared, err := producer.Prepare(ctx, artifact.PrepareRequest{ + Source: request.Source, + Name: request.Name, + Version: request.Version, + Kind: request.Kind, + Metadata: mergeMetadata(info.Defaults, request.Metadata), + Readme: request.Readme, + Changelog: request.Changelog, + Packaging: packaging, + OperationID: op.id, + PreparedAt: op.preparedAt, + }) + if err != nil { + return err + } + defer prepared.Close() + + in, err := newIncoming(prepared) + if err != nil { + return err + } + state.record = prepared.Record + fillPrepared(op, in) + if request.DryRun { + op.result.Outcome = artifact.OutcomeDryRun + op.info("dry-run", fmt.Sprintf("would publish %s to repository %q as %s packaging; no repository was contacted", op.result.Ref, request.Repository, packaging)) + return nil + } + repo, err := s.repos.Open(ctx, request.Repository) + if err != nil { + return err + } + pub := &publication{ + op: op, + tgt: &target{alias: request.Repository, repo: repo, info: &info}, + in: in, + force: request.Force, + state: state, + write: func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return repo.Publish(ctx, prepared, options) + }, + } + s.event(ctx, pub, PhasePrepared, 0, nil) + return s.commit(ctx, pub) +} + +// validatePublish checks the request before anything is staged or contacted. +func validatePublish(request *artifact.PublishRequest) error { + if err := requireRepository(request.Repository); err != nil { + return err + } + if request.Source == "" { + return fmt.Errorf("%w: a source is required", errUtils.ErrArtifactConfigInvalid) + } + if request.Kind == "" { + request.Kind = artifact.KindFiles + } + if err := artifact.ValidateKind(request.Kind); err != nil { + return err + } + if request.Name != "" { + if err := artifact.ValidateName(request.Name); err != nil { + return err + } + } + if request.Version != "" { + if err := artifact.ValidateVersion(request.Version); err != nil { + return err + } + } + return artifact.ValidateLabels(request.Metadata.Labels) +} + +// describeForWrite resolves the repository description offline and checks that it can be written. +func (s *Service) describeForWrite(alias, operation string) (RepositoryInfo, artifact.Packaging, error) { + info, err := s.repos.Describe(alias) + if err != nil { + return RepositoryInfo{}, "", err + } + if !info.Capabilities.Write { + return info, "", artifact.NewUnsupportedCapability(alias, operation, "write", "the repository is read-only") + } + packaging := info.Packaging + if packaging == "" && len(info.Capabilities.Packagings) > 0 { + packaging = info.Capabilities.Packagings[0] + } + if !info.Capabilities.HasPackaging(packaging) { + return info, packaging, artifact.NewUnsupportedCapability(alias, operation, "packaging/"+string(packaging), fmt.Sprintf("supported packagings: %v", info.Capabilities.Packagings)) + } + return info, packaging, nil +} + +// mergeMetadata layers explicit metadata over repository defaults: explicit labels override +// matching default keys, and an explicit description overrides the default. +func mergeMetadata(defaults, explicit artifact.Metadata) artifact.Metadata { + merged := artifact.Metadata{ + Description: explicit.Description, + Provenance: explicit.Provenance, + } + if merged.Description == "" { + merged.Description = defaults.Description + } + if len(defaults.Labels) > 0 || len(explicit.Labels) > 0 { + merged.Labels = make(map[string]string, len(defaults.Labels)+len(explicit.Labels)) + for key, value := range defaults.Labels { + merged.Labels[key] = value + } + for key, value := range explicit.Labels { + merged.Labels[key] = value + } + } + return merged +} + +// fillPrepared records the identity of the prepared artifact in the typed result. +func fillPrepared(op *operation, in *incoming) { + op.result.Ref = artifact.Ref{Name: in.record.Name, Version: in.record.Version}.String() + op.result.Requested = op.result.Ref + op.result.RecordDigest = in.recordDigest + op.result.ContentDigest = in.record.ContentDigest + op.result.Size = in.size +} + +// reportUnchanged fills the result from the EXISTING record; the current execution keeps its own +// operation ID and duration. +func (s *Service) reportUnchanged(pub *publication, cur *current) { + op, record := pub.op, cur.inspection.Record + pub.state.existing = true + if record != nil { + pub.state.record = *record + op.result.ContentDigest = record.ContentDigest + } + op.result.Outcome = artifact.OutcomeUnchanged + op.result.Location = cur.inspection.Representation.Locator + op.result.ResolvedDigest = cur.pinned.ResolvedDigest + op.result.RecordDigest = cur.pinned.RecordDigest + op.result.RepresentationDigest = cur.inspection.Representation.RepresentationDigest + op.result.Size = payloadSize(cur.inspection) + op.info("unchanged", fmt.Sprintf("an identical artifact already exists in repository %q; the existing record was returned unchanged", pub.tgt.alias)) +} + +func payloadSize(inspection *artifact.Inspection) int64 { + var total int64 + for index := range inspection.Representation.Payload { + total += inspection.Representation.Payload[index].Size + } + return total +} + +// reportWritten fills the result after a successful write. +func (s *Service) reportWritten(pub *publication, result *artifact.PublishResult, locks *lockSet, decision verdict, cur *current) { + op, in := pub.op, pub.in + if in.record != nil { + pub.state.record = *in.record + } + outcome := result.Outcome + if outcome == "" { + outcome = artifact.OutcomeCreated + if decision == verdictReplace || cur.unavailable { + outcome = artifact.OutcomeReplaced + } + } + op.result.Outcome = outcome + op.result.Location = result.Location + op.result.Aliases = result.Aliases + op.result.ResolvedDigest = result.Pinned.ResolvedDigest + op.result.RepresentationDigest = result.Representation.RepresentationDigest + op.result.LockScope = result.LockScope + if scope := locks.scope(); scope != "" { + op.result.LockScope = scope + } + if op.result.ResolvedDigest == "" { + op.result.ResolvedDigest = in.recordDigest + } +} diff --git a/pkg/artifact/service/publish_test.go b/pkg/artifact/service/publish_test.go new file mode 100644 index 00000000000..5a38342c700 --- /dev/null +++ b/pkg/artifact/service/publish_test.go @@ -0,0 +1,411 @@ +package service_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func TestPublish_CreatesAndReportsTypedResult(t *testing.T) { + e := newEnv(t, envConfig{}) + outcome := e.publish("dst", e.source("hello")) + + result := outcome.Operation + assert.Equal(t, artifact.ActionPublish, result.Action) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, "op-1", result.OperationID) + assert.Equal(t, "dst", result.Repository) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + assert.Equal(t, artifact.LockScopeHost, result.LockScope, "the actual lock scope is reported") + assert.NotEmpty(t, result.RecordDigest) + assert.Equal(t, outcome.Record.ContentDigest, result.ContentDigest) + assert.NotEmpty(t, result.Location) + assert.False(t, outcome.Existing) + assert.Equal(t, artifact.NewTimestamp(startTime), outcome.Record.CreatedAt) + assert.Empty(t, e.repos["dst"].HeldLocks(), "locks are released") + e.assertStagingClean() + assert.Equal(t, []service.JournalPhase{service.PhasePrepared, service.PhaseLocked, service.PhasePending, service.PhaseCommitted, service.PhaseReleased}, e.journal.phases()) + + // The interface method returns the same result. + again, err := e.svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) +} + +func TestPublish_IdenticalRepublishIsUnchangedEvenWhenTimeAndRunMetadataDiffer(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + first := e.publish("dst", src) + + e.clock.Advance(30 * time.Hour) + request := e.publishRequest("dst", src) + request.Metadata.Provenance = &artifact.Provenance{CI: &artifact.CIProvenance{Provider: "github", RunID: "999"}} + second, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + + assert.True(t, second.Existing) + assert.Equal(t, artifact.OutcomeUnchanged, second.Operation.Outcome) + firstBytes, err := first.Record.Canonical() + require.NoError(t, err) + secondBytes, err := second.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstBytes, secondBytes, "the EXISTING record is returned byte for byte") + assert.Equal(t, artifact.NewTimestamp(startTime), second.Record.CreatedAt, "created_at stays the original") + assert.Equal(t, first.Operation.RecordDigest, second.Operation.RecordDigest) + assert.Equal(t, "op-2", second.Operation.OperationID, "the current execution is reported separately") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "an unchanged no-op writes nothing") + assert.Nil(t, second.Record.Metadata.Provenance, "run metadata of the new execution never rewrites the stored record") +} + +func TestPublish_ChangedIntentConflictsWithoutForceAndReplacesWithIt(t *testing.T) { + tests := []struct { + name string + mutate func(t *testing.T, src string, request *artifact.PublishRequest) + reason string + }{ + {"content", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.WriteFile(t, src, "data.txt", "changed", 0o644) + }, "content"}, + {"mode", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.RequirePOSIXModes(t) + require.NoError(t, os.Chmod(filepath.Join(src, "data.txt"), 0o755)) + }, "content"}, + {"document", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "new readme\n", 0o644) + }, "content"}, + {"labels", func(_ *testing.T, _ string, request *artifact.PublishRequest) { + request.Metadata.Labels = map[string]string{"team": "platform"} + }, "labels"}, + {"description", func(_ *testing.T, _ string, request *artifact.PublishRequest) { + request.Metadata.Description = "new description" + }, "description"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + first := e.publish("dst", src) + request := e.publishRequest("dst", src) + tt.mutate(t, src, &request) + + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), tt.reason) + assert.NotNil(t, outcome, "a result is returned alongside the error") + assert.Equal(t, "op-2", outcome.Operation.OperationID) + current, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok) + assert.Equal(t, first.Operation.RecordDigest, current, "the stored artifact is untouched") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + request.Force = true + replaced, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, replaced.Operation.Outcome) + current, _ = e.repos["dst"].RecordDigestOf(exampleRef) + assert.Equal(t, replaced.Operation.RecordDigest.String(), current.String()) + e.assertStagingClean() + }) + } +} + +func TestPublish_ForceOnIdenticalContentStaysUnchanged(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.publish("dst", src) + request := e.publishRequest("dst", src) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) +} + +// pluginPolicy treats artifacts labeled format=plugin as immutable releases. +func pluginPolicy() service.ImmutabilityPolicy { + isPlugin := func(labels map[string]string) bool { return labels["format"] == "plugin" } + return service.ImmutabilityFunc(func(c service.Candidate) (bool, string) { + switch { + case c.Inspection != nil && c.Inspection.Record != nil && isPlugin(c.Inspection.Record.Metadata.Labels): + return true, "plugin format" + case c.Prepared != nil && isPlugin(c.Prepared.Record.Metadata.Labels): + return true, "plugin format" + } + return false, "" + }) +} + +func TestPublish_PluginFormatReleasesStayImmutableEvenWithForce(t *testing.T) { + plugin := map[string]string{"format": "plugin"} + tests := []struct { + name string + existing map[string]string + incoming map[string]string + wantImmutable bool + reason string + }{ + {"existing plugin", plugin, nil, true, "existing release"}, + {"incoming plugin", nil, plugin, true, "incoming release"}, + {"both plugin", plugin, map[string]string{"format": "plugin", "v": "2"}, true, "existing release"}, + {"neither", nil, map[string]string{"x": "y"}, false, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(pluginPolicy())}}) + src := e.source("hello") + request := e.publishRequest("dst", src) + request.Metadata.Labels = tt.existing + first, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + + request.Metadata.Labels = tt.incoming + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + if !tt.wantImmutable { + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, outcome.Operation.Outcome) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), tt.reason) + assert.Contains(t, err.Error(), "immutable") + current, _ := e.repos["dst"].RecordDigestOf(exampleRef) + assert.Equal(t, first.Operation.RecordDigest, current) + }) + } +} + +func TestPublish_IdenticalPluginReleaseRepublishIsStillAnUnchangedNoOp(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(pluginPolicy())}}) + src := e.source("hello") + request := e.publishRequest("dst", src) + request.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, outcome.Operation.Outcome) +} + +func TestPublish_ImmutabilityPolicyMockSeesBothSides(t *testing.T) { + ctrl := gomock.NewController(t) + policy := service.NewMockImmutabilityPolicy(ctrl) + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(policy)}}) + src := e.source("hello") + e.publish("dst", src) + + var sawExisting, sawIncoming bool + policy.EXPECT().Immutable(gomock.Any()).DoAndReturn(func(c service.Candidate) (bool, string) { + if c.Existing { + sawExisting = true + assert.NotNil(t, c.Inspection) + } else { + sawIncoming = true + require.NotNil(t, c.Inspection) + assert.Equal(t, "changed", c.Inspection.Record.Metadata.Description) + assert.NotNil(t, c.Prepared) + } + return false, "" + }).Times(2) + request := e.publishRequest("dst", src) + request.Force = true + request.Metadata.Description = "changed" + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.True(t, sawExisting && sawIncoming, "both the existing and the incoming artifact are admitted") +} + +func TestPublish_NameAndVersionResolutionHappensBeforeAnyWrite(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + tests := []struct { + name string + mutate func(*artifact.PublishRequest) + wantErr error + }{ + {"missing version", func(r *artifact.PublishRequest) { r.Version = "" }, errUtils.ErrArtifactRefInvalid}, + {"invalid version", func(r *artifact.PublishRequest) { r.Version = "bad version" }, errUtils.ErrArtifactRefInvalid}, + {"invalid name", func(r *artifact.PublishRequest) { r.Name = "Bad_Name" }, errUtils.ErrArtifactRefInvalid}, + {"traversal name", func(r *artifact.PublishRequest) { r.Name = "../x" }, errUtils.ErrArtifactUnsafePath}, + {"missing repository", func(r *artifact.PublishRequest) { r.Repository = "" }, errUtils.ErrArtifactConfigInvalid}, + {"missing source", func(r *artifact.PublishRequest) { r.Source = "" }, errUtils.ErrArtifactConfigInvalid}, + {"invalid kind", func(r *artifact.PublishRequest) { r.Kind = "Bad Kind" }, errUtils.ErrArtifactRecordInvalid}, + {"unregistered kind", func(r *artifact.PublishRequest) { r.Kind = "scaffold" }, errUtils.ErrArtifactConfigInvalid}, + {"bad labels", func(r *artifact.PublishRequest) { r.Metadata.Labels = map[string]string{"a=b": "c"} }, errUtils.ErrArtifactConfigInvalid}, + {"unknown repository", func(r *artifact.PublishRequest) { r.Repository = "nope" }, errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := e.publishRequest("dst", src) + tt.mutate(&request) + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, tt.wantErr) + require.NotNil(t, outcome) + assert.NotEmpty(t, outcome.Operation.OperationID) + assert.NotEmpty(t, outcome.Operation.Diagnostics, "the failure is reported in the result") + assert.Zero(t, e.repos["dst"].TotalCalls(), "nothing reached the repository") + e.assertStagingClean() + }) + } +} + +func TestPublish_BasenameNameFallbackAndInvalidBasename(t *testing.T) { + e := newEnv(t, envConfig{}) + good := filepath.Join(t.TempDir(), "my-app") + require.NoError(t, os.Mkdir(good, 0o755)) + fixtures.WriteFile(t, good, "a.txt", "a", 0o644) + outcome, err := e.svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: good, Version: "1"}) + require.NoError(t, err) + assert.Equal(t, "my-app", outcome.Record.Name) + + bad := filepath.Join(t.TempDir(), "My App") + require.NoError(t, os.Mkdir(bad, 0o755)) + fixtures.WriteFile(t, bad, "a.txt", "a", 0o644) + _, err = e.svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: bad, Version: "1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Contains(t, err.Error(), "explicit name") +} + +func TestPublish_MergesRepositoryDefaultMetadataUnderExplicitValues(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { + info.Defaults = artifact.Metadata{Description: "repo default", Labels: map[string]string{"team": "platform", "env": "dev"}} + }) + request := e.publishRequest("dst", e.source("hello")) + request.Metadata.Labels = map[string]string{"env": "prod"} + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "platform", "env": "prod"}, outcome.Record.Metadata.Labels) + assert.Equal(t, "repo default", outcome.Record.Metadata.Description) + + request = e.publishRequest("dst", e.source("other")) + request.Version = "2.0.0" + request.Metadata.Description = "explicit" + outcome, err = e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, "explicit", outcome.Record.Metadata.Description) +} + +func TestPublish_CapabilityAndLockGuaranteesAreCheckedBeforeMutation(t *testing.T) { + readOnly := artifacttest.DefaultCapabilities() + readOnly.Write = false + badPackaging := artifacttest.DefaultCapabilities() + badPackaging.Packagings = []artifact.Packaging{artifact.PackagingZip} + tests := []struct { + name string + caps artifact.Capabilities + info service.RepositoryInfo + }{ + {"read only", readOnly, service.RepositoryInfo{Packaging: artifact.PackagingNone}}, + {"unsupported packaging", badPackaging, service.RepositoryInfo{Packaging: artifact.PackagingTgz}}, + {"insufficient lock scope", artifacttest.DefaultCapabilities(), service.RepositoryInfo{Packaging: artifact.PackagingNone, RequiredLockScope: artifact.LockScopeRepository}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caps := tt.caps + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + info := tt.info + info.Capabilities = caps + e.static.Info["dst"] = info + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Zero(t, e.repos["dst"].TotalCalls(), "rejected before any repository call, including Lock") + assert.NotEmpty(t, outcome.Operation.OperationID) + e.assertStagingClean() + }) + } +} + +func TestPublish_RepositoryScopeLockIsReportedAsActual(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.LockScopes = []artifact.LockScope{artifact.LockScopeHost, artifact.LockScopeRepository} + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.RequiredLockScope = artifact.LockScopeRepository }) + outcome := e.publish("dst", e.source("hello")) + assert.Equal(t, artifact.LockScopeRepository, outcome.Operation.LockScope) +} + +func TestPublish_RegisteredKindProducerReceivesCapturedIdentity(t *testing.T) { + ctrl := gomock.NewController(t) + producer := artifact.NewMockProducer(ctrl) + producer.EXPECT().Kind().Return("scaffold").AnyTimes() + registry := artifact.NewProducerRegistry() + require.NoError(t, registry.Register(producer)) + + e := newEnv(t, envConfig{}) + svc, err := service.New(e.static, registry, service.WithClock(e.clock.Now), service.WithIDGenerator(func() string { return "fixed-op" }), service.WithStagingRoot(e.staging)) + require.NoError(t, err) + + producer.EXPECT().Prepare(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request artifact.PrepareRequest) (*artifact.PreparedArtifact, error) { + assert.Equal(t, "fixed-op", request.OperationID) + assert.True(t, startTime.Equal(request.PreparedAt)) + assert.Equal(t, time.UTC, request.PreparedAt.Location()) + assert.Equal(t, "scaffold", request.Kind) + assert.Equal(t, artifact.PackagingNone, request.Packaging) + return nil, errUtils.ErrArtifactPackagingInvalid + }) + request := e.publishRequest("dst", e.source("hello")) + request.Kind = "scaffold" + _, err = svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestPublish_DryRunPerformsNoRepositoryOrIdentityCalls(t *testing.T) { + t.Run("in-memory repository", func(t *testing.T) { + e := newEnv(t, envConfig{}) + request := e.publishRequest("dst", e.source("hello")) + request.DryRun = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDryRun, outcome.Operation.Outcome) + assert.NotEmpty(t, outcome.Operation.RecordDigest) + assert.NotEmpty(t, outcome.Record.Inventory) + assert.Zero(t, e.repos["dst"].TotalCalls()) + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + e.assertStagingClean() + assert.Empty(t, e.journal.phases(), "a dry run records no lifecycle") + }) + + t.Run("repositories mock asserts Open is never called", func(t *testing.T) { + ctrl := gomock.NewController(t) + repos := service.NewMockRepositories(ctrl) + caps := artifacttest.DefaultCapabilities() + repos.EXPECT().Describe("dst").Return(service.RepositoryInfo{Alias: "dst", Capabilities: caps, Packaging: artifact.PackagingTgz}, nil) + repos.EXPECT().Open(gomock.Any(), gomock.Any()).Times(0) + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "dst", Source: src, Name: "x/y", Version: "1", DryRun: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDryRun, result.Outcome) + }) +} + +func TestPublish_PackagingFollowsRepository(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = packaging }) + e.publish("dst", e.source("hello")) + result, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, packaging, result.Inspection.Representation.Packaging) + }) + } +} diff --git a/pkg/artifact/service/pull.go b/pkg/artifact/service/pull.go new file mode 100644 index 00000000000..fe125659cc1 --- /dev/null +++ b/pkg/artifact/service/pull.go @@ -0,0 +1,186 @@ +package service + +import ( + "context" + "fmt" + "os" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Pull implements [artifact.Service]. It resolves the reference once, verifies the pinned subject +// against policy, refuses to overlay a different non-empty destination without force, extracts +// into staging, and only then replaces the destination. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Pull(ctx context.Context, request artifact.PullRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Pull")() + + op := s.begin(artifact.ActionPull, request.Repository, request.Ref.String()) + err := s.pull(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func (s *Service) pull(ctx context.Context, op *operation, request *artifact.PullRequest) error { + if err := validatePull(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opPull) + if err != nil { + return err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + if inspection.Record == nil { + return artifact.NewUnsupportedCapability(request.Repository, opPull, "materialize/graph", "a native OCI graph without a portable Record cannot be materialized as a file tree") + } + if _, err := gate(ctx, request.Repository, repo, inspection, info.Policy); err != nil { + return err + } + unchanged, err := checkDestination(op, request, inspection.Record) + if err != nil || unchanged { + return err + } + return s.download(ctx, op, request, &pinnedSource{repo: repo, pinned: &pinned, inspection: inspection}) +} + +// pinnedSource is a repository together with the subject that was pinned and inspected in it. +type pinnedSource struct { + repo artifact.Repository + pinned *artifact.PinnedReference + inspection *artifact.Inspection +} + +// download reads the pinned content and materializes it, replacing the destination only after the +// whole tree was extracted and verified in staging. +func (s *Service) download(ctx context.Context, op *operation, request *artifact.PullRequest, source *pinnedSource) error { + content, err := source.repo.Read(ctx, *source.pinned) + if err != nil { + return fmt.Errorf("read %s from repository %q: %w", request.Ref, request.Repository, err) + } + defer content.Close() + if err := bindContent(content, source.inspection); err != nil { + return err + } + guard := materialize.WithReplaceGuard(func(dest *os.Root) error { + return recheckDestination(dest, request.Force, source.inspection.Record) + }) + if err := materialize.Materialize(ctx, content, request.Destination, guard); err != nil { + return err + } + op.result.Outcome = artifact.OutcomeRead + return nil +} + +func validatePull(request *artifact.PullRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if request.Destination == "" { + return fmt.Errorf("%w: a destination is required", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// checkDestination classifies the output directory. It reports unchanged when the destination +// already holds exactly the artifact and refuses a different non-empty destination without force. +func checkDestination(op *operation, request *artifact.PullRequest, record *artifact.Record) (bool, error) { + state, err := materialize.InspectDestination(request.Destination, record) + if err != nil { + return false, err + } + op.result.Location = request.Destination + switch { + case state == materialize.DestinationMatches: + op.result.Outcome = artifact.OutcomeUnchanged + op.info("unchanged", "the destination already holds exactly this artifact; nothing was downloaded") + return true, nil + case state == materialize.DestinationDiffers && !request.Force: + return false, fmt.Errorf("%w: destination %q already holds different content; use force to replace it", errUtils.ErrArtifactConflict, request.Destination) + } + return false, nil +} + +// recheckDestination repeats the conflict decision right before the destination is replaced, +// because files may have appeared in it while the artifact was downloading. An empty destination +// and a destination that already matches are always replaceable; different content needs force. +func recheckDestination(dest *os.Root, force bool, record *artifact.Record) error { + state, err := materialize.InspectBoundDestination(dest, record) + if err != nil { + return err + } + if state == materialize.DestinationDiffers && !force { + return fmt.Errorf("%w: destination %q changed during the download and now holds different content; use force to replace it", errUtils.ErrArtifactConflict, dest.Name()) + } + return nil +} + +// bindContent checks the content belongs to the inspected, pinned Record and fills a missing one. +func bindContent(content *artifact.Content, inspection *artifact.Inspection) error { + if content.Record == nil { + content.Record = inspection.Record + return nil + } + got, err := content.Record.RecordDigest() + if err != nil { + return err + } + want, err := inspection.Record.RecordDigest() + if err != nil { + return err + } + if got != want { + return fmt.Errorf("%w: the content record %s differs from the pinned record %s", errUtils.ErrArtifactIntegrityFailed, got, want) + } + return nil +} + +// openForRead describes the repository offline, requires read support, and opens it. +func (s *Service) openForRead(ctx context.Context, alias, operation string) (RepositoryInfo, artifact.Repository, error) { + info, err := s.repos.Describe(alias) + if err != nil { + return RepositoryInfo{}, nil, err + } + if !info.Capabilities.Read { + return info, nil, artifact.NewUnsupportedCapability(alias, operation, "read", "the repository is write-only") + } + repo, err := s.repos.Open(ctx, alias) + if err != nil { + return info, nil, err + } + return info, repo, nil +} + +// pinAndInspect resolves the reference exactly once and inspects the pinned subject. Everything +// after this point uses the returned pin, never the mutable reference. +func (s *Service) pinAndInspect(ctx context.Context, op *operation, repo artifact.Repository, alias string, ref artifact.Ref) (artifact.PinnedReference, *artifact.Inspection, error) { + pinned, err := repo.Resolve(ctx, ref) + if err != nil { + return pinned, nil, fmt.Errorf("resolve %s in repository %q: %w", ref, alias, err) + } + inspection, err := repo.Inspect(ctx, pinned) + if err != nil { + return pinned, nil, fmt.Errorf("inspect %s in repository %q: %w", ref, alias, err) + } + fillFromInspection(op, &pinned, inspection) + return pinned, inspection, nil +} + +// fillFromInspection records the pinned identity in the typed result. +func fillFromInspection(op *operation, pinned *artifact.PinnedReference, inspection *artifact.Inspection) { + op.result.ResolvedDigest = pinned.ResolvedDigest + op.result.RecordDigest = pinned.RecordDigest + op.result.ContentDigest = pinned.ContentDigest + op.result.RepresentationDigest = inspection.Representation.RepresentationDigest + op.result.Size = payloadSize(inspection) + if inspection.Record != nil { + op.result.Ref = artifact.Ref{Name: inspection.Record.Name, Version: inspection.Record.Version}.String() + op.result.ContentDigest = inspection.Record.ContentDigest + } +} diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go new file mode 100644 index 00000000000..172f0eac578 --- /dev/null +++ b/pkg/artifact/service/query.go @@ -0,0 +1,235 @@ +package service + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "sync" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Inspect implements [artifact.Service]. It reads metadata only. Verification is reported when +// policy requires or constrains it; a failed or unchecked verification is information here, not a +// reason to hide metadata, because inspect consumes no payload. +func (s *Service) Inspect(ctx context.Context, request artifact.InspectRequest) (*artifact.InspectResult, error) { + defer perf.Track(nil, "service.Service.Inspect")() + + op := s.begin(artifact.ActionInspect, request.Repository, request.Ref.String()) + result := &artifact.InspectResult{} + err := s.inspect(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) inspect(ctx context.Context, op *operation, request *artifact.InspectRequest, result *artifact.InspectResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opInspect) + if err != nil { + return err + } + _, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + result.Inspection = *inspection + op.result.Outcome = artifact.OutcomeRead + op.result.Location = inspection.Representation.Locator + verification, verifyErr := gate(ctx, request.Repository, repo, inspection, info.Policy) + if verification != nil && verification.Status != artifact.VerificationUnchecked { + result.Verification = verification + } + if verifyErr != nil { + op.warn("verification-failed", verifyErr.Error()) + } + return nil +} + +// ShowDocument implements [artifact.Service]. It never downloads the payload; the document bytes +// are bounded and verified against the Record descriptor. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) ShowDocument(ctx context.Context, request artifact.ShowDocumentRequest) (*artifact.DocumentResult, error) { + defer perf.Track(nil, "service.Service.ShowDocument")() + + op := s.begin(artifact.ActionShow, request.Repository, request.Ref.String()) + result := &artifact.DocumentResult{} + err := s.showDocument(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) showDocument(ctx context.Context, op *operation, request *artifact.ShowDocumentRequest, result *artifact.DocumentResult) error { + if err := validateShow(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opShow) + if err != nil { + return err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + if err := s.gatePinned(ctx, op, &target{alias: request.Repository, repo: repo, info: &info}, &pinned); err != nil { + return err + } + stream, err := repo.OpenDocument(ctx, pinned, request.Role) + if err != nil { + return fmt.Errorf("open %s of %s in repository %q: %w", request.Role, request.Ref, request.Repository, err) + } + defer stream.Reader.Close() + if !documentMatchesRecord(inspection.Record, request.Role, stream.Document) { + return fmt.Errorf("%w: document does not match the pinned record", errUtils.ErrArtifactIntegrityFailed) + } + content, err := readDocument(stream) + if err != nil { + return err + } + result.Document, result.Content = stream.Document, content + op.result.Outcome = artifact.OutcomeRead + op.result.Size = int64(len(content)) + return nil +} + +func documentMatchesRecord(record *artifact.Record, role artifact.DocumentRole, document artifact.Document) bool { + if record != nil { + for _, expected := range record.Documents { + if expected.Role == role && expected == document { + return true + } + } + } + return false +} + +func validateShow(request *artifact.ShowDocumentRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if request.Role != artifact.DocumentReadme && request.Role != artifact.DocumentChangelog { + return fmt.Errorf("%w: document role must be %q or %q", errUtils.ErrArtifactConfigInvalid, artifact.DocumentReadme, artifact.DocumentChangelog) + } + return nil +} + +// gatePinned enforces required verification on an already pinned subject. It inspects only when +// policy needs the subject, so unconstrained reads stay metadata-light. +func (s *Service) gatePinned(ctx context.Context, op *operation, tgt *target, pinned *artifact.PinnedReference) error { + if !verificationRequired(tgt.info.Policy) { + return nil + } + inspection, err := tgt.repo.Inspect(ctx, *pinned) + if err != nil { + return err + } + fillFromInspection(op, pinned, inspection) + _, err = gate(ctx, tgt.alias, tgt.repo, inspection, tgt.info.Policy) + return err +} + +// readDocument reads a document within the size bound and verifies it against its descriptor. +func readDocument(stream *artifact.DocumentStream) ([]byte, error) { + data, err := artifact.ReadBounded(stream.Reader, artifact.MaxDocumentSize) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != stream.Document.Size || digest.NewDigestFromBytes(digest.SHA256, sum[:]) != stream.Document.Digest { + return nil, fmt.Errorf("%w: the %s does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, stream.Document.Role) + } + return data, nil +} + +// List implements [artifact.Service]. +func (s *Service) List(ctx context.Context, request artifact.ListRequest) (*artifact.ListResult, error) { + defer perf.Track(nil, "service.Service.List")() + + return s.enumerate(ctx, artifact.ActionList, &enumeration{ + repositories: request.Repositories, name: request.Name, labels: request.Labels, refresh: request.Refresh, + }) +} + +// Search implements [artifact.Service]. The text is matched case-insensitively against names, +// kinds, descriptions, and label keys and values of the enumerated collection. +func (s *Service) Search(ctx context.Context, request artifact.SearchRequest) (*artifact.ListResult, error) { + defer perf.Track(nil, "service.Service.Search")() + + return s.enumerate(ctx, artifact.ActionSearch, &enumeration{ + repositories: request.Repositories, labels: request.Labels, refresh: request.Refresh, text: request.Query, + }) +} + +type enumeration struct { + repositories []string + name string + labels []artifact.Label + refresh bool + text string +} + +func (s *Service) enumerate(ctx context.Context, _ artifact.Action, request *enumeration) (*artifact.ListResult, error) { + result := &artifact.ListResult{Artifacts: []artifact.ListEntry{}} + err := s.runEnumeration(ctx, request, result) + return result, err +} + +// runEnumeration fans out over the selected repositories with bounded concurrency, collects +// per-repository outcomes, and fails only when an explicitly requested repository fails or none +// succeeded. Unsupported enumeration on one of several default repositories is reported, not fatal. +func (s *Service) runEnumeration(ctx context.Context, request *enumeration, result *artifact.ListResult) error { + aliases := request.repositories + explicit := len(aliases) > 0 + if !explicit { + aliases = s.repos.Aliases() + } + outcomes := make([]listOutcome, len(aliases)) + gates := make(chan struct{}, s.concurrency) + var wg sync.WaitGroup + for index := range aliases { + gates <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-gates }() + outcomes[index] = s.listRepository(ctx, aliases[index], request) + }() + } + wg.Wait() + var errs []error + succeeded := 0 + for index := range outcomes { + result.Artifacts = append(result.Artifacts, outcomes[index].entries...) + result.Repositories = append(result.Repositories, outcomes[index].listing) + if outcomes[index].err != nil { + errs = append(errs, outcomes[index].err) + continue + } + succeeded++ + } + sortEntries(result.Artifacts) + return enumerationError(ctx, explicit, succeeded, errs) +} + +// enumerationError decides whether the collected per-repository failures fail the listing. +// Cancellation is never partial success: a repository that stopped because the caller canceled left +// the listing incomplete, whatever the others returned. +func enumerationError(ctx context.Context, explicit bool, succeeded int, errs []error) error { + if len(errs) == 0 { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(append([]error{ctxErr}, errs...)...) + } + if explicit || succeeded == 0 { + return errors.Join(errs...) + } + return nil +} diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go new file mode 100644 index 00000000000..f4a31209561 --- /dev/null +++ b/pkg/artifact/service/repositories.go @@ -0,0 +1,170 @@ +package service + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate mockgen -package service -destination mock_repositories.go github.com/cloudposse/atmos/pkg/artifact/service Repositories,Journal,ImmutabilityPolicy,Repacker + +// RepositoryInfo is everything the service needs to know about a configured repository without +// touching the network or resolving credentials. +type RepositoryInfo struct { + // Alias is the configured repository alias. + Alias string + + // Capabilities describes the repository without network access. For a repository that + // delegates locking, it already includes the delegate's lock scopes. + Capabilities artifact.Capabilities + + // Packaging is the effective payload packaging mode for publications. + Packaging artifact.Packaging + + // Defaults is defaults.metadata; its labels sit under explicit publication labels. + Defaults artifact.Metadata + + // RequiredLockScope is locking.required_scope; empty means host. + RequiredLockScope artifact.LockScope + + // LockProvider is the alias of the repository that locking.repository delegates locks to. Empty + // (or the repository's own alias) means the repository locks for itself. + LockProvider string + + // Policy is the effective verification policy: the global and repository trust sets, both of + // which must be satisfied. + Policy artifact.VerificationPolicy +} + +// Clone returns a RepositoryInfo that shares no map, slice, or pointer with i, so a caller editing a +// description it received cannot change the configuration that produced it. +func (i *RepositoryInfo) Clone() RepositoryInfo { + defer perf.Track(nil, "service.RepositoryInfo.Clone")() + + out := *i + out.Capabilities = i.Capabilities.Clone() + out.Defaults = i.Defaults.Clone() + out.Policy = i.Policy.Clone() + return out +} + +// Repositories resolves repository aliases. Describe must be free of network access and +// credential resolution so dry runs never authenticate; Open may resolve identities. +type Repositories interface { + // Describe returns the repository description without authenticating. + Describe(alias string) (RepositoryInfo, error) + + // Open constructs (or returns a cached) repository, resolving credentials as needed. + Open(ctx context.Context, alias string) (artifact.Repository, error) + + // Aliases lists every configured repository alias, sorted. + Aliases() []string +} + +// JournalPhase names a step in the publication lifecycle. +type JournalPhase string + +// Journal phases, in lifecycle order. +const ( + // PhasePrepared means the source snapshot and Record were sealed. + PhasePrepared JournalPhase = "prepared" + + // PhaseLocked means the writer lock was acquired. + PhaseLocked JournalPhase = "locked" + + // PhasePending means the publication is about to be handed to the repository. + PhasePending JournalPhase = "pending" + + // PhaseCommitted means the repository confirmed completion. + PhaseCommitted JournalPhase = "committed" + + // PhaseReconciled means an ambiguous failure was resolved by reading the completed reference. + PhaseReconciled JournalPhase = "reconciled" + + // PhaseFailed means the operation failed without a committed publication. + PhaseFailed JournalPhase = "failed" + + // PhaseReleased means the writer lock was released. + PhaseReleased JournalPhase = "released" +) + +// JournalEntry is one lifecycle event. It is an implementation recovery record, not a catalog. +type JournalEntry struct { + // OperationID is the logical operation identifier, stable across retries. + OperationID string + + // Phase is the lifecycle step. + Phase JournalPhase + + // Repository is the target repository alias. + Repository string + + // Ref is the reference being mutated. + Ref string + + // IntentDigest is the intent digest of the artifact being published. + IntentDigest string + + // Attempt is the one-based publication attempt. + Attempt int + + // Err is the failure for PhaseFailed. + Err error +} + +// Journal receives publication lifecycle events. The service never fails because of it, but it +// calls Record inline and in lifecycle order, because a recovery record must not be reordered or +// dropped. A slow Record therefore delays the publication, so an implementation must return +// promptly and hand any slow work to its own buffer or goroutine. +type Journal interface { + // Record observes one lifecycle event. It must return promptly. + Record(ctx context.Context, entry *JournalEntry) +} + +// Candidate is an artifact being considered for the immutability policy. +type Candidate struct { + // Existing reports that the candidate is the artifact already stored (not the incoming one). + Existing bool + + // Repository is the repository alias involved. + Repository string + + // Inspection is the metadata view for every candidate, including prepared publications + // and native copies. Policies must classify through this view; Record may be nil for + // foreign graphs. Pinned is unset until a prepared publication is stored. + Inspection *artifact.Inspection + + // Prepared optionally provides staged content. It is nil for native copies and stored + // artifacts, so policies must not rely on it to classify an incoming release. + Prepared *artifact.PreparedArtifact +} + +// ImmutabilityPolicy is the admission seam for immutable release formats. The OCI adapter supplies +// the plugin-format detector; the service consults it for both the stored artifact and the +// incoming one, so a generic operation can never overwrite a plugin release, with or without force. +type ImmutabilityPolicy interface { + // Immutable reports whether the candidate is in an immutable release format and why. + Immutable(candidate Candidate) (bool, string) +} + +// ImmutabilityFunc adapts a function to [ImmutabilityPolicy]. +type ImmutabilityFunc func(candidate Candidate) (bool, string) + +// Immutable implements [ImmutabilityPolicy]. +func (f ImmutabilityFunc) Immutable(candidate Candidate) (bool, string) { + defer perf.Track(nil, "service.ImmutabilityFunc.Immutable")() + + return f(candidate) +} + +// StaticRepositories is a fixed alias-to-repository table. Describe reads Capabilities from each +// repository (which never authenticates) and Open returns it unchanged. It suits tests and +// composition roots that construct repositories up front. +type StaticRepositories struct { + // Repositories maps aliases to repositories. + Repositories map[string]artifact.Repository + + // Info optionally overrides the description of an alias; the zero fields fall back to defaults. + Info map[string]RepositoryInfo +} diff --git a/pkg/artifact/service/service.go b/pkg/artifact/service/service.go new file mode 100644 index 00000000000..1aef0758221 --- /dev/null +++ b/pkg/artifact/service/service.go @@ -0,0 +1,268 @@ +package service + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Operation names used in capability errors and lock requests. +const ( + opPublish = "publish" + opPull = "pull" + opCopy = "copy" + opDelete = "delete" + opSign = "sign" + opVerify = "verify" + opInspect = "inspect" + opShow = "show" + opPin = "pin" + opList = "list" +) + +// requireRepository rejects an empty repository alias: a missing required repository is an error. +func requireRepository(alias string) error { + if alias == "" { + return fmt.Errorf("%w: a repository is required", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// Defaults for retry behavior. Retries apply only to publications that failed with a temporary +// error and whose reconciliation proves nothing was committed. +const ( + // DefaultAttempts is the default number of publication attempts. + DefaultAttempts = 3 + + // DefaultBackoff is the default base delay between attempts; it doubles each attempt. + DefaultBackoff = 200 * time.Millisecond +) + +// Repacker rebuilds an existing Record for a different packaging. [produce.Producer] implements it. +type Repacker interface { + // Repack stages an existing Record from a materialized tree. + Repack(ctx context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) +} + +// Service implements [artifact.Service] over repositories. +type Service struct { + repos Repositories + producers *artifact.ProducerRegistry + repacker Repacker + signer artifact.Signer + journal Journal + immutable ImmutabilityPolicy + clock func() time.Time + newID func() string + sleep func(ctx context.Context, d time.Duration) error + owner string + stagingRoot string + concurrency int + attempts int + backoff time.Duration +} + +var _ artifact.Service = (*Service)(nil) + +// Option customizes a [Service]. +type Option func(*Service) + +// WithClock injects the clock. The service captures the time once per operation, in UTC. +func WithClock(clock func() time.Time) Option { + defer perf.Track(nil, "service.WithClock")() + + return func(s *Service) { s.clock = clock } +} + +// WithIDGenerator injects the operation ID generator. +func WithIDGenerator(newID func() string) Option { + defer perf.Track(nil, "service.WithIDGenerator")() + + return func(s *Service) { s.newID = newID } +} + +// WithConcurrency sets the transfer and discovery concurrency. The default is +// [artifact.DefaultConcurrency]. +func WithConcurrency(n int) Option { + defer perf.Track(nil, "service.WithConcurrency")() + + return func(s *Service) { s.concurrency = n } +} + +// WithRetry sets the number of publication attempts and the base backoff. +func WithRetry(attempts int, backoff time.Duration) Option { + defer perf.Track(nil, "service.WithRetry")() + + return func(s *Service) { + s.attempts = attempts + s.backoff = backoff + } +} + +// WithSleep injects the backoff sleep; tests use a no-op. +func WithSleep(sleep func(ctx context.Context, d time.Duration) error) Option { + defer perf.Track(nil, "service.WithSleep")() + + return func(s *Service) { s.sleep = sleep } +} + +// WithJournal registers a lifecycle observer. +func WithJournal(journal Journal) Option { + defer perf.Track(nil, "service.WithJournal")() + + return func(s *Service) { s.journal = journal } +} + +// WithImmutabilityPolicy registers the immutable-format admission seam. +func WithImmutabilityPolicy(policy ImmutabilityPolicy) Option { + defer perf.Track(nil, "service.WithImmutabilityPolicy")() + + return func(s *Service) { s.immutable = policy } +} + +// WithSigner supplies the signer used by Sign. +func WithSigner(signer artifact.Signer) Option { + defer perf.Track(nil, "service.WithSigner")() + + return func(s *Service) { s.signer = signer } +} + +// WithOwner sets the lock owner recorded in claims, for example host and process. +func WithOwner(owner string) Option { + defer perf.Track(nil, "service.WithOwner")() + + return func(s *Service) { s.owner = owner } +} + +// WithStagingRoot sets the directory for bounded disk staging (preparation and copy trees). +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "service.WithStagingRoot")() + + return func(s *Service) { s.stagingRoot = dir } +} + +// WithRepacker overrides the repacker used by Copy's staging fallback. +func WithRepacker(repacker Repacker) Option { + defer perf.Track(nil, "service.WithRepacker")() + + return func(s *Service) { s.repacker = repacker } +} + +// New returns a Service. With a nil producer registry, only the generic files producer is available. +func New(repos Repositories, producers *artifact.ProducerRegistry, opts ...Option) (*Service, error) { + defer perf.Track(nil, "service.New")() + + s := &Service{ + repos: repos, + producers: producers, + clock: time.Now, + newID: uuid.NewString, + sleep: sleepContext, + concurrency: artifact.DefaultConcurrency, + attempts: DefaultAttempts, + backoff: DefaultBackoff, + } + for _, opt := range opts { + if opt != nil { + opt(s) + } + } + if s.concurrency < 1 { + s.concurrency = artifact.DefaultConcurrency + } + if s.attempts < 1 { + s.attempts = 1 + } + if s.owner == "" { + s.owner = defaultOwner() + } + if s.producers == nil { + s.producers = artifact.NewProducerRegistry() + if err := produce.Register(s.producers, produce.WithStagingRoot(s.stagingRoot)); err != nil { + return nil, err + } + } + if s.repacker == nil { + s.repacker = produce.New(produce.WithStagingRoot(s.stagingRoot)) + } + return s, nil +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s:%d", host, os.Getpid()) +} + +func sleepContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// operation carries the identity of one logical operation: the ID and the UTC preparation time +// are captured exactly once, before the first attempt, and reused by every retry. +type operation struct { + id string + preparedAt time.Time + started time.Time + clock func() time.Time + result artifact.OperationResult +} + +func (s *Service) begin(action artifact.Action, repository, requested string) *operation { + now := s.clock().UTC() + return &operation{ + id: s.newID(), + preparedAt: now, + started: now, + clock: s.clock, + result: artifact.OperationResult{ + OperationID: "", + Action: action, + Repository: repository, + Requested: requested, + }, + } +} + +// done finalizes the typed result: the operation ID, the duration, and an error diagnostic on failure. +func (o *operation) done(err error) artifact.OperationResult { + o.result.OperationID = o.id + o.result.Duration = o.clock().Sub(o.started) + if err != nil { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "operation-failed", Message: err.Error()}) + } + return o.result +} + +func (o *operation) warn(code, message string) { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) +} + +func (o *operation) info(code, message string) { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityInfo, Code: code, Message: message}) +} + +// record forwards a lifecycle event to the journal, if any. The call is synchronous so events keep +// their order; the Journal contract requires implementations to return promptly. +func (s *Service) record(ctx context.Context, entry *JournalEntry) { + if s.journal != nil { + s.journal.Record(ctx, entry) + } +} diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go new file mode 100644 index 00000000000..b126bf60034 --- /dev/null +++ b/pkg/artifact/service/sign.go @@ -0,0 +1,161 @@ +package service + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Sign implements [artifact.Service]. It accepts digest references only, binds evidence to the +// resolved immutable subject, and reports that the subject is signed, not policy-verified. +func (s *Service) Sign(ctx context.Context, request artifact.SignRequest) (*artifact.SignResult, error) { + defer perf.Track(nil, "service.Service.Sign")() + + op := s.begin(artifact.ActionSign, request.Repository, request.Ref.String()) + result := &artifact.SignResult{} + err := s.sign(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) sign(ctx context.Context, op *operation, request *artifact.SignRequest, result *artifact.SignResult) error { + if err := s.validateSign(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opSign) + if err != nil { + return err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + subject, err := signSubject(&pinned, inspection, request.Ref) + if err != nil { + return err + } + if err := checkSigningCapability(request.Repository, &info, s.signer, subject); err != nil { + return err + } + resource := subjectLockResource(request.Ref.Name) + locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: resource}}) + if err != nil { + return err + } + defer func() { _ = locks.release(ctx, op) }() + + if err := verifyLockedSubject(ctx, repo, request.Ref, &pinned); err != nil { + return err + } + evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, resource)}) + if err != nil { + return fmt.Errorf("sign %s in repository %q: %w", request.Ref, request.Repository, err) + } + result.Subject, result.Evidence = subject, evidence + op.result.Outcome = artifact.OutcomeCreated + op.result.LockScope = locks.scope() + op.info("signed-not-verified", "the subject is signed; signing alone does not verify it against trust policy, use verify") + return nil +} + +func verifyLockedSubject(ctx context.Context, repo artifact.Repository, ref artifact.Ref, pinned *artifact.PinnedReference) error { + current, err := repo.Resolve(ctx, ref) + if err != nil { + return fmt.Errorf("resolve %s after locking: %w", ref, err) + } + if current.ResolvedDigest != pinned.ResolvedDigest || current.RecordDigest != pinned.RecordDigest { + return fmt.Errorf("%w: subject %s changed while acquiring its lock", errUtils.ErrArtifactConflict, ref) + } + return nil +} + +// validateSign requires a repository, a valid digest reference, and a configured signer. +func (s *Service) validateSign(request *artifact.SignRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if !request.Ref.IsDigest() { + return fmt.Errorf("%w: sign accepts digest references only (%s@sha256:...), not %s", errUtils.ErrArtifactRefInvalid, request.Ref.Name, request.Ref) + } + if s.signer == nil { + return artifact.NewUnsupportedCapability(request.Repository, opSign, "signer", "no signer is configured") + } + return nil +} + +// signSubject binds the digest reference to the immutable subject: the portable Record when the +// digest names the Record (or the resolved local/S3 digest), the original OCI root otherwise. +func signSubject(pinned *artifact.PinnedReference, inspection *artifact.Inspection, ref artifact.Ref) (artifact.SignatureSubject, error) { + subject, err := subjectOf(inspection) + if err != nil { + return subject, err + } + if ref.Digest != subject.Digest { + return subject, fmt.Errorf("%w: %s does not name the signable subject %s of the resolved artifact (resolved %s)", errUtils.ErrArtifactRefInvalid, ref, subject.Digest, pinned.ResolvedDigest) + } + return subject, subject.Validate() +} + +func checkSigningCapability(alias string, info *RepositoryInfo, signer artifact.Signer, subject artifact.SignatureSubject) error { + if !info.Capabilities.Write { + return artifact.NewUnsupportedCapability(alias, opSign, "write", "the repository cannot store signature evidence") + } + if !info.Capabilities.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(alias, opSign, "subject/"+string(subject.Kind), "the repository cannot bind evidence to this subject kind") + } + for _, format := range signer.Formats() { + if info.Capabilities.HasEvidenceFormat(format) { + return nil + } + } + return artifact.NewUnsupportedCapability(alias, opSign, "evidence/format", fmt.Sprintf("the signer produces %v but the repository stores %v", signer.Formats(), info.Capabilities.EvidenceFormats)) +} + +// Verify implements [artifact.Service]. An explicit verify always requires a trusted valid +// signature: with no usable trust configuration the result is unchecked and the call fails rather +// than returning a green result. +func (s *Service) Verify(ctx context.Context, request artifact.VerifyRequest) (*artifact.VerifyResult, error) { + defer perf.Track(nil, "service.Service.Verify")() + + op := s.begin(artifact.ActionVerify, request.Repository, request.Ref.String()) + result := &artifact.VerifyResult{} + err := s.verify(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) verify(ctx context.Context, op *operation, request *artifact.VerifyRequest, result *artifact.VerifyResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opVerify) + if err != nil { + return err + } + _, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + subject, err := subjectOf(inspection) + if err != nil { + return err + } + policy := info.Policy + policy.RequireSignature = true + if !policy.HasTrustConstraints() { + result.Verification = artifact.VerificationResult{Subject: subject, Status: artifact.VerificationUnchecked} + return fmt.Errorf("%w: repository %q has no trusted identities or keys configured, so nothing can be verified; configure artifacts.verification", errUtils.ErrArtifactConfigInvalid, request.Repository) + } + verification, err := verifySubject(ctx, request.Repository, repo, subject, policy) + if verification != nil { + result.Verification = *verification + } + if err != nil { + return err + } + op.result.Outcome = artifact.OutcomeRead + return nil +} diff --git a/pkg/artifact/service/static.go b/pkg/artifact/service/static.go new file mode 100644 index 00000000000..fcb64619ee1 --- /dev/null +++ b/pkg/artifact/service/static.go @@ -0,0 +1,59 @@ +package service + +import ( + "context" + "fmt" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +var _ Repositories = (*StaticRepositories)(nil) + +// Describe implements [Repositories]. +func (s *StaticRepositories) Describe(alias string) (RepositoryInfo, error) { + defer perf.Track(nil, "service.StaticRepositories.Describe")() + + repo, ok := s.Repositories[alias] + if !ok { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + // The table entry and the repository's capabilities are copied, so editing the returned + // description never reaches the table or the repository that every later call reads. + configured := s.Info[alias] + info := configured.Clone() + info.Alias = alias + if info.Capabilities.Packagings == nil && !info.Capabilities.Read && !info.Capabilities.Write { + reported := repo.Capabilities() + info.Capabilities = reported.Clone() + } + if info.Packaging == "" && len(info.Capabilities.Packagings) > 0 { + info.Packaging = info.Capabilities.Packagings[0] + } + return info, nil +} + +// Open implements [Repositories]. +func (s *StaticRepositories) Open(_ context.Context, alias string) (artifact.Repository, error) { + defer perf.Track(nil, "service.StaticRepositories.Open")() + + repo, ok := s.Repositories[alias] + if !ok { + return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + return repo, nil +} + +// Aliases implements [Repositories]. +func (s *StaticRepositories) Aliases() []string { + defer perf.Track(nil, "service.StaticRepositories.Aliases")() + + aliases := make([]string, 0, len(s.Repositories)) + for alias := range s.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/service/static_test.go b/pkg/artifact/service/static_test.go new file mode 100644 index 00000000000..9b5837c9e32 --- /dev/null +++ b/pkg/artifact/service/static_test.go @@ -0,0 +1,91 @@ +package service_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func staticPolicy() artifact.VerificationPolicy { + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: "release@example.com", Issuer: "https://issuer.example.com"}}, Keys: []string{"release.pub"}}, + }, + } +} + +func configuredInfo() service.RepositoryInfo { + return service.RepositoryInfo{ + Capabilities: artifact.Capabilities{Read: true, Packagings: []artifact.Packaging{artifact.PackagingTar, artifact.PackagingZip}, LockScopes: []artifact.LockScope{artifact.LockScopeHost}}, + Defaults: artifact.Metadata{ + Description: "defaults", + Labels: map[string]string{"team": "platform"}, + Provenance: &artifact.Provenance{CI: &artifact.CIProvenance{Provider: "github"}}, + }, + Policy: staticPolicy(), + } +} + +func reportedCapabilities() artifact.Capabilities { + return artifact.Capabilities{Read: true, Write: true, Packagings: []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingNone}, SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}} +} + +// TestStaticRepositories_DescribeReturnsIndependentCopies proves a caller editing what Describe +// returned cannot change the table or the repository, which every later Describe would serve. +func TestStaticRepositories_DescribeReturnsIndependentCopies(t *testing.T) { + tests := []struct { + name string + // info builds the table entry; an empty one falls back to what the repository reports. + info func() service.RepositoryInfo + mutate func(info *service.RepositoryInfo) + }{ + {"defaults labels and provenance", configuredInfo, func(info *service.RepositoryInfo) { + info.Defaults.Labels["team"] = "scribble" + info.Defaults.Provenance.CI.Provider = "scribble" + }}, + {"configured capabilities", configuredInfo, func(info *service.RepositoryInfo) { + info.Capabilities.Packagings[0] = artifact.PackagingNone + info.Capabilities.LockScopes[0] = artifact.LockScopeRepository + }}, + {"policy trust sets", configuredInfo, func(info *service.RepositoryInfo) { + info.Policy.TrustSets[0].Identities[0].Identity = "scribble" + info.Policy.TrustSets[0].Keys[0] = "scribble" + }}, + {"capabilities reported by the repository", func() service.RepositoryInfo { return service.RepositoryInfo{} }, func(info *service.RepositoryInfo) { + info.Capabilities.Packagings[0] = artifact.PackagingZip + info.Capabilities.SubjectKinds[0] = "scribble" + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // The repository hands out the same backing slices on every call, as a cached description would. + build := func() (*service.StaticRepositories, map[string]service.RepositoryInfo, *artifact.Capabilities) { + caps := reportedCapabilities() + repo := artifact.NewMockRepository(gomock.NewController(t)) + repo.EXPECT().Capabilities().DoAndReturn(func() artifact.Capabilities { return caps }).AnyTimes() + table := map[string]service.RepositoryInfo{"repo": tc.info()} + return &service.StaticRepositories{Repositories: map[string]artifact.Repository{"repo": repo}, Info: table}, table, &caps + } + untouched, _, _ := build() + want, err := untouched.Describe("repo") + require.NoError(t, err) + + static, table, repoCaps := build() + first, err := static.Describe("repo") + require.NoError(t, err) + tc.mutate(&first) + + second, err := static.Describe("repo") + require.NoError(t, err) + assert.Equal(t, want, second, "a later description is not affected by edits to an earlier one") + assert.Equal(t, tc.info(), table["repo"], "the configured table entry is untouched") + assert.Equal(t, reportedCapabilities(), *repoCaps, "the repository's own capabilities are untouched") + }) + } +} diff --git a/pkg/artifact/service/verify.go b/pkg/artifact/service/verify.go new file mode 100644 index 00000000000..8a059b54fd5 --- /dev/null +++ b/pkg/artifact/service/verify.go @@ -0,0 +1,159 @@ +package service + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// verificationRequired reports whether a policy requires, or constrains, verification. Without +// either, consumption reports "unchecked" and never "verified". +func verificationRequired(policy artifact.VerificationPolicy) bool { + return policy.RequireSignature || policy.HasTrustConstraints() +} + +// mergePolicies combines policies so that every trust set of every policy must be satisfied +// (global and repository sets are never unioned into a weaker policy). +func mergePolicies(policies ...artifact.VerificationPolicy) artifact.VerificationPolicy { + var merged artifact.VerificationPolicy + for _, policy := range policies { + merged.RequireSignature = merged.RequireSignature || policy.RequireSignature + merged.Offline = merged.Offline || policy.Offline + merged.TrustSets = append(merged.TrustSets, policy.TrustSets...) + } + return merged +} + +// subjectOf derives the immutable signature subject from an inspection: the original OCI root when +// a native graph is carried, otherwise the portable Record. Metadata claims never choose it. +func subjectOf(inspection *artifact.Inspection) (artifact.SignatureSubject, error) { + if root := inspection.Representation.OriginalRoot; root != nil { + return artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: root.Digest, MediaType: root.MediaType, Size: root.Size}, nil + } + if inspection.Record == nil { + return artifact.SignatureSubject{}, fmt.Errorf("%w: the artifact has neither a Record nor an OCI root to sign or verify", errUtils.ErrArtifactRecordInvalid) + } + canonical, err := inspection.Record.Canonical() + if err != nil { + return artifact.SignatureSubject{}, err + } + return artifact.SignatureSubject{ + Kind: artifact.SubjectRecord, + Digest: digest.FromBytes(canonical), + MediaType: artifact.MediaTypeConfig, + Size: int64(len(canonical)), + }, nil +} + +// verifySubject requires every trust set of policy to be satisfied by a valid permitted signature +// on subject. Each set is checked through its own repository Verify call, so the "both global and +// repository sets" rule holds regardless of the backend. Unsupported evidence fails closed. +func verifySubject(ctx context.Context, alias string, repo artifact.Repository, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + caps := repo.Capabilities() + if !caps.HasSubjectKind(subject.Kind) { + unsupported := artifact.NewUnsupportedCapability(alias, opVerify, "subject/"+string(subject.Kind), "the repository cannot bind or verify evidence for this subject kind") + return failedResult(subject, policy, unsupported), errors.Join(errUtils.ErrArtifactVerificationFailed, unsupported) + } + fingerprint, err := policy.Fingerprint() + if err != nil { + return nil, err + } + combined := &artifact.VerificationResult{Subject: subject, PolicyFingerprint: fingerprint, Status: artifact.VerificationVerified} + for _, call := range splitPolicy(policy) { + result, verifyErr := repo.Verify(ctx, subject, call) + if failure := checkVerifyResult(subject, result, verifyErr); failure != nil { + combined.Status = artifact.VerificationFailed + if result != nil { + combined.Diagnostics = append(combined.Diagnostics, result.Diagnostics...) + } + combined.Diagnostics = append(combined.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "verification-failed", Message: failure.Error()}) + return combined, errors.Join(errUtils.ErrArtifactVerificationFailed, failure) + } + mergeResult(combined, result) + } + return combined, nil +} + +// mergeResult folds a verified per-set result into the combined one without duplicating claims, +// evidence digests, or diagnostics that several sets share. +func mergeResult(combined, result *artifact.VerificationResult) { + for _, claim := range result.Claims { + if !slices.Contains(combined.Claims, claim) { + combined.Claims = append(combined.Claims, claim) + } + } + for _, evidence := range result.EvidenceDigests { + if !slices.Contains(combined.EvidenceDigests, evidence) { + combined.EvidenceDigests = append(combined.EvidenceDigests, evidence) + } + } + for _, diagnostic := range result.Diagnostics { + if !slices.Contains(combined.Diagnostics, diagnostic) { + combined.Diagnostics = append(combined.Diagnostics, diagnostic) + } + } +} + +// splitPolicy returns one single-trust-set policy per set, or the policy itself when it has none. +func splitPolicy(policy artifact.VerificationPolicy) []artifact.VerificationPolicy { + if len(policy.TrustSets) == 0 { + policy.RequireSignature = true + return []artifact.VerificationPolicy{policy} + } + calls := make([]artifact.VerificationPolicy, 0, len(policy.TrustSets)) + for _, set := range policy.TrustSets { + calls = append(calls, artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{set}, Offline: policy.Offline}) + } + return calls +} + +// checkVerifyResult returns a failure when the repository errored, returned nothing, returned a +// result for another subject, or did not report the verified status. +func checkVerifyResult(subject artifact.SignatureSubject, result *artifact.VerificationResult, err error) error { + switch { + case err != nil: + return err + case result == nil: + return fmt.Errorf("%w: the repository returned no verification result", errUtils.ErrArtifactVerificationFailed) + case result.Subject.Digest != subject.Digest || result.Subject.Kind != subject.Kind: + return fmt.Errorf("%w: the verification result is bound to %s, not the requested subject %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest, subject.Digest) + case result.Status != artifact.VerificationVerified: + return fmt.Errorf("%w: verification status is %q", errUtils.ErrArtifactVerificationFailed, result.Status) + } + return nil +} + +func failedResult(subject artifact.SignatureSubject, policy artifact.VerificationPolicy, cause error) *artifact.VerificationResult { + fingerprint, _ := policy.Fingerprint() + return &artifact.VerificationResult{ + Subject: subject, + PolicyFingerprint: fingerprint, + Status: artifact.VerificationFailed, + Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityError, Code: "verification-failed", Message: cause.Error()}}, + } +} + +// gate enforces the consumption policy. It returns an unchecked result when no policy requires or +// constrains verification (never "verified"), a verified result when every trust set is satisfied, +// and a failed result plus ErrArtifactVerificationFailed otherwise. Force never reaches this +// function. +func gate(ctx context.Context, alias string, repo artifact.Repository, inspection *artifact.Inspection, policies ...artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + merged := mergePolicies(policies...) + if !verificationRequired(merged) { + return &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil + } + subject, err := subjectOf(inspection) + if err != nil { + return nil, err + } + if merged.RequireSignature && !merged.HasTrustConstraints() { + return &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationUnchecked}, fmt.Errorf("%w: repository %q has no trusted identities or keys configured, so nothing can be verified; configure artifacts.verification", errUtils.ErrArtifactConfigInvalid, alias) + } + return verifySubject(ctx, alias, repo, subject, merged) +} diff --git a/pkg/artifact/service/verify_test.go b/pkg/artifact/service/verify_test.go new file mode 100644 index 00000000000..58f0d7c0be6 --- /dev/null +++ b/pkg/artifact/service/verify_test.go @@ -0,0 +1,269 @@ +package service_test + +import ( + "context" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +var releaseSigner = &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + +// signedEnv publishes the example artifact and signs it through the service. +func signedEnv(t *testing.T, config envConfig) (*env, *service.PublishOutcome) { + t.Helper() + config.options = append(config.options, service.WithSigner(releaseSigner)) + e := newEnv(t, config) + published := e.publish("dst", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + return e, published +} + +func TestSign_BindsEvidenceToTheDigestAndReportsSignedNotVerified(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + + result, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + assert.Equal(t, artifact.SubjectRecord, result.Subject.Kind) + assert.Equal(t, published.Operation.RecordDigest, result.Subject.Digest) + require.Len(t, result.Evidence, 1) + assert.Equal(t, artifact.OutcomeCreated, result.Operation.Outcome) + assert.Equal(t, artifact.LockScopeHost, result.Operation.LockScope) + var notVerified bool + for _, diagnostic := range result.Operation.Diagnostics { + notVerified = notVerified || diagnostic.Code == "signed-not-verified" + } + assert.True(t, notVerified, "signing alone reports signed, not policy-verified") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Nil(t, inspected.Verification, "with no trust configuration the artifact stays unchecked even though it is signed") + assert.Len(t, inspected.Inspection.Evidence, 1) +} + +func TestSign_Rejections(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + good := artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest} + tests := []struct { + name string + request artifact.SignRequest + want error + }{ + {"version reference", artifact.SignRequest{Repository: "dst", Ref: exampleRef}, errUtils.ErrArtifactRefInvalid}, + {"missing repository", artifact.SignRequest{Ref: good}, errUtils.ErrArtifactConfigInvalid}, + {"digest of another subject", artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: digest.FromString("other")}}, errUtils.ErrArtifactNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := e.svc.Sign(context.Background(), tt.request) + require.ErrorIs(t, err, tt.want) + require.NotNil(t, result) + }) + } + assert.Zero(t, e.repos["dst"].Calls("Sign")) +} + +func TestSign_CapabilityChecksPrecedeLocking(t *testing.T) { + noEvidence := artifacttest.DefaultCapabilities() + noEvidence.EvidenceFormats = nil + e := newEnv(t, envConfig{ + repos: map[string]artifacttest.Options{"dst": {Capabilities: &noEvidence}}, + options: []service.Option{service.WithSigner(releaseSigner)}, + }) + published := e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "no lock is taken for an unsupported signing request") + + noSigner := newEnv(t, envConfig{}) + published = noSigner.publish("dst", noSigner.source("hello")) + _, err = noSigner.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestVerify_RequiresATrustedValidSignature(t *testing.T) { + e, _ := signedEnv(t, envConfig{}) + request := artifact.VerifyRequest{Repository: "dst", Ref: exampleRef} + + t.Run("no trust configuration is not a green result", func(t *testing.T) { + result, err := e.svc.Verify(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Equal(t, artifact.VerificationUnchecked, result.Verification.Status) + assert.NotEqual(t, artifact.VerificationVerified, result.Verification.Status) + }) + + t.Run("trusted signer verifies", func(t *testing.T) { + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + result, err := e.svc.Verify(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Verification.Status) + assert.Equal(t, artifact.SubjectRecord, result.Verification.Subject.Kind) + assert.NotEmpty(t, result.Verification.EvidenceDigests) + assert.NotEmpty(t, result.Verification.PolicyFingerprint) + require.NotEmpty(t, result.Verification.Claims) + assert.Equal(t, "release@example.com", result.Verification.Claims[0].Identity) + }) + + t.Run("untrusted signer fails", func(t *testing.T) { + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.UntrustedPolicy() }) + result, err := e.svc.Verify(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Verification.Status) + }) +} + +func TestVerify_GlobalAndRepositoryTrustSetsBothMustBeSatisfied(t *testing.T) { + e, published := signedEnv(t, envConfig{}) + second := &artifacttest.FakeSigner{Identity: "security@example.com", Issuer: "https://issuer.example.com"} + policy := func(signers ...*artifacttest.FakeSigner) artifact.VerificationPolicy { + var merged artifact.VerificationPolicy + merged.RequireSignature = true + for _, s := range signers { + merged.TrustSets = append(merged.TrustSets, s.TrustedPolicy().TrustSets...) + } + return merged + } + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = policy(releaseSigner, second) }) + + _, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "satisfying one of two required trust sets is not enough; they are never unioned") + + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: published.Operation.RecordDigest, MediaType: artifact.MediaTypeConfig} + _, err = e.repos["dst"].Sign(context.Background(), subject, second, artifact.SignOptions{}) + require.NoError(t, err) + result, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Verification.Status) + assert.Len(t, result.Verification.Claims, 2) +} + +func TestConsumption_RequiredVerificationRunsBeforeMaterializationAndForceDoesNotBypassIt(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + reads := e.repos["dst"].Calls("Read") + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "force never bypasses verification") + assert.NotNil(t, result) + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "verification ran before the payload was read") + assert.True(t, missing(dest), "nothing was materialized") + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) +} + +func TestConsumption_RequireSignatureWithoutTrustConstraintsIsRejected(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = artifact.VerificationPolicy{RequireSignature: true} }) + verifies := e.repos["dst"].Calls("Verify") + + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a signature from any signer must not satisfy a policy that trusts no one") + assert.Equal(t, verifies, e.repos["dst"].Calls("Verify"), "the repository is never asked to verify") + assert.True(t, missing(dest), "nothing was materialized") +} + +func TestInspect_ReportsVerificationWhenPolicyConstrainsIt(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + unsigned, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err, "inspect consumes no payload, so a failed verification is information, not a hidden artifact") + require.NotNil(t, unsigned.Verification) + assert.Equal(t, artifact.VerificationFailed, unsigned.Verification.Status) + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + signed, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + require.NotNil(t, signed.Verification) + assert.Equal(t, artifact.VerificationVerified, signed.Verification.Status) +} + +func TestShowDocument_RequiredVerificationApplies(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + _, err := e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) +} + +func TestVerification_UnsupportedSubjectKindFailsClosed(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.SubjectKinds = nil + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "unsupported required evidence fails closed and says why") +} + +func TestVerification_RepositoryResultsAreNeverTrustedBlindly(t *testing.T) { + record := artifact.Record{Name: "tools/example", Version: "1", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + wrongSubject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("other")} + + tests := []struct { + name string + result *artifact.VerificationResult + err error + }{ + {"nil result", nil, nil}, + {"status unchecked", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil}, + {"result for another subject", &artifact.VerificationResult{Subject: wrongSubject, Status: artifact.VerificationVerified}, nil}, + {"verified but errored", &artifact.VerificationResult{Status: artifact.VerificationVerified}, errTransport}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + repo.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, subject artifact.SignatureSubject, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + if tt.result != nil && tt.result.Subject.Digest == "" { + tt.result.Subject = subject + } + return tt.result, tt.err + }, + ) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"dst": repo}, + Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone, Policy: releaseSigner.TrustedPolicy()}}, + } + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + result, err := svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.NotEqual(t, artifact.VerificationVerified, result.Verification.Status) + }) + } +} diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go new file mode 100644 index 00000000000..b35e1fe1bd7 --- /dev/null +++ b/pkg/artifact/snapshot/copy.go @@ -0,0 +1,228 @@ +package snapshot + +import ( + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// captureFile copies one regular file into staging while hashing it, detecting a file that was +// swapped, grew, shrank, or was modified while it was read. +func (c *capturer) captureFile(abs, rel string, before os.FileInfo) error { + if c.total+before.Size() > c.opts.limits.MaxBytes { + return fmt.Errorf("%w: staged content exceeds %d bytes", errUtils.ErrArtifactTooLarge, c.opts.limits.MaxBytes) + } + staged, sum, err := c.stageFile(abs, rel, before) + if err != nil { + return err + } + if hook := c.opts.onCaptured; hook != nil { + hook(rel) + } + entry := artifact.Entry{Path: rel, Type: artifact.EntryFile, Size: before.Size(), Mode: permBits(before), Digest: sum} + if err := c.addEntry(&entry); err != nil { + return err + } + c.total += before.Size() + c.staged[rel] = staged + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: before, digest: sum}) + return nil +} + +// stageFile copies abs into the staging data directory and returns the staged path and digest. +func (c *capturer) stageFile(abs, rel string, before os.FileInfo) (string, digest.Digest, error) { + src, err := openSame(abs, rel, before) + if err != nil { + return "", "", err + } + defer src.Close() + + c.counter++ + stagedPath := filepath.Join(c.dataDir, strconv.Itoa(c.counter)) + dst, err := os.OpenFile(stagedPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, stagedFilePerm) + if err != nil { + return "", "", fmt.Errorf("%w: stage %q: %w", errUtils.ErrArtifactPackagingInvalid, rel, err) + } + hasher := sha256.New() + copied, copyErr := copyContext(c.ctx, io.MultiWriter(dst, hasher), io.LimitReader(src, before.Size()+1)) + if closeErr := dst.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return "", "", fmt.Errorf("%w: stage %q: %w", errUtils.ErrArtifactPackagingInvalid, rel, copyErr) + } + after, err := src.Stat() + if err != nil { + return "", "", changedError(rel, err) + } + if copied != before.Size() || !sameFileState(before, after) { + return "", "", changedError(rel, errSizeOrTimeChanged) + } + return stagedPath, digest.NewDigest(digest.SHA256, hasher), nil +} + +// errSizeOrTimeChanged explains a file that changed while it was being read. +var errSizeOrTimeChanged = errors.New("changed size or modification time while it was being read") + +// errReplaced explains a path that was swapped for another file while it was being captured. +var errReplaced = errors.New("was replaced while it was being captured") + +// openSame opens abs and confirms it is still the regular file that was lstat'd. +func openSame(abs, rel string, before os.FileInfo) (*os.File, error) { + src, err := os.Open(abs) + if err != nil { + return nil, changedError(rel, err) + } + opened, err := src.Stat() + if err != nil { + _ = src.Close() + return nil, changedError(rel, err) + } + if !opened.Mode().IsRegular() || !os.SameFile(before, opened) { + _ = src.Close() + return nil, changedError(rel, errReplaced) + } + return src, nil +} + +// sameFileState reports whether two stats describe the same unmodified file state. +func sameFileState(a, b os.FileInfo) bool { + return a.Size() == b.Size() && a.ModTime().Equal(b.ModTime()) && a.Mode() == b.Mode() +} + +// recheck verifies every captured source entry still matches what was staged: type, size, +// modification time, directory listing, link target, and file digest. +func (c *capturer) recheck() error { + if err := c.recheckRoot(); err != nil { + return err + } + for index := range c.sources { + if err := c.ctx.Err(); err != nil { + return err + } + if err := c.recheckSource(&c.sources[index]); err != nil { + return err + } + } + for abs, names := range c.dirs { + current, err := readDirNames(abs) + if err != nil { + return err + } + if !slices.Equal(current, names) { + return fmt.Errorf("%w: directory %q gained or lost entries while it was being captured", errUtils.ErrArtifactSourceChanged, abs) + } + } + return nil +} + +// recheckRoot verifies that the source root is still the directory or file that was resolved. If another +// process replaced it, for example with a symlink, the children would have been read from the replacement +// and every per-entry check would agree with them, so the root's own identity is compared. +func (c *capturer) recheckRoot() error { + now, err := os.Lstat(c.root) + if err != nil { + return changedError(c.root, err) + } + if !os.SameFile(c.rootInfo, now) { + return fmt.Errorf("%w: the source %q was replaced while it was being captured", errUtils.ErrArtifactSourceChanged, c.root) + } + return nil +} + +func (c *capturer) recheckSource(record *sourceRecord) error { + now, err := os.Lstat(record.abs) + if err != nil { + return changedError(record.rel, err) + } + if now.Mode().Type() != record.info.Mode().Type() { + return fmt.Errorf("%w: %q changed type while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + if now.IsDir() && now.Mode() != record.info.Mode() { + return fmt.Errorf("%w: %q changed mode while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + switch { + case record.digest != "": + return c.recheckFile(record, now) + case record.target != "": + current, err := os.Readlink(record.abs) + if err != nil || filepath.ToSlash(current) != record.target { + return fmt.Errorf("%w: symlink %q changed while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + } + return nil +} + +// recheckFile compares stat state and then re-hashes the source file against the staged digest. +func (c *capturer) recheckFile(record *sourceRecord, now os.FileInfo) error { + if !sameFileState(record.info, now) { + return fmt.Errorf("%w: %q changed size, mode, or modification time while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + file, err := openSame(record.abs, record.rel, record.info) + if err != nil { + return err + } + defer file.Close() + opened, err := file.Stat() + if err != nil { + return changedError(record.rel, err) + } + if !sameFileState(record.info, opened) { + return changedError(record.rel, errSizeOrTimeChanged) + } + if err := c.rehash(file, record); err != nil { + return err + } + if hook := c.opts.onRehashed; hook != nil { + hook(record.rel) + } + // A mode change leaves the bytes and the digest alone, so the metadata is read once more. + final, err := file.Stat() + if err != nil { + return changedError(record.rel, err) + } + if !sameFileState(record.info, final) { + return changedError(record.rel, errSizeOrTimeChanged) + } + return recheckPath(record) +} + +// recheckPath rejects a replacement of the pathname while its original descriptor was hashed. +func recheckPath(record *sourceRecord) error { + current, err := os.Lstat(record.abs) + if err != nil { + return changedError(record.rel, err) + } + if !os.SameFile(record.info, current) || !sameFileState(record.info, current) { + return changedError(record.rel, errSizeOrTimeChanged) + } + return nil +} + +// rehash reads the source file again and compares its size and digest with the staged copy. +func (c *capturer) rehash(file *os.File, record *sourceRecord) error { + // Read at most one byte past the recorded size so a file that grew after the stat is rejected + // without being read in full. + hasher := sha256.New() + copied, err := copyContext(c.ctx, hasher, io.LimitReader(file, record.info.Size()+1)) + if err != nil { + return fmt.Errorf("%w: recheck %q: %w", errUtils.ErrArtifactSourceChanged, record.rel, err) + } + if copied != record.info.Size() { + return changedError(record.rel, errSizeOrTimeChanged) + } + if digest.NewDigest(digest.SHA256, hasher) != record.digest { + return fmt.Errorf("%w: %q content changed while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + return nil +} diff --git a/pkg/artifact/snapshot/doc.go b/pkg/artifact/snapshot/doc.go new file mode 100644 index 00000000000..91675669892 --- /dev/null +++ b/pkg/artifact/snapshot/doc.go @@ -0,0 +1,14 @@ +// Package snapshot captures a local source file or directory into bounded disk staging and +// produces a canonical inventory of it. +// +// Capture walks the source with lstat, copies regular file bytes into a private staging directory +// while hashing them, and then rechecks every source entry (type, size, modification time, and +// digest) so that an inventory is never published for bytes other than the ones staged. Files that +// change during capture fail the operation with ErrArtifactSourceChanged. +// +// Entry rules follow the artifact contracts: directories, file bytes, ordinary permission bits and +// contained relative symlinks are preserved; absolute or escaping symlinks, devices, sockets, and +// other special files are rejected; hard links become independent regular files; privileged mode +// bits are cleared; owner, group, and ACLs are never recorded. Owned temporary data is removed on +// success (via Close), error, and cancellation. +package snapshot diff --git a/pkg/artifact/snapshot/finish_test.go b/pkg/artifact/snapshot/finish_test.go new file mode 100644 index 00000000000..3bd635e7cd0 --- /dev/null +++ b/pkg/artifact/snapshot/finish_test.go @@ -0,0 +1,29 @@ +package snapshot + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func TestFinish_RejectsDuplicatesAfterUnicodeNormalization(t *testing.T) { + const digestOfEmpty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + composed := "café.txt" + decomposed := "café.txt" + require.NotEqual(t, composed, decomposed) + + c := &capturer{entries: []artifact.Entry{ + {Path: composed, Type: artifact.EntryFile, Mode: 0o644, Digest: digestOfEmpty}, + {Path: decomposed, Type: artifact.EntryFile, Mode: 0o644, Digest: digestOfEmpty}, + }} + err := c.finish() + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.Contains(t, err.Error(), "normalization") + + c.entries = c.entries[:1] + require.NoError(t, c.finish()) +} diff --git a/pkg/artifact/snapshot/reader.go b/pkg/artifact/snapshot/reader.go new file mode 100644 index 00000000000..9e79223cfc7 --- /dev/null +++ b/pkg/artifact/snapshot/reader.go @@ -0,0 +1,35 @@ +package snapshot + +import ( + "context" + "io" +) + +// copyChunkSize is the chunk size of context-aware copies. +const copyChunkSize = 32 * 1024 + +// copyContext copies src to dst in chunks and stops as soon as ctx is canceled, so large copies +// end promptly on cancellation. +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, copyChunkSize) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + n, readErr := src.Read(buffer) + if n > 0 { + written, writeErr := dst.Write(buffer[:n]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} diff --git a/pkg/artifact/snapshot/recheck_internal_test.go b/pkg/artifact/snapshot/recheck_internal_test.go new file mode 100644 index 00000000000..30cc6ae8566 --- /dev/null +++ b/pkg/artifact/snapshot/recheck_internal_test.go @@ -0,0 +1,174 @@ +package snapshot + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// withRehashHook registers a callback invoked after a source file was hashed again and before the +// recheck inspects its metadata one last time. +func withRehashHook(hook func(path string)) Option { + return func(o *options) { o.onRehashed = hook } +} + +func TestCapture_RejectsAModeChangeDuringTheFinalHash(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("file modes cannot be changed independently of content on %s", runtime.GOOS) + } + src := t.TempDir() + file := filepath.Join(src, "tool.sh") + require.NoError(t, os.WriteFile(file, []byte("#!/bin/sh\n"), 0o644)) + require.NoError(t, os.Chmod(file, 0o644)) + + // Byte count, digest, and modification time all stay the same: only the mode changes. + hook := func(rel string) { + assert.Equal(t, "tool.sh", rel) + require.NoError(t, os.Chmod(file, 0o755)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(hook)) + + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} + +// withResolvedHook registers a callback invoked once the source has been resolved and before it is read. +func withResolvedHook(hook func(path string)) Option { + return func(o *options) { o.onResolved = hook } +} + +// A source directory replaced by a symlink to another directory after it was resolved would be read +// through the link: every child record describes the link's target, so the per-file checks agree with +// each other. The root's own identity is retained and verified, so the capture is rejected. +func TestCapture_RejectsReplacementOfTheSourceRootBeforeItIsRead(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("a directory cannot be swapped for a symlink portably on %s", runtime.GOOS) + } + parent := t.TempDir() + src := filepath.Join(parent, "src") + require.NoError(t, os.Mkdir(src, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("selected"), 0o644)) + outside := filepath.Join(parent, "outside") + require.NoError(t, os.Mkdir(outside, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("not selected"), 0o644)) + + swapped := false + hook := func(string) { + swapped = true + require.NoError(t, os.Rename(src, filepath.Join(parent, "moved"))) + require.NoError(t, os.Symlink(outside, src)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withResolvedHook(hook)) + + require.True(t, swapped, "the hook ran") + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} + +func TestCapture_StaysQuietWhenNothingChangesDuringTheFinalHash(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("a"), 0o644)) + var seen []string + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(func(rel string) { seen = append(seen, rel) })) + require.NoError(t, err) + defer snap.Close() + assert.Equal(t, []string{"a.txt"}, seen) +} + +func TestCapture_RejectsCancellationBeforeAndAfterAnEmptyDirectory(t *testing.T) { + tests := []struct { + name string + do func(t *testing.T, stagingRoot string) error + }{ + {"already canceled", func(t *testing.T, stagingRoot string) error { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := Capture(ctx, t.TempDir(), WithStagingRoot(stagingRoot)) + return err + }}, + {"canceled after the last file was rechecked", func(t *testing.T, stagingRoot string) error { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("a"), 0o644)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + _, err := Capture(ctx, src, WithStagingRoot(stagingRoot), withRehashHook(func(string) { cancel() })) + return err + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stagingRoot := t.TempDir() + require.ErrorIs(t, tc.do(t, stagingRoot), context.Canceled) + leftovers, err := os.ReadDir(stagingRoot) + require.NoError(t, err) + assert.Empty(t, leftovers, "a canceled capture leaves no staging directory behind") + }) + } +} + +// The containment check reads the captured links, not the source: the source may change after the +// final recheck, and the snapshot still records the links as they were captured. +func TestCheckLinkContainment_UsesTheCapturedLinkTargets(t *testing.T) { + root := t.TempDir() + // On disk the directory link now points deeper, which would make the chain below look contained. + require.NoError(t, os.Mkdir(filepath.Join(root, "sub"), 0o755)) + if err := os.Symlink("sub", filepath.Join(root, "d")); err != nil { + t.Skipf("this platform cannot create symlinks: %v", err) + } + require.NoError(t, os.Symlink("d/../outside/missing", filepath.Join(root, "L"))) + + tests := []struct { + name string + sources []sourceRecord + wantErr error + }{ + {"the captured chain escapes", []sourceRecord{ + {abs: filepath.Join(root, "d"), rel: "d", target: "."}, + {abs: filepath.Join(root, "L"), rel: "L", target: "d/../outside/missing"}, + }, errUtils.ErrArtifactUnsafePath}, + {"the captured chain stays inside", []sourceRecord{ + {abs: filepath.Join(root, "d"), rel: "d", target: "sub"}, + {abs: filepath.Join(root, "L"), rel: "L", target: "d/../outside/missing"}, + }, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &capturer{root: root, sources: tc.sources} + err := c.checkLinkContainment() + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + assert.Contains(t, err.Error(), `"L"`) + return + } + require.NoError(t, err) + }) + } +} + +func TestCapture_RejectsPathReplacementDuringTheFinalHash(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows denies replacement while the source descriptor is open") + } + src := t.TempDir() + file := filepath.Join(src, "data.txt") + require.NoError(t, os.WriteFile(file, []byte("original"), 0o644)) + info, err := os.Stat(file) + require.NoError(t, err) + replacement := filepath.Join(t.TempDir(), "replacement") + require.NoError(t, os.WriteFile(replacement, []byte("original"), info.Mode())) + require.NoError(t, os.Chtimes(replacement, info.ModTime(), info.ModTime())) + hook := func(rel string) { + require.Equal(t, "data.txt", rel) + require.NoError(t, os.Rename(replacement, file)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(hook)) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go new file mode 100644 index 00000000000..413646e2b05 --- /dev/null +++ b/pkg/artifact/snapshot/snapshot.go @@ -0,0 +1,278 @@ +package snapshot + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "golang.org/x/text/unicode/norm" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Default capture limits. Both are overridable with [WithLimits]. +const ( + // DefaultMaxBytes bounds the total staged file bytes. + DefaultMaxBytes int64 = 16 << 30 + + // DefaultMaxEntries bounds the number of inventory entries. + DefaultMaxEntries = 100_000 +) + +const ( + stagingPattern = "atmos-artifact-*" + dataDirName = "data" + stagingDirPerm = 0o700 + stagedFilePerm = 0o600 +) + +// Limits bound a capture. Zero values select the defaults. +type Limits struct { + // MaxBytes is the maximum total size of staged file content. + MaxBytes int64 + + // MaxEntries is the maximum number of inventory entries. + MaxEntries int +} + +// Option customizes [Capture]. +type Option func(*options) + +type options struct { + stagingRoot string + limits Limits + onCaptured func(path string) + onRehashed func(path string) + onResolved func(path string) +} + +// WithStagingRoot sets the directory that holds the private staging directory. The default is the +// operating system's temporary directory. +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "snapshot.WithStagingRoot")() + + return func(o *options) { o.stagingRoot = dir } +} + +// WithLimits overrides the capture limits. +func WithLimits(limits Limits) Option { + defer perf.Track(nil, "snapshot.WithLimits")() + + return func(o *options) { o.limits = limits } +} + +// WithCaptureHook registers a callback invoked after each regular file has been copied and before +// the final recheck. Tests use it to mutate a source file mid-capture. +func WithCaptureHook(hook func(path string)) Option { + defer perf.Track(nil, "snapshot.WithCaptureHook")() + + return func(o *options) { o.onCaptured = hook } +} + +// Snapshot is a stable, staged copy of a source plus its sorted inventory. It owns a private +// staging directory; the caller must Close it. +type Snapshot struct { + // Root is the private staging directory. + Root string + + // Entries is the inventory, sorted by path. + Entries []artifact.Entry + + // TotalBytes is the total size of staged file content. + TotalBytes int64 + + // SourceIsFile reports that the source was a single file rather than a directory. + SourceIsFile bool + + staged map[string]string +} + +// Close removes the staging directory. It is idempotent. +func (s *Snapshot) Close() error { + defer perf.Track(nil, "snapshot.Snapshot.Close")() + + if s == nil || s.Root == "" { + return nil + } + err := os.RemoveAll(s.Root) + root := s.Root + s.Root = "" + s.staged = nil + if err != nil { + return fmt.Errorf("%w: remove staging directory %q: %w", errUtils.ErrArtifactPackagingInvalid, root, err) + } + return nil +} + +// StagedPath returns the staged copy of the regular file at the inventory path. +func (s *Snapshot) StagedPath(entryPath string) (string, bool) { + defer perf.Track(nil, "snapshot.Snapshot.StagedPath")() + + p, ok := s.staged[entryPath] + return p, ok +} + +// Open opens the staged copy of the regular file at the inventory path. +func (s *Snapshot) Open(entryPath string) (io.ReadCloser, error) { + defer perf.Track(nil, "snapshot.Snapshot.Open")() + + p, ok := s.staged[entryPath] + if !ok { + return nil, fmt.Errorf("%w: %q is not a staged file", errUtils.ErrArtifactNotFound, entryPath) + } + file, err := os.Open(p) + if err != nil { + return nil, fmt.Errorf("%w: open staged file %q: %w", errUtils.ErrArtifactPackagingInvalid, entryPath, err) + } + return file, nil +} + +// Lookup returns the inventory entry at path. +func (s *Snapshot) Lookup(entryPath string) (artifact.Entry, bool) { + defer perf.Track(nil, "snapshot.Snapshot.Lookup")() + + index := sort.Search(len(s.Entries), func(i int) bool { return s.Entries[i].Path >= entryPath }) + if index < len(s.Entries) && s.Entries[index].Path == entryPath { + return s.Entries[index], true + } + return artifact.Entry{}, false +} + +// Capture snapshots source into bounded disk staging. It fails, and removes all staged data, when +// the source contains unsafe entries, exceeds the limits, changes during capture, or ctx is +// canceled. +func Capture(ctx context.Context, source string, opts ...Option) (*Snapshot, error) { + defer perf.Track(nil, "snapshot.Capture")() + + var applied options + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.limits.MaxBytes <= 0 { + applied.limits.MaxBytes = DefaultMaxBytes + } + if applied.limits.MaxEntries <= 0 { + applied.limits.MaxEntries = DefaultMaxEntries + } + // Cancellation is final even when no entry would ever check it, such as an empty directory. + if err := ctx.Err(); err != nil { + return nil, classifyInterruption(err) + } + root, err := os.MkdirTemp(applied.stagingRoot, stagingPattern) + if err != nil { + return nil, fmt.Errorf("%w: create staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + snap, err := captureInto(ctx, source, root, &applied) + if err != nil { + _ = os.RemoveAll(root) + return nil, classifyInterruption(err) + } + return snap, nil +} + +// classifyInterruption gives a cancellation or deadline error the packaging sentinel so callers see +// a classified artifact error, and keeps the context error in the chain. Every other error, and one +// that already carries a sentinel, is returned unchanged. +func classifyInterruption(err error) error { + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + return err + } + if errors.Is(err, errUtils.ErrArtifactPackagingInvalid) { + return err + } + return fmt.Errorf("%w: capture interrupted: %w", errUtils.ErrArtifactPackagingInvalid, err) +} + +func captureInto(ctx context.Context, source, root string, opts *options) (*Snapshot, error) { + dataDir := filepath.Join(root, dataDirName) + if err := os.Mkdir(dataDir, stagingDirPerm); err != nil { + return nil, fmt.Errorf("%w: create staging data directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + resolved, info, err := resolveSource(source) + if err != nil { + return nil, err + } + if hook := opts.onResolved; hook != nil { + hook(resolved) + } + staging, err := os.Lstat(root) + if err != nil { + return nil, fmt.Errorf("%w: inspect staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + c := &capturer{staging: staging, ctx: ctx, opts: opts, root: resolved, rootInfo: info, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} + if info.IsDir() { + err = c.walkDir(resolved, "") + } else { + err = c.captureFile(resolved, filepath.Base(resolved), info) + } + if err != nil { + return nil, err + } + if err := c.verify(info.IsDir()); err != nil { + return nil, err + } + sort.Slice(c.entries, func(i, j int) bool { return c.entries[i].Path < c.entries[j].Path }) + return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), staged: c.staged}, nil +} + +// verify checks the finished capture: the source is unchanged, no link leaves it, the inventory is +// valid, and the capture was not canceled while it ran. +func (c *capturer) verify(isDir bool) error { + if err := c.recheck(); err != nil { + return err + } + if isDir { + if err := c.checkLinkContainment(); err != nil { + return err + } + } + if err := c.finish(); err != nil { + return err + } + return c.ctx.Err() +} + +// resolveSource resolves a user-chosen top-level source (which may itself be a symlink) and checks +// it is a directory or regular file. Links inside the source are never followed. +func resolveSource(source string) (string, os.FileInfo, error) { + if source == "" { + return "", nil, fmt.Errorf("%w: source is empty", errUtils.ErrArtifactPackagingInvalid) + } + resolved, err := filepath.EvalSymlinks(source) + if err != nil { + return "", nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + info, err := os.Lstat(resolved) + if err != nil { + return "", nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + if !info.IsDir() && !info.Mode().IsRegular() { + return "", nil, fmt.Errorf("%w: source %q must be a regular file or directory", errUtils.ErrArtifactPackagingInvalid, source) + } + return resolved, info, nil +} + +// finish validates every entry and rejects paths that collide after Unicode normalization. +func (c *capturer) finish() error { + seen := make(map[string]string, len(c.entries)) + for index := range c.entries { + entry := &c.entries[index] + if err := entry.Validate(); err != nil { + return err + } + normalized := norm.NFC.String(entry.Path) + if previous, duplicate := seen[normalized]; duplicate { + return fmt.Errorf("%w: paths %q and %q are duplicates after Unicode normalization", errUtils.ErrArtifactPackagingInvalid, previous, entry.Path) + } + seen[normalized] = entry.Path + } + return nil +} diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go new file mode 100644 index 00000000000..6ebf58bef8c --- /dev/null +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -0,0 +1,422 @@ +package snapshot_test + +import ( + "context" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +func entryPaths(entries []artifact.Entry) []string { + paths := make([]string, 0, len(entries)) + for _, e := range entries { + paths = append(paths, e.Path) + } + return paths +} + +func TestCapture_FilePackageInventory(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + + want := append(fixtures.FilePackagePaths(pkg.Symlinks), ".atmos", ".atmos/artifact", ".atmos/artifact/CHANGELOG.md", ".atmos/artifact/README.md") + assert.ElementsMatch(t, want, entryPaths(snap.Entries)) + assert.IsIncreasing(t, entryPaths(snap.Entries), "inventory must be sorted by path") + + helper, ok := snap.Lookup("bin/helper.sh") + require.True(t, ok) + assert.Equal(t, artifact.EntryFile, helper.Type) + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), helper.Mode, "executable bit preserved") + } + assert.Equal(t, int64(len(fixtures.HelperScript)), helper.Size) + assert.Equal(t, digest.FromString(fixtures.HelperScript), helper.Digest) + + empty, ok := snap.Lookup("empty.txt") + require.True(t, ok) + assert.Zero(t, empty.Size) + assert.Equal(t, digest.FromString(""), empty.Digest) + + dir, ok := snap.Lookup("emptydir") + require.True(t, ok) + assert.Equal(t, artifact.EntryDirectory, dir.Type) + + _, ok = snap.Lookup(fixtures.UnicodeName) + assert.True(t, ok, "Unicode filenames are preserved") + + if pkg.Symlinks { + link, ok := snap.Lookup("link-to-file") + require.True(t, ok) + assert.Equal(t, artifact.EntrySymlink, link.Type) + assert.Equal(t, "nested/deep/file.txt", link.Target) + assert.Zero(t, link.Mode) + } + + reader, err := snap.Open("nested/deep/file.txt") + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "deep content\n", string(data)) +} + +func TestCapture_SingleFileSource(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "bytes", 0o755) + snap, err := snapshot.Capture(context.Background(), filepath.Join(dir, "tool.bin"), snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + assert.True(t, snap.SourceIsFile) + require.Len(t, snap.Entries, 1) + assert.Equal(t, "tool.bin", snap.Entries[0].Path) + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), snap.Entries[0].Mode) + } +} + +func TestCapture_ClearsPrivilegedBitsAndIgnoresOwnership(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "x", 0o644) + path := filepath.Join(src, "a.txt") + if err := os.Chmod(path, 0o755|os.ModeSetuid); err != nil { + t.Skipf("setuid unavailable: %v", err) + } + info, err := os.Stat(path) + require.NoError(t, err) + if info.Mode()&os.ModeSetuid == 0 { + t.Skip("filesystem did not apply setuid") + } + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + entry, _ := snap.Lookup("a.txt") + assert.Zero(t, entry.Mode&^0o777, "privileged bits must be cleared") +} + +func TestCapture_HardLinksBecomeIndependentFiles(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "original.txt", "shared", 0o644) + if err := os.Link(filepath.Join(src, "original.txt"), filepath.Join(src, "hard.txt")); err != nil { + t.Skipf("hard links are unavailable here: %v", err) + } + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + original, _ := snap.Lookup("original.txt") + hard, _ := snap.Lookup("hard.txt") + assert.Equal(t, artifact.EntryFile, hard.Type) + assert.Equal(t, original.Digest, hard.Digest) + first, _ := snap.StagedPath("original.txt") + second, _ := snap.StagedPath("hard.txt") + assert.NotEqual(t, first, second, "each hard link is staged as its own file") +} + +func TestCapture_InvalidSources(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + want error + }{ + {"absolute symlink", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "abs", filepath.ToSlash(os.TempDir())) + }, errUtils.ErrArtifactUnsafePath}, + {"escaping relative symlink", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "sub") + return fixtures.Symlink(t, dir, "sub/escape", "../../outside") + }, errUtils.ErrArtifactUnsafePath}, + {"traversal target", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "up", "..") + }, errUtils.ErrArtifactUnsafePath}, + {"chained symlink escape through a dangling tail", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "a") + return fixtures.Symlink(t, dir, "a/b", "..") && fixtures.Symlink(t, dir, "c", "a/b/../../missing") + }, errUtils.ErrArtifactUnsafePath}, + {"chained symlink escape", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "a") + return fixtures.Symlink(t, dir, "a/b", "..") && fixtures.Symlink(t, dir, "c", "a/b/../..") + }, errUtils.ErrArtifactUnsafePath}, + {"special file", func(t *testing.T, dir string) bool { + return fixtures.SpecialFile(t, dir, "pipe") + }, errUtils.ErrArtifactPackagingInvalid}, + {"reserved namespace", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, artifact.ReservedNamespace+"/v1/refs/x.json", "{}", 0o644) + return true + }, errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + if !tt.build(t, src) { + t.Skip("fixture is unavailable on this platform") + } + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.Error(t, err) + assert.Nil(t, snap) + assert.ErrorIs(t, err, tt.want) + assertEmptyDir(t, staging) + }) + } +} + +func assertEmptyDir(t *testing.T, dir string) { + t.Helper() + items, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, items, "owned temporary data must be removed") +} + +func TestCapture_RejectsMissingAndNonRegularSource(t *testing.T) { + _, err := snapshot.Capture(context.Background(), filepath.Join(t.TempDir(), "absent"), snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + _, err = snapshot.Capture(context.Background(), "", snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestCapture_DetectsChangeDuringCapture(t *testing.T) { + tests := []struct { + name string + mutate func(t *testing.T, src string) + }{ + {"grown", func(t *testing.T, src string) { + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("original-and-more"), 0o644)) + }}, + {"same size new mtime", func(t *testing.T, src string) { + path := filepath.Join(src, "a.txt") + require.NoError(t, os.WriteFile(path, []byte("ORIGINAL"), 0o644)) + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Hour), time.Now().Add(time.Hour))) + }}, + {"same size same mtime different digest", func(t *testing.T, src string) { + path := filepath.Join(src, "a.txt") + info, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, []byte("ORIGINAL"), 0o644)) + require.NoError(t, os.Chtimes(path, info.ModTime(), info.ModTime())) + }}, + {"replaced by another file with identical state", func(t *testing.T, src string) { + if runtime.GOOS == "windows" { + t.Skip("replacing an open path is not portable on Windows") + } + path := filepath.Join(src, "a.txt") + info, err := os.Stat(path) + require.NoError(t, err) + replacement := filepath.Join(src, "replacement.tmp") + require.NoError(t, os.WriteFile(replacement, []byte("original"), 0o644)) + require.NoError(t, os.Chtimes(replacement, info.ModTime(), info.ModTime())) + require.NoError(t, os.Rename(replacement, path)) + }}, + {"file added", func(t *testing.T, src string) { + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + }}, + {"file removed", func(t *testing.T, src string) { + require.NoError(t, os.Remove(filepath.Join(src, "b.txt"))) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "original", 0o644) + fixtures.WriteFile(t, src, "b.txt", "other", 0o644) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, + snapshot.WithStagingRoot(staging), + snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + tt.mutate(t, src) + } + })) + require.Error(t, err) + assert.Nil(t, snap) + assert.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assertEmptyDir(t, staging) + }) + } +} + +func TestCapture_DetectsDirectoryModeChangeDuringCapture(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not tracked on Windows") + } + src := t.TempDir() + fixtures.MkdirAll(t, src, "0dir") + fixtures.WriteFile(t, src, "0dir/inner.txt", "inner", 0o644) + fixtures.WriteFile(t, src, "b.txt", "other", 0o644) + require.NoError(t, os.Chmod(filepath.Join(src, "0dir"), 0o755)) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, + snapshot.WithStagingRoot(staging), + snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + require.NoError(t, os.Chmod(filepath.Join(src, "0dir"), 0o700)) + } + })) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) + assertEmptyDir(t, staging) +} + +func TestCapture_StagingInsideTheSourceIsNotSourceContent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "alpha", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bravo", 0o644) + staging := filepath.Join(src, ".staging") + require.NoError(t, os.Mkdir(staging, 0o755)) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.NoError(t, err, "staged files must not be mistaken for a source change") + t.Cleanup(func() { _ = snap.Close() }) + assert.Equal(t, []string{".staging", "a.txt", "b.txt"}, entryPaths(snap.Entries), "the private staging directory is left out of the inventory") + assert.Equal(t, int64(10), snap.TotalBytes) +} + +func TestCapture_Limits(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + fixtures.WriteFile(t, src, "b.txt", "0123456789", 0o644) + + staging := t.TempDir() + _, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging), snapshot.WithLimits(snapshot.Limits{MaxBytes: 15})) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + assertEmptyDir(t, staging) + + _, err = snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithLimits(snapshot.Limits{MaxEntries: 1})) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithLimits(snapshot.Limits{MaxBytes: 20, MaxEntries: 2})) + require.NoError(t, err) + assert.Equal(t, int64(20), snap.TotalBytes) + require.NoError(t, snap.Close()) +} + +func TestCapture_CancellationRemovesStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + ctx, cancel := context.WithCancel(context.Background()) + staging := t.TempDir() + snap, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(staging), snapshot.WithCaptureHook(func(string) { cancel() })) + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, snap) + assertEmptyDir(t, staging) +} + +// Cancellation and deadline errors carry the packaging sentinel and keep the context error in the chain, +// whether the context ended before capture began or while it ran. +func TestCapture_CancellationIsClassifiedAndKeepsItsCause(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + + t.Run("canceled before capture", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.Canceled) + }) + t.Run("canceled during capture", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithCaptureHook(func(string) { cancel() })) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.Canceled) + }) + t.Run("deadline exceeded", func(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.DeadlineExceeded) + }) +} + +func TestSnapshot_CloseRemovesStagingAndIsIdempotent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.NoError(t, err) + require.NoError(t, snap.Close()) + require.NoError(t, snap.Close()) + assertEmptyDir(t, staging) + _, ok := snap.StagedPath("a.txt") + assert.False(t, ok) +} + +func TestSnapshot_OpenUnknownPath(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + _, err = snap.Open("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, ok := snap.Lookup("missing") + assert.False(t, ok) +} + +func TestSnapshot_OpenWrapsStagedFileFailure(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + staged, ok := snap.StagedPath("a.txt") + require.True(t, ok) + require.NoError(t, os.Remove(staged)) + + _, err = snap.Open("a.txt") + + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestSnapshot_CloseWrapsRemovalFailure(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("directory permissions do not block removal here") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + root := snap.Root + require.NoError(t, os.Chmod(root, 0o500)) + t.Cleanup(func() { _ = os.Chmod(root, 0o700) }) + + err = snap.Close() + + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrPermission) +} + +func TestCapture_RelativeSourceWithContainedSymlink(t *testing.T) { + base := t.TempDir() + src := filepath.Join(base, "src") + fixtures.WriteFile(t, src, "dir/target.txt", "target", 0o644) + if !fixtures.Symlink(t, src, "link.txt", "dir/target.txt") { + t.Skipf("symlinks are unavailable on this platform") + } + t.Chdir(base) + + snap, err := snapshot.Capture(context.Background(), "src", snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err, "a contained link must not be rejected because the source path is relative") + defer snap.Close() + link, ok := snap.Lookup("link.txt") + require.True(t, ok) + assert.Equal(t, artifact.EntrySymlink, link.Type) + assert.Equal(t, "dir/target.txt", link.Target) +} diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go new file mode 100644 index 00000000000..ab17aa1e5be --- /dev/null +++ b/pkg/artifact/snapshot/walk.go @@ -0,0 +1,171 @@ +package snapshot + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" +) + +// sourceRecord remembers one captured source entry for the final recheck. +type sourceRecord struct { + abs string + rel string + info os.FileInfo + digest digest.Digest + target string +} + +// capturer holds the state of one capture. +type capturer struct { + ctx context.Context + opts *options + root string + dataDir string + + // rootInfo identifies the source root as it was resolved, so a replacement is detected. + rootInfo os.FileInfo + entries []artifact.Entry + staged map[string]string + dirs map[string][]string + sources []sourceRecord + total int64 + counter int + + // staging identifies the private staging directory, which is skipped when it lies inside the source. + staging os.FileInfo +} + +// walkDir captures the directory at abs, whose slash-separated inventory path is rel ("" for the root). +func (c *capturer) walkDir(abs, rel string) error { + names, err := readDirNames(abs) + if err != nil { + return err + } + c.dirs[abs] = names + for _, name := range names { + if err := c.ctx.Err(); err != nil { + return err + } + childAbs := filepath.Join(abs, name) + childRel := name + if rel != "" { + childRel = rel + "/" + name + } + if err := c.captureChild(childAbs, childRel); err != nil { + return err + } + } + return nil +} + +// captureChild classifies one directory child with lstat and captures it. +func (c *capturer) captureChild(abs, rel string) error { + info, err := os.Lstat(abs) + if err != nil { + return changedError(rel, err) + } + if c.staging != nil && os.SameFile(c.staging, info) { + // The private staging directory changes while files are staged, so it is never source content. + return nil + } + mode := info.Mode() + switch { + case mode.IsDir(): + if err := c.addEntry(&artifact.Entry{Path: rel, Type: artifact.EntryDirectory, Mode: permBits(info)}); err != nil { + return err + } + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: info}) + return c.walkDir(abs, rel) + case mode.IsRegular(): + return c.captureFile(abs, rel, info) + case mode&os.ModeSymlink != 0: + return c.captureSymlink(abs, rel, info) + default: + return fmt.Errorf("%w: %q is a device, socket, pipe, or other special file (%s)", errUtils.ErrArtifactPackagingInvalid, rel, mode.Type()) + } +} + +// captureSymlink records a relative symlink without following it. Absolute or escaping targets +// are rejected when the entry is validated. +func (c *capturer) captureSymlink(abs, rel string, info os.FileInfo) error { + target, err := os.Readlink(abs) + if err != nil { + return changedError(rel, err) + } + if filepath.IsAbs(target) || filepath.VolumeName(target) != "" || strings.HasPrefix(target, "/") { + return fmt.Errorf("%w: symlink %q has absolute target %q", errUtils.ErrArtifactUnsafePath, rel, target) + } + target = filepath.ToSlash(target) + entry := artifact.Entry{Path: rel, Type: artifact.EntrySymlink, Target: target} + if err := entry.Validate(); err != nil { + return err + } + if err := c.addEntry(&entry); err != nil { + return err + } + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: info, target: target}) + return nil +} + +// addEntry appends an inventory entry, enforcing the entry limit. +func (c *capturer) addEntry(entry *artifact.Entry) error { + if len(c.entries) >= c.opts.limits.MaxEntries { + return fmt.Errorf("%w: more than %d entries", errUtils.ErrArtifactTooLarge, c.opts.limits.MaxEntries) + } + c.entries = append(c.entries, *entry) + return nil +} + +// permBits returns the ordinary permission bits of info with privileged bits cleared. +func permBits(info os.FileInfo) uint32 { + return uint32(info.Mode().Perm()) +} + +// readDirNames lists a directory's child names in bytewise order. +func readDirNames(dir string) ([]string, error) { + items, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("%w: read directory %q: %w", errUtils.ErrArtifactSourceChanged, dir, err) + } + names := make([]string, 0, len(items)) + for _, item := range items { + names = append(names, item.Name()) + } + slices.Sort(names) + return names, nil +} + +// checkLinkContainment resolves every captured symlink through the captured links and rejects any +// that leaves the source. A lexical check of each target is not enough, because a chain of links +// can clean to a contained path yet resolve outside it. A dangling link is resolved through its +// existing part plus the missing remainder, so it cannot escape once the missing path is created. +// The check reads the links as captured and never the source, which may have changed since. +func (c *capturer) checkLinkContainment() error { + links := make(map[string]string) + for index := range c.sources { + if record := &c.sources[index]; record.target != "" { + links[record.rel] = record.target + } + } + for index := range c.sources { + record := &c.sources[index] + if record.target != "" && linkpath.Escapes(links, record.rel) { + return fmt.Errorf("%w: symlink %q resolves outside the source", errUtils.ErrArtifactUnsafePath, record.rel) + } + } + return nil +} + +// changedError wraps a failure observed while a source entry was being captured. +func changedError(rel string, cause error) error { + return fmt.Errorf("%w: %q: %w", errUtils.ErrArtifactSourceChanged, rel, cause) +} diff --git a/pkg/auth/deferred/manager_test.go b/pkg/auth/deferred/manager_test.go index c3a7cd9bd88..73f62318ec9 100644 --- a/pkg/auth/deferred/manager_test.go +++ b/pkg/auth/deferred/manager_test.go @@ -173,7 +173,8 @@ func TestManagerDelegatesScopedIdentityOwnership(t *testing.T) { err := operation(ctx, &schema.AuthContext{Environment: map[string]string{"VAULT_TOKEN": "synthetic-reader"}}) events = append(events, "released") return err - }) + }, + ) err := manager.WithIsolatedIdentity(t.Context(), "reader", func(ctx context.Context, auth *schema.AuthContext) error { events = append(events, "operation") require.Same(t, t.Context(), ctx) diff --git a/pkg/oci/client.go b/pkg/oci/client.go index 14af87d279c..019d597a684 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -9,6 +9,9 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" @@ -101,73 +104,118 @@ func (c *Client) Get(ctx context.Context, ref name.Reference) (*remote.Descripto } // Head resolves the immutable descriptor without downloading artifact layers. +// It runs on the shared ORAS transport. func (c *Client) Head(ctx context.Context, ref name.Reference) (*v1.Descriptor, error) { defer perf.Track(c.config, "oci.Client.Head")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return nil, err } - result, err := remote.Head(ref, options...) + resolved, err := store.Resolve(ctx, ref.Identifier()) if err != nil { return nil, errors.Join(errUtils.ErrPullImage, err) } - return result, nil + hash, err := v1.NewHash(resolved.Digest.String()) + if err != nil { + return nil, errors.Join(errUtils.ErrPullImage, err) + } + return &v1.Descriptor{MediaType: types.MediaType(resolved.MediaType), Size: resolved.Size, Digest: hash}, nil } // Tags lists repository tags without interpreting them as versions. func (c *Client) Tags(ctx context.Context, repo name.Repository) ([]string, error) { defer perf.Track(c.config, "oci.Client.Tags")() - options, err := c.options(ctx, repo) + store, err := c.Repository(ctx, repositoryOf(repo)) if err != nil { return nil, err } - result, err := remote.List(repo, options...) - if err != nil { - return nil, errors.Join(errUtils.ErrOCIRegistry, err) - } - return result, nil + return store.Tags(ctx) } -// WriteImage uploads an image and its blobs. Rejected writes never retry anonymously. +// WriteImage uploads an image and its blobs, preserving the image's existing raw bytes. +// Rejected writes never retry anonymously, and an unknown outcome is reconciled by +// resolving the destination rather than writing again. Only OCI and Docker schema 2 images +// are supported: a schema 1 image is refused with an integrity error before any layer is read. func (c *Client) WriteImage(ctx context.Context, ref name.Reference, img v1.Image) error { defer perf.Track(c.config, "oci.Client.WriteImage")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return err } - if err := remote.Write(ref, img, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + graph, err := legacyImageGraph(ctx, img) + if err != nil { + return err } - return nil + defer graph.close() + _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) + return err } // WriteIndex uploads the complete index and its children without platform filtering. func (c *Client) WriteIndex(ctx context.Context, ref name.Reference, index v1.ImageIndex) error { defer perf.Track(c.config, "oci.Client.WriteIndex")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return err } - if err := remote.WriteIndex(ref, index, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + graph, err := legacyIndexGraph(ctx, index) + if err != nil { + return err } - return nil + defer graph.close() + _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) + return err } // Delete deletes only an explicitly selected digest, never a mutable tag. func (c *Client) Delete(ctx context.Context, ref name.Digest) error { defer perf.Track(c.config, "oci.Client.Delete")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return err } - if err := remote.Delete(ref, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + target, err := resolveDeleteTarget(ctx, store, ref) + if err != nil { + return err + } + if err := store.Delete(ctx, target); err != nil { + return c.reconcileDelete(ctx, store, ref, err) } return nil } + +// resolveDeleteTarget looks up the manifest descriptor to delete, because its media type lets the +// store also clean up referrers indexes. A credential that may delete but not read is refused the +// lookup with 401 or 403; the delete then goes out for the bare digest, which needs no read and +// matches what a delete-only credential is allowed to do. Any other lookup failure, including a +// missing digest, is returned unchanged and no DELETE is sent. +func resolveDeleteTarget(ctx context.Context, store RepositoryStore, ref name.Digest) (ocispec.Descriptor, error) { + target, err := store.Resolve(ctx, ref.DigestStr()) + if err == nil { + return target, nil + } + if status := statusOf(err); status != http.StatusUnauthorized && status != http.StatusForbidden { + return ocispec.Descriptor{}, err + } + parsed, parseErr := digest.Parse(ref.DigestStr()) + if parseErr != nil { + return ocispec.Descriptor{}, err + } + return ocispec.Descriptor{Digest: parsed}, nil +} + +// reconcileDelete treats a failed delete as complete only when the digest is gone. +func (c *Client) reconcileDelete(ctx context.Context, store RepositoryStore, ref name.Digest, cause error) error { + if !ambiguousWrite(ctx, cause) { + return cause + } + if _, err := store.Resolve(ctx, ref.DigestStr()); IsNotFound(err) { + return nil + } + return cause +} diff --git a/pkg/oci/client_test.go b/pkg/oci/client_test.go index 18afda53e18..96e70e030cd 100644 --- a/pkg/oci/client_test.go +++ b/pkg/oci/client_test.go @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/stretchr/testify/require" @@ -292,3 +293,31 @@ func TestExportIndexRejectsRegistryFailureAndOrdinaryImage(t *testing.T) { require.NoDirExists(t, destination, "failed export must not publish a local layout") } } + +func TestWriteImageUploadsAStreamingLayer(t *testing.T) { + client, ref, _ := credentialRegistry(t, "stream-test") + payload := strings.Repeat("streamed layer content ", 4096) + streamed := stream.NewLayer(io.NopCloser(strings.NewReader(payload))) + image, err := mutate.AppendLayers(empty.Image, streamed) + require.NoError(t, err) + + require.NoError(t, client.WriteImage(t.Context(), ref, image)) + + pulled, err := client.Get(t.Context(), ref) + require.NoError(t, err) + img, err := pulled.Image() + require.NoError(t, err) + layers, err := img.Layers() + require.NoError(t, err) + require.Len(t, layers, 1) + reader, err := layers[0].Compressed() + require.NoError(t, err) + defer reader.Close() + stored, err := io.ReadAll(reader) + require.NoError(t, err) + want, err := streamed.Digest() + require.NoError(t, err) + got, _, err := v1.SHA256(strings.NewReader(string(stored))) + require.NoError(t, err) + require.Equal(t, want, got, "the uploaded layer is the streamed content") +} diff --git a/pkg/oci/failure_test.go b/pkg/oci/failure_test.go index 272b0b03a84..86d68974251 100644 --- a/pkg/oci/failure_test.go +++ b/pkg/oci/failure_test.go @@ -227,3 +227,28 @@ func TestVerifiedBlobChecksExactSizeAndDigest(t *testing.T) { }) } } + +func TestBoundIdentityWithAnEmptyCredentialIsRejectedWhenUsed(t *testing.T) { + t.Parallel() + resource, err := name.NewRepository("registry.example.com/acme/plugin") + require.NoError(t, err) + for _, tc := range []struct { + name string + auth authn.Authenticator + }{ + {"empty config", authn.FromConfig(authn.AuthConfig{})}, + {"blank basic credential", authn.FromConfig(authn.AuthConfig{Auth: "Og=="})}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(tc.auth, true, nil) + client := NewClient(nil, WithAuthenticationResolver(resolver)) + actual, err := client.authentication(t.Context(), resource) + require.NoError(t, err) + _, err = actual.Authorization() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + }) + } +} diff --git a/pkg/oci/identity.go b/pkg/oci/identity.go index ad4bbd0c676..8c5da84d692 100644 --- a/pkg/oci/identity.go +++ b/pkg/oci/identity.go @@ -3,10 +3,13 @@ package oci import ( "context" "errors" + "fmt" "github.com/google/go-containerregistry/pkg/authn" + orasauth "oras.land/oras-go/v2/registry/remote/auth" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" ) // AuthenticationResolver resolves explicitly bound identities before ambient credentials. @@ -52,5 +55,47 @@ func (c *Client) boundAuthentication(ctx context.Context, resource authn.Resourc if matched && (auth == nil || auth == authn.Anonymous) { return nil, true, errUtils.ErrOCIRegistry } + if matched { + // An authenticator that resolves to an empty credential would otherwise send requests + // unauthenticated, so it is checked whenever it is asked to authorize. + return nonEmptyCredential{Authenticator: auth}, true, nil + } return auth, matched, nil } + +// nonEmptyCredential makes a matched identity fail closed: an authorization that carries no +// credential is an error, never an anonymous request. The check runs on every authorization, so a +// token that refreshes into an empty credential is rejected too. +type nonEmptyCredential struct { + authn.Authenticator +} + +func (n nonEmptyCredential) Authorization() (*authn.AuthConfig, error) { + defer perf.Track(nil, "oci.nonEmptyCredential.Authorization")() + + config, err := n.Authenticator.Authorization() + if err != nil { + return nil, errors.Join(errUtils.ErrOCIRegistry, err) + } + if config == nil { + return nil, fmt.Errorf("%w: the bound identity resolved to no credential", errUtils.ErrOCIRegistry) + } + credential, err := credentialFrom(authn.FromConfig(*config)) + if err != nil { + return nil, err + } + if credential == orasauth.EmptyCredential { + return nil, fmt.Errorf("%w: the bound identity resolved to an empty credential", errUtils.ErrOCIRegistry) + } + if !canSend(credential) { + return nil, fmt.Errorf("%w: the bound identity resolved to an incomplete credential: a token, or both a username and a password, is required", errUtils.ErrOCIRegistry) + } + return config, nil +} + +// canSend reports whether the registry client can turn the credential into an Authorization header: a +// token, or a username together with a password. A username or password on its own is rejected +// instead of letting the request go out unauthenticated. +func canSend(credential orasauth.Credential) bool { + return credential.AccessToken != "" || credential.RefreshToken != "" || (credential.Username != "" && credential.Password != "") +} diff --git a/pkg/oci/mock_index_test.go b/pkg/oci/mock_index_test.go new file mode 100644 index 00000000000..c4b5c017144 --- /dev/null +++ b/pkg/oci/mock_index_test.go @@ -0,0 +1,147 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: ImageIndex) +// +// Generated by this command: +// +// mockgen -destination mock_index_test.go -package oci github.com/google/go-containerregistry/pkg/v1 ImageIndex +// + +// Package oci is a generated GoMock package. +package oci + +import ( + reflect "reflect" + + v1 "github.com/google/go-containerregistry/pkg/v1" + types "github.com/google/go-containerregistry/pkg/v1/types" + gomock "go.uber.org/mock/gomock" +) + +// MockImageIndex is a mock of ImageIndex interface. +type MockImageIndex struct { + ctrl *gomock.Controller + recorder *MockImageIndexMockRecorder + isgomock struct{} +} + +// MockImageIndexMockRecorder is the mock recorder for MockImageIndex. +type MockImageIndexMockRecorder struct { + mock *MockImageIndex +} + +// NewMockImageIndex creates a new mock instance. +func NewMockImageIndex(ctrl *gomock.Controller) *MockImageIndex { + mock := &MockImageIndex{ctrl: ctrl} + mock.recorder = &MockImageIndexMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockImageIndex) EXPECT() *MockImageIndexMockRecorder { + return m.recorder +} + +// Digest mocks base method. +func (m *MockImageIndex) Digest() (v1.Hash, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Digest") + ret0, _ := ret[0].(v1.Hash) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Digest indicates an expected call of Digest. +func (mr *MockImageIndexMockRecorder) Digest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockImageIndex)(nil).Digest)) +} + +// Image mocks base method. +func (m *MockImageIndex) Image(arg0 v1.Hash) (v1.Image, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Image", arg0) + ret0, _ := ret[0].(v1.Image) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Image indicates an expected call of Image. +func (mr *MockImageIndexMockRecorder) Image(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Image", reflect.TypeOf((*MockImageIndex)(nil).Image), arg0) +} + +// ImageIndex mocks base method. +func (m *MockImageIndex) ImageIndex(arg0 v1.Hash) (v1.ImageIndex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageIndex", arg0) + ret0, _ := ret[0].(v1.ImageIndex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageIndex indicates an expected call of ImageIndex. +func (mr *MockImageIndexMockRecorder) ImageIndex(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageIndex", reflect.TypeOf((*MockImageIndex)(nil).ImageIndex), arg0) +} + +// IndexManifest mocks base method. +func (m *MockImageIndex) IndexManifest() (*v1.IndexManifest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexManifest") + ret0, _ := ret[0].(*v1.IndexManifest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IndexManifest indicates an expected call of IndexManifest. +func (mr *MockImageIndexMockRecorder) IndexManifest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexManifest", reflect.TypeOf((*MockImageIndex)(nil).IndexManifest)) +} + +// MediaType mocks base method. +func (m *MockImageIndex) MediaType() (types.MediaType, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MediaType") + ret0, _ := ret[0].(types.MediaType) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MediaType indicates an expected call of MediaType. +func (mr *MockImageIndexMockRecorder) MediaType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockImageIndex)(nil).MediaType)) +} + +// RawManifest mocks base method. +func (m *MockImageIndex) RawManifest() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RawManifest") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RawManifest indicates an expected call of RawManifest. +func (mr *MockImageIndexMockRecorder) RawManifest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RawManifest", reflect.TypeOf((*MockImageIndex)(nil).RawManifest)) +} + +// Size mocks base method. +func (m *MockImageIndex) Size() (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Size") + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Size indicates an expected call of Size. +func (mr *MockImageIndexMockRecorder) Size() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockImageIndex)(nil).Size)) +} diff --git a/pkg/oci/mock_transport.go b/pkg/oci/mock_transport.go new file mode 100644 index 00000000000..482a62479ba --- /dev/null +++ b/pkg/oci/mock_transport.go @@ -0,0 +1,394 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/oci (interfaces: ReadOnlyStore,Store,RepositoryStore,Transport) +// +// Generated by this command: +// +// mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci ReadOnlyStore,Store,RepositoryStore,Transport +// + +// Package oci is a generated GoMock package. +package oci + +import ( + context "context" + io "io" + reflect "reflect" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockReadOnlyStore is a mock of ReadOnlyStore interface. +type MockReadOnlyStore struct { + ctrl *gomock.Controller + recorder *MockReadOnlyStoreMockRecorder + isgomock struct{} +} + +// MockReadOnlyStoreMockRecorder is the mock recorder for MockReadOnlyStore. +type MockReadOnlyStoreMockRecorder struct { + mock *MockReadOnlyStore +} + +// NewMockReadOnlyStore creates a new mock instance. +func NewMockReadOnlyStore(ctrl *gomock.Controller) *MockReadOnlyStore { + mock := &MockReadOnlyStore{ctrl: ctrl} + mock.recorder = &MockReadOnlyStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockReadOnlyStore) EXPECT() *MockReadOnlyStoreMockRecorder { + return m.recorder +} + +// Exists mocks base method. +func (m *MockReadOnlyStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockReadOnlyStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockReadOnlyStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockReadOnlyStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockReadOnlyStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockReadOnlyStore)(nil).Fetch), ctx, target) +} + +// Resolve mocks base method. +func (m *MockReadOnlyStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockReadOnlyStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockReadOnlyStore)(nil).Resolve), ctx, reference) +} + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Exists mocks base method. +func (m *MockStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockStore)(nil).Fetch), ctx, target) +} + +// Push mocks base method. +func (m *MockStore) Push(ctx context.Context, expected v1.Descriptor, content io.Reader) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Push", ctx, expected, content) + ret0, _ := ret[0].(error) + return ret0 +} + +// Push indicates an expected call of Push. +func (mr *MockStoreMockRecorder) Push(ctx, expected, content any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Push", reflect.TypeOf((*MockStore)(nil).Push), ctx, expected, content) +} + +// Resolve mocks base method. +func (m *MockStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockStore)(nil).Resolve), ctx, reference) +} + +// Tag mocks base method. +func (m *MockStore) Tag(ctx context.Context, desc v1.Descriptor, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tag", ctx, desc, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// Tag indicates an expected call of Tag. +func (mr *MockStoreMockRecorder) Tag(ctx, desc, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tag", reflect.TypeOf((*MockStore)(nil).Tag), ctx, desc, reference) +} + +// MockRepositoryStore is a mock of RepositoryStore interface. +type MockRepositoryStore struct { + ctrl *gomock.Controller + recorder *MockRepositoryStoreMockRecorder + isgomock struct{} +} + +// MockRepositoryStoreMockRecorder is the mock recorder for MockRepositoryStore. +type MockRepositoryStoreMockRecorder struct { + mock *MockRepositoryStore +} + +// NewMockRepositoryStore creates a new mock instance. +func NewMockRepositoryStore(ctrl *gomock.Controller) *MockRepositoryStore { + mock := &MockRepositoryStore{ctrl: ctrl} + mock.recorder = &MockRepositoryStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepositoryStore) EXPECT() *MockRepositoryStoreMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockRepositoryStore) Delete(ctx context.Context, target v1.Descriptor) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, target) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockRepositoryStoreMockRecorder) Delete(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRepositoryStore)(nil).Delete), ctx, target) +} + +// Exists mocks base method. +func (m *MockRepositoryStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockRepositoryStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockRepositoryStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockRepositoryStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockRepositoryStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockRepositoryStore)(nil).Fetch), ctx, target) +} + +// Predecessors mocks base method. +func (m *MockRepositoryStore) Predecessors(ctx context.Context, node v1.Descriptor) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Predecessors", ctx, node) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Predecessors indicates an expected call of Predecessors. +func (mr *MockRepositoryStoreMockRecorder) Predecessors(ctx, node any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Predecessors", reflect.TypeOf((*MockRepositoryStore)(nil).Predecessors), ctx, node) +} + +// Push mocks base method. +func (m *MockRepositoryStore) Push(ctx context.Context, expected v1.Descriptor, content io.Reader) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Push", ctx, expected, content) + ret0, _ := ret[0].(error) + return ret0 +} + +// Push indicates an expected call of Push. +func (mr *MockRepositoryStoreMockRecorder) Push(ctx, expected, content any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Push", reflect.TypeOf((*MockRepositoryStore)(nil).Push), ctx, expected, content) +} + +// PushReference mocks base method. +func (m *MockRepositoryStore) PushReference(ctx context.Context, expected v1.Descriptor, content io.Reader, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PushReference", ctx, expected, content, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// PushReference indicates an expected call of PushReference. +func (mr *MockRepositoryStoreMockRecorder) PushReference(ctx, expected, content, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PushReference", reflect.TypeOf((*MockRepositoryStore)(nil).PushReference), ctx, expected, content, reference) +} + +// Referrers mocks base method. +func (m *MockRepositoryStore) Referrers(ctx context.Context, subject v1.Descriptor, artifactType string) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Referrers", ctx, subject, artifactType) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Referrers indicates an expected call of Referrers. +func (mr *MockRepositoryStoreMockRecorder) Referrers(ctx, subject, artifactType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Referrers", reflect.TypeOf((*MockRepositoryStore)(nil).Referrers), ctx, subject, artifactType) +} + +// Resolve mocks base method. +func (m *MockRepositoryStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockRepositoryStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockRepositoryStore)(nil).Resolve), ctx, reference) +} + +// Tag mocks base method. +func (m *MockRepositoryStore) Tag(ctx context.Context, desc v1.Descriptor, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tag", ctx, desc, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// Tag indicates an expected call of Tag. +func (mr *MockRepositoryStoreMockRecorder) Tag(ctx, desc, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tag", reflect.TypeOf((*MockRepositoryStore)(nil).Tag), ctx, desc, reference) +} + +// Tags mocks base method. +func (m *MockRepositoryStore) Tags(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tags", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Tags indicates an expected call of Tags. +func (mr *MockRepositoryStoreMockRecorder) Tags(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tags", reflect.TypeOf((*MockRepositoryStore)(nil).Tags), ctx) +} + +// MockTransport is a mock of Transport interface. +type MockTransport struct { + ctrl *gomock.Controller + recorder *MockTransportMockRecorder + isgomock struct{} +} + +// MockTransportMockRecorder is the mock recorder for MockTransport. +type MockTransportMockRecorder struct { + mock *MockTransport +} + +// NewMockTransport creates a new mock instance. +func NewMockTransport(ctrl *gomock.Controller) *MockTransport { + mock := &MockTransport{ctrl: ctrl} + mock.recorder = &MockTransportMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTransport) EXPECT() *MockTransportMockRecorder { + return m.recorder +} + +// Repository mocks base method. +func (m *MockTransport) Repository(ctx context.Context, repository Repository) (RepositoryStore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Repository", ctx, repository) + ret0, _ := ret[0].(RepositoryStore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Repository indicates an expected call of Repository. +func (mr *MockTransportMockRecorder) Repository(ctx, repository any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Repository", reflect.TypeOf((*MockTransport)(nil).Repository), ctx, repository) +} diff --git a/pkg/oci/ocitest/registry.go b/pkg/oci/ocitest/registry.go index 5a17abe6136..48854f61491 100644 --- a/pkg/oci/ocitest/registry.go +++ b/pkg/oci/ocitest/registry.go @@ -7,6 +7,8 @@ import ( "archive/zip" "bytes" "io" + "log" + "net/http" "net/http/httptest" "strings" "testing" @@ -151,3 +153,61 @@ func buildZip(t *testing.T, files map[string]string) []byte { require.NoError(t, zw.Close()) return buf.Bytes() } + +// Credential is the Basic-auth identity an authenticated test registry accepts. +type Credential struct{ Username, Password string } + +// Server is an in-process registry that can be shut down independently of the test. +type Server struct { + // Host is the "127.0.0.1:PORT" registry address. + Host string + server *httptest.Server +} + +// NewServer starts an empty in-process OCI registry with the native OCI 1.1 referrers API enabled. +// When credential is non-nil, every request must carry exactly that Basic identity; any other +// request receives a 401 challenge, so a client that downgraded to anonymous or reused +// another registry's credentials fails visibly. +func NewServer(t *testing.T, credential *Credential) *Server { + defer perf.Track(nil, "ocitest.NewServer")() + + t.Helper() + + return newServer(t, credential, true) +} + +// NewServerWithoutReferrers is NewServer for a registry without the OCI 1.1 referrers API, so attachment +// discovery has to use the referrers tag-schema fallback. Use it for fallback-only tests. +func NewServerWithoutReferrers(t *testing.T, credential *Credential) *Server { + defer perf.Track(nil, "ocitest.NewServerWithoutReferrers")() + + t.Helper() + + return newServer(t, credential, false) +} + +func newServer(t *testing.T, credential *Credential, referrers bool) *Server { + t.Helper() + + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0)), registry.WithReferrersSupport(referrers)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if credential != nil { + user, password, ok := r.BasicAuth() + if !ok || user != credential.Username || password != credential.Password { + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return &Server{Host: strings.TrimPrefix(srv.URL, "http://"), server: srv} +} + +// Close shuts the registry down now; calling it again is harmless. +func (s *Server) Close() { + defer perf.Track(nil, "ocitest.Server.Close")() + + s.server.Close() +} diff --git a/pkg/oci/referrers.go b/pkg/oci/referrers.go index ac16e750bb9..2816c5970aa 100644 --- a/pkg/oci/referrers.go +++ b/pkg/oci/referrers.go @@ -2,13 +2,12 @@ package oci import ( "context" - "errors" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" - errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" ) @@ -19,16 +18,22 @@ type ReferrerRegistry interface { } // Referrers uses the shared credentials and transport, including the standard -// referrers-tag fallback on registries without the OCI 1.1 API. +// referrers-tag fallback on registries without the OCI 1.1 API. Attachment +// descriptors are returned as an index; no content is fetched. func (c *Client) Referrers(ctx context.Context, ref name.Digest) (v1.ImageIndex, error) { defer perf.Track(c.config, "oci.Client.Referrers")() - options, err := c.options(ctx, ref.Context()) + + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return nil, err } - index, err := remote.Referrers(ref, options...) + found, err := store.Referrers(ctx, ocispec.Descriptor{Digest: digest.Digest(ref.DigestStr())}, "") if err != nil { - return nil, errors.Join(errUtils.ErrOCIRegistry, err) + return nil, err + } + index, err := newReferrersIndex(found) + if err != nil { + return nil, err } return index, nil } diff --git a/pkg/oci/referrers_fallback_test.go b/pkg/oci/referrers_fallback_test.go new file mode 100644 index 00000000000..c0b3b9c035f --- /dev/null +++ b/pkg/oci/referrers_fallback_test.go @@ -0,0 +1,225 @@ +package oci_test + +import ( + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" +) + +// registryClient returns a client that can present the identity for host. +func registryClient(t *testing.T, host string, credential *ocitest.Credential) *oci.Client { + t.Helper() + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + if credential != nil { + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: credential.Username, Password: credential.Password})) + } + return oci.NewClient(nil, oci.WithKeychain(chain)) +} + +// pushAttachment pushes a subject manifest and one attachment that names it, and returns both descriptors. +func pushAttachment(t *testing.T, store oci.RepositoryStore) (subject, attachment ocispec.Descriptor) { + t.Helper() + ctx := t.Context() + config := []byte("{}") + configDesc := oci.Descriptor("application/vnd.oci.empty.v1+json", config) + require.NoError(t, store.Push(ctx, configDesc, strings.NewReader(string(config)))) + push := func(subject *ocispec.Descriptor, note string) ocispec.Descriptor { + manifest := ocispec.Manifest{Config: configDesc, Layers: []ocispec.Descriptor{}, Subject: subject, Annotations: map[string]string{"note": note}} + manifest.SchemaVersion = 2 + manifest.MediaType = ocispec.MediaTypeImageManifest + raw, err := json.Marshal(manifest) + require.NoError(t, err) + desc := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, store.Push(ctx, desc, strings.NewReader(string(raw)))) + return desc + } + subject = push(nil, "subject") + attachment = push(&subject, "attachment") + return subject, attachment +} + +func referrerDigests(referrers []ocispec.Descriptor) []string { + digests := make([]string, 0, len(referrers)) + for _, referrer := range referrers { + digests = append(digests, referrer.Digest.String()) + } + return digests +} + +// The fixture serves the native OCI 1.1 referrers API by default and the tag-schema fallback on request; +// attachment discovery works against both. +func TestReferrersDiscoverAttachmentsOnNativeAndFallbackRegistries(t *testing.T) { + credential := &ocitest.Credential{Username: "publisher", Password: "s3cret"} + for name, start := range map[string]func(*testing.T, *ocitest.Credential) *ocitest.Server{ + "native": ocitest.NewServer, + "fallback": ocitest.NewServerWithoutReferrers, + } { + t.Run(name, func(t *testing.T) { + server := start(t, credential) + client := registryClient(t, server.Host, credential) + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + + referrers, err := store.Referrers(t.Context(), subject, "") + + require.NoError(t, err) + assert.Equal(t, []string{attachment.Digest.String()}, referrerDigests(referrers)) + }) + } +} + +// Some registries answer the referrers API with 400 or 406 instead of 404. That means the API is not +// available, so discovery falls back to the tag schema; other statuses stay errors. +func TestReferrersFallBackToTheTagSchemaOnBadRequestAndNotAcceptable(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusNotAcceptable} { + t.Run(http.StatusText(status), func(t *testing.T) { + var refuse atomic.Bool + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if refuse.Load() && strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(status) + return + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + refuse.Store(true) + // A new store has not learned whether the registry supports the referrers API, as in a later process. + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + + referrers, err := fresh.Referrers(t.Context(), subject, "") + + require.NoError(t, err) + assert.Equal(t, []string{attachment.Digest.String()}, referrerDigests(referrers)) + }) + } +} + +func TestReferrersKeepOtherStatusesAsErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + + _, err = store.Referrers(t.Context(), oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("x")), "") + + require.Error(t, err) +} + +// Predecessor discovery during an extended copy uses the same fallback as Referrers, so a registry that +// answers the referrers API with 400 still yields its attachments without a prior Referrers call. +func TestExtendedCopyDiscoversAttachmentsThroughTheReferrersFallback(t *testing.T) { + var refuse atomic.Bool + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if refuse.Load() && strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(http.StatusBadRequest) + return + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + require.NoError(t, store.Tag(t.Context(), subject, "1.0.0")) + refuse.Store(true) + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "layout")) + require.NoError(t, err) + + _, err = oci.ExtendedCopy(t.Context(), oci.GraphSource{Store: fresh, Reference: "1.0.0"}, oci.Destination{Store: layout, Reference: "1.0.0"}) + + require.NoError(t, err) + copied, err := layout.Exists(t.Context(), attachment) + require.NoError(t, err) + assert.True(t, copied, "the attachment was discovered through the tag-schema fallback") +} + +// Deleting an attachment needs the registry's referrers capability to keep the tag-schema index current. +// A 400 on the capability probe settles it through the fallback; the DELETE itself is sent exactly once. +func TestDeleteAttachmentSettlesTheReferrersCapabilityThroughTheFallback(t *testing.T) { + // The registry first answers the referrers API with 404, so pushes maintain the tag-schema index, and + // then with 400, the way a registry that rejects the endpoint does. + var referrersStatus atomic.Int32 + referrersStatus.Store(http.StatusNotFound) + var mu sync.Mutex + var deletes []string + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(int(referrersStatus.Load())) + return + } + if r.Method == http.MethodDelete { + mu.Lock() + deletes = append(deletes, r.URL.Path) + mu.Unlock() + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + _, attachment := pushAttachment(t, store) + referrersStatus.Store(http.StatusBadRequest) + ref, err := name.NewDigest(host + "/acme/plugin@" + attachment.Digest.String()) + require.NoError(t, err) + + require.NoError(t, client.Delete(t.Context(), ref)) + + mu.Lock() + defer mu.Unlock() + attachmentDeletes := 0 + for _, deleted := range deletes { + if strings.HasSuffix(deleted, "/manifests/"+attachment.Digest.String()) { + attachmentDeletes++ + } + } + assert.Equal(t, 1, attachmentDeletes, "the DELETE of the attachment is sent exactly once") + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + exists, err := fresh.Exists(t.Context(), attachment) + require.NoError(t, err) + assert.False(t, exists, "the attachment is deleted") +} diff --git a/pkg/oci/signature/plugin_error_test.go b/pkg/oci/signature/plugin_error_test.go new file mode 100644 index 00000000000..cdb14175910 --- /dev/null +++ b/pkg/oci/signature/plugin_error_test.go @@ -0,0 +1,35 @@ +package signature + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + sigcore "github.com/cloudposse/atmos/pkg/signature" +) + +func TestPluginErrorKeepsBothLegacyClassificationsForOfflineTrust(t *testing.T) { + cause := errors.New("trusted root cache is empty") + err := pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureOffline, Cause: cause}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid, "offline trust failures are still signature failures") + require.ErrorIs(t, err, cause) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureOffline}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureConfigInvalid}) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + require.NotErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + + // Production shape: the trust loader failure is wrapped as invalid, with offline in the cause chain. + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureInvalid, Cause: errUtils.ErrSignatureOffline}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureInvalid, Cause: errors.New("bad signature")}) + require.NotErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) +} diff --git a/pkg/oci/signature/signaturetest/signaturetest.go b/pkg/oci/signature/signaturetest/signaturetest.go new file mode 100644 index 00000000000..86d556b2667 --- /dev/null +++ b/pkg/oci/signature/signaturetest/signaturetest.go @@ -0,0 +1,20 @@ +// Package signaturetest keeps the historical import path for the isolated Sigstore test authority. +// The implementation lives in pkg/signature/signaturetest. +package signaturetest + +import ( + "testing" + + shared "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +// Authority owns isolated CA, CT and Rekor keys and exposes only public trust material. +type Authority = shared.Authority + +// New creates a fresh trust domain for one test. +// +//nolint:lintroller // Test helper delegating to the tracked shared implementation. +func New(t *testing.T) *Authority { + t.Helper() + return shared.New(t) +} diff --git a/pkg/oci/signature/verify.go b/pkg/oci/signature/verify.go index f70fe1483c1..0c3f6ada4b7 100644 --- a/pkg/oci/signature/verify.go +++ b/pkg/oci/signature/verify.go @@ -1,30 +1,27 @@ -// Package signature verifies Sigstore bundles against caller-selected identities. +// Package signature adapts the shared Sigstore verification primitives in pkg/signature to +// OCI attachments and plugin trust requirements. Verification itself lives in pkg/signature; this +// package keeps the OCI-specific transport (referrer fetch, offline evidence files) and the +// sentinel errors plugin callers already match. package signature import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" - "regexp" - "strings" - "github.com/sigstore/sigstore-go/pkg/bundle" "github.com/sigstore/sigstore-go/pkg/root" - "github.com/sigstore/sigstore-go/pkg/verify" errUtils "github.com/cloudposse/atmos/errors" - "github.com/cloudposse/atmos/pkg/perf" + sigcore "github.com/cloudposse/atmos/pkg/signature" ) -const MaxBundleSize = 4 << 20 +// MaxBundleSize bounds one Sigstore bundle. +const MaxBundleSize = sigcore.MaxBundleSize // Identity binds a certificate SAN (exact or * wildcard) to an exact OIDC issuer. -type Identity struct{ Identity, Issuer string } +type Identity = sigcore.Identity // Result contains authenticated certificate claims, never claims from unverified metadata. -type Result struct{ Identity, Issuer string } +type Result = sigcore.Result // Verifier uses a trusted root independent of the artifact and its publisher. // LoadRoot is injectable for private trust deployments and isolated cryptographic tests. @@ -32,87 +29,35 @@ type Verifier struct { LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) } -// Verify requires a certificate, SCT, transparency inclusion and authenticated -// signing time. The release digest is always supplied independently of the bundle. +// Verify requires a certificate, SCT, transparency inclusion and authenticated signing time. +// The release digest is always supplied independently of the bundle. Failures carry the plugin +// signature sentinels that existing callers match. +// +//nolint:lintroller // Delegates to the tracked shared verifier. func (v Verifier) Verify(ctx context.Context, raw []byte, digest string, identities []Identity, offline bool) (*Result, error) { - defer perf.Track(nil, "signature.Verifier.Verify")() - if err := ctx.Err(); err != nil { - return nil, err - } - hash, signed, err := parseBundle(raw, digest) - if err != nil { - return nil, err - } - if len(identities) == 0 { - return nil, errUtils.ErrPluginSignatureInvalid - } - options, err := identityOptions(identities) - if err != nil { - return nil, err - } - verifier, err := v.trustedVerifier(ctx, offline) - if err != nil { - return nil, err - } - result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), options...)) + result, err := sigcore.Verifier{LoadRoot: v.LoadRoot}.Verify(ctx, raw, digest, identities, offline) if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - if err := ctx.Err(); err != nil { - return nil, err - } - if result.Signature == nil || result.Signature.Certificate == nil { - return nil, errUtils.ErrPluginSignatureInvalid + return nil, pluginError(err) } - return &Result{Identity: result.Signature.Certificate.SubjectAlternativeName, Issuer: result.Signature.Certificate.Issuer}, nil + return result, nil } -func parseBundle(raw []byte, digest string) ([]byte, *bundle.Bundle, error) { - if len(raw) > MaxBundleSize || len(raw) == 0 { - return nil, nil, errUtils.ErrPluginSignatureInvalid +// pluginError translates shared sentinels to the plugin sentinels, preserving message text. +func pluginError(err error) error { + var shared *sigcore.Error + if !errors.As(err, &shared) { + return err } - hash, err := hex.DecodeString(strings.TrimPrefix(digest, "sha256:")) - if err != nil || !strings.HasPrefix(digest, "sha256:") || len(hash) != sha256.Size { - return nil, nil, errUtils.ErrPluginSignatureInvalid + legacy := errUtils.ErrPluginSignatureInvalid + switch { + case errors.Is(shared.Kind, errUtils.ErrSignatureConfigInvalid): + legacy = errUtils.ErrPluginInvalidConfig + case errors.Is(shared.Kind, errUtils.ErrSignatureOffline) || errors.Is(shared.Cause, errUtils.ErrSignatureOffline): + // Offline trust failures were always signature failures too; keep both classifications. + legacy = errors.Join(errUtils.ErrPluginOfflineArtifact, errUtils.ErrPluginSignatureInvalid) } - var signed bundle.Bundle - if err := json.Unmarshal(raw, &signed); err != nil { - return nil, nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - return hash, &signed, nil -} - -func (v Verifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { - loader := v.LoadRoot - if loader == nil { - loader = loadPublicRoot - } - trusted, err := loader(ctx, offline) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - // Require a certificate identity, a witnessed signing time and inclusion in - // a trusted transparency log. Bare-key bundles cannot establish the requested - // OIDC identity; the key-only rejection test guards this boundary. - verifier, err := verify.NewVerifier(trusted, verify.WithSignedCertificateTimestamps(1), verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - return verifier, nil -} - -func identityOptions(identities []Identity) ([]verify.PolicyOption, error) { - options := make([]verify.PolicyOption, 0, len(identities)) - for _, identity := range identities { - if identity.Identity == "" || identity.Issuer == "" { - return nil, errUtils.ErrPluginInvalidConfig - } - pattern := "^" + strings.ReplaceAll(regexp.QuoteMeta(identity.Identity), `\*`, ".*") + "$" - match, err := verify.NewShortCertificateIdentity(identity.Issuer, "", "", pattern) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginInvalidConfig, err) - } - options = append(options, verify.WithCertificateIdentity(match)) + if shared.Cause == nil { + return legacy } - return options, nil + return errors.Join(legacy, shared.Cause) } diff --git a/pkg/oci/signature/verify_test.go b/pkg/oci/signature/verify_test.go index 276f5df9278..afcb8f3e463 100644 --- a/pkg/oci/signature/verify_test.go +++ b/pkg/oci/signature/verify_test.go @@ -58,9 +58,3 @@ func TestVerifyRealSigstoreEvidence(t *testing.T) { _, err = verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) } - -func TestOfflineTrustCannotFetch(t *testing.T) { - t.Setenv("ATMOS_XDG_DATA_HOME", t.TempDir()) - _, err := loadPublicRoot(t.Context(), true) - require.Error(t, err) -} diff --git a/pkg/oci/status.go b/pkg/oci/status.go index 04ba99a7ddd..4cfec1bc969 100644 --- a/pkg/oci/status.go +++ b/pkg/oci/status.go @@ -5,12 +5,22 @@ import ( "net/http" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" ) // IsNotFound distinguishes a missing manifest from auth and transport failures. +// It recognizes errors from both the legacy and the ORAS-backed transport. // //nolint:lintroller // Trivial accessor, pure lookup, factory option, or delegation; exempt under repository tracking policy. func IsNotFound(err error) bool { var response *transport.Error - return errors.As(err, &response) && response.StatusCode == http.StatusNotFound + if errors.As(err, &response) { + return response.StatusCode == http.StatusNotFound + } + var remote *errcode.ErrorResponse + if errors.As(err, &remote) { + return remote.StatusCode == http.StatusNotFound + } + return errors.Is(err, errdef.ErrNotFound) } diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go new file mode 100644 index 00000000000..138020541f8 --- /dev/null +++ b/pkg/oci/transport.go @@ -0,0 +1,165 @@ +//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "io" + "math" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate go run go.uber.org/mock/mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci ReadOnlyStore,Store,RepositoryStore,Transport + +// Repository identifies one OCI repository independently of any client library. +// Registry is the host (with optional port) and Path the repository name. +type Repository struct { + Registry string + Path string + // Insecure selects plain HTTP for a registry that name.Insecure marked, so every operation on + // the repository uses the same scheme. Loopback registries use plain HTTP without it. + Insecure bool +} + +// ParseRepository validates "registry/path" using the same rules as the legacy client, +// so loopback registries still use plain HTTP and Docker Hub short names are normalized. +func ParseRepository(value string) (Repository, error) { + defer perf.Track(nil, "oci.ParseRepository")() + + parsed, err := name.NewRepository(value) + if err != nil { + return Repository{}, wrapRegistry(err) + } + return repositoryOf(parsed), nil +} + +func repositoryOf(repository name.Repository) Repository { + return Repository{Registry: repository.RegistryStr(), Path: repository.RepositoryStr(), Insecure: repository.Scheme() == "http"} +} + +// String returns "registry/path". +// +//nolint:lintroller // Trivial accessor. +func (r Repository) String() string { return r.Registry + "/" + r.Path } + +// legacy converts back to the type used for credential resolution. +func (r Repository) legacy() (name.Repository, error) { + var options []name.Option + if r.Insecure { + options = append(options, name.Insecure) + } + parsed, err := name.NewRepository(r.String(), options...) + if err != nil { + return name.Repository{}, wrapRegistry(err) + } + return parsed, nil +} + +// Fetcher streams content by descriptor. +type Fetcher interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) +} + +// Store is a content-addressed OCI graph store. Its method set deliberately mirrors +// the ORAS target contract so any implementation (registry, layout, in-memory, +// legacy adapter) can participate in graph copy. Only OCI image-spec descriptors +// and go-digest values appear in the signature. +type Store interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) + // Exists reports whether the content is already present. + Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) + // Push stores content that must match the expected descriptor's size and digest. + Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error + // Resolve resolves a tag or digest reference to a descriptor. + Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) + // Tag associates a reference with an existing manifest descriptor. + Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error +} + +// GraphStore additionally discovers the manifests that point at a descriptor, which +// is how referrers and signatures are found for extended copies. +type GraphStore interface { + Store + // Predecessors returns manifests whose subject or children include the node. + Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// ReferencePusher pushes a manifest directly under a tag or digest reference. +type ReferencePusher interface { + PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error +} + +// RepositoryStore is a registry repository opened with already-resolved credentials. +type RepositoryStore interface { + GraphStore + ReferencePusher + // Tags lists every tag without interpreting it as a version. + Tags(ctx context.Context) ([]string, error) + // Delete removes a manifest by its full descriptor. Tags pointing to that digest may stop resolving; shared blobs are not deleted. A + // descriptor with only a digest deletes the manifest without reading it first. + Delete(ctx context.Context, target ocispec.Descriptor) error + // Referrers lists attachments of the subject, optionally filtered by artifact type. + // Registries without the OCI 1.1 API use the referrers tag-schema fallback. + Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) +} + +// Transport opens repositories. Credentials are resolved when a repository is opened, +// so a failed identity fails before any network request and never downgrades to anonymous. +type Transport interface { + Repository(ctx context.Context, repository Repository) (RepositoryStore, error) +} + +// FetchBytes reads and verifies a small blob. The size limit is checked before and while +// reading; both the declared size and the digest must match, so oversized or tampered +// content never reaches callers. +func FetchBytes(ctx context.Context, store Fetcher, target ocispec.Descriptor, limit int64) ([]byte, error) { + defer perf.Track(nil, "oci.FetchBytes")() + + if limit <= 0 || target.Size < 0 || target.Size > limit { + return nil, errUtils.ErrOCIIntegrity + } + reader, err := store.Fetch(ctx, target) + if err != nil { + return nil, wrapRegistry(err) + } + defer reader.Close() + return readVerified(reader, target, limit) +} + +func readVerified(reader io.Reader, target ocispec.Descriptor, limit int64) ([]byte, error) { + // Read one byte past the declared size to detect oversized content. The declared size is already + // within limit, so the bound never depends on limit+1, which overflows for a MaxInt64 limit. + readLimit := target.Size + if readLimit < math.MaxInt64 { + readLimit++ + } + raw, err := io.ReadAll(io.LimitReader(reader, readLimit)) + if err != nil { + return nil, wrapIntegrity(err) + } + if int64(len(raw)) != target.Size || int64(len(raw)) > limit { + return nil, errUtils.ErrOCIIntegrity + } + if err := target.Digest.Validate(); err != nil { + return nil, wrapIntegrity(err) + } + verifier := target.Digest.Verifier() + if _, err := verifier.Write(raw); err != nil || !verifier.Verified() { + return nil, errUtils.ErrOCIIntegrity + } + return raw, nil +} + +// Descriptor builds a descriptor for raw bytes. +func Descriptor(mediaType string, raw []byte) ocispec.Descriptor { + defer perf.Track(nil, "oci.Descriptor")() + + return ocispec.Descriptor{MediaType: mediaType, Digest: digest.FromBytes(raw), Size: int64(len(raw))} +} diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go new file mode 100644 index 00000000000..32f1614094c --- /dev/null +++ b/pkg/oci/transport_behavior_test.go @@ -0,0 +1,566 @@ +package oci_test + +import ( + "context" + "encoding/json" + "errors" + "io" + "log" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" +) + +func memoryClient(t *testing.T, host, username, password string) *oci.Client { + t.Helper() + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: username, Password: password})) + return oci.NewClient(nil, oci.WithKeychain(chain)) +} + +// tokenRegistry fronts a registry with a bearer-token service. Only the identity "alice" with the +// password "alice-secret" receives a token; the token is valid for everybody who presents it, which +// is exactly what makes a shared token cache dangerous. +type tokenRegistry struct { + *httptest.Server + host string + tokenRequests map[string]*atomic.Int32 +} + +func newTokenRegistry(t *testing.T) *tokenRegistry { + t.Helper() + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + result := &tokenRegistry{tokenRequests: map[string]*atomic.Int32{"alice": {}, "mallory": {}}} + result.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + user, password, _ := r.BasicAuth() + if counter := result.tokenRequests[user]; counter != nil { + counter.Add(1) + } + if user != "alice" || password != "alice-secret" { + w.WriteHeader(http.StatusUnauthorized) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"token": "alice-token"})) + return + } + if r.Header.Get("Authorization") != "Bearer alice-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+result.URL+`/token",service="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(result.Close) + result.host = strings.TrimPrefix(result.URL, "http://") + return result +} + +func TestClientsDoNotShareCredentialCachesAcrossIdentities(t *testing.T) { + ctx := t.Context() + server := newTokenRegistry(t) + alice := memoryClient(t, server.host, "alice", "alice-secret") + mallory := memoryClient(t, server.host, "mallory", "guess") + repository := oci.Repository{Registry: server.host, Path: "acme/plugin"} + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + require.NoError(t, alice.WriteIndex(ctx, ref, testIndex(t))) + + // Alice's token is valid and cached while Mallory acts; Mallory must still be challenged and denied. + aliceStore, err := alice.Repository(ctx, repository) + require.NoError(t, err) + _, err = aliceStore.Resolve(ctx, "1.0.0") + require.NoError(t, err) + _, err = mallory.Head(ctx, ref) + require.ErrorIs(t, err, errUtils.ErrPullImage) + assert.Positive(t, server.tokenRequests["mallory"].Load(), "the second identity must authenticate on its own") + + // Reopening the same repository with Alice again must not pick up anything of Mallory's. + _, err = alice.Head(ctx, ref) + require.NoError(t, err) +} + +// flakyRegistry wraps a registry and lets a test intercept requests. +type flakyRegistry struct { + *httptest.Server + host string + mu sync.Mutex + requests []string +} + +func newFlakyRegistry(t *testing.T, intercept func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool) *flakyRegistry { + t.Helper() + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + result := &flakyRegistry{} + result.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + result.mu.Lock() + result.requests = append(result.requests, r.Method+" "+r.URL.Path) + result.mu.Unlock() + if intercept(w, r, handler) { + return + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(result.Close) + result.host = strings.TrimPrefix(result.URL, "http://") + return result +} + +// duplicates returns requests that were sent more than once, which would be a mutation retry. +func (f *flakyRegistry) duplicates(method string) []string { + f.mu.Lock() + defer f.mu.Unlock() + seen := map[string]bool{} + var repeated []string + for _, request := range f.requests { + if !strings.HasPrefix(request, method+" ") { + continue + } + if seen[request] { + repeated = append(repeated, request) + } + seen[request] = true + } + return repeated +} + +func (f *flakyRegistry) count(method, fragment string) int { + f.mu.Lock() + defer f.mu.Unlock() + total := 0 + for _, request := range f.requests { + if strings.HasPrefix(request, method+" ") && strings.Contains(request, fragment) { + total++ + } + } + return total +} + +func TestAmbiguousWriteIsReconciledWithoutRetryingTheMutation(t *testing.T) { + // The registry commits the manifest and then fails the response: the client cannot know. + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + serve.ServeHTTP(httptest.NewRecorder(), r) + http.Error(w, "gateway failure after commit", http.StatusBadGateway) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + want, err := index.Digest() + require.NoError(t, err) + + require.NoError(t, client.WriteIndex(t.Context(), ref, index), "a write that landed must be reported as success") + assert.Equal(t, 3, server.count(http.MethodPut, "/manifests/"), "two platform manifests and the index are each written exactly once") + assert.Empty(t, server.duplicates(http.MethodPut)) + head, err := client.Head(t.Context(), ref) + require.NoError(t, err) + assert.Equal(t, want, head.Digest) +} + +func TestAmbiguousWriteThatDidNotLandIsAnError(t *testing.T) { + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + + err = client.WriteIndex(t.Context(), ref, testIndex(t)) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.Empty(t, server.duplicates(http.MethodPut), "a failed mutation is never blindly retried") + assert.Positive(t, server.count(http.MethodPut, "/manifests/")) + _, err = client.Head(t.Context(), ref) + require.Error(t, err) +} + +func TestRejectedWriteIsNotReconciledOrRetriedAnonymously(t *testing.T) { + var anonymous, anonymousAfterRejection atomic.Int32 + var rejected atomic.Bool + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if _, _, ok := r.BasicAuth(); !ok { + anonymous.Add(1) + if rejected.Load() { + anonymousAfterRejection.Add(1) + } + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + return true + } + if r.Method == http.MethodPut { + rejected.Store(true) + w.WriteHeader(http.StatusForbidden) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + + err = client.WriteIndex(t.Context(), ref, testIndex(t)) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.Equal(t, 0, server.count(http.MethodGet, "/manifests/1.0.0")+server.count(http.MethodHead, "/manifests/1.0.0"), + "a definite rejection must not trigger destination reconciliation") + assert.Empty(t, server.duplicates(http.MethodPut), "a rejected mutation is not retried") + assert.Positive(t, anonymous.Load(), "the registry challenged the first request") + assert.True(t, rejected.Load(), "the authenticated write was rejected") + assert.Zero(t, anonymousAfterRejection.Load(), "no anonymous fallback after rejection") +} + +func TestDeleteIsReconciledByResolvingTheDigest(t *testing.T) { + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool { + if r.Method == http.MethodDelete { + serve.ServeHTTP(httptest.NewRecorder(), r) + http.Error(w, "gateway failure after commit", http.StatusGatewayTimeout) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + require.NoError(t, client.WriteIndex(t.Context(), ref, index)) + digest, err := index.Digest() + require.NoError(t, err) + + require.NoError(t, client.Delete(t.Context(), ref.Context().Digest(digest.String()))) + assert.Equal(t, 1, server.count(http.MethodDelete, "/manifests/")) + _, err = client.Head(t.Context(), ref.Context().Digest(digest.String())) + require.Error(t, err) +} + +func TestDeleteDoesNotNeedReadAccessAndOnlyFallsBackOnDeniedReads(t *testing.T) { + tests := []struct { + name string + readStatus int + wantDelete int + wantErr bool + }{ + {"read forbidden still deletes", http.StatusForbidden, 1, false}, + {"read unauthorized still deletes", http.StatusUnauthorized, 1, false}, + {"missing digest sends no delete", http.StatusNotFound, 0, true}, + {"server failure sends no delete", http.StatusInternalServerError, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var deny atomic.Bool + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if deny.Load() && (r.Method == http.MethodHead || r.Method == http.MethodGet) && strings.Contains(r.URL.Path, "/manifests/") { + w.WriteHeader(tt.readStatus) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + require.NoError(t, client.WriteIndex(t.Context(), ref, index)) + digest, err := index.Digest() + require.NoError(t, err) + + deny.Store(true) + err = client.Delete(t.Context(), ref.Context().Digest(digest.String())) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err, "a delete-only credential must reach the DELETE request") + } + assert.Equal(t, tt.wantDelete, server.count(http.MethodDelete, "/manifests/")) + }) + } +} + +func TestReadsAreRetriedButCredentialsAreNotDowngraded(t *testing.T) { + var failures atomic.Int32 + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/tags/list") && failures.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + require.NoError(t, client.WriteIndex(t.Context(), ref, testIndex(t))) + tags, err := client.Tags(t.Context(), ref.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"1.0.0"}, tags) + assert.Equal(t, 2, server.count(http.MethodGet, "/tags/list")) +} + +func TestFetchBytesIsBoundedAndDigestVerified(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + content := []byte(`{"hello":"world"}`) + descriptor := oci.Descriptor("application/json", content) + require.NoError(t, layout.Push(ctx, descriptor, strings.NewReader(string(content)))) + + got, err := oci.FetchBytes(ctx, layout, descriptor, int64(len(content))) + require.NoError(t, err) + assert.Equal(t, content, got) + + _, err = oci.FetchBytes(ctx, layout, descriptor, int64(len(content))-1) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity, "content larger than the limit is rejected before reading") + _, err = oci.FetchBytes(ctx, layout, descriptor, 0) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + + got, err = oci.FetchBytes(ctx, layout, descriptor, math.MaxInt64) + require.NoError(t, err, "a MaxInt64 limit must not overflow into an empty read") + assert.Equal(t, content, got) + + wrongSize := descriptor + wrongSize.Size++ + _, err = oci.FetchBytes(ctx, fetchOverride(t, map[string][]byte{descriptor.Digest.String(): content}), wrongSize, 1<<10) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + + tampered := fetchOverride(t, map[string][]byte{descriptor.Digest.String(): []byte(`{"hello":"WORLD"}`)}) + _, err = oci.FetchBytes(ctx, tampered, descriptor, 1<<10) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity, "same size, different bytes") + + _, err = oci.FetchBytes(ctx, layout, oci.Descriptor("application/json", []byte("absent")), 1<<10) + require.True(t, oci.IsNotFound(err)) + require.ErrorIs(t, err, errdef.ErrNotFound) +} + +// fetchOverride returns a generated source whose Fetch serves the supplied bytes by digest. +func fetchOverride(t *testing.T, contents map[string][]byte) *oci.MockReadOnlyStore { + t.Helper() + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(string(contents[target.Digest.String()]))), nil + }).AnyTimes() + return store +} + +// cancellingSource returns a generated source that delegates to base, cancels the copy after +// cancelAfter fetches, and counts how many nodes were read. +func cancellingSource(t *testing.T, base oci.ReadOnlyStore, cancelAfter int32, cancel context.CancelFunc) (*oci.MockReadOnlyStore, *atomic.Int32) { + t.Helper() + var fetches atomic.Int32 + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if fetches.Add(1) == cancelAfter { + cancel() + } + return base.Fetch(ctx, target) + }).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(base.Exists).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(base.Resolve).AnyTimes() + return store, &fetches +} + +func TestCancellationStopsGraphTraversalAndLeavesNoReference(t *testing.T) { + directory := filepath.Join(t.TempDir(), "source") + index := testIndex(t) + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + base, err := oci.OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + + // Establish how many nodes an uninterrupted copy fetches. + complete, completeFetches := cancellingSource(t, base, -1, nil) + full, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "full")) + require.NoError(t, err) + _, err = oci.Copy(t.Context(), oci.Source{Store: complete, Reference: digest.String()}, oci.Destination{Store: full, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) + require.NoError(t, err) + total := completeFetches.Load() + require.Greater(t, total, int32(3)) + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + interrupted, interruptedFetches := cancellingSource(t, base, 2, cancel) + target, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: interrupted, Reference: digest.String()}, oci.Destination{Store: target, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) + require.ErrorIs(t, err, context.Canceled) + assert.Less(t, interruptedFetches.Load(), total, "traversal must stop once the context ends") + assert.Empty(t, target.References(), "an interrupted copy must not publish the reference") + _, err = target.Resolve(t.Context(), "1.0.0") + require.Error(t, err) + + already, stop := context.WithCancel(t.Context()) + stop() + _, err = oci.Copy(already, oci.Source{Store: base, Reference: digest.String()}, oci.Destination{Store: target, Reference: "1.0.0"}) + require.ErrorIs(t, err, context.Canceled) +} + +// landingDestination returns a generated destination over layout whose reference push reports err +// after (optionally) completing it, and counts the push attempts. +func landingDestination(t *testing.T, layout *oci.Layout, land bool, pushErr error) (*oci.MockRepositoryStore, *atomic.Int32) { + t.Helper() + var pushes atomic.Int32 + store := oci.NewMockRepositoryStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(layout.Fetch).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(layout.Exists).AnyTimes() + store.EXPECT().Push(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(layout.Push).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(layout.Resolve).AnyTimes() + store.EXPECT().Tag(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(layout.Tag).AnyTimes() + store.EXPECT().PushReference(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + pushes.Add(1) + if land { + if err := layout.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + if err := layout.Tag(ctx, expected, reference); err != nil { + return err + } + } + return pushErr + }, + ).AnyTimes() + return store, &pushes +} + +// movingSource returns a generated source whose tag resolves once and then fails, as if it moved. +func movingSource(t *testing.T, base oci.ReadOnlyStore) (*oci.MockReadOnlyStore, *atomic.Int32) { + t.Helper() + var calls atomic.Int32 + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(base.Fetch).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(base.Exists).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, ref string) (ocispec.Descriptor, error) { + if calls.Add(1) > 1 { + return ocispec.Descriptor{}, errors.New("source tag moved") + } + return base.Resolve(ctx, ref) + }).AnyTimes() + return store, &calls +} + +func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { + directory := filepath.Join(t.TempDir(), "source") + index := testIndex(t) + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + source, err := oci.OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + + rejection := errors.Join(errors.New("rejected"), errcodeResponse(http.StatusForbidden)) + for _, tc := range []struct { + name string + land bool + err error + wantErr bool + }{ + {"landed despite unknown failure", true, errors.New("connection reset by peer"), false}, + {"did not land", false, errors.New("connection reset by peer"), true}, + {"landed but definitely rejected", true, rejection, true}, + } { + t.Run(tc.name, func(t *testing.T) { + layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) + require.NoError(t, err) + destination, pushes := landingDestination(t, layout, tc.land, tc.err) + moving, resolves := movingSource(t, source) + root, err := oci.Copy(t.Context(), oci.Source{Store: moving, Reference: digest.String()}, oci.Destination{Store: destination, Reference: "1.0.0"}) + assert.Equal(t, int32(1), pushes.Load(), "the mutation is attempted exactly once") + assert.Equal(t, int32(1), resolves.Load(), "reconciliation must use the selected root") + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, digest.String(), root.Digest.String()) + }) + } +} + +func TestCopyReconcilesAnAttachmentOnlyOnceItIsRegisteredUnderItsSubject(t *testing.T) { + ctx := t.Context() + source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) + require.NoError(t, err) + config := oci.Descriptor(ocispec.MediaTypeEmptyJSON, []byte("{}")) + require.NoError(t, source.Push(ctx, config, strings.NewReader("{}"))) + subject := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("subject manifest")) + raw, err := json.Marshal(ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: "application/vnd.example.signature", + Config: config, + Layers: []ocispec.Descriptor{}, + Subject: &subject, + }) + require.NoError(t, err) + attachment := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, source.Push(ctx, attachment, strings.NewReader(string(raw)))) + require.NoError(t, source.Tag(ctx, attachment, "sig")) + + for _, tc := range []struct { + name string + referrers []ocispec.Descriptor + wantErr bool + }{ + {"registered under the subject", []ocispec.Descriptor{attachment}, false}, + {"manifest exists but is not registered", nil, true}, + {"only another attachment is registered", []ocispec.Descriptor{oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("other"))}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + destination := oci.NewMockRepositoryStore(gomock.NewController(t)) + destination.EXPECT().Exists(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + destination.EXPECT().Push(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ ocispec.Descriptor, content io.Reader) error { + _, err := io.Copy(io.Discard, content) + return err + }, + ).AnyTimes() + destination.EXPECT().PushReference(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("connection reset by peer")).AnyTimes() + destination.EXPECT().Resolve(gomock.Any(), "1.0.0").Return(attachment, nil).AnyTimes() + destination.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, ocispec.Descriptor) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(string(raw))), nil + }, + ).AnyTimes() + destination.EXPECT().Predecessors(gomock.Any(), subject).Return(tc.referrers, nil).AnyTimes() + + got, err := oci.Copy(ctx, oci.Source{Store: source, Reference: "sig"}, oci.Destination{Store: destination, Reference: "1.0.0"}, oci.WithoutSubjectTraversal()) + + if tc.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + return + } + require.NoError(t, err) + assert.Equal(t, attachment.Digest, got.Digest) + }) + } +} + +func errcodeResponse(status int) error { + return &errcode.ErrorResponse{Method: http.MethodPut, StatusCode: status} +} diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go new file mode 100644 index 00000000000..0f8474eb698 --- /dev/null +++ b/pkg/oci/transport_copy.go @@ -0,0 +1,396 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "encoding/json" + "errors" + "io" + "slices" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ReadOnlyStore is the source side of a graph copy. +type ReadOnlyStore interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) + // Exists reports whether the content is present. + Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) + // Resolve resolves a tag or digest reference to a descriptor. + Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) +} + +// ReadOnlyGraphStore is a source that can also report predecessors (referrers). +type ReadOnlyGraphStore interface { + ReadOnlyStore + // Predecessors returns manifests whose subject or children include the node. + Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// Source names the content a copy reads: a store and a tag or digest reference within it. +type Source struct { + Store ReadOnlyStore + Reference string +} + +// GraphSource is a Source that can also report predecessors, for extended copies. +type GraphSource struct { + Store ReadOnlyGraphStore + Reference string +} + +// Destination names where a copy writes: a store and the reference the root is published under. +// An empty Reference reuses the source reference. +type Destination struct { + Store Store + Reference string +} + +const defaultCopyConcurrency = 3 + +type copyConfig struct { + concurrency int + onCopied func(ocispec.Descriptor) + noSubjects bool +} + +// CopyOption configures a graph copy. +type CopyOption func(*copyConfig) + +// WithCopyConcurrency bounds how many nodes transfer at once. Values below one use the default. +// +//nolint:lintroller // Functional option only. +func WithCopyConcurrency(concurrency int) CopyOption { + return func(config *copyConfig) { config.concurrency = concurrency } +} + +// WithCopyObserver reports every node that was actually transferred (not skipped as present). +// The observer may be called from several goroutines concurrently, so it must be safe for concurrent use. +// +//nolint:lintroller // Functional option only. +func WithCopyObserver(observe func(ocispec.Descriptor)) CopyOption { + return func(config *copyConfig) { config.onCopied = observe } +} + +// WithoutSubjectTraversal stops the copy at the selected manifest instead of also copying the +// manifest named by its subject field. Legacy writers (for example signature attachments) +// publish an attachment next to a subject that already exists at the destination. An extended +// copy still reaches the selected manifest through the subject edges of its referrers. +// +//nolint:lintroller // Functional option only. +func WithoutSubjectTraversal() CopyOption { + return func(config *copyConfig) { config.noSubjects = true } +} + +func newCopyConfig(options []CopyOption) copyConfig { + config := copyConfig{concurrency: defaultCopyConcurrency} + for _, option := range options { + option(&config) + } + if config.concurrency < 1 { + config.concurrency = defaultCopyConcurrency + } + return config +} + +func (config copyConfig) graphOptions() oras.CopyGraphOptions { + options := oras.CopyGraphOptions{Concurrency: config.concurrency} + if config.noSubjects { + options.FindSuccessors = successorsWithoutSubject + } + return config.withObserver(options) +} + +// extendedGraphOptions is graphOptions for an extended copy. Extended traversal starts from the +// outermost referrers and reaches the selected root through their subject edges, so the exclusion +// applies to the selected root alone: a referrer still lists the root it attaches to. +func (config copyConfig) extendedGraphOptions(selected *ocispec.Descriptor) oras.CopyGraphOptions { + options := oras.CopyGraphOptions{Concurrency: config.concurrency} + if config.noSubjects { + options.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if desc.Digest == selected.Digest { + return successorsWithoutSubject(ctx, fetcher, desc) + } + return content.Successors(ctx, fetcher, desc) + } + } + return config.withObserver(options) +} + +func (config copyConfig) withObserver(options oras.CopyGraphOptions) oras.CopyGraphOptions { + if config.onCopied != nil { + options.PostCopy = func(_ context.Context, desc ocispec.Descriptor) error { + config.onCopied(desc) + return nil + } + } + return options +} + +// successorsWithoutSubject is the default ORAS successor discovery minus the subject edge, +// which ORAS lists first for OCI image manifests and indexes. +func successorsWithoutSubject(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + successors, err := content.Successors(ctx, fetcher, desc) + hasSubject := desc.MediaType == ocispec.MediaTypeImageManifest || desc.MediaType == ocispec.MediaTypeImageIndex + if err != nil || !hasSubject || len(successors) == 0 { + return successors, err + } + raw, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, err + } + var manifest struct { + Subject *ocispec.Descriptor `json:"subject"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + if manifest.Subject != nil && successors[0].Digest == manifest.Subject.Digest { + return successors[1:], nil + } + return successors, nil +} + +// Copy transfers the graph below the source reference (a manifest, image, or index with every +// child) into the destination under its reference (the source reference when empty). Raw manifest and index bytes are streamed +// unchanged and every node is digest-verified by the receiving store, so identity +// survives registry, layout, and registry hops. Cancelling ctx stops traversal. +// +// If a write fails with an unknown outcome the destination is resolved before deciding: +// when the destination reference already addresses the source root the copy is complete. There are no blind +// mutation retries, and rejected credentials are never retried anonymously. +func Copy(ctx context.Context, src Source, dst Destination, options ...CopyOption) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Copy")() + + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if dst.Reference == "" { + dst.Reference = src.Reference + } + config := newCopyConfig(options) + var selected ocispec.Descriptor + root, err := oras.Copy(ctx, guardedSource{ReadOnlyStore: src.Store, selected: &selected}, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, + oras.CopyOptions{CopyGraphOptions: config.graphOptions()}) + if err != nil { + return reconcile(ctx, err, selected, dst) + } + return root, nil +} + +// ExtendedCopy is Copy plus every predecessor of the graph (signatures, SBOMs, and other +// referrers), discovered through the source's referrers API or tag-schema fallback. +func ExtendedCopy(ctx context.Context, src GraphSource, dst Destination, options ...CopyOption) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.ExtendedCopy")() + + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if dst.Reference == "" { + dst.Reference = src.Reference + } + config := newCopyConfig(options) + var selected ocispec.Descriptor + graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store, selected: &selected}, graph: src.Store} + root, err := oras.ExtendedCopy(ctx, graph, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, + oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.extendedGraphOptions(&selected)}}) + if err == nil { + return root, nil + } + want, reconcileErr := reconcile(ctx, err, selected, dst) + if reconcileErr != nil { + return want, reconcileErr + } + // The root alone does not prove an extended copy finished: every referrer must be there too. + if !referrersPresent(ctx, src.Store, dst.Store, want) { + return ocispec.Descriptor{}, classifyCopyError(err) + } + return want, nil +} + +// referrersPresent reports whether every predecessor of root in the source (signatures, SBOMs, and +// their own referrers) also exists at the destination. Any lookup failure counts as missing. +func referrersPresent(ctx context.Context, src ReadOnlyGraphStore, dst Store, root ocispec.Descriptor) bool { + visited := map[string]bool{root.Digest.String(): true} + pending := []ocispec.Descriptor{root} + for len(pending) > 0 { + node := pending[len(pending)-1] + pending = pending[:len(pending)-1] + predecessors, err := src.Predecessors(ctx, node) + if err != nil { + return false + } + for i := range predecessors { + key := predecessors[i].Digest.String() + if visited[key] { + continue + } + visited[key] = true + if exists, err := dst.Exists(ctx, predecessors[i]); err != nil || !exists { + return false + } + pending = append(pending, predecessors[i]) + } + } + return true +} + +// reconcile decides the outcome of a failed copy by looking at the destination. +func reconcile(ctx context.Context, cause error, want ocispec.Descriptor, dst Destination) (ocispec.Descriptor, error) { + failure := classifyCopyError(cause) + if !ambiguousWrite(ctx, failure) || want.Digest == "" { + return ocispec.Descriptor{}, failure + } + have, err := dst.Store.Resolve(ctx, dst.Reference) + if err != nil || have.Digest != want.Digest { + return ocispec.Descriptor{}, failure + } + // An attachment is complete only once it is registered under its subject; the manifest alone + // existing at the destination is not enough, so the original failure stands. + if !attachmentRegistered(ctx, dst.Store, want) { + return ocispec.Descriptor{}, failure + } + return want, nil +} + +// attachmentRegistered reports whether the manifest or index at desc is complete at the store: a +// manifest without a subject always is, and one with a subject must be listed among the subject's +// referrers. Any lookup failure, or a store that cannot list referrers, counts as not registered so +// a failed write is never reported as a success on weaker evidence. +func attachmentRegistered(ctx context.Context, store Store, desc ocispec.Descriptor) bool { + if desc.MediaType != ocispec.MediaTypeImageManifest && desc.MediaType != ocispec.MediaTypeImageIndex { + return true + } + subject, err := readSubject(ctx, store, &desc) + if err != nil { + return false + } + if subject == nil { + return true + } + graph, ok := store.(GraphStore) + if !ok { + return false + } + referrers, err := graph.Predecessors(ctx, *subject) + if err != nil { + return false + } + return slices.ContainsFunc(referrers, func(found ocispec.Descriptor) bool { return found.Digest == desc.Digest }) +} + +func classifyCopyError(err error) error { + if errors.Is(err, errUtils.ErrOCIRegistry) || errors.Is(err, errUtils.ErrOCILayout) { + return err + } + return wrapRegistry(err) +} + +// guardedSource aborts as soon as the context ends, even for stores that ignore it. +type guardedSource struct { + ReadOnlyStore + selected *ocispec.Descriptor +} + +func (g guardedSource) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + desc, err := g.ReadOnlyStore.Resolve(ctx, reference) + if err == nil && g.selected != nil { + *g.selected = desc + } + return desc, err +} + +func (g guardedSource) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return g.ReadOnlyStore.Fetch(ctx, target) +} + +func (g guardedSource) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return g.ReadOnlyStore.Exists(ctx, target) +} + +type guardedGraphSource struct { + guardedSource + graph ReadOnlyGraphStore +} + +func (g guardedGraphSource) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return g.graph.Predecessors(ctx, node) +} + +// guardedStore does the same for destinations, so cancellation never starts another write. +type guardedStore struct{ Store } + +func (g guardedStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return g.Store.Exists(ctx, target) +} + +func (g guardedStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + defer perf.Track(nil, "oci.guardedStore.Push")() + + if err := ctx.Err(); err != nil { + return err + } + err := g.Store.Push(ctx, expected, content) + if err == nil || errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + // The write may have landed even though it was reported as failed. Look at the + // destination before deciding; never write again. + if ambiguousWrite(ctx, err) { + if exists, existsErr := g.Store.Exists(ctx, expected); existsErr == nil && exists && attachmentRegistered(ctx, g.Store, expected) { + return nil + } + } + return err +} + +// PushReference forwards to the wrapped store when it can push a manifest under a reference, +// which lets registry destinations write the root manifest directly under its tag. +func (g guardedStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + defer perf.Track(nil, "oci.guardedStore.PushReference")() + + if err := ctx.Err(); err != nil { + return err + } + err := g.pushReference(ctx, expected, content, reference) + if err == nil || errors.Is(err, errdef.ErrAlreadyExists) || !ambiguousWrite(ctx, err) { + return err + } + if resolved, resolveErr := g.Resolve(ctx, reference); resolveErr == nil && resolved.Digest == expected.Digest && attachmentRegistered(ctx, g.Store, expected) { + return nil + } + return err +} + +func (g guardedStore) pushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + if pusher, ok := g.Store.(ReferencePusher); ok { + return pusher.PushReference(ctx, expected, content, reference) + } + if err := g.Store.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + // The push may have taken long enough for the copy to be canceled; never start the tag then. + if err := ctx.Err(); err != nil { + return err + } + return g.Tag(ctx, expected, reference) +} diff --git a/pkg/oci/transport_errors.go b/pkg/oci/transport_errors.go new file mode 100644 index 00000000000..124c3c85ecf --- /dev/null +++ b/pkg/oci/transport_errors.go @@ -0,0 +1,76 @@ +package oci + +import ( + "context" + "errors" + "net/http" + + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// wrapRegistry marks a failure as an OCI registry error while keeping the cause chain. +// HTTP failures also carry a legacy *transport.Error with the real status code so the +// existing status classifiers (IsNotFound, discovery availability, pull diagnostics) +// keep working for callers that still reason about go-containerregistry errors. +func wrapRegistry(err error) error { + if err == nil { + return nil + } + if legacy := legacyStatusError(err); legacy != nil && !errors.As(err, new(*transport.Error)) { + return errors.Join(errUtils.ErrOCIRegistry, err, legacy) + } + return errors.Join(errUtils.ErrOCIRegistry, err) +} + +func wrapIntegrity(err error) error { + return errors.Join(errUtils.ErrOCIIntegrity, err) +} + +// legacyStatusError synthesizes the status carrier from an ORAS error. +func legacyStatusError(err error) *transport.Error { + var response *errcode.ErrorResponse + if errors.As(err, &response) { + return &transport.Error{StatusCode: response.StatusCode} + } + if errors.Is(err, errdef.ErrNotFound) { + return &transport.Error{StatusCode: http.StatusNotFound} + } + return nil +} + +// statusOf returns the HTTP status behind an error, or zero when none was received. +func statusOf(err error) int { + var response *errcode.ErrorResponse + if errors.As(err, &response) { + return response.StatusCode + } + var legacy *transport.Error + if errors.As(err, &legacy) { + return legacy.StatusCode + } + if errors.Is(err, errdef.ErrNotFound) { + return http.StatusNotFound + } + return 0 +} + +// definiteRejection reports whether the destination clearly refused a write. Authentication, +// authorization, validation and conflict responses are final answers; everything else +// (timeouts, resets, 5xx) leaves the write outcome unknown. +func definiteRejection(err error) bool { + status := statusOf(err) + return status >= http.StatusBadRequest && status < http.StatusInternalServerError && + status != http.StatusRequestTimeout && status != http.StatusTooManyRequests +} + +// ambiguousWrite reports whether a failed mutation may nevertheless have taken effect. +func ambiguousWrite(ctx context.Context, err error) bool { + if err == nil || ctx.Err() != nil || errors.Is(err, context.Canceled) { + return false + } + return !definiteRejection(err) +} diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go new file mode 100644 index 00000000000..78d3408def5 --- /dev/null +++ b/pkg/oci/transport_hardening_test.go @@ -0,0 +1,316 @@ +package oci + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/registry/remote" + + errUtils "github.com/cloudposse/atmos/errors" +) + +//go:generate go run go.uber.org/mock/mockgen -destination mock_index_test.go -package oci github.com/google/go-containerregistry/pkg/v1 ImageIndex + +func TestLegacyIndexGraphVisitsASharedIndexOnceAndStopsOnCancel(t *testing.T) { + child, err := random.Index(64, 1, 1) + require.NoError(t, err) + childDesc, err := partial.Descriptor(child) + require.NoError(t, err) + manifest := &v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex, Manifests: []v1.Descriptor{*childDesc, *childDesc}} + raw, err := json.Marshal(manifest) + require.NoError(t, err) + + // The shared child counts how often its own children are listed and otherwise delegates. + listed := 0 + controller := gomock.NewController(t) + shared := NewMockImageIndex(controller) + shared.EXPECT().MediaType().DoAndReturn(child.MediaType).AnyTimes() + shared.EXPECT().Digest().DoAndReturn(child.Digest).AnyTimes() + shared.EXPECT().Size().DoAndReturn(child.Size).AnyTimes() + shared.EXPECT().RawManifest().DoAndReturn(child.RawManifest).AnyTimes() + shared.EXPECT().IndexManifest().DoAndReturn(func() (*v1.IndexManifest, error) { + listed++ + return child.IndexManifest() + }).AnyTimes() + shared.EXPECT().Image(gomock.Any()).DoAndReturn(child.Image).AnyTimes() + shared.EXPECT().ImageIndex(gomock.Any()).DoAndReturn(child.ImageIndex).AnyTimes() + + // The parent lists the shared child twice, as a subgraph shared by two parents appears. Only the + // requested child digest is served, so selecting any other child fails the test. + parent := NewMockImageIndex(controller) + parent.EXPECT().MediaType().Return(types.OCIImageIndex, nil).AnyTimes() + parent.EXPECT().RawManifest().Return(raw, nil).AnyTimes() + parent.EXPECT().IndexManifest().Return(manifest, nil).AnyTimes() + parent.EXPECT().ImageIndex(childDesc.Digest).Return(shared, nil).AnyTimes() + + graph, err := legacyIndexGraph(t.Context(), parent) + require.NoError(t, err) + assert.Equal(t, 1, listed, "a subgraph shared by two parents is traversed once") + childDigest, err := child.Digest() + require.NoError(t, err) + exists, err := graph.Exists(t.Context(), ocispec.Descriptor{Digest: digest.Digest(childDigest.String())}) + require.NoError(t, err) + assert.True(t, exists) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = legacyIndexGraph(ctx, parent) + require.ErrorIs(t, err, context.Canceled) + image, err := random.Image(64, 1) + require.NoError(t, err) + _, err = legacyImageGraph(ctx, image) + require.ErrorIs(t, err, context.Canceled) +} + +func TestLayoutTagKeepsMetadataAndDigestRoots(t *testing.T) { + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + tagged := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"1"}}`) + tagged.ArtifactType = "application/vnd.example" + tagged.Annotations = map[string]string{"org.example.note": "kept"} + require.NoError(t, layout.Tag(t.Context(), tagged, "release")) + assert.Equal(t, map[string]string{"org.example.note": "kept"}, tagged.Annotations, "the caller's descriptor is not mutated") + + byDigest := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"2"}}`) + require.NoError(t, layout.Tag(t.Context(), byDigest, byDigest.Digest.String())) + + reopened, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + named, err := reopened.Resolve(t.Context(), "release") + require.NoError(t, err) + assert.Equal(t, "application/vnd.example", named.ArtifactType) + assert.Equal(t, "kept", named.Annotations["org.example.note"]) + root, err := reopened.Resolve(t.Context(), byDigest.Digest.String()) + require.NoError(t, err) + assert.Equal(t, ocispec.MediaTypeImageManifest, root.MediaType, "a digest-only root keeps its manifest type after reopening") + assert.Equal(t, []string{"release"}, reopened.References(), "a digest root is listed but never named") +} + +func TestLayoutStateMatchesThePersistedIndexAfterAFailedWrite(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("directory write permissions are not enforced for this process") + } + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + desc := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[]}`) + + require.NoError(t, os.Chmod(dir, 0o500)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + require.Error(t, layout.Tag(t.Context(), desc, "release")) + assert.Empty(t, layout.References(), "a failed index write leaves the in-memory index unchanged") + + require.NoError(t, os.Chmod(dir, 0o755)) + require.NoError(t, layout.Tag(t.Context(), desc, "release"), "the retry repeats the index update") + assert.Equal(t, []string{"release"}, layout.References()) + raw, err := os.ReadFile(filepath.Join(dir, ocispec.ImageIndexFile)) + require.NoError(t, err) + var index ocispec.Index + require.NoError(t, json.Unmarshal(raw, &index)) + require.Len(t, index.Manifests, 1) + assert.Equal(t, "release", index.Manifests[0].Annotations[ocispec.AnnotationRefName]) +} + +func pushManifest(t *testing.T, layout *Layout, manifest string) ocispec.Descriptor { + t.Helper() + config := []byte("{}") + configDesc := Descriptor("application/vnd.oci.empty.v1+json", config) + err := layout.Push(t.Context(), configDesc, bytes.NewReader(config)) + if err != nil { + require.ErrorContains(t, err, "already exists") + } + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(manifest)) + require.NoError(t, layout.Push(t.Context(), desc, bytes.NewReader([]byte(manifest)))) + return desc +} + +func TestPushReferenceRejectsContentThatDoesNotMatchBeforeSending(t *testing.T) { + // The repository points at a closed port: a request would fail with a network error, not integrity. + repo, err := remote.NewRepository("127.0.0.1:1/acme/plugin") + require.NoError(t, err) + repo.PlainHTTP = true + store := ®istryStore{repo: repo} + expected := Descriptor(ocispec.MediaTypeImageManifest, []byte(`{"expected":true}`)) + + for name, content := range map[string][]byte{ + "different bytes": []byte(`{"expected":false}`), + "longer": []byte(`{"expected":true} `), + "shorter": []byte(`{}`), + } { + t.Run(name, func(t *testing.T) { + err := store.PushReference(t.Context(), expected, bytes.NewReader(content), "1.0.0") + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + }) + } + oversized := expected + oversized.Size = maxPushedManifest + 1 + require.ErrorIs(t, store.PushReference(t.Context(), oversized, bytes.NewReader(nil), "1.0.0"), errUtils.ErrOCIIntegrity) +} + +func TestRepositoryConversionKeepsTheInsecureScheme(t *testing.T) { + insecure, err := name.NewRepository("registry.example.com/acme/plugin", name.Insecure) + require.NoError(t, err) + converted := repositoryOf(insecure) + assert.True(t, converted.Insecure) + back, err := converted.legacy() + require.NoError(t, err) + assert.Equal(t, "http", back.Scheme(), "every operation on the repository keeps plain HTTP") + + secure, err := name.NewRepository("registry.example.com/acme/plugin") + require.NoError(t, err) + converted = repositoryOf(secure) + assert.False(t, converted.Insecure) + back, err = converted.legacy() + require.NoError(t, err) + assert.Equal(t, "https", back.Scheme()) +} + +func TestReferrersPresentRequiresEveryReferrerAtTheDestination(t *testing.T) { + src := memory.New() + dst := memory.New() + ctx := t.Context() + config := []byte("{}") + configDesc := Descriptor("application/vnd.oci.empty.v1+json", config) + rootRaw := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"` + configDesc.Digest.String() + `","size":2},"layers":[]}`) + root := Descriptor(ocispec.MediaTypeImageManifest, rootRaw) + sigRaw, err := json.Marshal(ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: ocispec.MediaTypeImageManifest, Config: configDesc, + Layers: []ocispec.Descriptor{}, Subject: &root, + }) + require.NoError(t, err) + signature := Descriptor(ocispec.MediaTypeImageManifest, sigRaw) + for _, store := range []*memory.Store{src, dst} { + require.NoError(t, store.Push(ctx, configDesc, bytes.NewReader(config))) + require.NoError(t, store.Push(ctx, root, bytes.NewReader(rootRaw))) + } + require.NoError(t, src.Push(ctx, signature, bytes.NewReader(sigRaw))) + + assert.False(t, referrersPresent(ctx, src, dst, root), "the root alone does not complete an extended copy") + require.NoError(t, dst.Push(ctx, signature, bytes.NewReader(sigRaw))) + assert.True(t, referrersPresent(ctx, src, dst, root)) +} + +func TestLayoutConcurrentTagsKeepResolverAndIndexInStep(t *testing.T) { + const writers = 16 + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + descriptors := make([]ocispec.Descriptor, writers) + for i := range descriptors { + descriptors[i] = pushManifest(t, layout, fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"%d"}}`, i)) + } + + var wg sync.WaitGroup + for i := range descriptors { + wg.Add(1) + go func() { + defer wg.Done() + assert.NoError(t, layout.Tag(t.Context(), descriptors[i], "release")) + }() + } + wg.Wait() + + resolved, err := layout.Resolve(t.Context(), "release") + require.NoError(t, err) + reopened, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + persisted, err := reopened.Resolve(t.Context(), "release") + require.NoError(t, err) + assert.Equal(t, resolved.Digest, persisted.Digest, "the persisted index names the same manifest the resolver returns") + assert.Equal(t, []string{"release"}, reopened.References()) +} + +func TestLegacyGraphSpoolsStreamingLayersAndCleansUp(t *testing.T) { + payload := "streamed layer content" + image, err := mutate.AppendLayers(empty.Image, stream.NewLayer(io.NopCloser(strings.NewReader(payload)))) + require.NoError(t, err) + + graph, err := legacyImageGraph(t.Context(), image) + require.NoError(t, err) + manifest, err := image.Manifest() + require.NoError(t, err) + require.Len(t, manifest.Layers, 1) + layer := ocispec.Descriptor{Digest: digest.Digest(manifest.Layers[0].Digest.String())} + var reads [][]byte + for range 2 { + reader, err := graph.Fetch(t.Context(), layer) + require.NoError(t, err) + content, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + reads = append(reads, content) + } + assert.NotEmpty(t, reads[0], "the spooled layer holds the streamed bytes") + assert.Equal(t, reads[0], reads[1], "a streamed layer can be read again after it was spooled") + assert.Equal(t, layer.Digest, digest.FromBytes(reads[0]), "the spooled bytes match the manifest digest") + + spoolDir := graph.spoolDir + require.DirExists(t, spoolDir) + graph.close() + assert.NoDirExists(t, spoolDir) + graph.close() +} + +// ORAS drops foreign and non-distributable layers from every graph copy, which is the legacy +// remote.Write behavior; this guards that the legacy graph keeps relying on it. +func TestLegacyCopyNeverUploadsNonDistributableLayers(t *testing.T) { + distributable := static.NewLayer([]byte("distributable layer"), types.OCILayer) + foreign := static.NewLayer([]byte("foreign layer"), types.DockerForeignLayer) + image, err := mutate.AppendLayers(empty.Image, distributable, foreign) + require.NoError(t, err) + rawManifest, err := image.RawManifest() + require.NoError(t, err) + foreignHash, err := foreign.Digest() + require.NoError(t, err) + distributableHash, err := distributable.Digest() + require.NoError(t, err) + + graph, err := legacyImageGraph(t.Context(), image) + require.NoError(t, err) + defer graph.close() + layout, err := OpenLayout(t.Context(), t.TempDir()) + require.NoError(t, err) + // ORAS invokes the observer from several goroutines, so the recorded set needs a lock. + var transferredMu sync.Mutex + transferred := map[digest.Digest]bool{} + root, err := Copy(t.Context(), Source{Store: graph, Reference: legacyRoot}, Destination{Store: layout, Reference: "latest"}, + WithoutSubjectTraversal(), WithCopyObserver(func(desc ocispec.Descriptor) { + transferredMu.Lock() + defer transferredMu.Unlock() + transferred[desc.Digest] = true + })) + require.NoError(t, err) + + transferredMu.Lock() + defer transferredMu.Unlock() + + assert.True(t, transferred[digest.Digest(distributableHash.String())], "distributable layers are transferred") + assert.False(t, transferred[digest.Digest(foreignHash.String())], "foreign layers are not uploaded, matching the legacy writer") + assert.Equal(t, digest.FromBytes(rawManifest), root.Digest, "the manifest bytes are unchanged") +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go new file mode 100644 index 00000000000..8f4631c40b5 --- /dev/null +++ b/pkg/oci/transport_internal_test.go @@ -0,0 +1,464 @@ +package oci + +import ( + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/errcode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" +) + +type failingAuthenticator struct{} + +func (failingAuthenticator) Authorization() (*authn.AuthConfig, error) { return nil, os.ErrPermission } + +func TestCredentialFromAdaptsEveryAuthenticatorShape(t *testing.T) { + encoded := base64.StdEncoding.EncodeToString([]byte("encoded-user:encoded-secret")) + for _, tc := range []struct { + name string + authenticator authn.Authenticator + want auth.Credential + err error + }{ + {"nil authenticator is anonymous", nil, auth.EmptyCredential, nil}, + {"anonymous", authn.Anonymous, auth.EmptyCredential, nil}, + {"basic", &authn.Basic{Username: "u", Password: "p"}, auth.Credential{Username: "u", Password: "p"}, nil}, + {"registry token", &authn.Bearer{Token: "access"}, auth.Credential{AccessToken: "access"}, nil}, + {"identity token", authn.FromConfig(authn.AuthConfig{IdentityToken: "refresh"}), auth.Credential{RefreshToken: "refresh"}, nil}, + {"base64 auth field", authn.FromConfig(authn.AuthConfig{Auth: encoded}), auth.Credential{Username: "encoded-user", Password: "encoded-secret"}, nil}, + {"explicit fields win over auth field", authn.FromConfig(authn.AuthConfig{Username: "u", Password: "p", Auth: encoded}), auth.Credential{Username: "u", Password: "p"}, nil}, + {"malformed auth field", authn.FromConfig(authn.AuthConfig{Auth: "!!"}), auth.EmptyCredential, errUtils.ErrOCIRegistry}, + {"auth field without separator", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte("nocolon"))}), auth.EmptyCredential, errUtils.ErrOCIRegistry}, + {"authenticator failure fails closed", failingAuthenticator{}, auth.EmptyCredential, os.ErrPermission}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := credentialFrom(tc.authenticator) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + assert.Equal(t, auth.EmptyCredential, got) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestCredentialsAreReleasedOnlyToTheirRegistry(t *testing.T) { + credential := hostScopedCredential("registry.example.com:5000", &authn.Basic{Username: "u", Password: "p"}) + for host, wantSecret := range map[string]bool{ + "registry.example.com:5000": true, + "REGISTRY.example.com:5000": true, + "registry.example.com": false, + "evil.example.com:5000": false, + "auth.example.com": false, + } { + got, err := credential(t.Context(), host) + require.NoError(t, err) + assert.Equal(t, wantSecret, got.Password == "p", host) + } + + hub := hostScopedCredential("docker.io", &authn.Basic{Username: "u", Password: "p"}) + got, err := hub(t.Context(), "registry-1.docker.io") + require.NoError(t, err) + assert.Equal(t, "p", got.Password, "Docker Hub traffic is redirected to its API endpoint") + assert.Equal(t, "docker.io", orasHost("index.docker.io")) + assert.Equal(t, "ghcr.io", orasHost("ghcr.io")) +} + +func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + repository := Repository{Registry: host, Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + + for _, tc := range []struct { + name string + auth authn.Authenticator + matched bool + err error + }{ + {"resolver error", nil, true, os.ErrPermission}, + {"matched identity without credentials", nil, true, nil}, + {"matched identity resolves to anonymous", authn.Anonymous, true, nil}, + {"bound authenticator cannot authorize", failingAuthenticator{}, true, nil}, + {"matched identity with an empty credential", authn.FromConfig(authn.AuthConfig{}), true, nil}, + {"matched identity with an empty basic credential", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte(":"))}), true, nil}, + {"matched identity with a username only", authn.FromConfig(authn.AuthConfig{Username: "identity"}), true, nil}, + {"matched identity with a password only", authn.FromConfig(authn.AuthConfig{Password: "bound"}), true, nil}, + {"matched identity with a username-only basic credential", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte("identity:"))}), true, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(tc.auth, tc.matched, tc.err) + // Neither ambient keyring credentials nor anonymous access may stand in for the identity. + store := keyring.NewMockKeyring(controller) + client := NewClient(nil, WithAuthenticationResolver(resolver), WithKeychain(CredentialKeychain{Store: store})) + opened, err := client.Repository(t.Context(), repository) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + } + assert.Nil(t, opened) + }) + } + assert.Zero(t, requests.Load(), "no request may reach the registry when the identity cannot be resolved") +} + +// A bound identity must be able to produce an Authorization header: a token, or a username with a +// password. These shapes are accepted. +func TestNonEmptyCredentialAcceptsSendableCredentials(t *testing.T) { + for name, config := range map[string]authn.AuthConfig{ + "username and password": {Username: "identity", Password: "bound"}, + "access token": {RegistryToken: "token"}, + "refresh token": {IdentityToken: "refresh"}, + "encoded basic": {Auth: base64.StdEncoding.EncodeToString([]byte("identity:bound"))}, + "username with encoded auth": {Username: "publisher", Auth: base64.StdEncoding.EncodeToString([]byte("publisher:secret"))}, + } { + t.Run(name, func(t *testing.T) { + got, err := nonEmptyCredential{Authenticator: authn.FromConfig(config)}.Authorization() + require.NoError(t, err) + assert.NotNil(t, got) + }) + } +} + +// An authorization failure from the bound identity carries the registry sentinel and its cause. +func TestNonEmptyCredentialClassifiesAuthorizationFailures(t *testing.T) { + _, err := nonEmptyCredential{Authenticator: failingAuthenticator{}}.Authorization() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.ErrorIs(t, err, os.ErrPermission) +} + +func TestBoundIdentityTakesPrecedenceOverAmbientCredentials(t *testing.T) { + var seen atomic.Value + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if user, password, ok := r.BasicAuth(); ok { + seen.Store(user + ":" + password) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"name":"acme/plugin","tags":["1.0.0"]}`)) + return + } + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + repository := Repository{Registry: host, Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(&authn.Basic{Username: "identity", Password: "bound"}, true, nil) + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := CredentialKeychain{Store: store} + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: "ambient", Password: "stored"})) + + client := NewClient(nil, WithAuthenticationResolver(resolver), WithKeychain(chain)) + opened, err := client.Repository(t.Context(), repository) + require.NoError(t, err) + tags, err := opened.Tags(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"1.0.0"}, tags) + assert.Equal(t, "identity:bound", seen.Load()) +} + +func TestParseRepositoryUsesLegacyRegistryRules(t *testing.T) { + repository, err := ParseRepository("ghcr.io/acme/plugin") + require.NoError(t, err) + assert.Equal(t, Repository{Registry: "ghcr.io", Path: "acme/plugin"}, repository) + assert.Equal(t, "ghcr.io/acme/plugin", repository.String()) + + hub, err := ParseRepository("library/nginx") + require.NoError(t, err) + assert.Equal(t, Repository{Registry: "index.docker.io", Path: "library/nginx"}, hub) + + _, err = ParseRepository("not a repository") + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + _, err = Repository{Registry: "bad registry", Path: "x"}.legacy() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) +} + +func TestNewReferrersIndexRoundTripsDescriptors(t *testing.T) { + found := []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageManifest, Size: 10, Digest: digest.FromString("a"), + ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json", Annotations: map[string]string{"k": "v"}, + Data: []byte("inline"), + Platform: &ocispec.Platform{Architecture: "arm64", OS: "linux", OSVersion: "6.1", OSFeatures: []string{"f1", "f2"}, Variant: "v8"}, + }} + index, err := newReferrersIndex(found) + require.NoError(t, err) + // Mutating the registry response afterwards must not reach the index (source to result isolation). + found[0].Data[0] = 'X' + found[0].Platform.OSFeatures[0] = "changed" + found[0].Annotations["k"] = "changed" + manifest, err := index.IndexManifest() + require.NoError(t, err) + require.Len(t, manifest.Manifests, 1) + assert.Equal(t, found[0].Digest.String(), manifest.Manifests[0].Digest.String()) + assert.Equal(t, found[0].ArtifactType, manifest.Manifests[0].ArtifactType) + assert.Equal(t, map[string]string{"k": "v"}, manifest.Manifests[0].Annotations) + assert.Equal(t, []byte("inline"), manifest.Manifests[0].Data, "embedded data survives the conversion") + require.NotNil(t, manifest.Manifests[0].Platform, "the platform survives the conversion") + assert.Equal(t, &v1.Platform{Architecture: "arm64", OS: "linux", OSVersion: "6.1", OSFeatures: []string{"f1", "f2"}, Variant: "v8"}, manifest.Manifests[0].Platform) + size, err := index.Size() + require.NoError(t, err) + raw, err := index.RawManifest() + require.NoError(t, err) + assert.Equal(t, int64(len(raw)), size) + hash, err := index.Digest() + require.NoError(t, err) + assert.Equal(t, digest.FromBytes(raw).String(), hash.String()) + _, err = index.Image(hash) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + _, err = index.ImageIndex(hash) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + + _, err = newReferrersIndex([]ocispec.Descriptor{{Digest: "not-a-digest"}}) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) +} + +func TestStatusClassification(t *testing.T) { + notFound := wrapRegistry(&errorResponse404) + assert.True(t, IsNotFound(notFound)) + assert.Equal(t, http.StatusNotFound, statusOf(notFound)) + assert.True(t, definiteRejection(wrapRegistry(&errorResponse403))) + assert.False(t, definiteRejection(wrapRegistry(errors.New("connection reset")))) + assert.False(t, definiteRejection(wrapRegistry(&errorResponse503))) + assert.False(t, IsNotFound(wrapRegistry(&errorResponse403))) + assert.False(t, IsNotFound(nil)) + assert.NoError(t, wrapRegistry(nil)) +} + +var ( + errorResponse404 = errcode.ErrorResponse{Method: http.MethodGet, StatusCode: http.StatusNotFound} + errorResponse403 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusForbidden} + errorResponse503 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusServiceUnavailable} +) + +func TestReadSubjectClassifiesScanFailuresAsIntegrity(t *testing.T) { + for _, tc := range []struct { + name string + manifest string + }{ + {"truncated manifest", `{"schemaVersion":2,"layers":[`}, + {"malformed subject", `{"schemaVersion":2,"subject":{"size":"x"}}`}, + {"not an object", `[1,2]`}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tc.manifest)), nil) + + subject, err := readSubject(t.Context(), store, &desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, subject) + }) + } +} + +func TestReadSubjectSkipsLongStringsAndFindsSubject(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + longString := strings.Repeat(`x\"y`, 1<<18) + for _, tc := range []struct { + name string + manifest string + want bool + }{ + {"long string value then subject", `{"annotations":{"k":"` + longString + `"},"subject":` + subject + `}`, true}, + {"long key is skipped", `{"` + strings.Repeat("k", 4096) + `":1,"subject":` + subject + `}`, true}, + {"escaped subject key", `{"subject":` + subject + `}`, true}, + {"scalars and nesting before subject", `{"a":1.5e3,"b":true,"c":null,"d":[{"e":"]}"}],"subject":` + subject + `}`, true}, + {"long string without subject", `{"annotations":{"k":"` + longString + `"},"n":7}`, false}, + {"empty object", `{}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tc.manifest)), nil) + + got, err := readSubject(t.Context(), store, &desc) + + require.NoError(t, err) + if !tc.want { + assert.Nil(t, got) + return + } + require.NotNil(t, got) + assert.Equal(t, int64(1), got.Size) + assert.Equal(t, ocispec.MediaTypeImageManifest, got.MediaType) + }) + } +} + +func TestReadSubjectVerifiesTheWholeBodyBeforeAcceptingEitherResult(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + declared := `{"subject":` + subject + `,"annotations":{"k":"v"}}` + tampered := `{"subject":` + subject + `,"annotations":{"k":"w"}}` + tests := []struct { + name string + desc ocispec.Descriptor + body string + }{ + {"subject found early but the tail is tampered", Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)), tampered}, + {"subject found early but the body is longer", Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)), declared + " "}, + {"no subject and the digest differs", Descriptor(ocispec.MediaTypeImageManifest, []byte(`{"a":1}`)), `{"a":2}`}, + {"no subject and the body is shorter than declared", ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString(`{"a":1} `), Size: 10}, `{"a":1}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + store.EXPECT().Fetch(gomock.Any(), tc.desc).Return(io.NopCloser(strings.NewReader(tc.body)), nil) + + got, err := readSubject(t.Context(), store, &tc.desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) + }) + } +} + +// cancelAfterFirstReadReader serves a body and cancels the context once the first chunk was read. It +// ignores the context otherwise, like ORAS local storage, and counts the bytes it served. +type cancelAfterFirstReadReader struct { + reader io.Reader + cancel context.CancelFunc + served int64 +} + +func (c *cancelAfterFirstReadReader) Read(p []byte) (int, error) { + n, err := c.reader.Read(p) + c.served += int64(n) + c.cancel() + return n, err +} + +func (c *cancelAfterFirstReadReader) Close() error { return nil } + +func TestReadSubjectStopsReadingWhenTheContextIsCanceled(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + padding := strings.Repeat("p", 4<<20) + tests := []struct { + name string + manifest string + }{ + {"subject found early with a large tail", `{"subject":` + subject + `,"annotations":{"k":"` + padding + `"}}`}, + {"no subject and a large body", `{"annotations":{"k":"` + padding + `"}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + body := &cancelAfterFirstReadReader{reader: strings.NewReader(tc.manifest), cancel: cancel} + store.EXPECT().Fetch(gomock.Any(), desc).Return(body, nil) + + got, err := readSubject(ctx, store, &desc) + + require.ErrorIs(t, err, context.Canceled) + assert.NotErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) + assert.Less(t, body.served, int64(len(tc.manifest)), "the canceled read must not consume the whole body") + }) + } +} + +func TestReadSubjectStillRejectsAWrongDigestWhenTheContextStaysActive(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + declared := `{"subject":` + subject + `,"annotations":{"k":"v"}}` + tampered := `{"subject":` + subject + `,"annotations":{"k":"w"}}` + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tampered)), nil) + + got, err := readSubject(t.Context(), store, &desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) +} + +func TestReadOnlyLayoutClassifiesEveryReadError(t *testing.T) { + store := NewMockRepositoryStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).Return(nil, os.ErrPermission) + store.EXPECT().Exists(gomock.Any(), gomock.Any()).Return(false, os.ErrPermission) + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(ocispec.Descriptor{}, os.ErrPermission) + store.EXPECT().Predecessors(gomock.Any(), gomock.Any()).Return(nil, os.ErrPermission) + layout := &readOnlyLayout{store: store} + _, fetchErr := layout.Fetch(t.Context(), ocispec.Descriptor{}) + _, existsErr := layout.Exists(t.Context(), ocispec.Descriptor{}) + _, resolveErr := layout.Resolve(t.Context(), "missing") + _, predecessorsErr := layout.Predecessors(t.Context(), ocispec.Descriptor{}) + for _, err := range []error{fetchErr, existsErr, resolveErr, predecessorsErr} { + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.ErrorIs(t, err, os.ErrPermission) + } +} + +func TestReadOnlyLayoutMissingReferenceIsNotRegistryFailure(t *testing.T) { + directory := t.TempDir() + _, err := OpenLayout(t.Context(), directory) + require.NoError(t, err) + source, err := OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + destination, err := OpenLayout(t.Context(), t.TempDir()) + require.NoError(t, err) + _, err = Copy(t.Context(), Source{Store: source, Reference: "missing"}, Destination{Store: destination, Reference: "copy"}) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.ErrorIs(t, err, errdef.ErrNotFound) + assert.NotErrorIs(t, err, errUtils.ErrOCIRegistry) +} + +func TestRepositoryRegistryTokenIgnoresMalformedBasicAuth(t *testing.T) { + repository := Repository{Registry: "registry.example.com", Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + resolver := NewMockAuthenticationResolver(gomock.NewController(t)) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(authn.FromConfig(authn.AuthConfig{RegistryToken: "valid-token", Auth: "!"}), true, nil) + client := NewClient(nil, WithAuthenticationResolver(resolver)) + opened, err := client.Repository(t.Context(), repository) + require.NoError(t, err) + store, ok := opened.(*registryStore) + require.True(t, ok) + authClient, ok := store.repo.Client.(*auth.Client) + require.True(t, ok) + credential, err := authClient.Credential(t.Context(), repository.Registry) + require.NoError(t, err) + assert.Equal(t, "valid-token", credential.AccessToken) + assert.Empty(t, credential.Username) + assert.Empty(t, credential.Password) +} diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go new file mode 100644 index 00000000000..6d80337e6e0 --- /dev/null +++ b/pkg/oci/transport_layout.go @@ -0,0 +1,373 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "encoding/json" + "errors" + "io" + "maps" + "os" + "path/filepath" + "slices" + "sync" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/filesystem" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + layoutIndexMode = 0o644 + maxAttachmentManifest = 4 << 20 +) + +// Layout is a writable OCI image layout that can hold any number of named roots. +// +// It keeps index.json limited to the roots a caller explicitly tagged. Child manifests +// and blobs live only under blobs/, exactly like the single-release layouts written by +// SaveIndex, so a layout that holds one release still satisfies LoadIndex. A layout with +// several references is intentionally rejected by LoadIndex; read it through Layout. +// +// Layout does not coordinate concurrent writers in other processes. +type Layout struct { + root string + store *oci.Store + mu sync.Mutex + index []ocispec.Descriptor +} + +var ( + _ GraphStore = (*Layout)(nil) + _ ReadOnlyGraphStore = (*readOnlyLayout)(nil) +) + +// OpenLayout opens or creates a layout directory. +func OpenLayout(ctx context.Context, directory string) (*Layout, error) { + defer perf.Track(nil, "oci.OpenLayout")() + + // Resolve the root once and give the same absolute path to the store and the layout: the store + // anchors itself, so a relative root here would let a later working-directory change send index + // commits to a different directory than the blobs. + root, err := filepath.Abs(directory) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + store, err := oci.NewWithContext(ctx, root) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + // The ORAS store would otherwise list every pushed manifest in index.json. + store.AutoSaveIndex = false + store.AutoGC = false + layout := &Layout{root: root, store: store} + if err := layout.loadIndex(); err != nil { + return nil, err + } + return layout, nil +} + +// OpenLayoutReadOnly opens an existing layout without ever writing to it, so committed +// fixtures and offline media stay untouched. +func OpenLayoutReadOnly(ctx context.Context, directory string) (ReadOnlyGraphStore, error) { + defer perf.Track(nil, "oci.OpenLayoutReadOnly")() + + store, err := oci.NewFromFS(ctx, os.DirFS(directory)) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + return &readOnlyLayout{store: store}, nil +} + +// readOnlyLayout preserves the layout error classification on every source operation. +type readOnlyLayout struct { + store ReadOnlyGraphStore +} + +func (l *readOnlyLayout) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := l.store.Fetch(ctx, target) + return reader, layoutError(err) +} + +func (l *readOnlyLayout) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := l.store.Exists(ctx, target) + return exists, layoutError(err) +} + +func (l *readOnlyLayout) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + desc, err := l.store.Resolve(ctx, reference) + return desc, layoutError(err) +} + +func (l *readOnlyLayout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + predecessors, err := l.store.Predecessors(ctx, node) + return predecessors, layoutError(err) +} + +func (l *Layout) indexPath() string { return filepath.Join(l.root, ocispec.ImageIndexFile) } + +func (l *Layout) loadIndex() error { + raw, err := os.ReadFile(l.indexPath()) + if err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + l.index = index.Manifests + return nil +} + +// commitIndex persists entries as index.json and only then adopts them as the in-memory state, so a +// failed write leaves the layout exactly as persisted and a retry repeats the whole update. Callers +// hold l.mu. +func (l *Layout) commitIndex(entries []ocispec.Descriptor) error { + if err := writeIndexFile(l.indexPath(), entries); err != nil { + return err + } + l.index = entries + return nil +} + +func writeIndexFile(indexPath string, entries []ocispec.Descriptor) error { + index := ocispec.Index{MediaType: ocispec.MediaTypeImageIndex, Manifests: entries} + index.SchemaVersion = 2 + if index.Manifests == nil { + index.Manifests = []ocispec.Descriptor{} + } + raw, err := json.Marshal(index) + if err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + if err := filesystem.NewOSFileSystem().WriteFileAtomic(indexPath, raw, layoutIndexMode); err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + return nil +} + +// Fetch streams a blob from the layout. +func (l *Layout) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := l.store.Fetch(ctx, target) + return reader, layoutError(err) +} + +// Exists reports whether the blob is in the layout. +func (l *Layout) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := l.store.Exists(ctx, target) + return exists, layoutError(err) +} + +// Push writes a digest-verified blob. Pushing existing content reports ErrAlreadyExists. +// A manifest that names a subject (a signature, SBOM, or other attachment) is also +// recorded as an unnamed root, because that is the only way its relationship to the +// subject survives reopening the layout. +func (l *Layout) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + defer perf.Track(nil, "oci.Layout.Push")() + + err := l.store.Push(ctx, expected, content) + if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return layoutError(err) + } + if recordErr := l.recordAttachment(ctx, expected); recordErr != nil { + return recordErr + } + return layoutError(err) +} + +func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) error { + if desc.MediaType != ocispec.MediaTypeImageManifest && desc.MediaType != ocispec.MediaTypeImageIndex { + return nil + } + subject, err := readSubject(ctx, l.store, &desc) + if err != nil { + return layoutError(err) + } + if subject == nil { + return nil + } + // The size cap protects attachment handling only; a large manifest without a subject is valid. + if desc.Size > maxAttachmentManifest { + return layoutError(errUtils.ErrOCIIntegrity) + } + return l.recordRoot(desc) +} + +// recordRoot lists desc as an unnamed root in index.json unless its digest is already listed, so the +// manifest and its graph edges still resolve after the layout is reopened. +func (l *Layout) recordRoot(desc ocispec.Descriptor) error { + l.mu.Lock() + defer l.mu.Unlock() + return l.recordRootLocked(desc) +} + +// recordRootLocked is recordRoot for a caller that already holds l.mu. +func (l *Layout) recordRootLocked(desc ocispec.Descriptor) error { + for i := range l.index { + if l.index[i].Digest == desc.Digest { + return nil + } + } + entries := append(slices.Clone(l.index), ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size}) + return l.commitIndex(entries) +} + +// Resolve resolves a reference name or digest to a descriptor. +func (l *Layout) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Layout.Resolve")() + + // The resolver and the committed index are observed under one lock, so a concurrent Tag cannot move the + // reference between the two reads and make an existing tag look unpersisted. + l.mu.Lock() + defer l.mu.Unlock() + resolved, err := l.store.Resolve(ctx, reference) + if err != nil { + return resolved, layoutError(err) + } + if !l.persistedLocked(reference, resolved.Digest) { + // The resolver is updated before index.json is written, so a failed write leaves a tag that + // resolves now and disappears when the layout is reopened. Never report such a tag as published. + return ocispec.Descriptor{}, layoutError(errdef.ErrNotFound) + } + return resolved, nil +} + +// persistedLocked reports whether a reference is backed by index.json. A digest reference always is, +// because it names content, not a tag; a named reference is only when the persisted index lists it for +// the digest it resolved to. The caller holds l.mu. +func (l *Layout) persistedLocked(reference string, resolved digest.Digest) bool { + if _, err := digest.Parse(reference); err == nil { + return true + } + for i := range l.index { + if l.index[i].Annotations[ocispec.AnnotationRefName] == reference && l.index[i].Digest == resolved { + return true + } + } + return false +} + +// Predecessors lists manifests in the layout that reference the node. +func (l *Layout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + found, err := l.store.Predecessors(ctx, node) + return found, layoutError(err) +} + +// Tag records reference as a named root in index.json. A digest reference is recorded as an +// unnamed root unless the digest is already listed. An unnamed root with the same digest is named +// in place, so a legacy single-release layout keeps exactly one entry. +func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + defer perf.Track(nil, "oci.Layout.Tag")() + + // The mutex covers the resolver update and the index commit so concurrent updates to the same + // reference are ordered: the last resolver write is also the last index write. + l.mu.Lock() + defer l.mu.Unlock() + if err := l.store.Tag(ctx, desc, reference); err != nil { + return layoutError(err) + } + if reference == desc.Digest.String() { + // A digest reference is never named, but the root must still be listed or reopening the + // layout loses its media type and children. + return l.recordRootLocked(desc) + } + // Copy the descriptor and its annotations so root metadata (artifact type, platform, other + // annotations) survives reopening without mutating the caller's descriptor. + named := desc + named.Annotations = make(map[string]string, len(desc.Annotations)+1) + maps.Copy(named.Annotations, desc.Annotations) + named.Annotations[ocispec.AnnotationRefName] = reference + displaced, moved := displacedByReference(l.index, reference, desc.Digest) + entries := withReference(slices.Clone(l.index), named) + if moved { + var err error + if entries, err = l.keepDisplacedAttachment(ctx, entries, displaced); err != nil { + return err + } + } + return l.commitIndex(entries) +} + +// displacedByReference returns the root that reference names when it points at a different digest than +// the one about to be tagged: moving the tag replaces that root's entry in index.json. +func displacedByReference(entries []ocispec.Descriptor, reference string, tagged digest.Digest) (ocispec.Descriptor, bool) { + for i := range entries { + if entries[i].Annotations[ocispec.AnnotationRefName] == reference && entries[i].Digest != tagged { + return entries[i], true + } + } + return ocispec.Descriptor{}, false +} + +// keepDisplacedAttachment lists a root that lost its tag as an unnamed root when it is an attachment, +// a manifest with a subject. Without its own entry the manifest would no longer be reachable from +// index.json, and reopening the layout would lose it as a referrer of its subject. A manifest without +// a subject, or one that is still listed, needs nothing. +func (l *Layout) keepDisplacedAttachment(ctx context.Context, entries []ocispec.Descriptor, displaced ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if displaced.MediaType != ocispec.MediaTypeImageManifest && displaced.MediaType != ocispec.MediaTypeImageIndex { + return entries, nil + } + for i := range entries { + if entries[i].Digest == displaced.Digest { + return entries, nil + } + } + subject, err := readSubject(ctx, l.store, &displaced) + if err != nil { + return nil, layoutError(err) + } + if subject == nil { + return entries, nil + } + return append(entries, ocispec.Descriptor{MediaType: displaced.MediaType, Digest: displaced.Digest, Size: displaced.Size}), nil +} + +func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []ocispec.Descriptor { + reference := named.Annotations[ocispec.AnnotationRefName] + for i := range entries { + if entries[i].Annotations[ocispec.AnnotationRefName] == reference { + entries[i] = named + // An unnamed root for the same index would leave two entries for it, so the tag takes its place. + return slices.DeleteFunc(entries, func(entry ocispec.Descriptor) bool { + return entry.Digest == named.Digest && entry.Annotations[ocispec.AnnotationRefName] == "" + }) + } + } + for i := range entries { + if entries[i].Digest == named.Digest && entries[i].Annotations[ocispec.AnnotationRefName] == "" { + entries[i] = named + return entries + } + } + return append(entries, named) +} + +// References returns the named roots in index.json in insertion order. +func (l *Layout) References() []string { + defer perf.Track(nil, "oci.Layout.References")() + + l.mu.Lock() + defer l.mu.Unlock() + var references []string + for i := range l.index { + if reference := l.index[i].Annotations[ocispec.AnnotationRefName]; reference != "" { + references = append(references, reference) + } + } + return references +} + +func layoutError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return errors.Join(errUtils.ErrOCILayout, err) +} diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go new file mode 100644 index 00000000000..bc3b3267129 --- /dev/null +++ b/pkg/oci/transport_legacy.go @@ -0,0 +1,412 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// legacyRoot is the only reference a legacyGraph resolves. +const legacyRoot = "root" + +// legacyGraph exposes the raw bytes of a go-containerregistry image or index as a +// read-only OCI graph. It serializes exactly what the legacy object already holds: +// manifests are never rebuilt (no ORAS PackManifest), so digests cannot drift. +type legacyGraph struct { + root ocispec.Descriptor + blobs map[digest.Digest]func() (io.ReadCloser, error) + + // indexes holds the descriptors of indexes whose children were already added, so a subgraph + // shared by several parents is traversed once. + indexes map[digest.Digest]ocispec.Descriptor + + // spooled maps a streaming layer's digest to the replayable file that holds its bytes, and + // spoolDir is the private directory that owns those files until close. + spooled map[digest.Digest]string + spoolDir string +} + +var _ ReadOnlyStore = (*legacyGraph)(nil) + +func newLegacyGraph() *legacyGraph { + return &legacyGraph{ + blobs: make(map[digest.Digest]func() (io.ReadCloser, error)), + indexes: make(map[digest.Digest]ocispec.Descriptor), + spooled: make(map[digest.Digest]string), + } +} + +// close removes the spooled streaming layers. It is safe to call more than once. +func (g *legacyGraph) close() { + if g == nil || g.spoolDir == "" { + return + } + _ = os.RemoveAll(g.spoolDir) + g.spoolDir = "" + g.spooled = map[digest.Digest]string{} +} + +// spoolStreamingLayers consumes every layer that is a stream not yet read (stream.NewLayer) into a +// replayable file. A stream's digest, size, and the image's manifest exist only after it was read +// once, and it can be read only once, so the bytes must be kept for the upload. +func (g *legacyGraph) spoolStreamingLayers(ctx context.Context, image v1.Image) error { + layers, err := image.Layers() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + for _, layer := range layers { + if _, err := layer.Digest(); !errors.Is(err, stream.ErrNotComputed) { + continue + } + if err := g.spoolLayer(ctx, layer); err != nil { + return err + } + } + return nil +} + +func (g *legacyGraph) spoolLayer(ctx context.Context, layer v1.Layer) error { + if err := ctx.Err(); err != nil { + return err + } + if g.spoolDir == "" { + dir, err := os.MkdirTemp("", "atmos-oci-spool-") + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.spoolDir = dir + } + reader, err := layer.Compressed() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + file, err := os.CreateTemp(g.spoolDir, "layer-*") + if err != nil { + _ = reader.Close() + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + _, copyErr := io.Copy(file, reader) + // Closing the stream is what computes its digest and size. + closeErr := errors.Join(reader.Close(), file.Close()) + if err := errors.Join(copyErr, closeErr); err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + hash, err := layer.Digest() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.spooled[digest.Digest(hash.String())] = file.Name() + return nil +} + +func legacyImageGraph(ctx context.Context, image v1.Image) (*legacyGraph, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + graph := newLegacyGraph() + root, err := graph.addImage(ctx, image) + if err != nil { + graph.close() + return nil, err + } + graph.root = root + return graph, nil +} + +func legacyIndexGraph(ctx context.Context, index v1.ImageIndex) (*legacyGraph, error) { + graph := newLegacyGraph() + root, err := graph.addIndex(ctx, index) + if err != nil { + graph.close() + return nil, err + } + graph.root = root + return graph, nil +} + +func (g *legacyGraph) addBytes(mediaType string, raw []byte) ocispec.Descriptor { + desc := Descriptor(mediaType, raw) + g.blobs[desc.Digest] = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(raw)), nil } + return desc +} + +func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Descriptor, error) { + mediaType, err := image.MediaType() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + if err := refuseSchema1(mediaType); err != nil { + return ocispec.Descriptor{}, err + } + if err := g.spoolStreamingLayers(ctx, image); err != nil { + return ocispec.Descriptor{}, err + } + raw, err := image.RawManifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + manifest, err := image.Manifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + rawConfig, err := image.RawConfigFile() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.blobs[digest.Digest(manifest.Config.Digest.String())] = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(rawConfig)), nil + } + g.registerLayers(image, manifest) + return g.addBytes(string(mediaType), raw), nil +} + +// refuseSchema1 rejects a schema 1 manifest. It has no config or layer descriptors in the shape this +// graph serves, so it is refused up front instead of failing later in Manifest() or after its layers +// were already read. +func refuseSchema1(mediaType types.MediaType) error { + if mediaType == types.DockerManifestSchema1 || mediaType == types.DockerManifestSchema1Signed { + return fmt.Errorf("%w: schema 1 images are not supported; convert the image to OCI or Docker schema 2", errUtils.ErrOCIIntegrity) + } + return nil +} + +// registerLayers serves each layer of an image, from its spool file when it was spooled and from the +// image otherwise. +func (g *legacyGraph) registerLayers(image v1.Image, manifest *v1.Manifest) { + for i := range manifest.Layers { + hash := manifest.Layers[i].Digest + if path, found := g.spooled[digest.Digest(hash.String())]; found { + g.blobs[digest.Digest(hash.String())] = func() (io.ReadCloser, error) { return os.Open(path) } + continue + } + g.blobs[digest.Digest(hash.String())] = func() (io.ReadCloser, error) { + layer, err := image.LayerByDigest(hash) + if err != nil { + return nil, err + } + return layer.Compressed() + } + } +} + +// addIndex records an index and, once per distinct index digest, every child below it. Cancellation +// is checked before each accessor call and child, so a canceled write stops preparing the graph. +// +// A child can be an image with a streaming layer, whose digest, and so the index manifest that lists +// it, exists only after the stream was read once. The children are therefore prepared first, through +// [partial.Manifests], which evaluates them lazily, and the index manifest is read afterwards. An index +// whose manifest is already computable is recognized beforehand, so a shared subgraph is traversed once. +func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispec.Descriptor, error) { + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if done, found := g.completedIndex(index); found { + return done, nil + } + if err := g.addChildren(ctx, index); err != nil { + return ocispec.Descriptor{}, err + } + raw, err := index.RawManifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + mediaType, err := index.MediaType() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + if done, found := g.indexes[digest.FromBytes(raw)]; found { + return done, nil + } + // The index is marked complete only after every child was added successfully. + desc := g.addBytes(string(mediaType), raw) + g.indexes[desc.Digest] = desc + return desc, nil +} + +// completedIndex returns the descriptor of an index that was already added, when its manifest is +// computable beforehand. +func (g *legacyGraph) completedIndex(index v1.ImageIndex) (ocispec.Descriptor, bool) { + raw, err := index.RawManifest() + if err != nil { + return ocispec.Descriptor{}, false + } + done, found := g.indexes[digest.FromBytes(raw)] + return done, found +} + +// addChildren prepares every child of an index, checking cancellation before each one. +func (g *legacyGraph) addChildren(ctx context.Context, index v1.ImageIndex) error { + children, err := partial.Manifests(index) + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + for _, child := range children { + if err := ctx.Err(); err != nil { + return err + } + if err := g.addChild(ctx, child); err != nil { + return err + } + } + return nil +} + +// addChild records one child of a legacy index: an image, a nested index, or a layer that is a blob in +// its own right. +func (g *legacyGraph) addChild(ctx context.Context, child partial.Describable) error { + switch typed := child.(type) { + case v1.ImageIndex: + _, err := g.addIndex(ctx, typed) + return err + case v1.Image: + _, err := g.addImage(ctx, typed) + return err + case v1.Layer: + return g.addLayer(ctx, typed) + default: + mediaType, _ := child.MediaType() + return fmt.Errorf("%w: unsupported child media type %q", errUtils.ErrOCIIntegrity, mediaType) + } +} + +// addLayer serves a layer that an index lists as a child. A streaming layer is spooled first, because +// its digest exists only after it was read. +func (g *legacyGraph) addLayer(ctx context.Context, layer v1.Layer) error { + hash, err := layer.Digest() + if errors.Is(err, stream.ErrNotComputed) { + if err := g.spoolLayer(ctx, layer); err != nil { + return err + } + hash, err = layer.Digest() + } + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + key := digest.Digest(hash.String()) + if path, found := g.spooled[key]; found { + g.blobs[key] = func() (io.ReadCloser, error) { return os.Open(path) } + return nil + } + g.blobs[key] = layer.Compressed + return nil +} + +// Fetch streams a known blob. +func (g *legacyGraph) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + open, found := g.blobs[target.Digest] + if !found { + return nil, fmt.Errorf("%s: %w", target.Digest, errdef.ErrNotFound) + } + return open() +} + +// Exists reports whether the legacy object holds the blob. +func (g *legacyGraph) Exists(_ context.Context, target ocispec.Descriptor) (bool, error) { + _, found := g.blobs[target.Digest] + return found, nil +} + +// Resolve returns the root for the only reference the graph knows. +func (g *legacyGraph) Resolve(_ context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.legacyGraph.Resolve")() + + if reference != legacyRoot && reference != g.root.Digest.String() { + return ocispec.Descriptor{}, fmt.Errorf("%s: %w", reference, errdef.ErrNotFound) + } + return g.root, nil +} + +// referrersIndex presents ORAS referrer descriptors as the legacy image index. +type referrersIndex struct { + manifest *v1.IndexManifest + raw []byte + hash v1.Hash +} + +var _ v1.ImageIndex = (*referrersIndex)(nil) + +func newReferrersIndex(found []ocispec.Descriptor) (*referrersIndex, error) { + manifest := &v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex, Manifests: make([]v1.Descriptor, 0, len(found))} + for i := range found { + hash, err := v1.NewHash(found[i].Digest.String()) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + manifest.Manifests = append(manifest.Manifests, v1.Descriptor{ + MediaType: types.MediaType(found[i].MediaType), Size: found[i].Size, Digest: hash, + Annotations: maps.Clone(found[i].Annotations), ArtifactType: found[i].ArtifactType, + URLs: slices.Clone(found[i].URLs), Data: slices.Clone(found[i].Data), Platform: legacyPlatform(found[i].Platform), + }) + } + raw, err := json.Marshal(manifest) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + hash, _, err := v1.SHA256(bytes.NewReader(raw)) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + return &referrersIndex{manifest: manifest, raw: raw, hash: hash}, nil +} + +// legacyPlatform converts an OCI platform to the legacy descriptor's, copying the feature list so +// the index never aliases the registry response. +func legacyPlatform(platform *ocispec.Platform) *v1.Platform { + if platform == nil { + return nil + } + return &v1.Platform{ + Architecture: platform.Architecture, OS: platform.OS, OSVersion: platform.OSVersion, + OSFeatures: slices.Clone(platform.OSFeatures), Variant: platform.Variant, + } +} + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) MediaType() (types.MediaType, error) { return types.OCIImageIndex, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) Digest() (v1.Hash, error) { return r.hash, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) Size() (int64, error) { return int64(len(r.raw)), nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) IndexManifest() (*v1.IndexManifest, error) { return r.manifest, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) RawManifest() ([]byte, error) { return r.raw, nil } + +// Image is unsupported: a referrers response holds descriptors, not content. +// +//nolint:lintroller // Constant error. +func (r *referrersIndex) Image(v1.Hash) (v1.Image, error) { + return nil, errUtils.ErrOCIRegistry +} + +// ImageIndex is unsupported: a referrers response holds descriptors, not content. +// +//nolint:lintroller // Constant error. +func (r *referrersIndex) ImageIndex(v1.Hash) (v1.ImageIndex, error) { + return nil, errUtils.ErrOCIRegistry +} diff --git a/pkg/oci/transport_legacy_children_test.go b/pkg/oci/transport_legacy_children_test.go new file mode 100644 index 00000000000..453653918e7 --- /dev/null +++ b/pkg/oci/transport_legacy_children_test.go @@ -0,0 +1,80 @@ +package oci + +import ( + "bytes" + "io" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// An index child that is an image with a streaming layer has no digest until the stream is read once, so +// every child is prepared before the index manifest, whose descriptors depend on it, is read. +func TestLegacyIndexGraphPreparesStreamingChildrenBeforeTheIndexManifest(t *testing.T) { + payload := bytes.Repeat([]byte("streamed layer bytes "), 64) + image, err := mutate.AppendLayers(empty.Image, stream.NewLayer(io.NopCloser(bytes.NewReader(payload)))) + require.NoError(t, err) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: image}) + + graph, err := legacyIndexGraph(t.Context(), index) + require.NoError(t, err) + t.Cleanup(graph.close) + + raw, err := index.RawManifest() + require.NoError(t, err, "the index manifest exists once its children were prepared") + assert.Equal(t, digest.FromBytes(raw), graph.root.Digest, "the root describes the manifest the index now serves") + layers, err := image.Layers() + require.NoError(t, err) + hash, err := layers[0].Digest() + require.NoError(t, err) + exists, err := graph.Exists(t.Context(), ocispec.Descriptor{Digest: digest.Digest(hash.String())}) + require.NoError(t, err) + assert.True(t, exists, "the streamed layer is served from its spool") +} + +// A child that is a layer rather than an image or an index is a blob the registry must receive too. +func TestLegacyIndexGraphServesLayerChildren(t *testing.T) { + blob := []byte("an artifact blob that is neither an image nor an index") + layer := static.NewLayer(blob, types.MediaType("application/vnd.example.blob.v1")) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: layer}) + + graph, err := legacyIndexGraph(t.Context(), index) + require.NoError(t, err) + t.Cleanup(graph.close) + + hash, err := layer.Digest() + require.NoError(t, err) + reader, err := graph.Fetch(t.Context(), ocispec.Descriptor{Digest: digest.Digest(hash.String())}) + require.NoError(t, err) + defer reader.Close() + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, blob, got) +} + +// Schema 1 images have no manifest the legacy graph can carry as OCI content, so they are refused with an +// explicit error instead of failing later in Manifest(). +func TestLegacyImageGraphRefusesSchemaOneImages(t *testing.T) { + image, err := random.Image(64, 1) + require.NoError(t, err) + for _, mediaType := range []types.MediaType{types.DockerManifestSchema1, types.DockerManifestSchema1Signed} { + t.Run(string(mediaType), func(t *testing.T) { + graph, err := legacyImageGraph(t.Context(), mutate.MediaType(image, mediaType)) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Contains(t, err.Error(), "schema 1") + assert.Nil(t, graph) + }) + } +} diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go new file mode 100644 index 00000000000..9cbf2987741 --- /dev/null +++ b/pkg/oci/transport_registry.go @@ -0,0 +1,316 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "io" + "net/http" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + ggcrremote "github.com/google/go-containerregistry/pkg/v1/remote" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/errcode" + "oras.land/oras-go/v2/registry/remote/retry" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + // The largest manifest pushed under a reference: the 4 MiB limit registries commonly enforce. + maxPushedManifest = 4 << 20 + + userAgent = "atmos" + dockerHubLegacy = "index.docker.io" + dockerHubRegistry = "docker.io" + dockerHubEndpoint = "registry-1.docker.io" +) + +// Repository opens a registry repository with credentials resolved up front. Resolution +// follows the same precedence as the legacy methods (bound identity, keychain, GHCR +// settings, fallback); an explicitly bound identity that fails stops here, before any +// request, and is never replaced by anonymous access. +// +// Every call builds a new credential cache, so tokens obtained for one identity can +// never be presented by another client or for another repository open. +func (c *Client) Repository(ctx context.Context, repository Repository) (RepositoryStore, error) { + defer perf.Track(c.config, "oci.Client.Repository")() + + resource, err := repository.legacy() + if err != nil { + return nil, err + } + authenticator, err := c.authentication(ctx, resource) + if err != nil { + return nil, err + } + // Resolve the bound credential once before any request, so a failing identity stops here + // instead of surfacing mid-operation or degrading to anonymous access. The callback below still + // re-invokes the authenticator on every challenge, so refreshing tokens keep refreshing. + if _, err := credentialFrom(authenticator); err != nil { + return nil, err + } + repo, err := c.remoteRepository(resource, authenticator) + if err != nil { + return nil, err + } + return ®istryStore{repo: repo}, nil +} + +func (c *Client) remoteRepository(resource name.Repository, authenticator authn.Authenticator) (*remote.Repository, error) { + host := orasHost(resource.RegistryStr()) + repo, err := remote.NewRepository(host + "/" + resource.RepositoryStr()) + if err != nil { + return nil, wrapRegistry(err) + } + repo.PlainHTTP = resource.Scheme() == "http" + repo.Client = &auth.Client{ + Client: &http.Client{Transport: newRegistryTransport(c.transport)}, + Header: http.Header{"User-Agent": {userAgent}}, + Cache: auth.NewCache(), + Credential: hostScopedCredential(host, authenticator), + } + return repo, nil +} + +func orasHost(registry string) string { + if strings.EqualFold(registry, dockerHubLegacy) { + return dockerHubRegistry + } + return registry +} + +// hostScopedCredential releases credentials only to the registry that the repository +// was opened for. ORAS also asks for credentials when it contacts a token realm; the +// registry-scoped credential is the one the legacy transport sent there as well. +func hostScopedCredential(registry string, authenticator authn.Authenticator) auth.CredentialFunc { + return func(_ context.Context, hostport string) (auth.Credential, error) { + if !hostMatches(registry, hostport) { + return auth.EmptyCredential, nil + } + return credentialFrom(authenticator) + } +} + +func hostMatches(registry, hostport string) bool { + if strings.EqualFold(registry, hostport) { + return true + } + return strings.EqualFold(registry, dockerHubRegistry) && strings.EqualFold(hostport, dockerHubEndpoint) +} + +// credentialFrom adapts a go-containerregistry authenticator. Authorization is invoked on +// every challenge, so authenticators that refresh tokens keep refreshing. +func credentialFrom(authenticator authn.Authenticator) (auth.Credential, error) { + if authenticator == nil || authenticator == authn.Anonymous { + return auth.EmptyCredential, nil + } + config, err := authenticator.Authorization() + if err != nil { + return auth.EmptyCredential, wrapRegistry(err) + } + if config == nil { + return auth.EmptyCredential, nil + } + credential := auth.Credential{ + Username: config.Username, + Password: config.Password, + RefreshToken: config.IdentityToken, + AccessToken: config.RegistryToken, + } + // A registry bearer token takes precedence over Basic credentials, even malformed Auth. + if credential.AccessToken != "" { + return credential, nil + } + // An encoded Auth fills whichever of the username and password is missing, the way the registry + // transport does; a username or password given explicitly keeps its value. + if config.Auth != "" && (credential.Username == "" || credential.Password == "") { + if err := fillFromEncodedAuth(&credential, config.Auth); err != nil { + return auth.EmptyCredential, err + } + } + return credential, nil +} + +// fillFromEncodedAuth decodes a base64 "user:password" value and fills the username and password the +// credential does not already carry. +func fillFromEncodedAuth(credential *auth.Credential, encoded string) error { + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return wrapRegistry(err) + } + user, password, found := strings.Cut(string(decoded), ":") + if !found { + return errUtils.ErrOCIRegistry + } + if credential.Username == "" { + credential.Username = user + } + if credential.Password == "" { + credential.Password = password + } + return nil +} + +// registryTransport retries only idempotent reads. Mutations are sent exactly once; +// an unknown outcome is reconciled by resolving the destination instead. +type registryTransport struct { + reads http.RoundTripper + base http.RoundTripper +} + +func newRegistryTransport(base http.RoundTripper) http.RoundTripper { + if base == nil { + // Keep the transport tuning (proxy, dialer, TLS defaults) the legacy client used. + base = ggcrremote.DefaultTransport + } + return registryTransport{reads: &retry.Transport{Base: base}, base: base} +} + +//nolint:lintroller // Hot-path transport wrapper that only selects a round tripper. +func (t registryTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.Method == http.MethodGet || request.Method == http.MethodHead { + return t.reads.RoundTrip(request) + } + return t.base.RoundTrip(request) +} + +// registryStore adapts one ORAS repository to RepositoryStore. +type registryStore struct { + repo *remote.Repository +} + +func (s *registryStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := s.repo.Fetch(ctx, target) + return reader, wrapRegistry(err) +} + +func (s *registryStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := s.repo.Exists(ctx, target) + return exists, wrapRegistry(err) +} + +func (s *registryStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + return wrapRegistry(s.repo.Push(ctx, expected, content)) +} + +// PushReference verifies the manifest bytes against expected before anything is sent, then pushes +// them under reference. A registry that omits Docker-Content-Digest therefore can never tag content +// other than the expected manifest. +func (s *registryStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + if expected.Size < 0 || expected.Size > maxPushedManifest { + return errUtils.ErrOCIIntegrity + } + raw, err := readVerified(content, expected, maxPushedManifest) + if err != nil { + return err + } + return wrapRegistry(s.repo.PushReference(ctx, expected, bytes.NewReader(raw), reference)) +} + +func (s *registryStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.registryStore.Resolve")() + + resolved, err := s.repo.Resolve(ctx, reference) + return resolved, wrapRegistry(err) +} + +func (s *registryStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + return wrapRegistry(s.repo.Tag(ctx, desc, reference)) +} + +func (s *registryStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + // Referrers applies the 400/406 tag-schema fallback, so discovery works without a prior Referrers call. + return s.Referrers(ctx, node, "") +} + +func (s *registryStore) Tags(ctx context.Context) ([]string, error) { + defer perf.Track(nil, "oci.registryStore.Tags")() + + var tags []string + err := s.repo.Tags(ctx, "", func(page []string) error { + tags = append(tags, page...) + return nil + }) + if err != nil { + return nil, wrapRegistry(err) + } + return tags, nil +} + +// Delete removes a manifest. A descriptor without a media type names the manifest by digest only: +// it is deleted with a single DELETE request that needs no read access and performs no referrers +// index cleanup, because the manifest is never fetched. +func (s *registryStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + if target.MediaType == "" { + return wrapRegistry(s.repo.Manifests().Delete(ctx, target)) + } + err := s.repo.Delete(ctx, target) + if err != nil && referrersProbeUnavailable(err) { + // Deleting an attachment first probes the referrers API to decide whether the tag-schema index needs + // updating. A 400 or 406 on that probe fails the call before any DELETE is sent, so settling the + // capability as unsupported and calling again never repeats a DELETE. If the capability was already + // settled the first answer stands. + if capabilityErr := s.repo.SetReferrersCapability(false); capabilityErr == nil { + err = s.repo.Delete(ctx, target) + } + } + return wrapRegistry(err) +} + +func (s *registryStore) Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { + found, err := s.listReferrers(ctx, subject, artifactType) + if err != nil && referrersAPIUnavailable(err) { + // The registry answered the referrers API with 400 or 406 instead of 404: the API is not available + // there. Mark it unsupported, which switches this repository to the tag schema, and list again. If the + // capability was already settled the first answer stands. + if capabilityErr := s.repo.SetReferrersCapability(false); capabilityErr == nil { + found, err = s.listReferrers(ctx, subject, artifactType) + } + } + if err != nil { + return nil, wrapRegistry(err) + } + return found, nil +} + +func (s *registryStore) listReferrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { + var found []ocispec.Descriptor + err := s.repo.Referrers(ctx, subject, artifactType, func(page []ocispec.Descriptor) error { + found = append(found, page...) + return nil + }) + return found, err +} + +// referrersAPIUnavailable reports whether err is a registry response that says the referrers API is not +// available: 400 Bad Request or 406 Not Acceptable. Every other error, including other statuses, is a failure. +func referrersAPIUnavailable(err error) bool { + var response *errcode.ErrorResponse + if !errors.As(err, &response) { + return false + } + return response.StatusCode == http.StatusBadRequest || response.StatusCode == http.StatusNotAcceptable +} + +// referrersProbeUnavailable reports whether err is a referrers API answer of 400 or 406 to a read of the +// referrers endpoint. A response to any other request, a DELETE included, never qualifies. +func referrersProbeUnavailable(err error) bool { + var response *errcode.ErrorResponse + if !errors.As(err, &response) || !referrersAPIUnavailable(err) { + return false + } + return response.Method == http.MethodGet && response.URL != nil && strings.Contains(response.URL.Path, "/referrers/") +} + +var ( + _ RepositoryStore = (*registryStore)(nil) + _ Transport = (*Client)(nil) +) diff --git a/pkg/oci/transport_roundtrip_test.go b/pkg/oci/transport_roundtrip_test.go new file mode 100644 index 00000000000..408365ba6f5 --- /dev/null +++ b/pkg/oci/transport_roundtrip_test.go @@ -0,0 +1,341 @@ +//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts. +package oci_test + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" + "github.com/cloudposse/atmos/pkg/oci/signature" + "github.com/cloudposse/atmos/pkg/oci/signature/signaturetest" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" +) + +// pluginReleaseIndexDigest is the pinned root digest of the committed A00 plugin release fixture +// (pkg/artifact/compat/testdata/plugin-release). It must never change. +const pluginReleaseIndexDigest = "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + +func pluginFixtureDirectory() string { + return filepath.Join("..", "artifact", "compat", "testdata", "plugin-release") +} + +// authenticatedRegistry starts a registry that accepts exactly one identity and returns a client +// that can only present that identity. +func authenticatedRegistry(t *testing.T, password string) (*ocitest.Server, *oci.Client) { + t.Helper() + server := ocitest.NewServerWithoutReferrers(t, &ocitest.Credential{Username: "publisher", Password: password}) + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + require.NoError(t, chain.Login(server.Host, &authn.AuthConfig{Username: "publisher", Password: password})) + return server, oci.NewClient(nil, oci.WithKeychain(chain)) +} + +func openRepository(t *testing.T, client *oci.Client, host string) oci.RepositoryStore { + t.Helper() + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + return store +} + +// readTree returns every blob below a layout, keyed by file name. +func readBlobs(t *testing.T, directory string) map[string][]byte { + t.Helper() + blobs := map[string][]byte{} + root := filepath.Join(directory, "blobs", "sha256") + require.NoError(t, filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + content, err := os.ReadFile(path) + blobs[entry.Name()] = content + return err + })) + require.NotEmpty(t, blobs) + return blobs +} + +// attachBundle publishes a Sigstore bundle as a subject-bound referrer, exactly like plugin signing. +func attachBundle(t *testing.T, client *oci.Client, repository name.Repository, subject *v1.Descriptor, raw []byte) { + t.Helper() + image, err := mutate.AppendLayers(empty.Image, static.NewLayer(raw, types.MediaType("application/json"))) + require.NoError(t, err) + image = mutate.MediaType(image, types.OCIManifestSchema1) + image = mutate.ConfigMediaType(image, types.MediaType("application/vnd.dev.sigstore.bundle.v0.3+json")) + image = mutate.Subject(image, *subject).(v1.Image) + digest, err := image.Digest() + require.NoError(t, err) + require.NoError(t, client.WriteImage(t.Context(), repository.Digest(digest.String()), image)) +} + +type roundTripCase struct { + name string + prepare func(t *testing.T) (directory, digest string) + inspectors func(t *testing.T, client *oci.Client, repository name.Repository, digest string) any +} + +func twoPlatformLayout(t *testing.T) (string, string) { + t.Helper() + index := testIndex(t) + directory := filepath.Join(t.TempDir(), "source") + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + return directory, digest.String() +} + +// inspectedRelease captures what plugin verification derives from a release. +func inspectedRelease(t *testing.T, client *oci.Client, repository name.Repository, digest string) any { + t.Helper() + descriptor, err := client.Get(t.Context(), repository.Digest(digest)) + require.NoError(t, err) + index, err := descriptor.ImageIndex() + require.NoError(t, err) + release, err := pluginoci.Inspect(index, digest) + require.NoError(t, err) + platforms := map[string]string{} + for platform, image := range release.Platforms { + platforms[platform] = image.Digest.String() + } + return map[string]any{"name": release.Name, "version": release.Version, "digest": release.Digest, "platforms": platforms} +} + +func manifestDigests(t *testing.T, client *oci.Client, repository name.Repository, digest string) any { + t.Helper() + descriptor, err := client.Get(t.Context(), repository.Digest(digest)) + require.NoError(t, err) + index, err := descriptor.ImageIndex() + require.NoError(t, err) + manifest, err := index.IndexManifest() + require.NoError(t, err) + var digests []string + for i := range manifest.Manifests { + digests = append(digests, manifest.Manifests[i].Digest.String()+"/"+manifest.Manifests[i].Platform.Architecture) + } + return digests +} + +// TestRegistryLayoutRegistryRoundTripKeepsDigestsAndVerification moves releases registry A -> +// OCI layout -> registry B. The registries accept different credentials and A is shut down +// before the final hop, so the layout alone must carry the complete graph, including the signature +// referrer. Digests, blobs, release inspection, and signature verification must not change. +func TestRegistryLayoutRegistryRoundTripKeepsDigestsAndVerification(t *testing.T) { + ctx := t.Context() + for _, tc := range []roundTripCase{ + {"plugin release fixture", func(t *testing.T) (string, string) { return pluginFixtureDirectory(), pluginReleaseIndexDigest }, inspectedRelease}, + {"two-platform index", twoPlatformLayout, manifestDigests}, + } { + t.Run(tc.name, func(t *testing.T) { + sourceDirectory, digest := tc.prepare(t) + sourceBlobs := readBlobs(t, sourceDirectory) + source, err := oci.OpenLayoutReadOnly(ctx, sourceDirectory) + require.NoError(t, err) + rootDescriptor, err := source.Resolve(ctx, digest) + require.NoError(t, err) + require.Equal(t, digest, rootDescriptor.Digest.String()) + + // Registry A: publish the release and a Sigstore bundle bound to its digest. + serverA, clientA := authenticatedRegistry(t, "secret-a") + storeA := openRepository(t, clientA, serverA.Host) + _, err = oci.Copy(ctx, oci.Source{Store: source, Reference: digest}, oci.Destination{Store: storeA, Reference: "1.2.3"}) + require.NoError(t, err) + repositoryA, err := name.NewRepository(serverA.Host + "/acme/plugin") + require.NoError(t, err) + head, err := clientA.Head(ctx, repositoryA.Tag("1.2.3")) + require.NoError(t, err) + require.Equal(t, digest, head.Digest.String(), "pushing must not alter the release digest") + + rawRoot, err := oci.FetchBytes(ctx, source, rootDescriptor, 1<<20) + require.NoError(t, err) + authority := signaturetest.New(t) + identity := signature.Identity{Identity: "https://github.com/acme/plugin/.github/workflows/release.yml@refs/tags/v1.2.3", Issuer: "https://token.actions.githubusercontent.com"} + bundle := authority.Sign(t, identity.Identity, identity.Issuer, rawRoot) + verifier := signature.Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil }} + want, err := verifier.Verify(ctx, bundle, digest, []signature.Identity{identity}, true) + require.NoError(t, err) + attachBundle(t, clientA, repositoryA, head, bundle) + before := tc.inspectors(t, clientA, repositoryA, digest) + + // Hop 1: registry A -> layout, with referrers. + layoutDirectory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, layoutDirectory) + require.NoError(t, err) + copied, err := oci.ExtendedCopy(ctx, oci.GraphSource{Store: storeA, Reference: "1.2.3"}, oci.Destination{Store: layout, Reference: "1.2.3"}) + require.NoError(t, err) + require.Equal(t, digest, copied.Digest.String()) + require.Equal(t, []string{"1.2.3"}, layout.References()) + layoutBlobs := readBlobs(t, layoutDirectory) + for name, content := range sourceBlobs { + require.Equal(t, content, layoutBlobs[name], "blob %s changed in transit", name) + } + require.Len(t, layoutBlobs, len(sourceBlobs)+3, "signature manifest, config, and bundle layer travel with the release") + + // Registry A disappears; registry B has different credentials. + serverA.Close() + serverB, clientB := authenticatedRegistry(t, "secret-b") + storeB := openRepository(t, clientB, serverB.Host) + + // Hop 2 reads the layout from disk again, proving the on-disk state is self-sufficient. + reopened, err := oci.OpenLayout(ctx, layoutDirectory) + require.NoError(t, err) + _, err = oci.ExtendedCopy(ctx, oci.GraphSource{Store: reopened, Reference: "1.2.3"}, oci.Destination{Store: storeB, Reference: "1.2.3"}) + require.NoError(t, err) + + repositoryB, err := name.NewRepository(serverB.Host + "/acme/plugin") + require.NoError(t, err) + after, err := clientB.Head(ctx, repositoryB.Tag("1.2.3")) + require.NoError(t, err) + require.Equal(t, digest, after.Digest.String()) + require.Equal(t, before, tc.inspectors(t, clientB, repositoryB, digest)) + tags, err := clientB.Tags(ctx, repositoryB) + require.NoError(t, err) + // The registry fixture lacks the OCI-Subject header, so the attachment is also indexed + // under the standard referrers tag-schema fallback tag. + require.ElementsMatch(t, []string{"1.2.3", "sha256-" + strings.TrimPrefix(digest, "sha256:")}, tags) + + bundles, err := signature.Fetch(ctx, clientB, repositoryB.Digest(digest)) + require.NoError(t, err) + require.Equal(t, [][]byte{bundle}, bundles) + got, err := verifier.Verify(ctx, bundles[0], digest, []signature.Identity{identity}, true) + require.NoError(t, err) + require.Equal(t, want, got, "verification result must survive the move") + + // Registry B's credentials never worked on A, and A's never work on B. + _, err = oci.Copy(ctx, oci.Source{Store: reopened, Reference: "1.2.3"}, oci.Destination{Store: openRepository(t, clientA, serverB.Host), Reference: "again"}) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + }) + } +} + +func TestLayoutHoldsSeveralReferencesWithoutWeakeningLoadIndex(t *testing.T) { + ctx := t.Context() + twoPlatformDirectory, twoPlatformDigest := twoPlatformLayout(t) + releaseStore, err := oci.OpenLayoutReadOnly(ctx, pluginFixtureDirectory()) + require.NoError(t, err) + otherStore, err := oci.OpenLayoutReadOnly(ctx, twoPlatformDirectory) + require.NoError(t, err) + + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: releaseStore, Reference: pluginReleaseIndexDigest}, oci.Destination{Store: layout, Reference: "plugin-1.2.3"}) + require.NoError(t, err) + + // One release in a layout still satisfies the single-release validation. + index, err := oci.LoadIndex(directory, pluginReleaseIndexDigest) + require.NoError(t, err) + digest, err := index.Digest() + require.NoError(t, err) + assert.Equal(t, pluginReleaseIndexDigest, digest.String()) + + _, err = oci.Copy(ctx, oci.Source{Store: otherStore, Reference: twoPlatformDigest}, oci.Destination{Store: layout, Reference: "two-platform"}) + require.NoError(t, err) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, layout.References()) + _, err = oci.LoadIndex(directory, "") + require.ErrorIs(t, err, errUtils.ErrOCILayout, "several references must still be rejected by LoadIndex") + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, reopened.References()) + for reference, want := range map[string]string{"plugin-1.2.3": pluginReleaseIndexDigest, "two-platform": twoPlatformDigest} { + resolved, err := reopened.Resolve(ctx, reference) + require.NoError(t, err) + assert.Equal(t, want, resolved.Digest.String(), reference) + } + + // Re-tagging an existing reference replaces it rather than adding a duplicate entry. + resolved, err := reopened.Resolve(ctx, "two-platform") + require.NoError(t, err) + require.NoError(t, reopened.Tag(ctx, resolved, "two-platform")) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, reopened.References()) +} + +func TestLayoutMovingATagOntoAnUnnamedRootLeavesASingleEntry(t *testing.T) { + ctx := t.Context() + twoPlatformDirectory, twoPlatformDigest := twoPlatformLayout(t) + releaseStore, err := oci.OpenLayoutReadOnly(ctx, pluginFixtureDirectory()) + require.NoError(t, err) + otherStore, err := oci.OpenLayoutReadOnly(ctx, twoPlatformDirectory) + require.NoError(t, err) + + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: releaseStore, Reference: pluginReleaseIndexDigest}, oci.Destination{Store: layout, Reference: "release"}) + require.NoError(t, err) + // The second index is listed as an unnamed root first, then the tag moves onto it. + _, err = oci.Copy(ctx, oci.Source{Store: otherStore, Reference: twoPlatformDigest}, oci.Destination{Store: layout, Reference: twoPlatformDigest}) + require.NoError(t, err) + moved, err := otherStore.Resolve(ctx, twoPlatformDigest) + require.NoError(t, err) + require.NoError(t, layout.Tag(ctx, moved, "release")) + + index, err := oci.LoadIndex(directory, twoPlatformDigest) + require.NoError(t, err, "the moved tag leaves exactly one root entry") + require.NotNil(t, index) + + server, client := authenticatedRegistry(t, "secret") + ref, err := name.NewTag(server.Host + "/acme/plugin:moved") + require.NoError(t, err) + require.NoError(t, oci.ImportIndex(ctx, client, directory, ref)) +} + +// TestLegacyWriteIndexPublishesTheExistingReleaseBytes proves the bridge serializes a legacy +// index's raw bytes: publishing the committed plugin release through the old Registry API must yield +// byte-identical blobs and the pinned digest, never a rebuilt release. +func TestLegacyWriteIndexPublishesTheExistingReleaseBytes(t *testing.T) { + ctx := t.Context() + index, err := oci.LoadIndex(pluginFixtureDirectory(), pluginReleaseIndexDigest) + require.NoError(t, err) + server, client := authenticatedRegistry(t, "secret") + ref, err := name.NewTag(server.Host + "/acme/plugin:1.2.3") + require.NoError(t, err) + require.NoError(t, client.WriteIndex(ctx, ref, index)) + + head, err := client.Head(ctx, ref) + require.NoError(t, err) + require.Equal(t, pluginReleaseIndexDigest, head.Digest.String()) + + // Pull the registry's content back into a layout: every blob must equal the committed fixture. + directory := filepath.Join(t.TempDir(), "pulled") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: openRepository(t, client, server.Host), Reference: "1.2.3"}, oci.Destination{Store: layout, Reference: "1.2.3"}) + require.NoError(t, err) + want := readBlobs(t, pluginFixtureDirectory()) + got := readBlobs(t, directory) + require.Len(t, got, len(want)) + for name, content := range want { + require.Equal(t, content, got[name], "blob %s differs from the committed release", name) + } + + descriptor, err := client.Get(ctx, ref) + require.NoError(t, err) + assert.Equal(t, pluginReleaseIndexDigest, descriptor.Digest.String()) + assert.NotNil(t, inspectedRelease(t, client, mustRepository(t, server.Host), pluginReleaseIndexDigest)) +} + +func mustRepository(t *testing.T, host string) name.Repository { + t.Helper() + repository, err := name.NewRepository(host + "/acme/plugin") + require.NoError(t, err) + return repository +} diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go new file mode 100644 index 00000000000..df83494d5e1 --- /dev/null +++ b/pkg/oci/transport_subject.go @@ -0,0 +1,321 @@ +package oci + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxManifestKeyBytes bounds how much of an object key the subject scan retains. A longer key can +// never be the "subject" field, so the remainder is skipped without being kept. +const maxManifestKeyBytes = 64 + +// readSubject streams a stored manifest or index and returns its subject, or nil when it has none. +// The scan is bounded by the descriptor size and the whole body is verified against its size and +// digest before any result is returned. It keeps constant memory: every other top-level field, +// including arbitrarily long strings, is skipped byte by byte without being buffered. A large +// manifest without a subject is therefore never rejected. +func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { + if desc.Size < 0 { + return nil, errUtils.ErrOCIIntegrity + } + if err := desc.Digest.Validate(); err != nil { + return nil, wrapIntegrity(err) + } + reader, err := store.Fetch(ctx, *desc) + if err != nil { + return nil, err + } + defer reader.Close() + // Some stores, such as ORAS local storage, ignore the fetch context, so every read is guarded here. + subject, err := verifySubject(ctx, contextReader{ctx: ctx, reader: reader}, desc) + if err != nil { + return nil, canceledOr(ctx, err) + } + // A cancellation that raced the final reads must not be reported as a verified result. + if err := ctx.Err(); err != nil { + return nil, err + } + return subject, nil +} + +// verifySubject scans the body for the subject and then reads the remainder so the size and digest +// cover every byte. +func verifySubject(ctx context.Context, reader io.Reader, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { + check := &bodyCheck{verifier: desc.Digest.Verifier()} + scanner := bufio.NewReader(io.TeeReader(io.LimitReader(reader, desc.Size), check)) + if err := expectByte(scanner, '{'); err != nil { + return nil, integrityError(err) + } + subject, err := scanForSubject(scanner) + if err != nil { + return nil, err + } + // The scan may stop at the subject or at the closing brace; either way the rest of the body is read + // so the size and digest cover every byte before the result is trusted. + if _, err := io.Copy(io.Discard, scanner); err != nil { + return nil, integrityError(err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if err := check.finish(reader, desc.Size); err != nil { + return nil, err + } + return subject, nil +} + +// canceledOr returns the context error when the context ended, since a canceled read is not an +// integrity failure; otherwise it returns err unchanged. +func canceledOr(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("reading the manifest subject: %w", ctxErr) + } + return err +} + +// contextReader fails a read as soon as the context is done. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (c contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "oci.contextReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.reader.Read(p) +} + +// bodyCheck accumulates the size and digest of the bytes a scan consumed. +type bodyCheck struct { + verifier digest.Verifier + size int64 +} + +func (b *bodyCheck) Write(p []byte) (int, error) { + defer perf.Track(nil, "oci.bodyCheck.Write")() + + b.size += int64(len(p)) + return b.verifier.Write(p) +} + +// finish reports an integrity failure unless exactly want bytes were read, nothing follows them in +// source, and they match the descriptor digest. +func (b *bodyCheck) finish(source io.Reader, want int64) error { + if b.size != want { + return fmt.Errorf("%w: the manifest has %d bytes but %d were declared", errUtils.ErrOCIIntegrity, b.size, want) + } + var extra [1]byte + for { + n, err := source.Read(extra[:]) + if n > 0 { + return fmt.Errorf("%w: the manifest is longer than the declared %d bytes", errUtils.ErrOCIIntegrity, want) + } + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return integrityError(err) + } + } + if !b.verifier.Verified() { + return fmt.Errorf("%w: the manifest does not match its digest", errUtils.ErrOCIIntegrity) + } + return nil +} + +// scanForSubject reads the members of a manifest object, after its opening brace, until it finds +// the subject descriptor or the object ends. +func scanForSubject(scanner *bufio.Reader) (*ocispec.Descriptor, error) { + for { + next, err := nextNonSpace(scanner) + if err != nil { + return nil, integrityError(err) + } + switch next { + case '}': + return nil, nil + case ',': + continue + case '"': + default: + return nil, fmt.Errorf("%w: unexpected %q in the manifest object", errUtils.ErrOCIIntegrity, next) + } + subject, found, err := scanField(scanner) + if err != nil { + return nil, integrityError(err) + } + if found { + return subject, nil + } + } +} + +// scanField consumes one object member whose opening quote was already read. It decodes the member +// only when it is the subject; every other value is skipped. +func scanField(scanner *bufio.Reader) (*ocispec.Descriptor, bool, error) { + name, err := readKey(scanner) + if err != nil { + return nil, false, err + } + if err := expectByte(scanner, ':'); err != nil { + return nil, false, err + } + if !strings.EqualFold(name, "subject") { + return nil, false, skipValue(scanner) + } + var subject *ocispec.Descriptor + if err := json.NewDecoder(scanner).Decode(&subject); err != nil { + return nil, false, err + } + return subject, true, nil +} + +// integrityError classifies a manifest scan failure with ErrOCIIntegrity and keeps the cause. An +// error that already carries the sentinel is returned unchanged. +func integrityError(err error) error { + if errors.Is(err, errUtils.ErrOCIIntegrity) { + return err + } + return fmt.Errorf("%w: reading the manifest subject: %w", errUtils.ErrOCIIntegrity, err) +} + +// nextNonSpace returns the next byte that is not JSON whitespace. +func nextNonSpace(scanner *bufio.Reader) (byte, error) { + for { + next, err := scanner.ReadByte() + if err != nil { + return 0, err + } + switch next { + case ' ', '\t', '\r', '\n': + default: + return next, nil + } + } +} + +func expectByte(scanner *bufio.Reader, want byte) error { + next, err := nextNonSpace(scanner) + if err != nil { + return err + } + if next != want { + return fmt.Errorf("%w: expected %q in the manifest but found %q", errUtils.ErrOCIIntegrity, want, next) + } + return nil +} + +// readKey reads an object key whose opening quote was consumed. It retains at most +// maxManifestKeyBytes of the key; an overlong key comes back empty because it cannot be "subject". +func readKey(scanner *bufio.Reader) (string, error) { + var raw bytes.Buffer + overflow, err := scanString(scanner, &raw, maxManifestKeyBytes) + if err != nil || overflow { + return "", err + } + if !bytes.ContainsRune(raw.Bytes(), '\\') { + return raw.String(), nil + } + var name string + if err := json.Unmarshal([]byte(`"`+raw.String()+`"`), &name); err != nil { + return "", err + } + return name, nil +} + +// scanString consumes a JSON string whose opening quote was already read. It copies at most limit +// bytes into keep (when keep is not nil) and reports whether the string was longer. Memory stays +// constant however long the string is. +func scanString(scanner *bufio.Reader, keep *bytes.Buffer, limit int) (bool, error) { + overflow := false + escaped := false + for { + next, err := scanner.ReadByte() + if err != nil { + return overflow, err + } + if !escaped && next == '"' { + return overflow, nil + } + escaped = !escaped && next == '\\' + if keep == nil { + continue + } + if keep.Len() >= limit { + overflow = true + continue + } + keep.WriteByte(next) + } +} + +// skipValue consumes one JSON value, however deeply nested or long, without buffering it. +func skipValue(scanner *bufio.Reader) error { + first, err := nextNonSpace(scanner) + if err != nil { + return err + } + switch first { + case '"': + _, err := scanString(scanner, nil, 0) + return err + case '{', '[': + return skipContainer(scanner) + default: + return skipScalar(scanner) + } +} + +// skipContainer consumes the rest of an object or array whose opening bracket was already read. +func skipContainer(scanner *bufio.Reader) error { + depth := 1 + for depth > 0 { + next, err := scanner.ReadByte() + if err != nil { + return err + } + switch next { + case '"': + if _, err := scanString(scanner, nil, 0); err != nil { + return err + } + case '{', '[': + depth++ + case '}', ']': + depth-- + } + } + return nil +} + +// skipScalar consumes the rest of a number, boolean, or null, leaving the delimiter unread. +func skipScalar(scanner *bufio.Reader) error { + for { + next, err := scanner.ReadByte() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + switch next { + case ',', '}', ']', ' ', '\t', '\r', '\n': + return scanner.UnreadByte() + } + } +} diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go new file mode 100644 index 00000000000..9f5becb534e --- /dev/null +++ b/pkg/oci/transport_subject_test.go @@ -0,0 +1,249 @@ +package oci_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/oci" +) + +// layoutRoots reads the digests listed in a layout's index.json. +func layoutRoots(t *testing.T, directory string) []string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + var index ocispec.Index + require.NoError(t, json.Unmarshal(raw, &index)) + roots := make([]string, 0, len(index.Manifests)) + for _, manifest := range index.Manifests { + roots = append(roots, manifest.Digest.String()) + } + return roots +} + +func TestLayoutPush_RecordsOnlyManifestsThatNameASubject(t *testing.T) { + const fourMiB = 4 << 20 + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + padding := strings.Repeat("p", fourMiB+1024) + tests := []struct { + name string + manifest string + wantRecord bool + wantErr error + }{ + {"attachment is recorded", `{"schemaVersion":2,"subject":` + subject + `}`, true, nil}, + {"subject after other fields is found", `{"schemaVersion":2,"layers":[{"a":[1,{"b":null}]}],"annotations":{"k":"v"},"subject":` + subject + `}`, true, nil}, + {"manifest without a subject is not recorded", `{"schemaVersion":2,"layers":[]}`, false, nil}, + {"null subject is not recorded", `{"schemaVersion":2,"subject":null}`, false, nil}, + {"large manifest without a subject is accepted", `{"schemaVersion":2,"annotations":{"padding":"` + padding + `"}}`, false, nil}, + {"large attachment is refused", `{"schemaVersion":2,"annotations":{"padding":"` + padding + `"},"subject":` + subject + `}`, false, errUtils.ErrOCIIntegrity}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + descriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(tt.manifest)) + + err = layout.Push(ctx, descriptor, strings.NewReader(tt.manifest)) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + return + } + require.NoError(t, err) + exists, err := layout.Exists(ctx, descriptor) + require.NoError(t, err) + assert.True(t, exists) + if tt.wantRecord { + assert.Equal(t, []string{descriptor.Digest.String()}, layoutRoots(t, directory)) + } else { + assert.Empty(t, layoutRoots(t, directory)) + } + + // Pushing the same manifest again reports it exists and keeps the index free of duplicates. + before := layoutRoots(t, directory) + err = layout.Push(ctx, descriptor, strings.NewReader(tt.manifest)) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.Equal(t, before, layoutRoots(t, directory)) + }) + } +} + +// pushSubjectManifest pushes an image manifest that optionally names a subject and returns its descriptor. +func pushSubjectManifest(t *testing.T, layout *oci.Layout, subject *ocispec.Descriptor, note string) ocispec.Descriptor { + t.Helper() + ctx := t.Context() + config := []byte("{}") + configDesc := oci.Descriptor("application/vnd.oci.empty.v1+json", config) + if exists, err := layout.Exists(ctx, configDesc); err == nil && !exists { + require.NoError(t, layout.Push(ctx, configDesc, strings.NewReader(string(config)))) + } + manifest := ocispec.Manifest{Config: configDesc, Layers: []ocispec.Descriptor{}, Subject: subject, Annotations: map[string]string{"note": note}} + manifest.SchemaVersion = 2 + manifest.MediaType = ocispec.MediaTypeImageManifest + raw, err := json.Marshal(manifest) + require.NoError(t, err) + desc := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, layout.Push(ctx, desc, strings.NewReader(string(raw)))) + return desc +} + +// Moving a tag off an attachment manifest must not drop the attachment from the layout: it stays an +// unnamed root, so reopening the layout still finds it as a referrer of its subject. +func TestTagMovePreservesTheDisplacedAttachmentRoot(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + subject := pushSubjectManifest(t, layout, nil, "subject") + attachment := pushSubjectManifest(t, layout, &subject, "attachment") + replacement := pushSubjectManifest(t, layout, nil, "replacement") + require.NoError(t, layout.Tag(ctx, attachment, "moving")) + + require.NoError(t, layout.Tag(ctx, replacement, "moving")) + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + resolved, err := reopened.Resolve(ctx, "moving") + require.NoError(t, err) + assert.Equal(t, replacement.Digest, resolved.Digest, "the tag moved to the new manifest") + referrers, err := reopened.Predecessors(ctx, subject) + require.NoError(t, err) + digests := make([]string, 0, len(referrers)) + for _, referrer := range referrers { + digests = append(digests, referrer.Digest.String()) + } + assert.Contains(t, digests, attachment.Digest.String(), "the displaced attachment is still discoverable after reopening") + assert.Contains(t, layoutRoots(t, directory), attachment.Digest.String()) +} + +// A tag that could not be written to index.json would vanish when the layout is reopened, so it must not +// resolve: otherwise a recovery check would report success for a tag that was never persisted. +func TestResolveDoesNotTrustATagThatWasNotPersisted(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + root := pushSubjectManifest(t, layout, nil, "root") + + // Replace index.json with a non-empty directory so the atomic write cannot land on any platform. + indexPath := filepath.Join(directory, "index.json") + require.NoError(t, os.Remove(indexPath)) + require.NoError(t, os.MkdirAll(filepath.Join(indexPath, "blocker"), 0o755)) + + require.Error(t, layout.Tag(ctx, root, "1.0.0"), "the index write fails") + _, err = layout.Resolve(ctx, "1.0.0") + require.Error(t, err, "an unpersisted tag does not resolve") + require.ErrorIs(t, err, errdef.ErrNotFound) + + resolved, err := layout.Resolve(ctx, root.Digest.String()) + require.NoError(t, err, "a digest reference still resolves") + assert.Equal(t, root.Digest, resolved.Digest) +} + +// A relative layout root is anchored when the layout is opened: changing the working directory afterwards +// must not send index commits to a different directory than the one holding the blobs. +func TestOpenLayoutAnchorsARelativeRoot(t *testing.T) { + ctx := t.Context() + base := t.TempDir() + other := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(other, "layout"), 0o755)) + t.Chdir(base) + layout, err := oci.OpenLayout(ctx, "layout") + require.NoError(t, err) + root := pushSubjectManifest(t, layout, nil, "root") + + t.Chdir(other) + require.NoError(t, layout.Tag(ctx, root, "1.0.0")) + + assert.Contains(t, layoutRoots(t, filepath.Join(base, "layout")), root.Digest.String(), "the tag landed in the layout that was opened") + _, err = os.Stat(filepath.Join(other, "layout", "index.json")) + assert.ErrorIs(t, err, os.ErrNotExist, "an unrelated layout in the new working directory is untouched") +} + +// A reference that exists throughout a concurrent Tag keeps resolving: the resolver and the committed index +// are read under one lock, so the tag moving between two digests never looks unpersisted. +func TestResolveIsAtomicWithConcurrentTags(t *testing.T) { + ctx := t.Context() + layout, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "layout")) + require.NoError(t, err) + first := pushSubjectManifest(t, layout, nil, "first") + second := pushSubjectManifest(t, layout, nil, "second") + require.NoError(t, layout.Tag(ctx, first, "moving")) + + const rounds = 300 + var wg sync.WaitGroup + failures := make(chan error, rounds) + wg.Add(2) + go func() { + defer wg.Done() + for i := range rounds { + target := first + if i%2 == 0 { + target = second + } + if err := layout.Tag(ctx, target, "moving"); err != nil { + failures <- err + return + } + } + }() + go func() { + defer wg.Done() + for range rounds { + if _, err := layout.Resolve(ctx, "moving"); err != nil { + failures <- err + return + } + } + }() + wg.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } +} + +func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers(t *testing.T) { + ctx := t.Context() + source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) + require.NoError(t, err) + grandSubject := pushSubjectManifest(t, source, nil, "grand-subject") + root := pushSubjectManifest(t, source, &grandSubject, "root") + signature := pushSubjectManifest(t, source, &root, "signature") + require.NoError(t, source.Tag(ctx, root, "1.0.0")) + + dest, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "dest")) + require.NoError(t, err) + got, err := oci.ExtendedCopy(ctx, oci.GraphSource{Store: source, Reference: "1.0.0"}, oci.Destination{Store: dest, Reference: "1.0.0"}, oci.WithoutSubjectTraversal()) + require.NoError(t, err) + assert.Equal(t, root.Digest, got.Digest) + + for name, want := range map[string]struct { + desc ocispec.Descriptor + wanted bool + }{ + "the selected root": {root, true}, + "the referrer of the root": {signature, true}, + "the subject of the root": {grandSubject, false}, + } { + exists, err := dest.Exists(ctx, want.desc) + require.NoError(t, err, name) + assert.Equal(t, want.wanted, exists, name) + } + resolved, err := dest.Resolve(ctx, "1.0.0") + require.NoError(t, err) + assert.Equal(t, root.Digest, resolved.Digest) +} diff --git a/pkg/plugin/sign/sign.go b/pkg/plugin/sign/sign.go index 2fb2b7f31a9..9746e8bae85 100644 --- a/pkg/plugin/sign/sign.go +++ b/pkg/plugin/sign/sign.go @@ -2,9 +2,7 @@ package sign import ( - "bytes" "context" - "crypto/sha256" "encoding/json" "errors" "strings" @@ -15,15 +13,14 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/static" "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/sigstore/sigstore-go/pkg/bundle" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/oci" - "github.com/cloudposse/atmos/pkg/oci/signature" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/plugin/extension" pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" pluginpolicy "github.com/cloudposse/atmos/pkg/plugin/policy" + sigcore "github.com/cloudposse/atmos/pkg/signature" ) // BlobSigner produces a standard Sigstore bundle for the exact supplied bytes. @@ -94,19 +91,15 @@ func (s Service) signIndex(ctx context.Context, ref name.Digest, descriptor *v1. } func validateBundle(raw, artifact []byte) error { - if len(raw) == 0 || len(raw) > signature.MaxBundleSize { - return errUtils.ErrPluginSignatureInvalid + err := sigcore.CheckBundleSubject(raw, artifact) + if err == nil { + return nil } - var signed bundle.Bundle - if err := json.Unmarshal(raw, &signed); err != nil { - return errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - digest := sha256.Sum256(artifact) - message := signed.GetMessageSignature() - if message == nil || message.MessageDigest == nil || message.MessageDigest.Algorithm.String() != "SHA2_256" || !bytes.Equal(message.MessageDigest.Digest, digest[:]) { - return errUtils.ErrPluginSignatureInvalid + var shared *sigcore.Error + if errors.As(err, &shared) && shared.Cause != nil { + return errors.Join(errUtils.ErrPluginSignatureInvalid, shared.Cause) } - return nil + return errUtils.ErrPluginSignatureInvalid } func (s Service) attach(ctx context.Context, ref name.Digest, subject *v1.Descriptor, raw []byte) error { diff --git a/pkg/signature/boundary_test.go b/pkg/signature/boundary_test.go new file mode 100644 index 00000000000..2cece67492e --- /dev/null +++ b/pkg/signature/boundary_test.go @@ -0,0 +1,46 @@ +package signature + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPackageDependencyBoundary keeps the shared primitives free of transport and product layers. +func TestPackageDependencyBoundary(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/internal/exec", + "github.com/cloudposse/atmos/pkg/oci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/google/go-containerregistry", + "oras.land/oras-go", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/signature/doc.go b/pkg/signature/doc.go new file mode 100644 index 00000000000..ca6acbcaa52 --- /dev/null +++ b/pkg/signature/doc.go @@ -0,0 +1,9 @@ +// Package signature holds the reusable Sigstore verification and signing primitives shared by +// every artifact repository and the plugin installer: bundle verification against trusted +// certificate identities, a separate explicitly configured public-key bundle path, the trusted +// root, and signer-output validation. +// +// It performs no registry, plugin, CI, or command-line work. Adapters fetch and store evidence and +// then call these verifiers, so attachment transport stays outside this package. No custom +// cryptography lives here; everything delegates to the Sigstore libraries. +package signature diff --git a/pkg/signature/errors.go b/pkg/signature/errors.go new file mode 100644 index 00000000000..e1e604d4e30 --- /dev/null +++ b/pkg/signature/errors.go @@ -0,0 +1,35 @@ +package signature + +import ( + "errors" +) + +// Error carries a shared sentinel (Kind) separately from its cause, so adapters can translate the +// sentinel to a legacy one without changing the rest of the message. +type Error struct { + // Kind is one of the ErrSignature* sentinels. + Kind error + // Cause is the underlying failure, if any. + Cause error +} + +// Error joins the sentinel and cause exactly like errors.Join. +// +//nolint:lintroller // Error formatting only. +func (e *Error) Error() string { + return errors.Join(e.Kind, e.Cause).Error() +} + +// Unwrap exposes both the sentinel and the cause to errors.Is and errors.As. +// +//nolint:lintroller // Trivial accessor. +func (e *Error) Unwrap() []error { + if e.Cause == nil { + return []error{e.Kind} + } + return []error{e.Kind, e.Cause} +} + +func fail(kind, cause error) error { + return &Error{Kind: kind, Cause: cause} +} diff --git a/pkg/signature/key.go b/pkg/signature/key.go new file mode 100644 index 00000000000..d636450e132 --- /dev/null +++ b/pkg/signature/key.go @@ -0,0 +1,148 @@ +package signature + +import ( + "context" + "crypto" + "fmt" + "time" + + "github.com/sigstore/sigstore-go/pkg/bundle" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/sigstore/sigstore-go/pkg/verify" + sigstore "github.com/sigstore/sigstore/pkg/signature" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// TrustedKey is a public key an operator explicitly configured as a trusted signer. +type TrustedKey struct { + // Hint is the key identifier a bundle carries in its verification material. + Hint string + // PublicKey is the verification key. + PublicKey crypto.PublicKey +} + +// KeyResult names the configured key that authenticated a bundle. It carries no certificate claims. +type KeyResult struct{ Hint string } + +// KeyVerifier verifies Sigstore bundles signed with an explicitly configured public key. +// +// It is a separate type from Verifier on purpose: its result is a key hint, never an OIDC +// identity or issuer, so a public-key signature cannot be mistaken for, or satisfy, an identity +// requirement. It requires the bundle to be signed with a key (certificate bundles are rejected), +// inclusion in a trusted transparency log, and an authenticated signing time. +type KeyVerifier struct { + // Keys are the only keys that can authenticate a bundle. An empty set fails closed. + Keys []TrustedKey + // LoadRoot supplies transparency-log trust. It defaults to the Sigstore public-good root. + LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) +} + +// Verify checks raw against the independently supplied subject digest. When several configured keys +// share a hint, each of them is tried; the first that verifies the bundle wins. +func (v KeyVerifier) Verify(ctx context.Context, raw []byte, digest string, offline bool) (*KeyResult, error) { + defer perf.Track(nil, "signature.KeyVerifier.Verify")() + + if err := ctx.Err(); err != nil { + return nil, err + } + if len(v.Keys) == 0 { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + hash, signed, err := parseBundle(raw, digest) + if err != nil { + return nil, err + } + rounds, err := v.keyRounds() + if err != nil { + return nil, err + } + logs, err := v.loadLogs(ctx, offline) + if err != nil { + return nil, err + } + policy := verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), verify.WithKey()) + return verifyRounds(ctx, rounds, logs, signed, policy) +} + +// verifyRounds tries each key round in order and returns the hint of the first key that verifies +// the bundle. When no round verifies it, the last verification error is reported. +func verifyRounds(ctx context.Context, rounds []root.TrustedMaterial, logs root.TrustedMaterial, signed *bundle.Bundle, policy verify.PolicyBuilder) (*KeyResult, error) { + var last error + for _, keys := range rounds { + verifier, err := verify.NewVerifier(root.TrustedMaterialCollection{keys, logs}, verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + result, err := verifier.Verify(signed, policy) + if err != nil { + last = err + continue + } + if err := ctx.Err(); err != nil { + return nil, err + } + if result.Signature == nil || result.Signature.PublicKeyID == nil { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + return &KeyResult{Hint: string(*result.Signature.PublicKeyID)}, nil + } + return nil, fail(errUtils.ErrSignatureInvalid, last) +} + +// loadLogs loads transparency-log trust and strips any public keys it carries, so only the +// configured keys can authenticate a key-signed bundle. +func (v KeyVerifier) loadLogs(ctx context.Context, offline bool) (root.TrustedMaterial, error) { + loader := v.LoadRoot + if loader == nil { + loader = LoadPublicRoot + } + logs, err := loader(ctx, offline) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + return transparencyOnly{TrustedMaterial: logs}, nil +} + +// transparencyOnly exposes transparency-log and timestamp trust but never resolves a signer key. +type transparencyOnly struct{ root.TrustedMaterial } + +//nolint:lintroller // Constant rejection with no work to track. +func (transparencyOnly) PublicKeyVerifier(hint string) (root.TimeConstrainedVerifier, error) { + return nil, fmt.Errorf("%w: key %q is not a configured trusted key", errUtils.ErrSignatureInvalid, hint) +} + +// keyRounds groups the configured keys so each round holds at most one key per hint. A bundle names +// a single hint, so trying every round tries every configured key for that hint, and a key never +// overwrites another that shares its hint. Certificate authorities from other trust material cannot +// validate a key-signed bundle. +func (v KeyVerifier) keyRounds() ([]root.TrustedMaterial, error) { + var rounds []map[string]*root.ExpiringKey + seen := make(map[string]int, len(v.Keys)) + for _, key := range v.Keys { + if key.Hint == "" || key.PublicKey == nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + loaded, err := sigstore.LoadVerifier(key.PublicKey, crypto.SHA256) + if err != nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, err) + } + // Message signatures made with Ed25519 can only be verified against the full artifact, and + // this path holds only its digest, so the key would pass here and then reject every bundle. + if _, pure := loaded.(*sigstore.ED25519Verifier); pure { + return nil, fail(errUtils.ErrSignatureConfigInvalid, fmt.Errorf("%w: ed25519 signatures need the full artifact", errUtils.ErrSignatureKeyUnsupported)) + } + round := seen[key.Hint] + seen[key.Hint]++ + if round == len(rounds) { + rounds = append(rounds, make(map[string]*root.ExpiringKey)) + } + rounds[round][key.Hint] = root.NewExpiringKey(loaded, time.Time{}, time.Time{}) + } + materials := make([]root.TrustedMaterial, 0, len(rounds)) + for _, keys := range rounds { + materials = append(materials, root.NewTrustedPublicKeyMaterialFromMapping(keys)) + } + return materials, nil +} diff --git a/pkg/signature/key_test.go b/pkg/signature/key_test.go new file mode 100644 index 00000000000..f62b981e41d --- /dev/null +++ b/pkg/signature/key_test.go @@ -0,0 +1,178 @@ +package signature + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "strings" + "testing" + "time" + + "github.com/sigstore/sigstore-go/pkg/root" + sigstore "github.com/sigstore/sigstore/pkg/signature" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func logsOnly(authority *signaturetest.Authority) func(context.Context, bool) (root.TrustedMaterial, error) { + return func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil } +} + +func TestKeyVerifierAcceptsOnlyConfiguredKeys(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + configured := TrustedKey{Hint: signed.Hint, PublicKey: signed.PublicKey} + verifier := KeyVerifier{Keys: []TrustedKey{configured}, LoadRoot: logsOnly(authority)} + + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, &KeyResult{Hint: signaturetest.KeyHint}, result) + + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + for _, tc := range []struct { + name string + verifier KeyVerifier + digest string + raw []byte + kind error + }{ + {"different key under the same hint", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint, PublicKey: &stranger.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"hint not configured", KeyVerifier{Keys: []TrustedKey{{Hint: "other", PublicKey: signed.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"wrong subject digest", verifier, "sha256:" + strings.Repeat("b", 64), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"malformed bundle", verifier, digestOf(artifact), []byte("{}"), errUtils.ErrSignatureInvalid}, + {"no configured keys", KeyVerifier{LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"key without hint", KeyVerifier{Keys: []TrustedKey{{PublicKey: signed.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"key without material", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"unsupported key type", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint, PublicKey: "not a key"}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := tc.verifier.Verify(t.Context(), tc.raw, tc.digest, true) + require.ErrorIs(t, err, tc.kind) + require.Nil(t, result) + }) + } + + failing := KeyVerifier{Keys: []TrustedKey{configured}, LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { + return nil, errUtils.ErrSignatureOffline + }} + _, err = failing.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = verifier.Verify(ctx, signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, context.Canceled) +} + +// TestKeyAndIdentityPathsNeverSubstituteForEachOther is the trust-boundary guarantee: neither +// verifier accepts the other's evidence, and the key path never reports an OIDC identity. +func TestKeyAndIdentityPathsNeverSubstituteForEachOther(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("subject") + identity := Identity{Identity: "builder@example.com", Issuer: "https://issuer.example.com"} + certificateBundle := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + keySigned := authority.SignWithKey(t, artifact) + trust := logsOnly(authority) + + // A public-key signature cannot satisfy an identity requirement, even when the key is trusted. + _, err := Verifier{LoadRoot: trust}.Verify(t.Context(), keySigned.Bundle, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + // A certificate-signed bundle cannot be accepted by the key path, whatever keys are configured. + keys := KeyVerifier{Keys: []TrustedKey{{Hint: keySigned.Hint, PublicKey: keySigned.PublicKey}}, LoadRoot: trust} + _, err = keys.Verify(t.Context(), certificateBundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) +} + +// leakingRoot is transparency-log trust that also resolves a signer key, as a loader might. +type leakingRoot struct { + *signaturetest.Authority + key root.TimeConstrainedVerifier +} + +func (l leakingRoot) PublicKeyVerifier(string) (root.TimeConstrainedVerifier, error) { + return l.key, nil +} + +func TestKeyVerifierIgnoresSignerKeysFromTheLoader(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + loaded, err := sigstore.LoadVerifier(signed.PublicKey, crypto.SHA256) + require.NoError(t, err) + leaking := func(context.Context, bool) (root.TrustedMaterial, error) { + return leakingRoot{Authority: authority, key: root.NewExpiringKey(loaded, time.Time{}, time.Time{})}, nil + } + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + // The loader's key would verify the bundle, but it is not configured, so verification fails. + verifier := KeyVerifier{Keys: []TrustedKey{{Hint: "other", PublicKey: &stranger.PublicKey}}, LoadRoot: leaking} + _, err = verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + // The loader's transparency-log trust is still used when the signer key is configured. + verifier.Keys = []TrustedKey{{Hint: signed.Hint, PublicKey: signed.PublicKey}} + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, signaturetest.KeyHint, result.Hint) +} + +func TestKeyVerifierTriesEveryKeyThatSharesAHint(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + right := TrustedKey{Hint: signed.Hint, PublicKey: signed.PublicKey} + wrong := TrustedKey{Hint: signed.Hint, PublicKey: &stranger.PublicKey} + + for name, keys := range map[string][]TrustedKey{"wrong key first": {wrong, right}, "right key first": {right, wrong}} { + t.Run(name, func(t *testing.T) { + result, err := KeyVerifier{Keys: keys, LoadRoot: logsOnly(authority)}.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, signaturetest.KeyHint, result.Hint) + }) + } + + // Two non-matching keys under the hint still fail. + other, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + _, err = KeyVerifier{Keys: []TrustedKey{wrong, {Hint: signed.Hint, PublicKey: &other.PublicKey}}, LoadRoot: logsOnly(authority)}.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) +} + +// Ed25519 message signatures can only be verified against the full artifact, which the digest-based +// verification path never has, so such a key is a configuration error rather than a key that +// validates and then fails every bundle. +func TestKeyVerifierRejectsEd25519KeysAtConfiguration(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + edPublic, _, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + + for _, tc := range []struct { + name string + keys []TrustedKey + }{ + {"ed25519 key alone", []TrustedKey{{Hint: signed.Hint, PublicKey: edPublic}}}, + {"ed25519 key beside a supported key", []TrustedKey{{Hint: signed.Hint, PublicKey: signed.PublicKey}, {Hint: "ed", PublicKey: edPublic}}}, + } { + t.Run(tc.name, func(t *testing.T) { + verifier := KeyVerifier{Keys: tc.keys, LoadRoot: logsOnly(authority)} + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureConfigInvalid) + assert.Contains(t, err.Error(), "ed25519") + assert.Nil(t, result) + }) + } +} diff --git a/pkg/signature/mock_signer.go b/pkg/signature/mock_signer.go new file mode 100644 index 00000000000..109c9353818 --- /dev/null +++ b/pkg/signature/mock_signer.go @@ -0,0 +1,56 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/signature (interfaces: BlobSigner) +// +// Generated by this command: +// +// mockgen -destination=mock_signer.go -package=signature . BlobSigner +// + +// Package signature is a generated GoMock package. +package signature + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockBlobSigner is a mock of BlobSigner interface. +type MockBlobSigner struct { + ctrl *gomock.Controller + recorder *MockBlobSignerMockRecorder + isgomock struct{} +} + +// MockBlobSignerMockRecorder is the mock recorder for MockBlobSigner. +type MockBlobSignerMockRecorder struct { + mock *MockBlobSigner +} + +// NewMockBlobSigner creates a new mock instance. +func NewMockBlobSigner(ctrl *gomock.Controller) *MockBlobSigner { + mock := &MockBlobSigner{ctrl: ctrl} + mock.recorder = &MockBlobSignerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBlobSigner) EXPECT() *MockBlobSignerMockRecorder { + return m.recorder +} + +// Sign mocks base method. +func (m *MockBlobSigner) Sign(ctx context.Context, artifact []byte) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, artifact) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockBlobSignerMockRecorder) Sign(ctx, artifact any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockBlobSigner)(nil).Sign), ctx, artifact) +} diff --git a/pkg/oci/signature/signaturetest/bundle.go b/pkg/signature/signaturetest/bundle.go similarity index 100% rename from pkg/oci/signature/signaturetest/bundle.go rename to pkg/signature/signaturetest/bundle.go diff --git a/pkg/oci/signature/signaturetest/dsse.go b/pkg/signature/signaturetest/dsse.go similarity index 100% rename from pkg/oci/signature/signaturetest/dsse.go rename to pkg/signature/signaturetest/dsse.go diff --git a/pkg/oci/signature/signaturetest/public_key.go b/pkg/signature/signaturetest/public_key.go similarity index 69% rename from pkg/oci/signature/signaturetest/public_key.go rename to pkg/signature/signaturetest/public_key.go index d50217322b0..e6ad2aaed1b 100644 --- a/pkg/oci/signature/signaturetest/public_key.go +++ b/pkg/signature/signaturetest/public_key.go @@ -25,11 +25,41 @@ import ( "github.com/cloudposse/atmos/pkg/perf" ) +// KeySignature is a genuine key-only Sigstore bundle together with the public half of the key. +type KeySignature struct { + // Bundle is the raw Sigstore bundle. + Bundle []byte + // Hint is the key identifier embedded in the bundle. + Hint string + // PublicKey verifies the signature; an operator would configure it as a trusted key. + PublicKey crypto.PublicKey + // Logs is the transparency-log trust material for this test authority. + Logs root.TrustedMaterial +} + +// KeyHint is the identifier SignWithKey embeds in every bundle. +const KeyHint = "fixture-key" + // SignWithPublicKey generates a genuine key-only signature and Rekor evidence. // This intentionally unsupported signing mode must never satisfy a certificate policy. func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, root.TrustedMaterial) { defer perf.Track(nil, "signaturetest.Authority.SignWithPublicKey")() + t.Helper() + signed := a.SignWithKey(t, artifact) + verifier, err := sig.LoadVerifier(signed.PublicKey, crypto.SHA256) + require.NoError(t, err) + keyTrust := root.NewTrustedPublicKeyMaterial(func(string) (root.TimeConstrainedVerifier, error) { + return root.NewExpiringKey(verifier, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)), nil + }) + return signed.Bundle, root.TrustedMaterialCollection{a.TrustedMaterial, keyTrust} +} + +// SignWithKey generates a genuine key-only signature with a fresh key and returns the public key +// so tests can configure it explicitly as trusted. +func (a *Authority) SignWithKey(t *testing.T, artifact []byte) KeySignature { + defer perf.Track(nil, "signaturetest.Authority.SignWithKey")() + t.Helper() key := newKey(t) hash := sha256.Sum256(artifact) @@ -52,7 +82,7 @@ func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, ro bundle := &protobundle.Bundle{ MediaType: "application/vnd.dev.sigstore.bundle.v0.3+json", VerificationMaterial: &protobundle.VerificationMaterial{ - Content: &protobundle.VerificationMaterial_PublicKey{PublicKey: &common.PublicKeyIdentifier{Hint: "fixture-key"}}, + Content: &protobundle.VerificationMaterial_PublicKey{PublicKey: &common.PublicKeyIdentifier{Hint: KeyHint}}, TlogEntries: []*rekor.TransparencyLogEntry{a.logEntry(t, body)}, }, Content: &protobundle.Bundle_MessageSignature{MessageSignature: &common.MessageSignature{ @@ -61,10 +91,5 @@ func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, ro } raw, err := protojson.Marshal(bundle) require.NoError(t, err) - verifier, err := sig.LoadECDSAVerifier(&key.PublicKey, crypto.SHA256) - require.NoError(t, err) - keyTrust := root.NewTrustedPublicKeyMaterial(func(string) (root.TimeConstrainedVerifier, error) { - return root.NewExpiringKey(verifier, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)), nil - }) - return raw, root.TrustedMaterialCollection{a.TrustedMaterial, keyTrust} + return KeySignature{Bundle: raw, Hint: KeyHint, PublicKey: &key.PublicKey, Logs: a.TrustedMaterial} } diff --git a/pkg/signature/signer.go b/pkg/signature/signer.go new file mode 100644 index 00000000000..acbdaa70f7a --- /dev/null +++ b/pkg/signature/signer.go @@ -0,0 +1,57 @@ +package signature + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + + "github.com/sigstore/sigstore-go/pkg/bundle" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// BlobSigner produces a standard Sigstore bundle for the exact supplied bytes. Implementations +// (for example the managed Cosign authoring path) own signing identity; registry credentials are +// unrelated to it. +// +//go:generate go run go.uber.org/mock/mockgen -destination=mock_signer.go -package=signature . BlobSigner +type BlobSigner interface { + Sign(ctx context.Context, artifact []byte) ([]byte, error) +} + +// SignBlob signs artifact and refuses output that is oversized, malformed, or bound to different bytes. +func SignBlob(ctx context.Context, signer BlobSigner, artifact []byte) ([]byte, error) { + defer perf.Track(nil, "signature.SignBlob")() + + signed, err := signer.Sign(ctx, artifact) + if err != nil { + return nil, errors.Join(errUtils.ErrSignatureSigningFailed, err) + } + if err := CheckBundleSubject(signed, artifact); err != nil { + return nil, err + } + return signed, nil +} + +// CheckBundleSubject confirms a message-signature bundle commits to the SHA-256 of artifact. +// This is a structural check on freshly produced evidence; verification still goes through a Verifier. +func CheckBundleSubject(raw, artifact []byte) error { + defer perf.Track(nil, "signature.CheckBundleSubject")() + + if len(raw) == 0 || len(raw) > MaxBundleSize { + return fail(errUtils.ErrSignatureInvalid, nil) + } + var signed bundle.Bundle + if err := json.Unmarshal(raw, &signed); err != nil { + return fail(errUtils.ErrSignatureInvalid, err) + } + digest := sha256.Sum256(artifact) + message := signed.GetMessageSignature() + if message == nil || message.MessageDigest == nil || message.MessageDigest.Algorithm.String() != "SHA2_256" || !bytes.Equal(message.MessageDigest.Digest, digest[:]) { + return fail(errUtils.ErrSignatureInvalid, nil) + } + return nil +} diff --git a/pkg/signature/signer_test.go b/pkg/signature/signer_test.go new file mode 100644 index 00000000000..74fb47c96c1 --- /dev/null +++ b/pkg/signature/signer_test.go @@ -0,0 +1,59 @@ +package signature + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func TestSignBlobAcceptsOnlyEvidenceBoundToTheSignedBytes(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("release index bytes") + valid := authority.Sign(t, "builder@example.com", "https://issuer.example.com", artifact) + otherArtifact := authority.Sign(t, "builder@example.com", "https://issuer.example.com", []byte("different bytes")) + failure := errors.New("signing backend unavailable") + + for _, tc := range []struct { + name string + bundle []byte + err error + want error + }{ + {"matching bundle", valid, nil, nil}, + {"bundle for other bytes", otherArtifact, nil, errUtils.ErrSignatureInvalid}, + {"malformed bundle", []byte("not json"), nil, errUtils.ErrSignatureInvalid}, + {"empty bundle", nil, nil, errUtils.ErrSignatureInvalid}, + {"oversized bundle", make([]byte, MaxBundleSize+1), nil, errUtils.ErrSignatureInvalid}, + {"signer failure", nil, failure, errUtils.ErrSignatureSigningFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + signer := NewMockBlobSigner(gomock.NewController(t)) + signer.EXPECT().Sign(gomock.Any(), artifact).Return(tc.bundle, tc.err) + got, err := SignBlob(t.Context(), signer, artifact) + if tc.want != nil { + require.ErrorIs(t, err, tc.want) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + } + assert.Nil(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, tc.bundle, got) + }) + } +} + +func TestCheckBundleSubjectRejectsEnvelopeBundles(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("statement subject") + statement := authority.SignStatement(t, "builder@example.com", "https://issuer.example.com", artifact) + // DSSE statements are verified by Verifier; SignBlob only vouches for message signatures. + require.ErrorIs(t, CheckBundleSubject(statement, artifact), errUtils.ErrSignatureInvalid) +} diff --git a/pkg/oci/signature/root.go b/pkg/signature/trust.go similarity index 82% rename from pkg/oci/signature/root.go rename to pkg/signature/trust.go index 9bb3c2472b2..6b21ae2e9b3 100644 --- a/pkg/oci/signature/root.go +++ b/pkg/signature/trust.go @@ -18,9 +18,12 @@ import ( const trustDirectoryMode = 0o700 -// Offline mode may use only unexpired, TUF-authenticated cached trust material. -// Never accept roots supplied by a plugin or signature bundle. -func loadPublicRoot(ctx context.Context, offline bool) (root.TrustedMaterial, error) { +// LoadPublicRoot returns the Sigstore public-good trusted root. Offline mode may use only +// unexpired, TUF-authenticated cached trust material. Never accept roots supplied by an artifact +// or signature bundle. +func LoadPublicRoot(ctx context.Context, offline bool) (root.TrustedMaterial, error) { + defer perf.Track(nil, "signature.LoadPublicRoot")() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() directory, err := xdg.GetXDGDataDir("sigstore", trustDirectoryMode) @@ -47,7 +50,7 @@ type offlineFetcher struct{} //nolint:lintroller // Pure fail-closed fetcher: never performs I/O. func (offlineFetcher) DownloadFile(_ string, _ int64, _ time.Duration) ([]byte, error) { - return nil, errUtils.ErrPluginOfflineArtifact + return nil, errUtils.ErrSignatureOffline } type contextClient struct { diff --git a/pkg/oci/signature/root_transport_test.go b/pkg/signature/trust_test.go similarity index 67% rename from pkg/oci/signature/root_transport_test.go rename to pkg/signature/trust_test.go index f292f84b1d0..d340c9158cc 100644 --- a/pkg/oci/signature/root_transport_test.go +++ b/pkg/signature/trust_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -18,7 +19,7 @@ func TestPublicTrustFailsWhenCacheDirectoryCannotBeCreated(t *testing.T) { path := filepath.Join(t.TempDir(), "file") require.NoError(t, os.WriteFile(path, nil, 0o600)) t.Setenv("ATMOS_XDG_DATA_HOME", path) - trusted, err := loadPublicRoot(t.Context(), true) + trusted, err := LoadPublicRoot(t.Context(), true) require.Error(t, err) require.Nil(t, trusted) } @@ -49,5 +50,23 @@ func TestTrustHTTPClientUsesVerificationContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) require.Nil(t, response) _, err = (offlineFetcher{}).DownloadFile(server.URL, 1024, 0) - require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrSignatureOffline) +} + +type countingTransport struct{ requests atomic.Int32 } + +func (t *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + t.requests.Add(1) + return nil, errUtils.ErrSignatureOffline +} + +func TestOfflineTrustCannotFetch(t *testing.T) { + transport := &countingTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + t.Setenv("ATMOS_XDG_DATA_HOME", t.TempDir()) + _, err := LoadPublicRoot(t.Context(), true) + require.Error(t, err) + require.Zero(t, transport.requests.Load()) } diff --git a/pkg/signature/verify.go b/pkg/signature/verify.go new file mode 100644 index 00000000000..2c517b67086 --- /dev/null +++ b/pkg/signature/verify.go @@ -0,0 +1,121 @@ +package signature + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "regexp" + "strings" + + "github.com/sigstore/sigstore-go/pkg/bundle" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/sigstore/sigstore-go/pkg/verify" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MaxBundleSize bounds one Sigstore bundle. +const MaxBundleSize = 4 << 20 + +// Identity binds a certificate SAN (exact or * wildcard) to an exact OIDC issuer. +type Identity struct{ Identity, Issuer string } + +// Result contains authenticated certificate claims, never claims from unverified metadata. +type Result struct{ Identity, Issuer string } + +// Verifier uses a trusted root independent of the artifact and its publisher. +// LoadRoot is injectable for private trust deployments and isolated cryptographic tests. +// +// It verifies certificate-based (keyless) bundles only. A bundle signed with a bare public key can +// never satisfy an identity requirement here; KeyVerifier is the separate path for that. +type Verifier struct { + LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) +} + +// Verify requires a certificate, SCT, transparency inclusion and authenticated +// signing time. The subject digest is always supplied independently of the bundle. +func (v Verifier) Verify(ctx context.Context, raw []byte, digest string, identities []Identity, offline bool) (*Result, error) { + defer perf.Track(nil, "signature.Verifier.Verify")() + + if err := ctx.Err(); err != nil { + return nil, err + } + hash, signed, err := parseBundle(raw, digest) + if err != nil { + return nil, err + } + if len(identities) == 0 { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + options, err := identityOptions(identities) + if err != nil { + return nil, err + } + verifier, err := v.trustedVerifier(ctx, offline) + if err != nil { + return nil, err + } + result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), options...)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if result.Signature == nil || result.Signature.Certificate == nil { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + return &Result{Identity: result.Signature.Certificate.SubjectAlternativeName, Issuer: result.Signature.Certificate.Issuer}, nil +} + +func parseBundle(raw []byte, digest string) ([]byte, *bundle.Bundle, error) { + if len(raw) > MaxBundleSize || len(raw) == 0 { + return nil, nil, fail(errUtils.ErrSignatureInvalid, nil) + } + hash, err := hex.DecodeString(strings.TrimPrefix(digest, "sha256:")) + if err != nil || !strings.HasPrefix(digest, "sha256:") || len(hash) != sha256.Size { + return nil, nil, fail(errUtils.ErrSignatureInvalid, nil) + } + var signed bundle.Bundle + if err := json.Unmarshal(raw, &signed); err != nil { + return nil, nil, fail(errUtils.ErrSignatureInvalid, err) + } + return hash, &signed, nil +} + +func (v Verifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { + loader := v.LoadRoot + if loader == nil { + loader = LoadPublicRoot + } + trusted, err := loader(ctx, offline) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + // Require a certificate identity, a witnessed signing time and inclusion in + // a trusted transparency log. Bare-key bundles cannot establish the requested + // OIDC identity; the key-only rejection test guards this boundary. + verifier, err := verify.NewVerifier(trusted, verify.WithSignedCertificateTimestamps(1), verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + return verifier, nil +} + +func identityOptions(identities []Identity) ([]verify.PolicyOption, error) { + options := make([]verify.PolicyOption, 0, len(identities)) + for _, identity := range identities { + if identity.Identity == "" || identity.Issuer == "" { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + pattern := "^" + strings.ReplaceAll(regexp.QuoteMeta(identity.Identity), `\*`, ".*") + "$" + match, err := verify.NewShortCertificateIdentity(identity.Issuer, "", "", pattern) + if err != nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, err) + } + options = append(options, verify.WithCertificateIdentity(match)) + } + return options, nil +} diff --git a/pkg/signature/verify_test.go b/pkg/signature/verify_test.go new file mode 100644 index 00000000000..a85e5c4e549 --- /dev/null +++ b/pkg/signature/verify_test.go @@ -0,0 +1,123 @@ +package signature + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func digestOf(artifact []byte) string { + sum := sha256.Sum256(artifact) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func TestVerifyRealSigstoreEvidence(t *testing.T) { + authority := signaturetest.New(t) + verifier := Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil }} + artifact := []byte("immutable subject") + reference := digestOf(artifact) + identity := Identity{Identity: "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + raw := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + + result, err := verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) + require.NoError(t, err) + assert.Equal(t, &Result{Identity: identity.Identity, Issuer: identity.Issuer}, result) + + wildcard := Identity{Identity: "https://github.com/acme/*", Issuer: identity.Issuer} + _, err = verifier.Verify(t.Context(), raw, reference, []Identity{wildcard}, true) + require.NoError(t, err) + + for _, tc := range []struct { + name, digest string + identity Identity + raw []byte + }{ + {"wrong digest", "sha256:" + strings.Repeat("a", 64), identity, raw}, + {"wrong issuer", reference, Identity{Identity: identity.Identity, Issuer: "https://wrong.example"}, raw}, + {"wrong identity", reference, Identity{Identity: "https://github.com/other/*", Issuer: identity.Issuer}, raw}, + {"identity is anchored", reference, Identity{Identity: "github.com/acme/*", Issuer: identity.Issuer}, raw}, + {"malformed bundle", reference, identity, []byte("{}")}, + {"empty bundle", reference, identity, nil}, + {"oversized bundle", reference, identity, make([]byte, MaxBundleSize+1)}, + {"non-sha256 digest", "sha512:" + strings.Repeat("a", 128), identity, raw}, + {"short digest", "sha256:abcd", identity, raw}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := verifier.Verify(t.Context(), tc.raw, tc.digest, []Identity{tc.identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + }) + } + + var changed map[string]any + require.NoError(t, json.Unmarshal(raw, &changed)) + changed["verificationMaterial"].(map[string]any)["tlogEntries"] = []any{} + tampered, err := json.Marshal(changed) + require.NoError(t, err) + _, err = verifier.Verify(t.Context(), tampered, reference, []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "a bundle without transparency inclusion is rejected") + + other := signaturetest.New(t) + verifier.LoadRoot = func(context.Context, bool) (root.TrustedMaterial, error) { return other, nil } + _, err = verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "a root that did not issue the certificate cannot verify it") +} + +func TestVerifyRejectsUnusablePolicyAndTrust(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("subject") + identity := Identity{Identity: "builder@example.com", Issuer: "https://issuer.example.com"} + raw := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + trusted := func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil } + + _, err := Verifier{LoadRoot: trusted}.Verify(t.Context(), raw, digestOf(artifact), nil, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "verification without a required identity is refused") + + for _, bad := range []Identity{{Issuer: identity.Issuer}, {Identity: identity.Identity}} { + _, err = Verifier{LoadRoot: trusted}.Verify(t.Context(), raw, digestOf(artifact), []Identity{bad}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureConfigInvalid) + require.NotErrorIs(t, err, errUtils.ErrSignatureInvalid) + } + + failing := func(context.Context, bool) (root.TrustedMaterial, error) { return nil, errUtils.ErrSignatureOffline } + _, err = Verifier{LoadRoot: failing}.Verify(t.Context(), raw, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorIs(t, err, errUtils.ErrSignatureOffline, "the trust failure stays inspectable") + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = Verifier{LoadRoot: trusted}.Verify(ctx, raw, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, context.Canceled) +} + +func TestVerifyRejectsPublicKeyOnlySignatureForIdentityRequirements(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("release signed without a certificate") + raw, trusted := authority.SignWithPublicKey(t, artifact) + verifier := Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return trusted, nil }} + result, err := verifier.Verify(t.Context(), raw, digestOf(artifact), []Identity{{ + Identity: "builder@example.com", Issuer: "https://issuer.example.com", + }}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorContains(t, err, "SCTs required but bundle is signed with a public key") + require.Nil(t, result) +} + +func TestErrorExposesSentinelAndCause(t *testing.T) { + cause := context.DeadlineExceeded + err := fail(errUtils.ErrSignatureInvalid, cause) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), errUtils.ErrSignatureInvalid.Error()) + assert.Contains(t, err.Error(), cause.Error()) + assert.Equal(t, errUtils.ErrSignatureInvalid.Error(), fail(errUtils.ErrSignatureInvalid, nil).Error()) +} diff --git a/pkg/store/authbridge/resolver_scoped_test.go b/pkg/store/authbridge/resolver_scoped_test.go index a64c59cc705..4cd57c2b844 100644 --- a/pkg/store/authbridge/resolver_scoped_test.go +++ b/pkg/store/authbridge/resolver_scoped_test.go @@ -40,7 +40,8 @@ func TestScopedResolverKeepsIdentityOwnership(t *testing.T) { err := operation(ctx, tt.auth) events = append(events, "released") return errors.Join(err, tt.cleanupError) - }) + }, + ) resolver := NewResolver(manager, &schema.ConfigAndStacksInfo{}) err := resolver.WithEnvironmentAuthContext(t.Context(), "reader", func(ctx context.Context, auth *store.EnvironmentAuthConfig) error { events = append(events, "operation") diff --git a/pkg/store/providers/hashicorp_vault_lifetime_test.go b/pkg/store/providers/hashicorp_vault_lifetime_test.go index f481ea77a36..b6c503b48ab 100644 --- a/pkg/store/providers/hashicorp_vault_lifetime_test.go +++ b/pkg/store/providers/hashicorp_vault_lifetime_test.go @@ -25,7 +25,8 @@ func scopedVault(t *testing.T, cleanup error) (*VaultStore, *fakeVaultKV, *int) err := operation(ctx, &store.EnvironmentAuthConfig{Environment: map[string]string{"VAULT_TOKEN": "synthetic-identity"}}) *releases++ return errors.Join(err, cleanup) - }).AnyTimes() + }, + ).AnyTimes() s.SetAuthContext(resolver, "openbao") return s, fake, releases } @@ -130,7 +131,8 @@ func TestVaultOperationLifetimeRejectsMissingCredentials(t *testing.T) { return operation(ctx, nil) } return operation(ctx, &store.EnvironmentAuthConfig{Environment: environment}) - }) + }, + ) s.SetAuthContext(resolver, "openbao") value, err := s.GetKey("key") require.ErrorIs(t, err, store.ErrAuthContextNotAvailable)