From 418623022389d4ac97d0931528f2f5675721e7a4 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 08:46:04 -0500 Subject: [PATCH 01/52] feat(artifacts): add ORAS-backed OCI transport and shared pkg/signature (A02) Co-Authored-By: Claude Sonnet 5.5 --- errors/signature.go | 16 + go.mod | 2 +- pkg/oci/client.go | 65 ++- pkg/oci/mock_transport.go | 325 ++++++++++++++ pkg/oci/ocitest/registry.go | 44 ++ pkg/oci/referrers.go | 19 +- .../signature/signaturetest/signaturetest.go | 20 + pkg/oci/signature/verify.go | 118 ++--- pkg/oci/signature/verify_test.go | 6 - pkg/oci/status.go | 12 +- pkg/oci/transport.go | 150 +++++++ pkg/oci/transport_behavior_test.go | 422 ++++++++++++++++++ pkg/oci/transport_copy.go | 290 ++++++++++++ pkg/oci/transport_errors.go | 76 ++++ pkg/oci/transport_internal_test.go | 221 +++++++++ pkg/oci/transport_layout.go | 239 ++++++++++ pkg/oci/transport_legacy.go | 228 ++++++++++ pkg/oci/transport_registry.go | 225 ++++++++++ pkg/oci/transport_roundtrip_test.go | 311 +++++++++++++ pkg/plugin/sign/sign.go | 23 +- pkg/signature/boundary_test.go | 46 ++ pkg/signature/doc.go | 9 + pkg/signature/errors.go | 35 ++ pkg/signature/key.go | 107 +++++ pkg/signature/key_test.go | 89 ++++ pkg/signature/mock_signer.go | 56 +++ .../signature/signaturetest/bundle.go | 0 pkg/{oci => }/signature/signaturetest/dsse.go | 0 .../signature/signaturetest/public_key.go | 39 +- pkg/signature/signer.go | 56 +++ pkg/signature/signer_test.go | 56 +++ .../signature/root.go => signature/trust.go} | 11 +- .../trust_test.go} | 10 +- pkg/signature/verify.go | 121 +++++ pkg/signature/verify_test.go | 123 +++++ 35 files changed, 3419 insertions(+), 151 deletions(-) create mode 100644 errors/signature.go create mode 100644 pkg/oci/mock_transport.go create mode 100644 pkg/oci/signature/signaturetest/signaturetest.go create mode 100644 pkg/oci/transport.go create mode 100644 pkg/oci/transport_behavior_test.go create mode 100644 pkg/oci/transport_copy.go create mode 100644 pkg/oci/transport_errors.go create mode 100644 pkg/oci/transport_internal_test.go create mode 100644 pkg/oci/transport_layout.go create mode 100644 pkg/oci/transport_legacy.go create mode 100644 pkg/oci/transport_registry.go create mode 100644 pkg/oci/transport_roundtrip_test.go create mode 100644 pkg/signature/boundary_test.go create mode 100644 pkg/signature/doc.go create mode 100644 pkg/signature/errors.go create mode 100644 pkg/signature/key.go create mode 100644 pkg/signature/key_test.go create mode 100644 pkg/signature/mock_signer.go rename pkg/{oci => }/signature/signaturetest/bundle.go (100%) rename pkg/{oci => }/signature/signaturetest/dsse.go (100%) rename pkg/{oci => }/signature/signaturetest/public_key.go (69%) create mode 100644 pkg/signature/signer.go create mode 100644 pkg/signature/signer_test.go rename pkg/{oci/signature/root.go => signature/trust.go} (82%) rename pkg/{oci/signature/root_transport_test.go => signature/trust_test.go} (84%) create mode 100644 pkg/signature/verify.go create mode 100644 pkg/signature/verify_test.go diff --git a/errors/signature.go b/errors/signature.go new file mode 100644 index 00000000000..ac3e4eef484 --- /dev/null +++ b/errors/signature.go @@ -0,0 +1,16 @@ +package errors + +import "errors" + +// Shared signature-verification sentinels used by pkg/signature. Adapters that predate the shared +// package translate these to their legacy sentinels so existing callers keep matching. +var ( + // ErrSignatureInvalid indicates signature evidence is malformed, untrusted, or does not match the subject. + ErrSignatureInvalid = errors.New("signature does not satisfy the trust policy") + + // ErrSignatureConfigInvalid indicates the verification policy itself is unusable (for example an empty identity). + ErrSignatureConfigInvalid = errors.New("invalid signature verification configuration") + + // ErrSignatureOffline indicates trust material cannot be fetched while verifying offline. + ErrSignatureOffline = errors.New("signature trust material is unavailable offline") +) diff --git a/go.mod b/go.mod index 11bee90ea65..9dac93c69cd 100644 --- a/go.mod +++ b/go.mod @@ -447,7 +447,7 @@ require ( modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/xurls/v2 v2.6.0 // indirect - oras.land/oras-go/v2 v2.6.2 // indirect + oras.land/oras-go/v2 v2.6.2 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect diff --git a/pkg/oci/client.go b/pkg/oci/client.go index 14af87d279c..574524d74a7 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -9,6 +9,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" @@ -101,73 +102,95 @@ func (c *Client) Get(ctx context.Context, ref name.Reference) (*remote.Descripto } // Head resolves the immutable descriptor without downloading artifact layers. +// It runs on the shared ORAS transport. func (c *Client) Head(ctx context.Context, ref name.Reference) (*v1.Descriptor, error) { defer perf.Track(c.config, "oci.Client.Head")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return nil, err } - result, err := remote.Head(ref, options...) + resolved, err := store.Resolve(ctx, ref.Identifier()) if err != nil { return nil, errors.Join(errUtils.ErrPullImage, err) } - return result, nil + hash, err := v1.NewHash(resolved.Digest.String()) + if err != nil { + return nil, errors.Join(errUtils.ErrPullImage, err) + } + return &v1.Descriptor{MediaType: types.MediaType(resolved.MediaType), Size: resolved.Size, Digest: hash}, nil } // Tags lists repository tags without interpreting them as versions. func (c *Client) Tags(ctx context.Context, repo name.Repository) ([]string, error) { defer perf.Track(c.config, "oci.Client.Tags")() - options, err := c.options(ctx, repo) + store, err := c.Repository(ctx, repositoryOf(repo)) if err != nil { return nil, err } - result, err := remote.List(repo, options...) - if err != nil { - return nil, errors.Join(errUtils.ErrOCIRegistry, err) - } - return result, nil + return store.Tags(ctx) } -// WriteImage uploads an image and its blobs. Rejected writes never retry anonymously. +// WriteImage uploads an image and its blobs, preserving the image's existing raw bytes. +// Rejected writes never retry anonymously, and an unknown outcome is reconciled by +// resolving the destination rather than writing again. func (c *Client) WriteImage(ctx context.Context, ref name.Reference, img v1.Image) error { defer perf.Track(c.config, "oci.Client.WriteImage")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return err } - if err := remote.Write(ref, img, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + graph, err := legacyImageGraph(img) + if err != nil { + return err } - return nil + _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) + return err } // WriteIndex uploads the complete index and its children without platform filtering. func (c *Client) WriteIndex(ctx context.Context, ref name.Reference, index v1.ImageIndex) error { defer perf.Track(c.config, "oci.Client.WriteIndex")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return err } - if err := remote.WriteIndex(ref, index, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + graph, err := legacyIndexGraph(index) + if err != nil { + return err } - return nil + _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) + return err } // Delete deletes only an explicitly selected digest, never a mutable tag. func (c *Client) Delete(ctx context.Context, ref name.Digest) error { defer perf.Track(c.config, "oci.Client.Delete")() - options, err := c.options(ctx, ref.Context()) + store, err := c.Repository(ctx, repositoryOf(ref.Context())) + if err != nil { + return err + } + target, err := store.Resolve(ctx, ref.DigestStr()) if err != nil { return err } - if err := remote.Delete(ref, options...); err != nil { - return errors.Join(errUtils.ErrOCIRegistry, err) + if err := store.Delete(ctx, target); err != nil { + return c.reconcileDelete(ctx, store, ref, err) } return nil } + +// reconcileDelete treats a failed delete as complete only when the digest is gone. +func (c *Client) reconcileDelete(ctx context.Context, store RepositoryStore, ref name.Digest, cause error) error { + if !ambiguousWrite(ctx, cause) { + return cause + } + if _, err := store.Resolve(ctx, ref.DigestStr()); IsNotFound(err) { + return nil + } + return cause +} diff --git a/pkg/oci/mock_transport.go b/pkg/oci/mock_transport.go new file mode 100644 index 00000000000..d4b3799d805 --- /dev/null +++ b/pkg/oci/mock_transport.go @@ -0,0 +1,325 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/oci (interfaces: Store,RepositoryStore,Transport) +// +// Generated by this command: +// +// mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci Store,RepositoryStore,Transport +// + +// Package oci is a generated GoMock package. +package oci + +import ( + context "context" + io "io" + reflect "reflect" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Exists mocks base method. +func (m *MockStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockStore)(nil).Fetch), ctx, target) +} + +// Push mocks base method. +func (m *MockStore) Push(ctx context.Context, expected v1.Descriptor, content io.Reader) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Push", ctx, expected, content) + ret0, _ := ret[0].(error) + return ret0 +} + +// Push indicates an expected call of Push. +func (mr *MockStoreMockRecorder) Push(ctx, expected, content any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Push", reflect.TypeOf((*MockStore)(nil).Push), ctx, expected, content) +} + +// Resolve mocks base method. +func (m *MockStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockStore)(nil).Resolve), ctx, reference) +} + +// Tag mocks base method. +func (m *MockStore) Tag(ctx context.Context, desc v1.Descriptor, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tag", ctx, desc, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// Tag indicates an expected call of Tag. +func (mr *MockStoreMockRecorder) Tag(ctx, desc, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tag", reflect.TypeOf((*MockStore)(nil).Tag), ctx, desc, reference) +} + +// MockRepositoryStore is a mock of RepositoryStore interface. +type MockRepositoryStore struct { + ctrl *gomock.Controller + recorder *MockRepositoryStoreMockRecorder + isgomock struct{} +} + +// MockRepositoryStoreMockRecorder is the mock recorder for MockRepositoryStore. +type MockRepositoryStoreMockRecorder struct { + mock *MockRepositoryStore +} + +// NewMockRepositoryStore creates a new mock instance. +func NewMockRepositoryStore(ctrl *gomock.Controller) *MockRepositoryStore { + mock := &MockRepositoryStore{ctrl: ctrl} + mock.recorder = &MockRepositoryStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepositoryStore) EXPECT() *MockRepositoryStoreMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockRepositoryStore) Delete(ctx context.Context, target v1.Descriptor) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, target) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockRepositoryStoreMockRecorder) Delete(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRepositoryStore)(nil).Delete), ctx, target) +} + +// Exists mocks base method. +func (m *MockRepositoryStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockRepositoryStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockRepositoryStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockRepositoryStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockRepositoryStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockRepositoryStore)(nil).Fetch), ctx, target) +} + +// Predecessors mocks base method. +func (m *MockRepositoryStore) Predecessors(ctx context.Context, node v1.Descriptor) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Predecessors", ctx, node) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Predecessors indicates an expected call of Predecessors. +func (mr *MockRepositoryStoreMockRecorder) Predecessors(ctx, node any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Predecessors", reflect.TypeOf((*MockRepositoryStore)(nil).Predecessors), ctx, node) +} + +// Push mocks base method. +func (m *MockRepositoryStore) Push(ctx context.Context, expected v1.Descriptor, content io.Reader) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Push", ctx, expected, content) + ret0, _ := ret[0].(error) + return ret0 +} + +// Push indicates an expected call of Push. +func (mr *MockRepositoryStoreMockRecorder) Push(ctx, expected, content any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Push", reflect.TypeOf((*MockRepositoryStore)(nil).Push), ctx, expected, content) +} + +// PushReference mocks base method. +func (m *MockRepositoryStore) PushReference(ctx context.Context, expected v1.Descriptor, content io.Reader, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PushReference", ctx, expected, content, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// PushReference indicates an expected call of PushReference. +func (mr *MockRepositoryStoreMockRecorder) PushReference(ctx, expected, content, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PushReference", reflect.TypeOf((*MockRepositoryStore)(nil).PushReference), ctx, expected, content, reference) +} + +// Referrers mocks base method. +func (m *MockRepositoryStore) Referrers(ctx context.Context, subject v1.Descriptor, artifactType string) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Referrers", ctx, subject, artifactType) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Referrers indicates an expected call of Referrers. +func (mr *MockRepositoryStoreMockRecorder) Referrers(ctx, subject, artifactType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Referrers", reflect.TypeOf((*MockRepositoryStore)(nil).Referrers), ctx, subject, artifactType) +} + +// Resolve mocks base method. +func (m *MockRepositoryStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockRepositoryStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockRepositoryStore)(nil).Resolve), ctx, reference) +} + +// Tag mocks base method. +func (m *MockRepositoryStore) Tag(ctx context.Context, desc v1.Descriptor, reference string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tag", ctx, desc, reference) + ret0, _ := ret[0].(error) + return ret0 +} + +// Tag indicates an expected call of Tag. +func (mr *MockRepositoryStoreMockRecorder) Tag(ctx, desc, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tag", reflect.TypeOf((*MockRepositoryStore)(nil).Tag), ctx, desc, reference) +} + +// Tags mocks base method. +func (m *MockRepositoryStore) Tags(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tags", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Tags indicates an expected call of Tags. +func (mr *MockRepositoryStoreMockRecorder) Tags(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tags", reflect.TypeOf((*MockRepositoryStore)(nil).Tags), ctx) +} + +// MockTransport is a mock of Transport interface. +type MockTransport struct { + ctrl *gomock.Controller + recorder *MockTransportMockRecorder + isgomock struct{} +} + +// MockTransportMockRecorder is the mock recorder for MockTransport. +type MockTransportMockRecorder struct { + mock *MockTransport +} + +// NewMockTransport creates a new mock instance. +func NewMockTransport(ctrl *gomock.Controller) *MockTransport { + mock := &MockTransport{ctrl: ctrl} + mock.recorder = &MockTransportMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTransport) EXPECT() *MockTransportMockRecorder { + return m.recorder +} + +// Repository mocks base method. +func (m *MockTransport) Repository(ctx context.Context, repository Repository) (RepositoryStore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Repository", ctx, repository) + ret0, _ := ret[0].(RepositoryStore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Repository indicates an expected call of Repository. +func (mr *MockTransportMockRecorder) Repository(ctx, repository any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Repository", reflect.TypeOf((*MockTransport)(nil).Repository), ctx, repository) +} diff --git a/pkg/oci/ocitest/registry.go b/pkg/oci/ocitest/registry.go index 5a17abe6136..ef882d5d521 100644 --- a/pkg/oci/ocitest/registry.go +++ b/pkg/oci/ocitest/registry.go @@ -7,6 +7,8 @@ import ( "archive/zip" "bytes" "io" + "log" + "net/http" "net/http/httptest" "strings" "testing" @@ -151,3 +153,45 @@ func buildZip(t *testing.T, files map[string]string) []byte { require.NoError(t, zw.Close()) return buf.Bytes() } + +// Credential is the Basic-auth identity an authenticated test registry accepts. +type Credential struct{ Username, Password string } + +// Server is an in-process registry that can be shut down independently of the test. +type Server struct { + // Host is the "127.0.0.1:PORT" registry address. + Host string + server *httptest.Server +} + +// NewServer starts an empty in-process OCI registry (OCI 1.1 referrers included). When +// credential is non-nil, every request must carry exactly that Basic identity; any other +// request receives a 401 challenge, so a client that downgraded to anonymous or reused +// another registry's credentials fails visibly. +func NewServer(t *testing.T, credential *Credential) *Server { + defer perf.Track(nil, "ocitest.NewServer")() + + t.Helper() + + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if credential != nil { + user, password, ok := r.BasicAuth() + if !ok || user != credential.Username || password != credential.Password { + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return &Server{Host: strings.TrimPrefix(srv.URL, "http://"), server: srv} +} + +// Close shuts the registry down now; calling it again is harmless. +func (s *Server) Close() { + defer perf.Track(nil, "ocitest.Server.Close")() + + s.server.Close() +} diff --git a/pkg/oci/referrers.go b/pkg/oci/referrers.go index ac16e750bb9..2816c5970aa 100644 --- a/pkg/oci/referrers.go +++ b/pkg/oci/referrers.go @@ -2,13 +2,12 @@ package oci import ( "context" - "errors" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" - errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" ) @@ -19,16 +18,22 @@ type ReferrerRegistry interface { } // Referrers uses the shared credentials and transport, including the standard -// referrers-tag fallback on registries without the OCI 1.1 API. +// referrers-tag fallback on registries without the OCI 1.1 API. Attachment +// descriptors are returned as an index; no content is fetched. func (c *Client) Referrers(ctx context.Context, ref name.Digest) (v1.ImageIndex, error) { defer perf.Track(c.config, "oci.Client.Referrers")() - options, err := c.options(ctx, ref.Context()) + + store, err := c.Repository(ctx, repositoryOf(ref.Context())) if err != nil { return nil, err } - index, err := remote.Referrers(ref, options...) + found, err := store.Referrers(ctx, ocispec.Descriptor{Digest: digest.Digest(ref.DigestStr())}, "") if err != nil { - return nil, errors.Join(errUtils.ErrOCIRegistry, err) + return nil, err + } + index, err := newReferrersIndex(found) + if err != nil { + return nil, err } return index, nil } diff --git a/pkg/oci/signature/signaturetest/signaturetest.go b/pkg/oci/signature/signaturetest/signaturetest.go new file mode 100644 index 00000000000..86d556b2667 --- /dev/null +++ b/pkg/oci/signature/signaturetest/signaturetest.go @@ -0,0 +1,20 @@ +// Package signaturetest keeps the historical import path for the isolated Sigstore test authority. +// The implementation lives in pkg/signature/signaturetest. +package signaturetest + +import ( + "testing" + + shared "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +// Authority owns isolated CA, CT and Rekor keys and exposes only public trust material. +type Authority = shared.Authority + +// New creates a fresh trust domain for one test. +// +//nolint:lintroller // Test helper delegating to the tracked shared implementation. +func New(t *testing.T) *Authority { + t.Helper() + return shared.New(t) +} diff --git a/pkg/oci/signature/verify.go b/pkg/oci/signature/verify.go index f70fe1483c1..1b13a529259 100644 --- a/pkg/oci/signature/verify.go +++ b/pkg/oci/signature/verify.go @@ -1,30 +1,27 @@ -// Package signature verifies Sigstore bundles against caller-selected identities. +// Package signature adapts the shared Sigstore verification primitives in pkg/signature to +// OCI attachments and plugin trust requirements. Verification itself lives in pkg/signature; this +// package keeps the OCI-specific transport (referrer fetch, offline evidence files) and the +// sentinel errors plugin callers already match. package signature import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" - "regexp" - "strings" - "github.com/sigstore/sigstore-go/pkg/bundle" "github.com/sigstore/sigstore-go/pkg/root" - "github.com/sigstore/sigstore-go/pkg/verify" errUtils "github.com/cloudposse/atmos/errors" - "github.com/cloudposse/atmos/pkg/perf" + sigcore "github.com/cloudposse/atmos/pkg/signature" ) -const MaxBundleSize = 4 << 20 +// MaxBundleSize bounds one Sigstore bundle. +const MaxBundleSize = sigcore.MaxBundleSize // Identity binds a certificate SAN (exact or * wildcard) to an exact OIDC issuer. -type Identity struct{ Identity, Issuer string } +type Identity = sigcore.Identity // Result contains authenticated certificate claims, never claims from unverified metadata. -type Result struct{ Identity, Issuer string } +type Result = sigcore.Result // Verifier uses a trusted root independent of the artifact and its publisher. // LoadRoot is injectable for private trust deployments and isolated cryptographic tests. @@ -32,87 +29,34 @@ type Verifier struct { LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) } -// Verify requires a certificate, SCT, transparency inclusion and authenticated -// signing time. The release digest is always supplied independently of the bundle. +// Verify requires a certificate, SCT, transparency inclusion and authenticated signing time. +// The release digest is always supplied independently of the bundle. Failures carry the plugin +// signature sentinels that existing callers match. +// +//nolint:lintroller // Delegates to the tracked shared verifier. func (v Verifier) Verify(ctx context.Context, raw []byte, digest string, identities []Identity, offline bool) (*Result, error) { - defer perf.Track(nil, "signature.Verifier.Verify")() - if err := ctx.Err(); err != nil { - return nil, err - } - hash, signed, err := parseBundle(raw, digest) - if err != nil { - return nil, err - } - if len(identities) == 0 { - return nil, errUtils.ErrPluginSignatureInvalid - } - options, err := identityOptions(identities) - if err != nil { - return nil, err - } - verifier, err := v.trustedVerifier(ctx, offline) - if err != nil { - return nil, err - } - result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), options...)) + result, err := sigcore.Verifier{LoadRoot: v.LoadRoot}.Verify(ctx, raw, digest, identities, offline) if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - if err := ctx.Err(); err != nil { - return nil, err - } - if result.Signature == nil || result.Signature.Certificate == nil { - return nil, errUtils.ErrPluginSignatureInvalid + return nil, pluginError(err) } - return &Result{Identity: result.Signature.Certificate.SubjectAlternativeName, Issuer: result.Signature.Certificate.Issuer}, nil + return result, nil } -func parseBundle(raw []byte, digest string) ([]byte, *bundle.Bundle, error) { - if len(raw) > MaxBundleSize || len(raw) == 0 { - return nil, nil, errUtils.ErrPluginSignatureInvalid +// pluginError translates shared sentinels to the plugin sentinels, preserving message text. +func pluginError(err error) error { + var shared *sigcore.Error + if !errors.As(err, &shared) { + return err } - hash, err := hex.DecodeString(strings.TrimPrefix(digest, "sha256:")) - if err != nil || !strings.HasPrefix(digest, "sha256:") || len(hash) != sha256.Size { - return nil, nil, errUtils.ErrPluginSignatureInvalid + legacy := errUtils.ErrPluginSignatureInvalid + switch { + case errors.Is(shared.Kind, errUtils.ErrSignatureConfigInvalid): + legacy = errUtils.ErrPluginInvalidConfig + case errors.Is(shared.Kind, errUtils.ErrSignatureOffline): + legacy = errUtils.ErrPluginOfflineArtifact } - var signed bundle.Bundle - if err := json.Unmarshal(raw, &signed); err != nil { - return nil, nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - return hash, &signed, nil -} - -func (v Verifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { - loader := v.LoadRoot - if loader == nil { - loader = loadPublicRoot - } - trusted, err := loader(ctx, offline) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - // Require a certificate identity, a witnessed signing time and inclusion in - // a trusted transparency log. Bare-key bundles cannot establish the requested - // OIDC identity; the key-only rejection test guards this boundary. - verifier, err := verify.NewVerifier(trusted, verify.WithSignedCertificateTimestamps(1), verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - return verifier, nil -} - -func identityOptions(identities []Identity) ([]verify.PolicyOption, error) { - options := make([]verify.PolicyOption, 0, len(identities)) - for _, identity := range identities { - if identity.Identity == "" || identity.Issuer == "" { - return nil, errUtils.ErrPluginInvalidConfig - } - pattern := "^" + strings.ReplaceAll(regexp.QuoteMeta(identity.Identity), `\*`, ".*") + "$" - match, err := verify.NewShortCertificateIdentity(identity.Issuer, "", "", pattern) - if err != nil { - return nil, errors.Join(errUtils.ErrPluginInvalidConfig, err) - } - options = append(options, verify.WithCertificateIdentity(match)) + if shared.Cause == nil { + return legacy } - return options, nil + return errors.Join(legacy, shared.Cause) } diff --git a/pkg/oci/signature/verify_test.go b/pkg/oci/signature/verify_test.go index 276f5df9278..afcb8f3e463 100644 --- a/pkg/oci/signature/verify_test.go +++ b/pkg/oci/signature/verify_test.go @@ -58,9 +58,3 @@ func TestVerifyRealSigstoreEvidence(t *testing.T) { _, err = verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) } - -func TestOfflineTrustCannotFetch(t *testing.T) { - t.Setenv("ATMOS_XDG_DATA_HOME", t.TempDir()) - _, err := loadPublicRoot(t.Context(), true) - require.Error(t, err) -} diff --git a/pkg/oci/status.go b/pkg/oci/status.go index 04ba99a7ddd..4cfec1bc969 100644 --- a/pkg/oci/status.go +++ b/pkg/oci/status.go @@ -5,12 +5,22 @@ import ( "net/http" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" ) // IsNotFound distinguishes a missing manifest from auth and transport failures. +// It recognizes errors from both the legacy and the ORAS-backed transport. // //nolint:lintroller // Trivial accessor, pure lookup, factory option, or delegation; exempt under repository tracking policy. func IsNotFound(err error) bool { var response *transport.Error - return errors.As(err, &response) && response.StatusCode == http.StatusNotFound + if errors.As(err, &response) { + return response.StatusCode == http.StatusNotFound + } + var remote *errcode.ErrorResponse + if errors.As(err, &remote) { + return remote.StatusCode == http.StatusNotFound + } + return errors.Is(err, errdef.ErrNotFound) } diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go new file mode 100644 index 00000000000..1a3f746c9d5 --- /dev/null +++ b/pkg/oci/transport.go @@ -0,0 +1,150 @@ +//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "io" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate go run go.uber.org/mock/mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci Store,RepositoryStore,Transport + +// Repository identifies one OCI repository independently of any client library. +// Registry is the host (with optional port) and Path the repository name. +type Repository struct { + Registry string + Path string +} + +// ParseRepository validates "registry/path" using the same rules as the legacy client, +// so loopback registries still use plain HTTP and Docker Hub short names are normalized. +func ParseRepository(value string) (Repository, error) { + defer perf.Track(nil, "oci.ParseRepository")() + + parsed, err := name.NewRepository(value) + if err != nil { + return Repository{}, wrapRegistry(err) + } + return repositoryOf(parsed), nil +} + +func repositoryOf(repository name.Repository) Repository { + return Repository{Registry: repository.RegistryStr(), Path: repository.RepositoryStr()} +} + +// String returns "registry/path". +// +//nolint:lintroller // Trivial accessor. +func (r Repository) String() string { return r.Registry + "/" + r.Path } + +// legacy converts back to the type used for credential resolution. +func (r Repository) legacy() (name.Repository, error) { + parsed, err := name.NewRepository(r.String()) + if err != nil { + return name.Repository{}, wrapRegistry(err) + } + return parsed, nil +} + +// Fetcher streams content by descriptor. +type Fetcher interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) +} + +// Store is a content-addressed OCI graph store. Its method set deliberately mirrors +// the ORAS target contract so any implementation (registry, layout, in-memory, +// legacy adapter) can participate in graph copy. Only OCI image-spec descriptors +// and go-digest values appear in the signature. +type Store interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) + // Exists reports whether the content is already present. + Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) + // Push stores content that must match the expected descriptor's size and digest. + Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error + // Resolve resolves a tag or digest reference to a descriptor. + Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) + // Tag associates a reference with an existing manifest descriptor. + Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error +} + +// GraphStore additionally discovers the manifests that point at a descriptor, which +// is how referrers and signatures are found for extended copies. +type GraphStore interface { + Store + // Predecessors returns manifests whose subject or children include the node. + Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// ReferencePusher pushes a manifest directly under a tag or digest reference. +type ReferencePusher interface { + PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error +} + +// RepositoryStore is a registry repository opened with already-resolved credentials. +type RepositoryStore interface { + GraphStore + ReferencePusher + // Tags lists every tag without interpreting it as a version. + Tags(ctx context.Context) ([]string, error) + // Delete removes a manifest by its full descriptor. It never deletes tags or shared blobs. + Delete(ctx context.Context, target ocispec.Descriptor) error + // Referrers lists attachments of the subject, optionally filtered by artifact type. + // Registries without the OCI 1.1 API use the referrers tag-schema fallback. + Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) +} + +// Transport opens repositories. Credentials are resolved when a repository is opened, +// so a failed identity fails before any network request and never downgrades to anonymous. +type Transport interface { + Repository(ctx context.Context, repository Repository) (RepositoryStore, error) +} + +// FetchBytes reads and verifies a small blob. The size limit is checked before and while +// reading; both the declared size and the digest must match, so oversized or tampered +// content never reaches callers. +func FetchBytes(ctx context.Context, store Fetcher, target ocispec.Descriptor, limit int64) ([]byte, error) { + defer perf.Track(nil, "oci.FetchBytes")() + + if limit <= 0 || target.Size < 0 || target.Size > limit { + return nil, errUtils.ErrOCIIntegrity + } + reader, err := store.Fetch(ctx, target) + if err != nil { + return nil, wrapRegistry(err) + } + defer reader.Close() + return readVerified(reader, target, limit) +} + +func readVerified(reader io.Reader, target ocispec.Descriptor, limit int64) ([]byte, error) { + raw, err := io.ReadAll(io.LimitReader(reader, limit+1)) + if err != nil { + return nil, wrapIntegrity(err) + } + if int64(len(raw)) != target.Size || int64(len(raw)) > limit { + return nil, errUtils.ErrOCIIntegrity + } + if err := target.Digest.Validate(); err != nil { + return nil, wrapIntegrity(err) + } + verifier := target.Digest.Verifier() + if _, err := verifier.Write(raw); err != nil || !verifier.Verified() { + return nil, errUtils.ErrOCIIntegrity + } + return raw, nil +} + +// Descriptor builds a descriptor for raw bytes. +func Descriptor(mediaType string, raw []byte) ocispec.Descriptor { + defer perf.Track(nil, "oci.Descriptor")() + + return ocispec.Descriptor{MediaType: mediaType, Digest: digest.FromBytes(raw), Size: int64(len(raw))} +} diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go new file mode 100644 index 00000000000..e64af66b385 --- /dev/null +++ b/pkg/oci/transport_behavior_test.go @@ -0,0 +1,422 @@ +//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts. +package oci_test + +import ( + "context" + "encoding/json" + "errors" + "io" + "log" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" +) + +func memoryClient(t *testing.T, host, username, password string) *oci.Client { + t.Helper() + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: username, Password: password})) + return oci.NewClient(nil, oci.WithKeychain(chain)) +} + +// tokenRegistry fronts a registry with a bearer-token service. Only the identity "alice" with the +// password "alice-secret" receives a token; the token is valid for everybody who presents it, which +// is exactly what makes a shared token cache dangerous. +type tokenRegistry struct { + *httptest.Server + host string + tokenRequests map[string]*atomic.Int32 +} + +func newTokenRegistry(t *testing.T) *tokenRegistry { + t.Helper() + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + result := &tokenRegistry{tokenRequests: map[string]*atomic.Int32{"alice": {}, "mallory": {}}} + result.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + user, password, _ := r.BasicAuth() + if counter := result.tokenRequests[user]; counter != nil { + counter.Add(1) + } + if user != "alice" || password != "alice-secret" { + w.WriteHeader(http.StatusUnauthorized) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"token": "alice-token"})) + return + } + if r.Header.Get("Authorization") != "Bearer alice-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+result.URL+`/token",service="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(result.Close) + result.host = strings.TrimPrefix(result.URL, "http://") + return result +} + +func TestClientsDoNotShareCredentialCachesAcrossIdentities(t *testing.T) { + ctx := t.Context() + server := newTokenRegistry(t) + alice := memoryClient(t, server.host, "alice", "alice-secret") + mallory := memoryClient(t, server.host, "mallory", "guess") + repository := oci.Repository{Registry: server.host, Path: "acme/plugin"} + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + require.NoError(t, alice.WriteIndex(ctx, ref, testIndex(t))) + + // Alice's token is valid and cached while Mallory acts; Mallory must still be challenged and denied. + aliceStore, err := alice.Repository(ctx, repository) + require.NoError(t, err) + _, err = aliceStore.Resolve(ctx, "1.0.0") + require.NoError(t, err) + _, err = mallory.Head(ctx, ref) + require.ErrorIs(t, err, errUtils.ErrPullImage) + assert.Positive(t, server.tokenRequests["mallory"].Load(), "the second identity must authenticate on its own") + + // Reopening the same repository with Alice again must not pick up anything of Mallory's. + _, err = alice.Head(ctx, ref) + require.NoError(t, err) +} + +// flakyRegistry wraps a registry and lets a test intercept requests. +type flakyRegistry struct { + *httptest.Server + host string + mu sync.Mutex + requests []string +} + +func newFlakyRegistry(t *testing.T, intercept func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool) *flakyRegistry { + t.Helper() + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + result := &flakyRegistry{} + result.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + result.mu.Lock() + result.requests = append(result.requests, r.Method+" "+r.URL.Path) + result.mu.Unlock() + if intercept(w, r, handler) { + return + } + handler.ServeHTTP(w, r) + })) + t.Cleanup(result.Close) + result.host = strings.TrimPrefix(result.URL, "http://") + return result +} + +// duplicates returns requests that were sent more than once, which would be a mutation retry. +func (f *flakyRegistry) duplicates(method string) []string { + f.mu.Lock() + defer f.mu.Unlock() + seen := map[string]bool{} + var repeated []string + for _, request := range f.requests { + if !strings.HasPrefix(request, method+" ") { + continue + } + if seen[request] { + repeated = append(repeated, request) + } + seen[request] = true + } + return repeated +} + +func (f *flakyRegistry) count(method, fragment string) int { + f.mu.Lock() + defer f.mu.Unlock() + total := 0 + for _, request := range f.requests { + if strings.HasPrefix(request, method+" ") && strings.Contains(request, fragment) { + total++ + } + } + return total +} + +func TestAmbiguousWriteIsReconciledWithoutRetryingTheMutation(t *testing.T) { + // The registry commits the manifest and then fails the response: the client cannot know. + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + serve.ServeHTTP(httptest.NewRecorder(), r) + http.Error(w, "gateway failure after commit", http.StatusBadGateway) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + want, err := index.Digest() + require.NoError(t, err) + + require.NoError(t, client.WriteIndex(t.Context(), ref, index), "a write that landed must be reported as success") + assert.Equal(t, 3, server.count(http.MethodPut, "/manifests/"), "two platform manifests and the index are each written exactly once") + assert.Empty(t, server.duplicates(http.MethodPut)) + head, err := client.Head(t.Context(), ref) + require.NoError(t, err) + assert.Equal(t, want, head.Digest) +} + +func TestAmbiguousWriteThatDidNotLandIsAnError(t *testing.T) { + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + + err = client.WriteIndex(t.Context(), ref, testIndex(t)) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.Empty(t, server.duplicates(http.MethodPut), "a failed mutation is never blindly retried") + assert.Positive(t, server.count(http.MethodPut, "/manifests/")) + _, err = client.Head(t.Context(), ref) + require.Error(t, err) +} + +func TestRejectedWriteIsNotReconciledOrRetriedAnonymously(t *testing.T) { + var anonymous atomic.Int32 + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if _, _, ok := r.BasicAuth(); !ok { + anonymous.Add(1) + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + return true + } + if r.Method == http.MethodPut { + w.WriteHeader(http.StatusForbidden) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + + err = client.WriteIndex(t.Context(), ref, testIndex(t)) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.Equal(t, 0, server.count(http.MethodGet, "/manifests/1.0.0")+server.count(http.MethodHead, "/manifests/1.0.0"), + "a definite rejection must not trigger destination reconciliation") + assert.Empty(t, server.duplicates(http.MethodPut), "a rejected mutation is not retried") + assert.Positive(t, anonymous.Load(), "the registry challenged the first request") +} + +func TestDeleteIsReconciledByResolvingTheDigest(t *testing.T) { + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, serve http.Handler) bool { + if r.Method == http.MethodDelete { + serve.ServeHTTP(httptest.NewRecorder(), r) + http.Error(w, "gateway failure after commit", http.StatusGatewayTimeout) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + require.NoError(t, client.WriteIndex(t.Context(), ref, index)) + digest, err := index.Digest() + require.NoError(t, err) + + require.NoError(t, client.Delete(t.Context(), ref.Context().Digest(digest.String()))) + assert.Equal(t, 1, server.count(http.MethodDelete, "/manifests/")) + _, err = client.Head(t.Context(), ref.Context().Digest(digest.String())) + require.Error(t, err) +} + +func TestReadsAreRetriedButCredentialsAreNotDowngraded(t *testing.T) { + var failures atomic.Int32 + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/tags/list") && failures.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + require.NoError(t, client.WriteIndex(t.Context(), ref, testIndex(t))) + tags, err := client.Tags(t.Context(), ref.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"1.0.0"}, tags) + assert.Equal(t, 2, server.count(http.MethodGet, "/tags/list")) +} + +func TestFetchBytesIsBoundedAndDigestVerified(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + content := []byte(`{"hello":"world"}`) + descriptor := oci.Descriptor("application/json", content) + require.NoError(t, layout.Push(ctx, descriptor, strings.NewReader(string(content)))) + + got, err := oci.FetchBytes(ctx, layout, descriptor, int64(len(content))) + require.NoError(t, err) + assert.Equal(t, content, got) + + _, err = oci.FetchBytes(ctx, layout, descriptor, int64(len(content))-1) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity, "content larger than the limit is rejected before reading") + _, err = oci.FetchBytes(ctx, layout, descriptor, 0) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + + wrongSize := descriptor + wrongSize.Size++ + _, err = oci.FetchBytes(ctx, fetchOverride{descriptor.Digest.String(): content}, wrongSize, 1<<10) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + + tampered := fetchOverride{descriptor.Digest.String(): []byte(`{"hello":"WORLD"}`)} + _, err = oci.FetchBytes(ctx, tampered, descriptor, 1<<10) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity, "same size, different bytes") + + _, err = oci.FetchBytes(ctx, layout, oci.Descriptor("application/json", []byte("absent")), 1<<10) + require.True(t, oci.IsNotFound(err)) + require.ErrorIs(t, err, errdef.ErrNotFound) +} + +type fetchOverride map[string][]byte + +func (f fetchOverride) Fetch(_ context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(string(f[target.Digest.String()]))), nil +} + +// cancellingSource cancels the copy after a number of fetches and counts how many nodes were read. +type cancellingSource struct { + oci.ReadOnlyStore + fetches atomic.Int32 + cancelAfter int32 + cancel context.CancelFunc +} + +func (c *cancellingSource) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if c.fetches.Add(1) == c.cancelAfter { + c.cancel() + } + return c.ReadOnlyStore.Fetch(ctx, target) +} + +func TestCancellationStopsGraphTraversalAndLeavesNoReference(t *testing.T) { + directory := filepath.Join(t.TempDir(), "source") + index := testIndex(t) + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + base, err := oci.OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + + // Establish how many nodes an uninterrupted copy fetches. + complete := &cancellingSource{ReadOnlyStore: base, cancelAfter: -1} + full, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "full")) + require.NoError(t, err) + _, err = oci.Copy(t.Context(), oci.Source{Store: complete, Reference: digest.String()}, oci.Destination{Store: full, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) + require.NoError(t, err) + total := complete.fetches.Load() + require.Greater(t, total, int32(3)) + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + interrupted := &cancellingSource{ReadOnlyStore: base, cancelAfter: 2, cancel: cancel} + target, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: interrupted, Reference: digest.String()}, oci.Destination{Store: target, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) + require.ErrorIs(t, err, context.Canceled) + assert.Less(t, interrupted.fetches.Load(), total, "traversal must stop once the context ends") + assert.Empty(t, target.References(), "an interrupted copy must not publish the reference") + _, err = target.Resolve(t.Context(), "1.0.0") + require.Error(t, err) + + already, stop := context.WithCancel(t.Context()) + stop() + _, err = oci.Copy(already, oci.Source{Store: base, Reference: digest.String()}, oci.Destination{Store: target, Reference: "1.0.0"}) + require.ErrorIs(t, err, context.Canceled) +} + +// landingDestination reports a failed reference push after (optionally) completing it. +type landingDestination struct { + oci.Store + land bool + err error + pushes atomic.Int32 +} + +func (l *landingDestination) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + l.pushes.Add(1) + if l.land { + if err := l.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + if err := l.Tag(ctx, expected, reference); err != nil { + return err + } + } + return l.err +} + +func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { + directory := filepath.Join(t.TempDir(), "source") + index := testIndex(t) + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + source, err := oci.OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + + rejection := errors.Join(errors.New("rejected"), errcodeResponse(http.StatusForbidden)) + for _, tc := range []struct { + name string + land bool + err error + wantErr bool + }{ + {"landed despite unknown failure", true, errors.New("connection reset by peer"), false}, + {"did not land", false, errors.New("connection reset by peer"), true}, + {"landed but definitely rejected", true, rejection, true}, + } { + t.Run(tc.name, func(t *testing.T) { + layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) + require.NoError(t, err) + destination := &landingDestination{Store: layout, land: tc.land, err: tc.err} + root, err := oci.Copy(t.Context(), oci.Source{Store: source, Reference: digest.String()}, oci.Destination{Store: destination, Reference: "1.0.0"}) + assert.Equal(t, int32(1), destination.pushes.Load(), "the mutation is attempted exactly once") + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, digest.String(), root.Digest.String()) + }) + } +} + +func errcodeResponse(status int) error { + return &errcode.ErrorResponse{Method: http.MethodPut, StatusCode: status} +} diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go new file mode 100644 index 00000000000..5d2c9b74b63 --- /dev/null +++ b/pkg/oci/transport_copy.go @@ -0,0 +1,290 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "encoding/json" + "errors" + "io" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ReadOnlyStore is the source side of a graph copy. +type ReadOnlyStore interface { + // Fetch streams the content identified by the descriptor. + Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) + // Exists reports whether the content is present. + Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) + // Resolve resolves a tag or digest reference to a descriptor. + Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) +} + +// ReadOnlyGraphStore is a source that can also report predecessors (referrers). +type ReadOnlyGraphStore interface { + ReadOnlyStore + // Predecessors returns manifests whose subject or children include the node. + Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// Source names the content a copy reads: a store and a tag or digest reference within it. +type Source struct { + Store ReadOnlyStore + Reference string +} + +// GraphSource is a Source that can also report predecessors, for extended copies. +type GraphSource struct { + Store ReadOnlyGraphStore + Reference string +} + +// Destination names where a copy writes: a store and the reference the root is published under. +// An empty Reference reuses the source reference. +type Destination struct { + Store Store + Reference string +} + +const defaultCopyConcurrency = 3 + +type copyConfig struct { + concurrency int + onCopied func(ocispec.Descriptor) + noSubjects bool +} + +// CopyOption configures a graph copy. +type CopyOption func(*copyConfig) + +// WithCopyConcurrency bounds how many nodes transfer at once. Values below one use the default. +// +//nolint:lintroller // Functional option only. +func WithCopyConcurrency(concurrency int) CopyOption { + return func(config *copyConfig) { config.concurrency = concurrency } +} + +// WithCopyObserver reports every node that was actually transferred (not skipped as present). +// +//nolint:lintroller // Functional option only. +func WithCopyObserver(observe func(ocispec.Descriptor)) CopyOption { + return func(config *copyConfig) { config.onCopied = observe } +} + +// WithoutSubjectTraversal stops the copy at the selected manifest instead of also copying the +// manifest named by its subject field. Legacy writers (for example signature attachments) +// publish an attachment next to a subject that already exists at the destination. +// +//nolint:lintroller // Functional option only. +func WithoutSubjectTraversal() CopyOption { + return func(config *copyConfig) { config.noSubjects = true } +} + +func newCopyConfig(options []CopyOption) copyConfig { + config := copyConfig{concurrency: defaultCopyConcurrency} + for _, option := range options { + option(&config) + } + if config.concurrency < 1 { + config.concurrency = defaultCopyConcurrency + } + return config +} + +func (config copyConfig) graphOptions() oras.CopyGraphOptions { + options := oras.CopyGraphOptions{Concurrency: config.concurrency} + if config.noSubjects { + options.FindSuccessors = successorsWithoutSubject + } + if config.onCopied != nil { + options.PostCopy = func(_ context.Context, desc ocispec.Descriptor) error { + config.onCopied(desc) + return nil + } + } + return options +} + +// successorsWithoutSubject is the default ORAS successor discovery minus the subject edge, +// which ORAS lists first for OCI image manifests and indexes. +func successorsWithoutSubject(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + successors, err := content.Successors(ctx, fetcher, desc) + hasSubject := desc.MediaType == ocispec.MediaTypeImageManifest || desc.MediaType == ocispec.MediaTypeImageIndex + if err != nil || !hasSubject || len(successors) == 0 { + return successors, err + } + raw, err := content.FetchAll(ctx, fetcher, desc) + if err != nil { + return nil, err + } + var manifest struct { + Subject *ocispec.Descriptor `json:"subject"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + if manifest.Subject != nil && successors[0].Digest == manifest.Subject.Digest { + return successors[1:], nil + } + return successors, nil +} + +// Copy transfers the graph below the source reference (a manifest, image, or index with every +// child) into the destination under its reference (the source reference when empty). Raw manifest and index bytes are streamed +// unchanged and every node is digest-verified by the receiving store, so identity +// survives registry, layout, and registry hops. Cancelling ctx stops traversal. +// +// If a write fails with an unknown outcome the destination is resolved before deciding: +// when the destination reference already addresses the source root the copy is complete. There are no blind +// mutation retries, and rejected credentials are never retried anonymously. +func Copy(ctx context.Context, src Source, dst Destination, options ...CopyOption) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Copy")() + + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if dst.Reference == "" { + dst.Reference = src.Reference + } + config := newCopyConfig(options) + root, err := oras.Copy(ctx, guardedSource{ReadOnlyStore: src.Store}, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, + oras.CopyOptions{CopyGraphOptions: config.graphOptions()}) + if err != nil { + return reconcile(ctx, err, src.Store, src.Reference, dst) + } + return root, nil +} + +// ExtendedCopy is Copy plus every predecessor of the graph (signatures, SBOMs, and other +// referrers), discovered through the source's referrers API or tag-schema fallback. +func ExtendedCopy(ctx context.Context, src GraphSource, dst Destination, options ...CopyOption) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.ExtendedCopy")() + + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if dst.Reference == "" { + dst.Reference = src.Reference + } + config := newCopyConfig(options) + graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store}, graph: src.Store} + root, err := oras.ExtendedCopy(ctx, graph, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, + oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.graphOptions()}}) + if err != nil { + return reconcile(ctx, err, src.Store, src.Reference, dst) + } + return root, nil +} + +// reconcile decides the outcome of a failed copy by looking at the destination. +func reconcile(ctx context.Context, cause error, src ReadOnlyStore, srcRef string, dst Destination) (ocispec.Descriptor, error) { + failure := classifyCopyError(cause) + if !ambiguousWrite(ctx, failure) { + return ocispec.Descriptor{}, failure + } + want, err := src.Resolve(ctx, srcRef) + if err != nil { + return ocispec.Descriptor{}, failure + } + have, err := dst.Store.Resolve(ctx, dst.Reference) + if err != nil || have.Digest != want.Digest { + return ocispec.Descriptor{}, failure + } + return want, nil +} + +func classifyCopyError(err error) error { + if errors.Is(err, errUtils.ErrOCIRegistry) || errors.Is(err, errUtils.ErrOCILayout) { + return err + } + return wrapRegistry(err) +} + +// guardedSource aborts as soon as the context ends, even for stores that ignore it. +type guardedSource struct{ ReadOnlyStore } + +func (g guardedSource) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return g.ReadOnlyStore.Fetch(ctx, target) +} + +func (g guardedSource) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return g.ReadOnlyStore.Exists(ctx, target) +} + +type guardedGraphSource struct { + guardedSource + graph ReadOnlyGraphStore +} + +func (g guardedGraphSource) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return g.graph.Predecessors(ctx, node) +} + +// guardedStore does the same for destinations, so cancellation never starts another write. +type guardedStore struct{ Store } + +func (g guardedStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return g.Store.Exists(ctx, target) +} + +func (g guardedStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + err := g.Store.Push(ctx, expected, content) + if err == nil || errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + // The write may have landed even though it was reported as failed. Look at the + // destination before deciding; never write again. + if ambiguousWrite(ctx, err) { + if exists, existsErr := g.Store.Exists(ctx, expected); existsErr == nil && exists { + return nil + } + } + return err +} + +// PushReference forwards to the wrapped store when it can push a manifest under a reference, +// which lets registry destinations write the root manifest directly under its tag. +func (g guardedStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + if err := ctx.Err(); err != nil { + return err + } + err := g.pushReference(ctx, expected, content, reference) + if err == nil || errors.Is(err, errdef.ErrAlreadyExists) || !ambiguousWrite(ctx, err) { + return err + } + if resolved, resolveErr := g.Resolve(ctx, reference); resolveErr == nil && resolved.Digest == expected.Digest { + return nil + } + return err +} + +func (g guardedStore) pushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + if pusher, ok := g.Store.(ReferencePusher); ok { + return pusher.PushReference(ctx, expected, content, reference) + } + if err := g.Store.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + return g.Tag(ctx, expected, reference) +} diff --git a/pkg/oci/transport_errors.go b/pkg/oci/transport_errors.go new file mode 100644 index 00000000000..124c3c85ecf --- /dev/null +++ b/pkg/oci/transport_errors.go @@ -0,0 +1,76 @@ +package oci + +import ( + "context" + "errors" + "net/http" + + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/registry/remote/errcode" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// wrapRegistry marks a failure as an OCI registry error while keeping the cause chain. +// HTTP failures also carry a legacy *transport.Error with the real status code so the +// existing status classifiers (IsNotFound, discovery availability, pull diagnostics) +// keep working for callers that still reason about go-containerregistry errors. +func wrapRegistry(err error) error { + if err == nil { + return nil + } + if legacy := legacyStatusError(err); legacy != nil && !errors.As(err, new(*transport.Error)) { + return errors.Join(errUtils.ErrOCIRegistry, err, legacy) + } + return errors.Join(errUtils.ErrOCIRegistry, err) +} + +func wrapIntegrity(err error) error { + return errors.Join(errUtils.ErrOCIIntegrity, err) +} + +// legacyStatusError synthesizes the status carrier from an ORAS error. +func legacyStatusError(err error) *transport.Error { + var response *errcode.ErrorResponse + if errors.As(err, &response) { + return &transport.Error{StatusCode: response.StatusCode} + } + if errors.Is(err, errdef.ErrNotFound) { + return &transport.Error{StatusCode: http.StatusNotFound} + } + return nil +} + +// statusOf returns the HTTP status behind an error, or zero when none was received. +func statusOf(err error) int { + var response *errcode.ErrorResponse + if errors.As(err, &response) { + return response.StatusCode + } + var legacy *transport.Error + if errors.As(err, &legacy) { + return legacy.StatusCode + } + if errors.Is(err, errdef.ErrNotFound) { + return http.StatusNotFound + } + return 0 +} + +// definiteRejection reports whether the destination clearly refused a write. Authentication, +// authorization, validation and conflict responses are final answers; everything else +// (timeouts, resets, 5xx) leaves the write outcome unknown. +func definiteRejection(err error) bool { + status := statusOf(err) + return status >= http.StatusBadRequest && status < http.StatusInternalServerError && + status != http.StatusRequestTimeout && status != http.StatusTooManyRequests +} + +// ambiguousWrite reports whether a failed mutation may nevertheless have taken effect. +func ambiguousWrite(ctx context.Context, err error) bool { + if err == nil || ctx.Err() != nil || errors.Is(err, context.Canceled) { + return false + } + return !definiteRejection(err) +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go new file mode 100644 index 00000000000..bdab1c7f4cb --- /dev/null +++ b/pkg/oci/transport_internal_test.go @@ -0,0 +1,221 @@ +package oci + +import ( + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/errcode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" +) + +type failingAuthenticator struct{} + +func (failingAuthenticator) Authorization() (*authn.AuthConfig, error) { return nil, os.ErrPermission } + +func TestCredentialFromAdaptsEveryAuthenticatorShape(t *testing.T) { + encoded := base64.StdEncoding.EncodeToString([]byte("encoded-user:encoded-secret")) + for _, tc := range []struct { + name string + authenticator authn.Authenticator + want auth.Credential + err error + }{ + {"nil authenticator is anonymous", nil, auth.EmptyCredential, nil}, + {"anonymous", authn.Anonymous, auth.EmptyCredential, nil}, + {"basic", &authn.Basic{Username: "u", Password: "p"}, auth.Credential{Username: "u", Password: "p"}, nil}, + {"registry token", &authn.Bearer{Token: "access"}, auth.Credential{AccessToken: "access"}, nil}, + {"identity token", authn.FromConfig(authn.AuthConfig{IdentityToken: "refresh"}), auth.Credential{RefreshToken: "refresh"}, nil}, + {"base64 auth field", authn.FromConfig(authn.AuthConfig{Auth: encoded}), auth.Credential{Username: "encoded-user", Password: "encoded-secret"}, nil}, + {"explicit fields win over auth field", authn.FromConfig(authn.AuthConfig{Username: "u", Password: "p", Auth: encoded}), auth.Credential{Username: "u", Password: "p"}, nil}, + {"malformed auth field", authn.FromConfig(authn.AuthConfig{Auth: "!!"}), auth.EmptyCredential, errUtils.ErrOCIRegistry}, + {"auth field without separator", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte("nocolon"))}), auth.EmptyCredential, errUtils.ErrOCIRegistry}, + {"authenticator failure fails closed", failingAuthenticator{}, auth.EmptyCredential, os.ErrPermission}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := credentialFrom(tc.authenticator) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + assert.Equal(t, auth.EmptyCredential, got) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestCredentialsAreReleasedOnlyToTheirRegistry(t *testing.T) { + credential := hostScopedCredential("registry.example.com:5000", &authn.Basic{Username: "u", Password: "p"}) + for host, wantSecret := range map[string]bool{ + "registry.example.com:5000": true, + "REGISTRY.example.com:5000": true, + "registry.example.com": false, + "evil.example.com:5000": false, + "auth.example.com": false, + } { + got, err := credential(t.Context(), host) + require.NoError(t, err) + assert.Equal(t, wantSecret, got.Password == "p", host) + } + + hub := hostScopedCredential("docker.io", &authn.Basic{Username: "u", Password: "p"}) + got, err := hub(t.Context(), "registry-1.docker.io") + require.NoError(t, err) + assert.Equal(t, "p", got.Password, "Docker Hub traffic is redirected to its API endpoint") + assert.Equal(t, "docker.io", orasHost("index.docker.io")) + assert.Equal(t, "ghcr.io", orasHost("ghcr.io")) +} + +func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + repository := Repository{Registry: host, Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + + for _, tc := range []struct { + name string + auth authn.Authenticator + matched bool + err error + }{ + {"resolver error", nil, true, os.ErrPermission}, + {"matched identity without credentials", nil, true, nil}, + {"matched identity resolves to anonymous", authn.Anonymous, true, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(tc.auth, tc.matched, tc.err) + // Neither ambient keyring credentials nor anonymous access may stand in for the identity. + store := keyring.NewMockKeyring(controller) + client := NewClient(nil, WithAuthenticationResolver(resolver), WithKeychain(CredentialKeychain{Store: store})) + opened, err := client.Repository(t.Context(), repository) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + } + assert.Nil(t, opened) + }) + } + assert.Zero(t, requests.Load(), "no request may reach the registry when the identity cannot be resolved") +} + +func TestBoundIdentityTakesPrecedenceOverAmbientCredentials(t *testing.T) { + var seen atomic.Value + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if user, password, ok := r.BasicAuth(); ok { + seen.Store(user + ":" + password) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"name":"acme/plugin","tags":["1.0.0"]}`)) + return + } + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + repository := Repository{Registry: host, Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(&authn.Basic{Username: "identity", Password: "bound"}, true, nil) + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := CredentialKeychain{Store: store} + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: "ambient", Password: "stored"})) + + client := NewClient(nil, WithAuthenticationResolver(resolver), WithKeychain(chain)) + opened, err := client.Repository(t.Context(), repository) + require.NoError(t, err) + tags, err := opened.Tags(t.Context()) + require.NoError(t, err) + assert.Equal(t, []string{"1.0.0"}, tags) + assert.Equal(t, "identity:bound", seen.Load()) +} + +func TestParseRepositoryUsesLegacyRegistryRules(t *testing.T) { + repository, err := ParseRepository("ghcr.io/acme/plugin") + require.NoError(t, err) + assert.Equal(t, Repository{Registry: "ghcr.io", Path: "acme/plugin"}, repository) + assert.Equal(t, "ghcr.io/acme/plugin", repository.String()) + + hub, err := ParseRepository("library/nginx") + require.NoError(t, err) + assert.Equal(t, Repository{Registry: "index.docker.io", Path: "library/nginx"}, hub) + + _, err = ParseRepository("not a repository") + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + _, err = Repository{Registry: "bad registry", Path: "x"}.legacy() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) +} + +func TestNewReferrersIndexRoundTripsDescriptors(t *testing.T) { + found := []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageManifest, Size: 10, Digest: digest.FromString("a"), + ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json", Annotations: map[string]string{"k": "v"}, + }} + index, err := newReferrersIndex(found) + require.NoError(t, err) + manifest, err := index.IndexManifest() + require.NoError(t, err) + require.Len(t, manifest.Manifests, 1) + assert.Equal(t, found[0].Digest.String(), manifest.Manifests[0].Digest.String()) + assert.Equal(t, found[0].ArtifactType, manifest.Manifests[0].ArtifactType) + assert.Equal(t, found[0].Annotations, manifest.Manifests[0].Annotations) + size, err := index.Size() + require.NoError(t, err) + raw, err := index.RawManifest() + require.NoError(t, err) + assert.Equal(t, int64(len(raw)), size) + hash, err := index.Digest() + require.NoError(t, err) + assert.Equal(t, digest.FromBytes(raw).String(), hash.String()) + _, err = index.Image(hash) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + _, err = index.ImageIndex(hash) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + + _, err = newReferrersIndex([]ocispec.Descriptor{{Digest: "not-a-digest"}}) + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) +} + +func TestStatusClassification(t *testing.T) { + notFound := wrapRegistry(&errorResponse404) + assert.True(t, IsNotFound(notFound)) + assert.Equal(t, http.StatusNotFound, statusOf(notFound)) + assert.True(t, definiteRejection(wrapRegistry(&errorResponse403))) + assert.False(t, definiteRejection(wrapRegistry(errors.New("connection reset")))) + assert.False(t, definiteRejection(wrapRegistry(&errorResponse503))) + assert.False(t, IsNotFound(wrapRegistry(&errorResponse403))) + assert.False(t, IsNotFound(nil)) + assert.NoError(t, wrapRegistry(nil)) +} + +var ( + errorResponse404 = errcode.ErrorResponse{Method: http.MethodGet, StatusCode: http.StatusNotFound} + errorResponse403 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusForbidden} + errorResponse503 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusServiceUnavailable} +) diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go new file mode 100644 index 00000000000..ee43ade328c --- /dev/null +++ b/pkg/oci/transport_layout.go @@ -0,0 +1,239 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sync" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/filesystem" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + layoutIndexMode = 0o644 + maxAttachmentManifest = 4 << 20 +) + +// Layout is a writable OCI image layout that can hold any number of named roots. +// +// It keeps index.json limited to the roots a caller explicitly tagged. Child manifests +// and blobs live only under blobs/, exactly like the single-release layouts written by +// SaveIndex, so a layout that holds one release still satisfies LoadIndex. A layout with +// several references is intentionally rejected by LoadIndex; read it through Layout. +// +// Layout does not coordinate concurrent writers in other processes. +type Layout struct { + root string + store *oci.Store + mu sync.Mutex + index []ocispec.Descriptor +} + +var ( + _ GraphStore = (*Layout)(nil) + _ ReadOnlyGraphStore = (*oci.ReadOnlyStore)(nil) +) + +// OpenLayout opens or creates a layout directory. +func OpenLayout(ctx context.Context, directory string) (*Layout, error) { + defer perf.Track(nil, "oci.OpenLayout")() + + store, err := oci.NewWithContext(ctx, directory) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + // The ORAS store would otherwise list every pushed manifest in index.json. + store.AutoSaveIndex = false + store.AutoGC = false + layout := &Layout{root: filepath.Clean(directory), store: store} + if err := layout.loadIndex(); err != nil { + return nil, err + } + return layout, nil +} + +// OpenLayoutReadOnly opens an existing layout without ever writing to it, so committed +// fixtures and offline media stay untouched. +func OpenLayoutReadOnly(ctx context.Context, directory string) (ReadOnlyGraphStore, error) { + defer perf.Track(nil, "oci.OpenLayoutReadOnly")() + + store, err := oci.NewFromFS(ctx, os.DirFS(directory)) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + return store, nil +} + +func (l *Layout) indexPath() string { return filepath.Join(l.root, ocispec.ImageIndexFile) } + +func (l *Layout) loadIndex() error { + raw, err := os.ReadFile(l.indexPath()) + if err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + l.index = index.Manifests + return nil +} + +func (l *Layout) writeIndex() error { + index := ocispec.Index{MediaType: ocispec.MediaTypeImageIndex, Manifests: l.index} + index.SchemaVersion = 2 + if index.Manifests == nil { + index.Manifests = []ocispec.Descriptor{} + } + raw, err := json.Marshal(index) + if err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + if err := filesystem.NewOSFileSystem().WriteFileAtomic(l.indexPath(), raw, layoutIndexMode); err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + return nil +} + +// Fetch streams a blob from the layout. +func (l *Layout) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := l.store.Fetch(ctx, target) + return reader, layoutError(err) +} + +// Exists reports whether the blob is in the layout. +func (l *Layout) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := l.store.Exists(ctx, target) + return exists, layoutError(err) +} + +// Push writes a digest-verified blob. Pushing existing content reports ErrAlreadyExists. +// A manifest that names a subject (a signature, SBOM, or other attachment) is also +// recorded as an unnamed root, because that is the only way its relationship to the +// subject survives reopening the layout. +func (l *Layout) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + err := l.store.Push(ctx, expected, content) + if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return layoutError(err) + } + if recordErr := l.recordAttachment(ctx, expected); recordErr != nil { + return recordErr + } + return layoutError(err) +} + +func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) error { + if desc.MediaType != ocispec.MediaTypeImageManifest && desc.MediaType != ocispec.MediaTypeImageIndex { + return nil + } + raw, err := FetchBytes(ctx, l.store, desc, maxAttachmentManifest) + if err != nil { + return layoutError(err) + } + var manifest struct { + Subject *ocispec.Descriptor `json:"subject"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + return errors.Join(errUtils.ErrOCILayout, err) + } + if manifest.Subject == nil { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + for i := range l.index { + if l.index[i].Digest == desc.Digest { + return nil + } + } + l.index = append(l.index, ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size}) + return l.writeIndex() +} + +// Resolve resolves a reference name or digest to a descriptor. +func (l *Layout) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Layout.Resolve")() + + resolved, err := l.store.Resolve(ctx, reference) + return resolved, layoutError(err) +} + +// Predecessors lists manifests in the layout that reference the node. +func (l *Layout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + found, err := l.store.Predecessors(ctx, node) + return found, layoutError(err) +} + +// Tag records reference as a named root in index.json. Digest references are implicit +// and never listed. An unnamed root with the same digest is named in place, so a legacy +// single-release layout keeps exactly one entry. +func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + defer perf.Track(nil, "oci.Layout.Tag")() + + if err := l.store.Tag(ctx, desc, reference); err != nil { + return layoutError(err) + } + if reference == desc.Digest.String() { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + named := ocispec.Descriptor{ + MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size, + Annotations: map[string]string{ocispec.AnnotationRefName: reference}, + } + l.index = withReference(l.index, named) + return l.writeIndex() +} + +func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []ocispec.Descriptor { + reference := named.Annotations[ocispec.AnnotationRefName] + for i := range entries { + if entries[i].Annotations[ocispec.AnnotationRefName] == reference { + entries[i] = named + return entries + } + } + for i := range entries { + if entries[i].Digest == named.Digest && entries[i].Annotations[ocispec.AnnotationRefName] == "" { + entries[i] = named + return entries + } + } + return append(entries, named) +} + +// References returns the named roots in index.json in insertion order. +func (l *Layout) References() []string { + defer perf.Track(nil, "oci.Layout.References")() + + l.mu.Lock() + defer l.mu.Unlock() + var references []string + for i := range l.index { + if reference := l.index[i].Annotations[ocispec.AnnotationRefName]; reference != "" { + references = append(references, reference) + } + } + return references +} + +func layoutError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return errors.Join(errUtils.ErrOCILayout, err) +} diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go new file mode 100644 index 00000000000..9f5b2160821 --- /dev/null +++ b/pkg/oci/transport_legacy.go @@ -0,0 +1,228 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/errdef" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// legacyRoot is the only reference a legacyGraph resolves. +const legacyRoot = "root" + +// legacyGraph exposes the raw bytes of a go-containerregistry image or index as a +// read-only OCI graph. It serializes exactly what the legacy object already holds: +// manifests are never rebuilt (no ORAS PackManifest), so digests cannot drift. +type legacyGraph struct { + root ocispec.Descriptor + blobs map[digest.Digest]func() (io.ReadCloser, error) +} + +var _ ReadOnlyStore = (*legacyGraph)(nil) + +func newLegacyGraph() *legacyGraph { + return &legacyGraph{blobs: make(map[digest.Digest]func() (io.ReadCloser, error))} +} + +func legacyImageGraph(image v1.Image) (*legacyGraph, error) { + graph := newLegacyGraph() + root, err := graph.addImage(image) + if err != nil { + return nil, err + } + graph.root = root + return graph, nil +} + +func legacyIndexGraph(index v1.ImageIndex) (*legacyGraph, error) { + graph := newLegacyGraph() + root, err := graph.addIndex(index) + if err != nil { + return nil, err + } + graph.root = root + return graph, nil +} + +func (g *legacyGraph) addBytes(mediaType string, raw []byte) ocispec.Descriptor { + desc := Descriptor(mediaType, raw) + g.blobs[desc.Digest] = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(raw)), nil } + return desc +} + +func (g *legacyGraph) addImage(image v1.Image) (ocispec.Descriptor, error) { + raw, err := image.RawManifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + mediaType, err := image.MediaType() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + manifest, err := image.Manifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + rawConfig, err := image.RawConfigFile() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.blobs[digest.Digest(manifest.Config.Digest.String())] = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(rawConfig)), nil + } + for i := range manifest.Layers { + hash := manifest.Layers[i].Digest + g.blobs[digest.Digest(hash.String())] = func() (io.ReadCloser, error) { + layer, err := image.LayerByDigest(hash) + if err != nil { + return nil, err + } + return layer.Compressed() + } + } + return g.addBytes(string(mediaType), raw), nil +} + +func (g *legacyGraph) addIndex(index v1.ImageIndex) (ocispec.Descriptor, error) { + raw, err := index.RawManifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + mediaType, err := index.MediaType() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + manifest, err := index.IndexManifest() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + for i := range manifest.Manifests { + if err := g.addChild(index, &manifest.Manifests[i]); err != nil { + return ocispec.Descriptor{}, err + } + } + return g.addBytes(string(mediaType), raw), nil +} + +// addChild records one child manifest or nested index of a legacy index. +func (g *legacyGraph) addChild(index v1.ImageIndex, child *v1.Descriptor) error { + switch { + case child.MediaType.IsIndex(): + nested, err := index.ImageIndex(child.Digest) + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + _, err = g.addIndex(nested) + return err + case child.MediaType.IsImage(): + nested, err := index.Image(child.Digest) + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + _, err = g.addImage(nested) + return err + default: + return fmt.Errorf("%w: unsupported child media type %q", errUtils.ErrOCIIntegrity, child.MediaType) + } +} + +// Fetch streams a known blob. +func (g *legacyGraph) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + open, found := g.blobs[target.Digest] + if !found { + return nil, fmt.Errorf("%s: %w", target.Digest, errdef.ErrNotFound) + } + return open() +} + +// Exists reports whether the legacy object holds the blob. +func (g *legacyGraph) Exists(_ context.Context, target ocispec.Descriptor) (bool, error) { + _, found := g.blobs[target.Digest] + return found, nil +} + +// Resolve returns the root for the only reference the graph knows. +func (g *legacyGraph) Resolve(_ context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.legacyGraph.Resolve")() + + if reference != legacyRoot && reference != g.root.Digest.String() { + return ocispec.Descriptor{}, fmt.Errorf("%s: %w", reference, errdef.ErrNotFound) + } + return g.root, nil +} + +// referrersIndex presents ORAS referrer descriptors as the legacy image index. +type referrersIndex struct { + manifest *v1.IndexManifest + raw []byte + hash v1.Hash +} + +var _ v1.ImageIndex = (*referrersIndex)(nil) + +func newReferrersIndex(found []ocispec.Descriptor) (*referrersIndex, error) { + manifest := &v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex, Manifests: make([]v1.Descriptor, 0, len(found))} + for i := range found { + hash, err := v1.NewHash(found[i].Digest.String()) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + manifest.Manifests = append(manifest.Manifests, v1.Descriptor{ + MediaType: types.MediaType(found[i].MediaType), Size: found[i].Size, Digest: hash, + Annotations: found[i].Annotations, ArtifactType: found[i].ArtifactType, URLs: found[i].URLs, + }) + } + raw, err := json.Marshal(manifest) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + hash, _, err := v1.SHA256(bytes.NewReader(raw)) + if err != nil { + return nil, errors.Join(errUtils.ErrOCIIntegrity, err) + } + return &referrersIndex{manifest: manifest, raw: raw, hash: hash}, nil +} + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) MediaType() (types.MediaType, error) { return types.OCIImageIndex, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) Digest() (v1.Hash, error) { return r.hash, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) Size() (int64, error) { return int64(len(r.raw)), nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) IndexManifest() (*v1.IndexManifest, error) { return r.manifest, nil } + +//nolint:lintroller // Trivial getters of an in-memory value. +func (r *referrersIndex) RawManifest() ([]byte, error) { return r.raw, nil } + +// Image is unsupported: a referrers response holds descriptors, not content. +// +//nolint:lintroller // Constant error. +func (r *referrersIndex) Image(v1.Hash) (v1.Image, error) { + return nil, errUtils.ErrOCIRegistry +} + +// ImageIndex is unsupported: a referrers response holds descriptors, not content. +// +//nolint:lintroller // Constant error. +func (r *referrersIndex) ImageIndex(v1.Hash) (v1.ImageIndex, error) { + return nil, errUtils.ErrOCIRegistry +} diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go new file mode 100644 index 00000000000..b977c970129 --- /dev/null +++ b/pkg/oci/transport_registry.go @@ -0,0 +1,225 @@ +//nolint:gocritic,lintroller // Descriptor-by-value parameters and thin store delegators mirror the ORAS content contracts so stores stay interchangeable. +package oci + +import ( + "context" + "encoding/base64" + "io" + "net/http" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + ggcrremote "github.com/google/go-containerregistry/pkg/v1/remote" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + userAgent = "atmos" + dockerHubLegacy = "index.docker.io" + dockerHubRegistry = "docker.io" + dockerHubEndpoint = "registry-1.docker.io" +) + +// Repository opens a registry repository with credentials resolved up front. Resolution +// follows the same precedence as the legacy methods (bound identity, keychain, GHCR +// settings, fallback); an explicitly bound identity that fails stops here, before any +// request, and is never replaced by anonymous access. +// +// Every call builds a new credential cache, so tokens obtained for one identity can +// never be presented by another client or for another repository open. +func (c *Client) Repository(ctx context.Context, repository Repository) (RepositoryStore, error) { + defer perf.Track(c.config, "oci.Client.Repository")() + + resource, err := repository.legacy() + if err != nil { + return nil, err + } + authenticator, err := c.authentication(ctx, resource) + if err != nil { + return nil, err + } + repo, err := c.remoteRepository(resource, authenticator) + if err != nil { + return nil, err + } + return ®istryStore{repo: repo}, nil +} + +func (c *Client) remoteRepository(resource name.Repository, authenticator authn.Authenticator) (*remote.Repository, error) { + host := orasHost(resource.RegistryStr()) + repo, err := remote.NewRepository(host + "/" + resource.RepositoryStr()) + if err != nil { + return nil, wrapRegistry(err) + } + repo.PlainHTTP = resource.Scheme() == "http" + repo.Client = &auth.Client{ + Client: &http.Client{Transport: newRegistryTransport(c.transport)}, + Header: http.Header{"User-Agent": {userAgent}}, + Cache: auth.NewCache(), + Credential: hostScopedCredential(host, authenticator), + } + return repo, nil +} + +func orasHost(registry string) string { + if strings.EqualFold(registry, dockerHubLegacy) { + return dockerHubRegistry + } + return registry +} + +// hostScopedCredential releases credentials only to the registry that the repository +// was opened for. ORAS also asks for credentials when it contacts a token realm; the +// registry-scoped credential is the one the legacy transport sent there as well. +func hostScopedCredential(registry string, authenticator authn.Authenticator) auth.CredentialFunc { + return func(_ context.Context, hostport string) (auth.Credential, error) { + if !hostMatches(registry, hostport) { + return auth.EmptyCredential, nil + } + return credentialFrom(authenticator) + } +} + +func hostMatches(registry, hostport string) bool { + if strings.EqualFold(registry, hostport) { + return true + } + return strings.EqualFold(registry, dockerHubRegistry) && strings.EqualFold(hostport, dockerHubEndpoint) +} + +// credentialFrom adapts a go-containerregistry authenticator. Authorization is invoked on +// every challenge, so authenticators that refresh tokens keep refreshing. +func credentialFrom(authenticator authn.Authenticator) (auth.Credential, error) { + if authenticator == nil || authenticator == authn.Anonymous { + return auth.EmptyCredential, nil + } + config, err := authenticator.Authorization() + if err != nil { + return auth.EmptyCredential, wrapRegistry(err) + } + if config == nil { + return auth.EmptyCredential, nil + } + credential := auth.Credential{ + Username: config.Username, + Password: config.Password, + RefreshToken: config.IdentityToken, + AccessToken: config.RegistryToken, + } + if config.Auth != "" && credential.Username == "" && credential.Password == "" { + decoded, err := base64.StdEncoding.DecodeString(config.Auth) + if err != nil { + return auth.EmptyCredential, wrapRegistry(err) + } + user, password, found := strings.Cut(string(decoded), ":") + if !found { + return auth.EmptyCredential, errUtils.ErrOCIRegistry + } + credential.Username, credential.Password = user, password + } + return credential, nil +} + +// registryTransport retries only idempotent reads. Mutations are sent exactly once; +// an unknown outcome is reconciled by resolving the destination instead. +type registryTransport struct { + reads http.RoundTripper + base http.RoundTripper +} + +func newRegistryTransport(base http.RoundTripper) http.RoundTripper { + if base == nil { + // Keep the transport tuning (proxy, dialer, TLS defaults) the legacy client used. + base = ggcrremote.DefaultTransport + } + return registryTransport{reads: &retry.Transport{Base: base}, base: base} +} + +//nolint:lintroller // Hot-path transport wrapper that only selects a round tripper. +func (t registryTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.Method == http.MethodGet || request.Method == http.MethodHead { + return t.reads.RoundTrip(request) + } + return t.base.RoundTrip(request) +} + +// registryStore adapts one ORAS repository to RepositoryStore. +type registryStore struct { + repo *remote.Repository +} + +func (s *registryStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := s.repo.Fetch(ctx, target) + return reader, wrapRegistry(err) +} + +func (s *registryStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := s.repo.Exists(ctx, target) + return exists, wrapRegistry(err) +} + +func (s *registryStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + return wrapRegistry(s.repo.Push(ctx, expected, content)) +} + +func (s *registryStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + return wrapRegistry(s.repo.PushReference(ctx, expected, content, reference)) +} + +func (s *registryStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.registryStore.Resolve")() + + resolved, err := s.repo.Resolve(ctx, reference) + return resolved, wrapRegistry(err) +} + +func (s *registryStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + return wrapRegistry(s.repo.Tag(ctx, desc, reference)) +} + +func (s *registryStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + found, err := s.repo.Predecessors(ctx, node) + return found, wrapRegistry(err) +} + +func (s *registryStore) Tags(ctx context.Context) ([]string, error) { + defer perf.Track(nil, "oci.registryStore.Tags")() + + var tags []string + err := s.repo.Tags(ctx, "", func(page []string) error { + tags = append(tags, page...) + return nil + }) + if err != nil { + return nil, wrapRegistry(err) + } + return tags, nil +} + +func (s *registryStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + return wrapRegistry(s.repo.Delete(ctx, target)) +} + +func (s *registryStore) Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { + var found []ocispec.Descriptor + err := s.repo.Referrers(ctx, subject, artifactType, func(page []ocispec.Descriptor) error { + found = append(found, page...) + return nil + }) + if err != nil { + return nil, wrapRegistry(err) + } + return found, nil +} + +var ( + _ RepositoryStore = (*registryStore)(nil) + _ Transport = (*Client)(nil) +) diff --git a/pkg/oci/transport_roundtrip_test.go b/pkg/oci/transport_roundtrip_test.go new file mode 100644 index 00000000000..f1e32120543 --- /dev/null +++ b/pkg/oci/transport_roundtrip_test.go @@ -0,0 +1,311 @@ +//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts. +package oci_test + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" + "github.com/cloudposse/atmos/pkg/oci/signature" + "github.com/cloudposse/atmos/pkg/oci/signature/signaturetest" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" +) + +// pluginReleaseIndexDigest is the pinned root digest of the committed A00 plugin release fixture +// (pkg/artifact/compat/testdata/plugin-release). It must never change. +const pluginReleaseIndexDigest = "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + +func pluginFixtureDirectory() string { + return filepath.Join("..", "artifact", "compat", "testdata", "plugin-release") +} + +// authenticatedRegistry starts a registry that accepts exactly one identity and returns a client +// that can only present that identity. +func authenticatedRegistry(t *testing.T, password string) (*ocitest.Server, *oci.Client) { + t.Helper() + server := ocitest.NewServer(t, &ocitest.Credential{Username: "publisher", Password: password}) + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + require.NoError(t, chain.Login(server.Host, &authn.AuthConfig{Username: "publisher", Password: password})) + return server, oci.NewClient(nil, oci.WithKeychain(chain)) +} + +func openRepository(t *testing.T, client *oci.Client, host string) oci.RepositoryStore { + t.Helper() + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + return store +} + +// readTree returns every blob below a layout, keyed by file name. +func readBlobs(t *testing.T, directory string) map[string][]byte { + t.Helper() + blobs := map[string][]byte{} + root := filepath.Join(directory, "blobs", "sha256") + require.NoError(t, filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + content, err := os.ReadFile(path) + blobs[entry.Name()] = content + return err + })) + require.NotEmpty(t, blobs) + return blobs +} + +// attachBundle publishes a Sigstore bundle as a subject-bound referrer, exactly like plugin signing. +func attachBundle(t *testing.T, client *oci.Client, repository name.Repository, subject *v1.Descriptor, raw []byte) { + t.Helper() + image, err := mutate.AppendLayers(empty.Image, static.NewLayer(raw, types.MediaType("application/json"))) + require.NoError(t, err) + image = mutate.MediaType(image, types.OCIManifestSchema1) + image = mutate.ConfigMediaType(image, types.MediaType("application/vnd.dev.sigstore.bundle.v0.3+json")) + image = mutate.Subject(image, *subject).(v1.Image) + digest, err := image.Digest() + require.NoError(t, err) + require.NoError(t, client.WriteImage(t.Context(), repository.Digest(digest.String()), image)) +} + +type roundTripCase struct { + name string + prepare func(t *testing.T) (directory, digest string) + inspectors func(t *testing.T, client *oci.Client, repository name.Repository, digest string) any +} + +func twoPlatformLayout(t *testing.T) (string, string) { + t.Helper() + index := testIndex(t) + directory := filepath.Join(t.TempDir(), "source") + require.NoError(t, oci.SaveIndex(directory, index)) + digest, err := index.Digest() + require.NoError(t, err) + return directory, digest.String() +} + +// inspectedRelease captures what plugin verification derives from a release. +func inspectedRelease(t *testing.T, client *oci.Client, repository name.Repository, digest string) any { + t.Helper() + descriptor, err := client.Get(t.Context(), repository.Digest(digest)) + require.NoError(t, err) + index, err := descriptor.ImageIndex() + require.NoError(t, err) + release, err := pluginoci.Inspect(index, digest) + require.NoError(t, err) + platforms := map[string]string{} + for platform, image := range release.Platforms { + platforms[platform] = image.Digest.String() + } + return map[string]any{"name": release.Name, "version": release.Version, "digest": release.Digest, "platforms": platforms} +} + +func manifestDigests(t *testing.T, client *oci.Client, repository name.Repository, digest string) any { + t.Helper() + descriptor, err := client.Get(t.Context(), repository.Digest(digest)) + require.NoError(t, err) + index, err := descriptor.ImageIndex() + require.NoError(t, err) + manifest, err := index.IndexManifest() + require.NoError(t, err) + var digests []string + for i := range manifest.Manifests { + digests = append(digests, manifest.Manifests[i].Digest.String()+"/"+manifest.Manifests[i].Platform.Architecture) + } + return digests +} + +// TestRegistryLayoutRegistryRoundTripKeepsDigestsAndVerification moves releases registry A -> +// OCI layout -> registry B. The registries accept different credentials and A is shut down +// before the final hop, so the layout alone must carry the complete graph, including the signature +// referrer. Digests, blobs, release inspection, and signature verification must not change. +func TestRegistryLayoutRegistryRoundTripKeepsDigestsAndVerification(t *testing.T) { + ctx := t.Context() + for _, tc := range []roundTripCase{ + {"plugin release fixture", func(t *testing.T) (string, string) { return pluginFixtureDirectory(), pluginReleaseIndexDigest }, inspectedRelease}, + {"two-platform index", twoPlatformLayout, manifestDigests}, + } { + t.Run(tc.name, func(t *testing.T) { + sourceDirectory, digest := tc.prepare(t) + sourceBlobs := readBlobs(t, sourceDirectory) + source, err := oci.OpenLayoutReadOnly(ctx, sourceDirectory) + require.NoError(t, err) + rootDescriptor, err := source.Resolve(ctx, digest) + require.NoError(t, err) + require.Equal(t, digest, rootDescriptor.Digest.String()) + + // Registry A: publish the release and a Sigstore bundle bound to its digest. + serverA, clientA := authenticatedRegistry(t, "secret-a") + storeA := openRepository(t, clientA, serverA.Host) + _, err = oci.Copy(ctx, oci.Source{Store: source, Reference: digest}, oci.Destination{Store: storeA, Reference: "1.2.3"}) + require.NoError(t, err) + repositoryA, err := name.NewRepository(serverA.Host + "/acme/plugin") + require.NoError(t, err) + head, err := clientA.Head(ctx, repositoryA.Tag("1.2.3")) + require.NoError(t, err) + require.Equal(t, digest, head.Digest.String(), "pushing must not alter the release digest") + + rawRoot, err := oci.FetchBytes(ctx, source, rootDescriptor, 1<<20) + require.NoError(t, err) + authority := signaturetest.New(t) + identity := signature.Identity{Identity: "https://github.com/acme/plugin/.github/workflows/release.yml@refs/tags/v1.2.3", Issuer: "https://token.actions.githubusercontent.com"} + bundle := authority.Sign(t, identity.Identity, identity.Issuer, rawRoot) + verifier := signature.Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil }} + want, err := verifier.Verify(ctx, bundle, digest, []signature.Identity{identity}, true) + require.NoError(t, err) + attachBundle(t, clientA, repositoryA, head, bundle) + before := tc.inspectors(t, clientA, repositoryA, digest) + + // Hop 1: registry A -> layout, with referrers. + layoutDirectory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, layoutDirectory) + require.NoError(t, err) + copied, err := oci.ExtendedCopy(ctx, oci.GraphSource{Store: storeA, Reference: "1.2.3"}, oci.Destination{Store: layout, Reference: "1.2.3"}) + require.NoError(t, err) + require.Equal(t, digest, copied.Digest.String()) + require.Equal(t, []string{"1.2.3"}, layout.References()) + layoutBlobs := readBlobs(t, layoutDirectory) + for name, content := range sourceBlobs { + require.Equal(t, content, layoutBlobs[name], "blob %s changed in transit", name) + } + require.Len(t, layoutBlobs, len(sourceBlobs)+3, "signature manifest, config, and bundle layer travel with the release") + + // Registry A disappears; registry B has different credentials. + serverA.Close() + serverB, clientB := authenticatedRegistry(t, "secret-b") + storeB := openRepository(t, clientB, serverB.Host) + + // Hop 2 reads the layout from disk again, proving the on-disk state is self-sufficient. + reopened, err := oci.OpenLayout(ctx, layoutDirectory) + require.NoError(t, err) + _, err = oci.ExtendedCopy(ctx, oci.GraphSource{Store: reopened, Reference: "1.2.3"}, oci.Destination{Store: storeB, Reference: "1.2.3"}) + require.NoError(t, err) + + repositoryB, err := name.NewRepository(serverB.Host + "/acme/plugin") + require.NoError(t, err) + after, err := clientB.Head(ctx, repositoryB.Tag("1.2.3")) + require.NoError(t, err) + require.Equal(t, digest, after.Digest.String()) + require.Equal(t, before, tc.inspectors(t, clientB, repositoryB, digest)) + tags, err := clientB.Tags(ctx, repositoryB) + require.NoError(t, err) + // The registry fixture lacks the OCI-Subject header, so the attachment is also indexed + // under the standard referrers tag-schema fallback tag. + require.ElementsMatch(t, []string{"1.2.3", "sha256-" + strings.TrimPrefix(digest, "sha256:")}, tags) + + bundles, err := signature.Fetch(ctx, clientB, repositoryB.Digest(digest)) + require.NoError(t, err) + require.Equal(t, [][]byte{bundle}, bundles) + got, err := verifier.Verify(ctx, bundles[0], digest, []signature.Identity{identity}, true) + require.NoError(t, err) + require.Equal(t, want, got, "verification result must survive the move") + + // Registry B's credentials never worked on A, and A's never work on B. + _, err = oci.Copy(ctx, oci.Source{Store: reopened, Reference: "1.2.3"}, oci.Destination{Store: openRepository(t, clientA, serverB.Host), Reference: "again"}) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + }) + } +} + +func TestLayoutHoldsSeveralReferencesWithoutWeakeningLoadIndex(t *testing.T) { + ctx := t.Context() + twoPlatformDirectory, twoPlatformDigest := twoPlatformLayout(t) + releaseStore, err := oci.OpenLayoutReadOnly(ctx, pluginFixtureDirectory()) + require.NoError(t, err) + otherStore, err := oci.OpenLayoutReadOnly(ctx, twoPlatformDirectory) + require.NoError(t, err) + + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: releaseStore, Reference: pluginReleaseIndexDigest}, oci.Destination{Store: layout, Reference: "plugin-1.2.3"}) + require.NoError(t, err) + + // One release in a layout still satisfies the single-release validation. + index, err := oci.LoadIndex(directory, pluginReleaseIndexDigest) + require.NoError(t, err) + digest, err := index.Digest() + require.NoError(t, err) + assert.Equal(t, pluginReleaseIndexDigest, digest.String()) + + _, err = oci.Copy(ctx, oci.Source{Store: otherStore, Reference: twoPlatformDigest}, oci.Destination{Store: layout, Reference: "two-platform"}) + require.NoError(t, err) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, layout.References()) + _, err = oci.LoadIndex(directory, "") + require.ErrorIs(t, err, errUtils.ErrOCILayout, "several references must still be rejected by LoadIndex") + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, reopened.References()) + for reference, want := range map[string]string{"plugin-1.2.3": pluginReleaseIndexDigest, "two-platform": twoPlatformDigest} { + resolved, err := reopened.Resolve(ctx, reference) + require.NoError(t, err) + assert.Equal(t, want, resolved.Digest.String(), reference) + } + + // Re-tagging an existing reference replaces it rather than adding a duplicate entry. + resolved, err := reopened.Resolve(ctx, "two-platform") + require.NoError(t, err) + require.NoError(t, reopened.Tag(ctx, resolved, "two-platform")) + assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, reopened.References()) +} + +// TestLegacyWriteIndexPublishesTheExistingReleaseBytes proves the bridge serializes a legacy +// index's raw bytes: publishing the committed plugin release through the old Registry API must yield +// byte-identical blobs and the pinned digest, never a rebuilt release. +func TestLegacyWriteIndexPublishesTheExistingReleaseBytes(t *testing.T) { + ctx := t.Context() + index, err := oci.LoadIndex(pluginFixtureDirectory(), pluginReleaseIndexDigest) + require.NoError(t, err) + server, client := authenticatedRegistry(t, "secret") + ref, err := name.NewTag(server.Host + "/acme/plugin:1.2.3") + require.NoError(t, err) + require.NoError(t, client.WriteIndex(ctx, ref, index)) + + head, err := client.Head(ctx, ref) + require.NoError(t, err) + require.Equal(t, pluginReleaseIndexDigest, head.Digest.String()) + + // Pull the registry's content back into a layout: every blob must equal the committed fixture. + directory := filepath.Join(t.TempDir(), "pulled") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: openRepository(t, client, server.Host), Reference: "1.2.3"}, oci.Destination{Store: layout, Reference: "1.2.3"}) + require.NoError(t, err) + want := readBlobs(t, pluginFixtureDirectory()) + got := readBlobs(t, directory) + require.Len(t, got, len(want)) + for name, content := range want { + require.Equal(t, content, got[name], "blob %s differs from the committed release", name) + } + + descriptor, err := client.Get(ctx, ref) + require.NoError(t, err) + assert.Equal(t, pluginReleaseIndexDigest, descriptor.Digest.String()) + assert.NotNil(t, inspectedRelease(t, client, mustRepository(t, server.Host), pluginReleaseIndexDigest)) +} + +func mustRepository(t *testing.T, host string) name.Repository { + t.Helper() + repository, err := name.NewRepository(host + "/acme/plugin") + require.NoError(t, err) + return repository +} diff --git a/pkg/plugin/sign/sign.go b/pkg/plugin/sign/sign.go index 2fb2b7f31a9..9746e8bae85 100644 --- a/pkg/plugin/sign/sign.go +++ b/pkg/plugin/sign/sign.go @@ -2,9 +2,7 @@ package sign import ( - "bytes" "context" - "crypto/sha256" "encoding/json" "errors" "strings" @@ -15,15 +13,14 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/static" "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/sigstore/sigstore-go/pkg/bundle" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/oci" - "github.com/cloudposse/atmos/pkg/oci/signature" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/plugin/extension" pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" pluginpolicy "github.com/cloudposse/atmos/pkg/plugin/policy" + sigcore "github.com/cloudposse/atmos/pkg/signature" ) // BlobSigner produces a standard Sigstore bundle for the exact supplied bytes. @@ -94,19 +91,15 @@ func (s Service) signIndex(ctx context.Context, ref name.Digest, descriptor *v1. } func validateBundle(raw, artifact []byte) error { - if len(raw) == 0 || len(raw) > signature.MaxBundleSize { - return errUtils.ErrPluginSignatureInvalid + err := sigcore.CheckBundleSubject(raw, artifact) + if err == nil { + return nil } - var signed bundle.Bundle - if err := json.Unmarshal(raw, &signed); err != nil { - return errors.Join(errUtils.ErrPluginSignatureInvalid, err) - } - digest := sha256.Sum256(artifact) - message := signed.GetMessageSignature() - if message == nil || message.MessageDigest == nil || message.MessageDigest.Algorithm.String() != "SHA2_256" || !bytes.Equal(message.MessageDigest.Digest, digest[:]) { - return errUtils.ErrPluginSignatureInvalid + var shared *sigcore.Error + if errors.As(err, &shared) && shared.Cause != nil { + return errors.Join(errUtils.ErrPluginSignatureInvalid, shared.Cause) } - return nil + return errUtils.ErrPluginSignatureInvalid } func (s Service) attach(ctx context.Context, ref name.Digest, subject *v1.Descriptor, raw []byte) error { diff --git a/pkg/signature/boundary_test.go b/pkg/signature/boundary_test.go new file mode 100644 index 00000000000..2cece67492e --- /dev/null +++ b/pkg/signature/boundary_test.go @@ -0,0 +1,46 @@ +package signature + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPackageDependencyBoundary keeps the shared primitives free of transport and product layers. +func TestPackageDependencyBoundary(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/internal/exec", + "github.com/cloudposse/atmos/pkg/oci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/google/go-containerregistry", + "oras.land/oras-go", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/signature/doc.go b/pkg/signature/doc.go new file mode 100644 index 00000000000..ca6acbcaa52 --- /dev/null +++ b/pkg/signature/doc.go @@ -0,0 +1,9 @@ +// Package signature holds the reusable Sigstore verification and signing primitives shared by +// every artifact repository and the plugin installer: bundle verification against trusted +// certificate identities, a separate explicitly configured public-key bundle path, the trusted +// root, and signer-output validation. +// +// It performs no registry, plugin, CI, or command-line work. Adapters fetch and store evidence and +// then call these verifiers, so attachment transport stays outside this package. No custom +// cryptography lives here; everything delegates to the Sigstore libraries. +package signature diff --git a/pkg/signature/errors.go b/pkg/signature/errors.go new file mode 100644 index 00000000000..e1e604d4e30 --- /dev/null +++ b/pkg/signature/errors.go @@ -0,0 +1,35 @@ +package signature + +import ( + "errors" +) + +// Error carries a shared sentinel (Kind) separately from its cause, so adapters can translate the +// sentinel to a legacy one without changing the rest of the message. +type Error struct { + // Kind is one of the ErrSignature* sentinels. + Kind error + // Cause is the underlying failure, if any. + Cause error +} + +// Error joins the sentinel and cause exactly like errors.Join. +// +//nolint:lintroller // Error formatting only. +func (e *Error) Error() string { + return errors.Join(e.Kind, e.Cause).Error() +} + +// Unwrap exposes both the sentinel and the cause to errors.Is and errors.As. +// +//nolint:lintroller // Trivial accessor. +func (e *Error) Unwrap() []error { + if e.Cause == nil { + return []error{e.Kind} + } + return []error{e.Kind, e.Cause} +} + +func fail(kind, cause error) error { + return &Error{Kind: kind, Cause: cause} +} diff --git a/pkg/signature/key.go b/pkg/signature/key.go new file mode 100644 index 00000000000..462bfb8fb4a --- /dev/null +++ b/pkg/signature/key.go @@ -0,0 +1,107 @@ +package signature + +import ( + "context" + "crypto" + "time" + + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/sigstore/sigstore-go/pkg/verify" + sigstore "github.com/sigstore/sigstore/pkg/signature" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// TrustedKey is a public key an operator explicitly configured as a trusted signer. +type TrustedKey struct { + // Hint is the key identifier a bundle carries in its verification material. + Hint string + // PublicKey is the verification key. + PublicKey crypto.PublicKey +} + +// KeyResult names the configured key that authenticated a bundle. It carries no certificate claims. +type KeyResult struct{ Hint string } + +// KeyVerifier verifies Sigstore bundles signed with an explicitly configured public key. +// +// It is a separate type from Verifier on purpose: its result is a key hint, never an OIDC +// identity or issuer, so a public-key signature cannot be mistaken for, or satisfy, an identity +// requirement. It requires the bundle to be signed with a key (certificate bundles are rejected), +// inclusion in a trusted transparency log, and an authenticated signing time. +type KeyVerifier struct { + // Keys are the only keys that can authenticate a bundle. An empty set fails closed. + Keys []TrustedKey + // LoadRoot supplies transparency-log trust. It defaults to the Sigstore public-good root. + LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) +} + +// Verify checks raw against the independently supplied subject digest. +func (v KeyVerifier) Verify(ctx context.Context, raw []byte, digest string, offline bool) (*KeyResult, error) { + defer perf.Track(nil, "signature.KeyVerifier.Verify")() + + if err := ctx.Err(); err != nil { + return nil, err + } + if len(v.Keys) == 0 { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + hash, signed, err := parseBundle(raw, digest) + if err != nil { + return nil, err + } + verifier, err := v.trustedVerifier(ctx, offline) + if err != nil { + return nil, err + } + result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), verify.WithKey())) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if result.Signature == nil || result.Signature.PublicKeyID == nil { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + return &KeyResult{Hint: string(*result.Signature.PublicKeyID)}, nil +} + +// trustedVerifier combines the configured keys with transparency-log trust. +func (v KeyVerifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { + keys, err := v.keyMaterial() + if err != nil { + return nil, err + } + loader := v.LoadRoot + if loader == nil { + loader = LoadPublicRoot + } + logs, err := loader(ctx, offline) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + verifier, err := verify.NewVerifier(root.TrustedMaterialCollection{keys, logs}, verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + return verifier, nil +} + +// keyMaterial exposes only the configured keys. Certificate authorities from other trust +// material cannot validate a key-signed bundle. +func (v KeyVerifier) keyMaterial() (root.TrustedMaterial, error) { + verifiers := make(map[string]*root.ExpiringKey, len(v.Keys)) + for _, key := range v.Keys { + if key.Hint == "" || key.PublicKey == nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + loaded, err := sigstore.LoadVerifier(key.PublicKey, crypto.SHA256) + if err != nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, err) + } + verifiers[key.Hint] = root.NewExpiringKey(loaded, time.Time{}, time.Time{}) + } + return root.NewTrustedPublicKeyMaterialFromMapping(verifiers), nil +} diff --git a/pkg/signature/key_test.go b/pkg/signature/key_test.go new file mode 100644 index 00000000000..4e796c3ac35 --- /dev/null +++ b/pkg/signature/key_test.go @@ -0,0 +1,89 @@ +package signature + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "strings" + "testing" + + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func logsOnly(authority *signaturetest.Authority) func(context.Context, bool) (root.TrustedMaterial, error) { + return func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil } +} + +func TestKeyVerifierAcceptsOnlyConfiguredKeys(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + configured := TrustedKey{Hint: signed.Hint, PublicKey: signed.PublicKey} + verifier := KeyVerifier{Keys: []TrustedKey{configured}, LoadRoot: logsOnly(authority)} + + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, &KeyResult{Hint: signaturetest.KeyHint}, result) + + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + for _, tc := range []struct { + name string + verifier KeyVerifier + digest string + raw []byte + kind error + }{ + {"different key under the same hint", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint, PublicKey: &stranger.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"hint not configured", KeyVerifier{Keys: []TrustedKey{{Hint: "other", PublicKey: signed.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"wrong subject digest", verifier, "sha256:" + strings.Repeat("b", 64), signed.Bundle, errUtils.ErrSignatureInvalid}, + {"malformed bundle", verifier, digestOf(artifact), []byte("{}"), errUtils.ErrSignatureInvalid}, + {"no configured keys", KeyVerifier{LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"key without hint", KeyVerifier{Keys: []TrustedKey{{PublicKey: signed.PublicKey}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"key without material", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + {"unsupported key type", KeyVerifier{Keys: []TrustedKey{{Hint: signed.Hint, PublicKey: "not a key"}}, LoadRoot: logsOnly(authority)}, digestOf(artifact), signed.Bundle, errUtils.ErrSignatureConfigInvalid}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := tc.verifier.Verify(t.Context(), tc.raw, tc.digest, true) + require.ErrorIs(t, err, tc.kind) + require.Nil(t, result) + }) + } + + failing := KeyVerifier{Keys: []TrustedKey{configured}, LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { + return nil, errUtils.ErrSignatureOffline + }} + _, err = failing.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = verifier.Verify(ctx, signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, context.Canceled) +} + +// TestKeyAndIdentityPathsNeverSubstituteForEachOther is the trust-boundary guarantee: neither +// verifier accepts the other's evidence, and the key path never reports an OIDC identity. +func TestKeyAndIdentityPathsNeverSubstituteForEachOther(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("subject") + identity := Identity{Identity: "builder@example.com", Issuer: "https://issuer.example.com"} + certificateBundle := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + keySigned := authority.SignWithKey(t, artifact) + trust := logsOnly(authority) + + // A public-key signature cannot satisfy an identity requirement, even when the key is trusted. + _, err := Verifier{LoadRoot: trust}.Verify(t.Context(), keySigned.Bundle, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + // A certificate-signed bundle cannot be accepted by the key path, whatever keys are configured. + keys := KeyVerifier{Keys: []TrustedKey{{Hint: keySigned.Hint, PublicKey: keySigned.PublicKey}}, LoadRoot: trust} + _, err = keys.Verify(t.Context(), certificateBundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) +} diff --git a/pkg/signature/mock_signer.go b/pkg/signature/mock_signer.go new file mode 100644 index 00000000000..109c9353818 --- /dev/null +++ b/pkg/signature/mock_signer.go @@ -0,0 +1,56 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/signature (interfaces: BlobSigner) +// +// Generated by this command: +// +// mockgen -destination=mock_signer.go -package=signature . BlobSigner +// + +// Package signature is a generated GoMock package. +package signature + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockBlobSigner is a mock of BlobSigner interface. +type MockBlobSigner struct { + ctrl *gomock.Controller + recorder *MockBlobSignerMockRecorder + isgomock struct{} +} + +// MockBlobSignerMockRecorder is the mock recorder for MockBlobSigner. +type MockBlobSignerMockRecorder struct { + mock *MockBlobSigner +} + +// NewMockBlobSigner creates a new mock instance. +func NewMockBlobSigner(ctrl *gomock.Controller) *MockBlobSigner { + mock := &MockBlobSigner{ctrl: ctrl} + mock.recorder = &MockBlobSignerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBlobSigner) EXPECT() *MockBlobSignerMockRecorder { + return m.recorder +} + +// Sign mocks base method. +func (m *MockBlobSigner) Sign(ctx context.Context, artifact []byte) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, artifact) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockBlobSignerMockRecorder) Sign(ctx, artifact any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockBlobSigner)(nil).Sign), ctx, artifact) +} diff --git a/pkg/oci/signature/signaturetest/bundle.go b/pkg/signature/signaturetest/bundle.go similarity index 100% rename from pkg/oci/signature/signaturetest/bundle.go rename to pkg/signature/signaturetest/bundle.go diff --git a/pkg/oci/signature/signaturetest/dsse.go b/pkg/signature/signaturetest/dsse.go similarity index 100% rename from pkg/oci/signature/signaturetest/dsse.go rename to pkg/signature/signaturetest/dsse.go diff --git a/pkg/oci/signature/signaturetest/public_key.go b/pkg/signature/signaturetest/public_key.go similarity index 69% rename from pkg/oci/signature/signaturetest/public_key.go rename to pkg/signature/signaturetest/public_key.go index d50217322b0..e6ad2aaed1b 100644 --- a/pkg/oci/signature/signaturetest/public_key.go +++ b/pkg/signature/signaturetest/public_key.go @@ -25,11 +25,41 @@ import ( "github.com/cloudposse/atmos/pkg/perf" ) +// KeySignature is a genuine key-only Sigstore bundle together with the public half of the key. +type KeySignature struct { + // Bundle is the raw Sigstore bundle. + Bundle []byte + // Hint is the key identifier embedded in the bundle. + Hint string + // PublicKey verifies the signature; an operator would configure it as a trusted key. + PublicKey crypto.PublicKey + // Logs is the transparency-log trust material for this test authority. + Logs root.TrustedMaterial +} + +// KeyHint is the identifier SignWithKey embeds in every bundle. +const KeyHint = "fixture-key" + // SignWithPublicKey generates a genuine key-only signature and Rekor evidence. // This intentionally unsupported signing mode must never satisfy a certificate policy. func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, root.TrustedMaterial) { defer perf.Track(nil, "signaturetest.Authority.SignWithPublicKey")() + t.Helper() + signed := a.SignWithKey(t, artifact) + verifier, err := sig.LoadVerifier(signed.PublicKey, crypto.SHA256) + require.NoError(t, err) + keyTrust := root.NewTrustedPublicKeyMaterial(func(string) (root.TimeConstrainedVerifier, error) { + return root.NewExpiringKey(verifier, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)), nil + }) + return signed.Bundle, root.TrustedMaterialCollection{a.TrustedMaterial, keyTrust} +} + +// SignWithKey generates a genuine key-only signature with a fresh key and returns the public key +// so tests can configure it explicitly as trusted. +func (a *Authority) SignWithKey(t *testing.T, artifact []byte) KeySignature { + defer perf.Track(nil, "signaturetest.Authority.SignWithKey")() + t.Helper() key := newKey(t) hash := sha256.Sum256(artifact) @@ -52,7 +82,7 @@ func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, ro bundle := &protobundle.Bundle{ MediaType: "application/vnd.dev.sigstore.bundle.v0.3+json", VerificationMaterial: &protobundle.VerificationMaterial{ - Content: &protobundle.VerificationMaterial_PublicKey{PublicKey: &common.PublicKeyIdentifier{Hint: "fixture-key"}}, + Content: &protobundle.VerificationMaterial_PublicKey{PublicKey: &common.PublicKeyIdentifier{Hint: KeyHint}}, TlogEntries: []*rekor.TransparencyLogEntry{a.logEntry(t, body)}, }, Content: &protobundle.Bundle_MessageSignature{MessageSignature: &common.MessageSignature{ @@ -61,10 +91,5 @@ func (a *Authority) SignWithPublicKey(t *testing.T, artifact []byte) ([]byte, ro } raw, err := protojson.Marshal(bundle) require.NoError(t, err) - verifier, err := sig.LoadECDSAVerifier(&key.PublicKey, crypto.SHA256) - require.NoError(t, err) - keyTrust := root.NewTrustedPublicKeyMaterial(func(string) (root.TimeConstrainedVerifier, error) { - return root.NewExpiringKey(verifier, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)), nil - }) - return raw, root.TrustedMaterialCollection{a.TrustedMaterial, keyTrust} + return KeySignature{Bundle: raw, Hint: KeyHint, PublicKey: &key.PublicKey, Logs: a.TrustedMaterial} } diff --git a/pkg/signature/signer.go b/pkg/signature/signer.go new file mode 100644 index 00000000000..a72bfc449c0 --- /dev/null +++ b/pkg/signature/signer.go @@ -0,0 +1,56 @@ +package signature + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + + "github.com/sigstore/sigstore-go/pkg/bundle" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// BlobSigner produces a standard Sigstore bundle for the exact supplied bytes. Implementations +// (for example the managed Cosign authoring path) own signing identity; registry credentials are +// unrelated to it. +// +//go:generate go run go.uber.org/mock/mockgen -destination=mock_signer.go -package=signature . BlobSigner +type BlobSigner interface { + Sign(ctx context.Context, artifact []byte) ([]byte, error) +} + +// SignBlob signs artifact and refuses output that is oversized, malformed, or bound to different bytes. +func SignBlob(ctx context.Context, signer BlobSigner, artifact []byte) ([]byte, error) { + defer perf.Track(nil, "signature.SignBlob")() + + signed, err := signer.Sign(ctx, artifact) + if err != nil { + return nil, err + } + if err := CheckBundleSubject(signed, artifact); err != nil { + return nil, err + } + return signed, nil +} + +// CheckBundleSubject confirms a message-signature bundle commits to the SHA-256 of artifact. +// This is a structural check on freshly produced evidence; verification still goes through a Verifier. +func CheckBundleSubject(raw, artifact []byte) error { + defer perf.Track(nil, "signature.CheckBundleSubject")() + + if len(raw) == 0 || len(raw) > MaxBundleSize { + return fail(errUtils.ErrSignatureInvalid, nil) + } + var signed bundle.Bundle + if err := json.Unmarshal(raw, &signed); err != nil { + return fail(errUtils.ErrSignatureInvalid, err) + } + digest := sha256.Sum256(artifact) + message := signed.GetMessageSignature() + if message == nil || message.MessageDigest == nil || message.MessageDigest.Algorithm.String() != "SHA2_256" || !bytes.Equal(message.MessageDigest.Digest, digest[:]) { + return fail(errUtils.ErrSignatureInvalid, nil) + } + return nil +} diff --git a/pkg/signature/signer_test.go b/pkg/signature/signer_test.go new file mode 100644 index 00000000000..97014348737 --- /dev/null +++ b/pkg/signature/signer_test.go @@ -0,0 +1,56 @@ +package signature + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func TestSignBlobAcceptsOnlyEvidenceBoundToTheSignedBytes(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("release index bytes") + valid := authority.Sign(t, "builder@example.com", "https://issuer.example.com", artifact) + otherArtifact := authority.Sign(t, "builder@example.com", "https://issuer.example.com", []byte("different bytes")) + failure := errors.New("signing backend unavailable") + + for _, tc := range []struct { + name string + bundle []byte + err error + want error + }{ + {"matching bundle", valid, nil, nil}, + {"bundle for other bytes", otherArtifact, nil, errUtils.ErrSignatureInvalid}, + {"malformed bundle", []byte("not json"), nil, errUtils.ErrSignatureInvalid}, + {"empty bundle", nil, nil, errUtils.ErrSignatureInvalid}, + {"oversized bundle", make([]byte, MaxBundleSize+1), nil, errUtils.ErrSignatureInvalid}, + {"signer failure", nil, failure, failure}, + } { + t.Run(tc.name, func(t *testing.T) { + signer := NewMockBlobSigner(gomock.NewController(t)) + signer.EXPECT().Sign(gomock.Any(), artifact).Return(tc.bundle, tc.err) + got, err := SignBlob(t.Context(), signer, artifact) + if tc.want != nil { + require.ErrorIs(t, err, tc.want) + assert.Nil(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, tc.bundle, got) + }) + } +} + +func TestCheckBundleSubjectRejectsEnvelopeBundles(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("statement subject") + statement := authority.SignStatement(t, "builder@example.com", "https://issuer.example.com", artifact) + // DSSE statements are verified by Verifier; SignBlob only vouches for message signatures. + require.ErrorIs(t, CheckBundleSubject(statement, artifact), errUtils.ErrSignatureInvalid) +} diff --git a/pkg/oci/signature/root.go b/pkg/signature/trust.go similarity index 82% rename from pkg/oci/signature/root.go rename to pkg/signature/trust.go index 9bb3c2472b2..6b21ae2e9b3 100644 --- a/pkg/oci/signature/root.go +++ b/pkg/signature/trust.go @@ -18,9 +18,12 @@ import ( const trustDirectoryMode = 0o700 -// Offline mode may use only unexpired, TUF-authenticated cached trust material. -// Never accept roots supplied by a plugin or signature bundle. -func loadPublicRoot(ctx context.Context, offline bool) (root.TrustedMaterial, error) { +// LoadPublicRoot returns the Sigstore public-good trusted root. Offline mode may use only +// unexpired, TUF-authenticated cached trust material. Never accept roots supplied by an artifact +// or signature bundle. +func LoadPublicRoot(ctx context.Context, offline bool) (root.TrustedMaterial, error) { + defer perf.Track(nil, "signature.LoadPublicRoot")() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() directory, err := xdg.GetXDGDataDir("sigstore", trustDirectoryMode) @@ -47,7 +50,7 @@ type offlineFetcher struct{} //nolint:lintroller // Pure fail-closed fetcher: never performs I/O. func (offlineFetcher) DownloadFile(_ string, _ int64, _ time.Duration) ([]byte, error) { - return nil, errUtils.ErrPluginOfflineArtifact + return nil, errUtils.ErrSignatureOffline } type contextClient struct { diff --git a/pkg/oci/signature/root_transport_test.go b/pkg/signature/trust_test.go similarity index 84% rename from pkg/oci/signature/root_transport_test.go rename to pkg/signature/trust_test.go index f292f84b1d0..81c790867d0 100644 --- a/pkg/oci/signature/root_transport_test.go +++ b/pkg/signature/trust_test.go @@ -18,7 +18,7 @@ func TestPublicTrustFailsWhenCacheDirectoryCannotBeCreated(t *testing.T) { path := filepath.Join(t.TempDir(), "file") require.NoError(t, os.WriteFile(path, nil, 0o600)) t.Setenv("ATMOS_XDG_DATA_HOME", path) - trusted, err := loadPublicRoot(t.Context(), true) + trusted, err := LoadPublicRoot(t.Context(), true) require.Error(t, err) require.Nil(t, trusted) } @@ -49,5 +49,11 @@ func TestTrustHTTPClientUsesVerificationContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) require.Nil(t, response) _, err = (offlineFetcher{}).DownloadFile(server.URL, 1024, 0) - require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrSignatureOffline) +} + +func TestOfflineTrustCannotFetch(t *testing.T) { + t.Setenv("ATMOS_XDG_DATA_HOME", t.TempDir()) + _, err := LoadPublicRoot(t.Context(), true) + require.Error(t, err) } diff --git a/pkg/signature/verify.go b/pkg/signature/verify.go new file mode 100644 index 00000000000..2c517b67086 --- /dev/null +++ b/pkg/signature/verify.go @@ -0,0 +1,121 @@ +package signature + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "regexp" + "strings" + + "github.com/sigstore/sigstore-go/pkg/bundle" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/sigstore/sigstore-go/pkg/verify" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MaxBundleSize bounds one Sigstore bundle. +const MaxBundleSize = 4 << 20 + +// Identity binds a certificate SAN (exact or * wildcard) to an exact OIDC issuer. +type Identity struct{ Identity, Issuer string } + +// Result contains authenticated certificate claims, never claims from unverified metadata. +type Result struct{ Identity, Issuer string } + +// Verifier uses a trusted root independent of the artifact and its publisher. +// LoadRoot is injectable for private trust deployments and isolated cryptographic tests. +// +// It verifies certificate-based (keyless) bundles only. A bundle signed with a bare public key can +// never satisfy an identity requirement here; KeyVerifier is the separate path for that. +type Verifier struct { + LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) +} + +// Verify requires a certificate, SCT, transparency inclusion and authenticated +// signing time. The subject digest is always supplied independently of the bundle. +func (v Verifier) Verify(ctx context.Context, raw []byte, digest string, identities []Identity, offline bool) (*Result, error) { + defer perf.Track(nil, "signature.Verifier.Verify")() + + if err := ctx.Err(); err != nil { + return nil, err + } + hash, signed, err := parseBundle(raw, digest) + if err != nil { + return nil, err + } + if len(identities) == 0 { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + options, err := identityOptions(identities) + if err != nil { + return nil, err + } + verifier, err := v.trustedVerifier(ctx, offline) + if err != nil { + return nil, err + } + result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), options...)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if result.Signature == nil || result.Signature.Certificate == nil { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + return &Result{Identity: result.Signature.Certificate.SubjectAlternativeName, Issuer: result.Signature.Certificate.Issuer}, nil +} + +func parseBundle(raw []byte, digest string) ([]byte, *bundle.Bundle, error) { + if len(raw) > MaxBundleSize || len(raw) == 0 { + return nil, nil, fail(errUtils.ErrSignatureInvalid, nil) + } + hash, err := hex.DecodeString(strings.TrimPrefix(digest, "sha256:")) + if err != nil || !strings.HasPrefix(digest, "sha256:") || len(hash) != sha256.Size { + return nil, nil, fail(errUtils.ErrSignatureInvalid, nil) + } + var signed bundle.Bundle + if err := json.Unmarshal(raw, &signed); err != nil { + return nil, nil, fail(errUtils.ErrSignatureInvalid, err) + } + return hash, &signed, nil +} + +func (v Verifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { + loader := v.LoadRoot + if loader == nil { + loader = LoadPublicRoot + } + trusted, err := loader(ctx, offline) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + // Require a certificate identity, a witnessed signing time and inclusion in + // a trusted transparency log. Bare-key bundles cannot establish the requested + // OIDC identity; the key-only rejection test guards this boundary. + verifier, err := verify.NewVerifier(trusted, verify.WithSignedCertificateTimestamps(1), verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + return verifier, nil +} + +func identityOptions(identities []Identity) ([]verify.PolicyOption, error) { + options := make([]verify.PolicyOption, 0, len(identities)) + for _, identity := range identities { + if identity.Identity == "" || identity.Issuer == "" { + return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) + } + pattern := "^" + strings.ReplaceAll(regexp.QuoteMeta(identity.Identity), `\*`, ".*") + "$" + match, err := verify.NewShortCertificateIdentity(identity.Issuer, "", "", pattern) + if err != nil { + return nil, fail(errUtils.ErrSignatureConfigInvalid, err) + } + options = append(options, verify.WithCertificateIdentity(match)) + } + return options, nil +} diff --git a/pkg/signature/verify_test.go b/pkg/signature/verify_test.go new file mode 100644 index 00000000000..a85e5c4e549 --- /dev/null +++ b/pkg/signature/verify_test.go @@ -0,0 +1,123 @@ +package signature + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +func digestOf(artifact []byte) string { + sum := sha256.Sum256(artifact) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func TestVerifyRealSigstoreEvidence(t *testing.T) { + authority := signaturetest.New(t) + verifier := Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil }} + artifact := []byte("immutable subject") + reference := digestOf(artifact) + identity := Identity{Identity: "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + raw := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + + result, err := verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) + require.NoError(t, err) + assert.Equal(t, &Result{Identity: identity.Identity, Issuer: identity.Issuer}, result) + + wildcard := Identity{Identity: "https://github.com/acme/*", Issuer: identity.Issuer} + _, err = verifier.Verify(t.Context(), raw, reference, []Identity{wildcard}, true) + require.NoError(t, err) + + for _, tc := range []struct { + name, digest string + identity Identity + raw []byte + }{ + {"wrong digest", "sha256:" + strings.Repeat("a", 64), identity, raw}, + {"wrong issuer", reference, Identity{Identity: identity.Identity, Issuer: "https://wrong.example"}, raw}, + {"wrong identity", reference, Identity{Identity: "https://github.com/other/*", Issuer: identity.Issuer}, raw}, + {"identity is anchored", reference, Identity{Identity: "github.com/acme/*", Issuer: identity.Issuer}, raw}, + {"malformed bundle", reference, identity, []byte("{}")}, + {"empty bundle", reference, identity, nil}, + {"oversized bundle", reference, identity, make([]byte, MaxBundleSize+1)}, + {"non-sha256 digest", "sha512:" + strings.Repeat("a", 128), identity, raw}, + {"short digest", "sha256:abcd", identity, raw}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := verifier.Verify(t.Context(), tc.raw, tc.digest, []Identity{tc.identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + }) + } + + var changed map[string]any + require.NoError(t, json.Unmarshal(raw, &changed)) + changed["verificationMaterial"].(map[string]any)["tlogEntries"] = []any{} + tampered, err := json.Marshal(changed) + require.NoError(t, err) + _, err = verifier.Verify(t.Context(), tampered, reference, []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "a bundle without transparency inclusion is rejected") + + other := signaturetest.New(t) + verifier.LoadRoot = func(context.Context, bool) (root.TrustedMaterial, error) { return other, nil } + _, err = verifier.Verify(t.Context(), raw, reference, []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "a root that did not issue the certificate cannot verify it") +} + +func TestVerifyRejectsUnusablePolicyAndTrust(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("subject") + identity := Identity{Identity: "builder@example.com", Issuer: "https://issuer.example.com"} + raw := authority.Sign(t, identity.Identity, identity.Issuer, artifact) + trusted := func(context.Context, bool) (root.TrustedMaterial, error) { return authority, nil } + + _, err := Verifier{LoadRoot: trusted}.Verify(t.Context(), raw, digestOf(artifact), nil, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid, "verification without a required identity is refused") + + for _, bad := range []Identity{{Issuer: identity.Issuer}, {Identity: identity.Identity}} { + _, err = Verifier{LoadRoot: trusted}.Verify(t.Context(), raw, digestOf(artifact), []Identity{bad}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureConfigInvalid) + require.NotErrorIs(t, err, errUtils.ErrSignatureInvalid) + } + + failing := func(context.Context, bool) (root.TrustedMaterial, error) { return nil, errUtils.ErrSignatureOffline } + _, err = Verifier{LoadRoot: failing}.Verify(t.Context(), raw, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorIs(t, err, errUtils.ErrSignatureOffline, "the trust failure stays inspectable") + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = Verifier{LoadRoot: trusted}.Verify(ctx, raw, digestOf(artifact), []Identity{identity}, true) + require.ErrorIs(t, err, context.Canceled) +} + +func TestVerifyRejectsPublicKeyOnlySignatureForIdentityRequirements(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("release signed without a certificate") + raw, trusted := authority.SignWithPublicKey(t, artifact) + verifier := Verifier{LoadRoot: func(context.Context, bool) (root.TrustedMaterial, error) { return trusted, nil }} + result, err := verifier.Verify(t.Context(), raw, digestOf(artifact), []Identity{{ + Identity: "builder@example.com", Issuer: "https://issuer.example.com", + }}, true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorContains(t, err, "SCTs required but bundle is signed with a public key") + require.Nil(t, result) +} + +func TestErrorExposesSentinelAndCause(t *testing.T) { + cause := context.DeadlineExceeded + err := fail(errUtils.ErrSignatureInvalid, cause) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + require.ErrorIs(t, err, cause) + assert.Contains(t, err.Error(), errUtils.ErrSignatureInvalid.Error()) + assert.Contains(t, err.Error(), cause.Error()) + assert.Equal(t, errUtils.ErrSignatureInvalid.Error(), fail(errUtils.ErrSignatureInvalid, nil).Error()) +} From 0cde54ac6eaea84ab9f42dcae70a1beaa0c920f0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 09:22:28 -0500 Subject: [PATCH 02/52] feat(artifacts): add artifact service, producers, snapshotting, and conformance harness (A03) Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 12 + pkg/archive/deterministic.go | 240 +++++++ pkg/archive/deterministic_test.go | 171 +++++ pkg/artifact/artifacttest/conformance.go | 461 +++++++++++++ pkg/artifact/artifacttest/conformance_test.go | 71 ++ pkg/artifact/artifacttest/doc.go | 11 + pkg/artifact/artifacttest/lock.go | 150 +++++ pkg/artifact/artifacttest/read.go | 253 ++++++++ pkg/artifact/artifacttest/repository.go | 284 ++++++++ pkg/artifact/artifacttest/sign.go | 222 +++++++ pkg/artifact/artifacttest/write.go | 202 ++++++ pkg/artifact/evidence.go | 20 + pkg/artifact/fixtures/fixtures.go | 153 +++++ pkg/artifact/fixtures/fixtures_other.go | 11 + pkg/artifact/fixtures/fixtures_unix.go | 27 + pkg/artifact/materialize/archive.go | 334 ++++++++++ pkg/artifact/materialize/doc.go | 10 + pkg/artifact/materialize/extract.go | 151 +++++ pkg/artifact/materialize/links.go | 44 ++ pkg/artifact/materialize/materialize_test.go | 396 ++++++++++++ pkg/artifact/materialize/replace.go | 240 +++++++ pkg/artifact/materialize/tree.go | 121 ++++ pkg/artifact/mock_evidence.go | 95 +++ pkg/artifact/produce/assemble.go | 199 ++++++ pkg/artifact/produce/doc.go | 10 + pkg/artifact/produce/documents.go | 247 +++++++ pkg/artifact/produce/files.go | 228 +++++++ pkg/artifact/produce/identity.go | 97 +++ pkg/artifact/produce/produce_test.go | 610 ++++++++++++++++++ pkg/artifact/produce/repack.go | 139 ++++ pkg/artifact/produce/repack_test.go | 111 ++++ pkg/artifact/producers.go | 72 +++ pkg/artifact/producers_test.go | 38 ++ pkg/artifact/service/boundary_test.go | 45 ++ pkg/artifact/service/commit.go | 219 +++++++ pkg/artifact/service/config_repositories.go | 138 ++++ .../service/config_repositories_test.go | 120 ++++ pkg/artifact/service/consume_test.go | 327 ++++++++++ pkg/artifact/service/copy.go | 311 +++++++++ pkg/artifact/service/copy_test.go | 273 ++++++++ pkg/artifact/service/doc.go | 18 + pkg/artifact/service/edge_test.go | 436 +++++++++++++ pkg/artifact/service/enumerate.go | 128 ++++ pkg/artifact/service/execute.go | 165 +++++ pkg/artifact/service/helpers_test.go | 196 ++++++ pkg/artifact/service/internal_helpers_test.go | 10 + pkg/artifact/service/internal_test.go | 172 +++++ pkg/artifact/service/interrupt_test.go | 309 +++++++++ pkg/artifact/service/locks.go | 140 ++++ pkg/artifact/service/mock_repositories.go | 201 ++++++ pkg/artifact/service/mutate.go | 104 +++ pkg/artifact/service/mutate_test.go | 229 +++++++ pkg/artifact/service/publish.go | 246 +++++++ pkg/artifact/service/publish_test.go | 409 ++++++++++++ pkg/artifact/service/pull.go | 168 +++++ pkg/artifact/service/query.go | 208 ++++++ pkg/artifact/service/repositories.go | 148 +++++ pkg/artifact/service/service.go | 267 ++++++++ pkg/artifact/service/sign.go | 146 +++++ pkg/artifact/service/static.go | 55 ++ pkg/artifact/service/verify.go | 156 +++++ pkg/artifact/service/verify_test.go | 254 ++++++++ pkg/artifact/snapshot/copy.go | 164 +++++ pkg/artifact/snapshot/doc.go | 14 + pkg/artifact/snapshot/finish_test.go | 29 + pkg/artifact/snapshot/reader.go | 35 + pkg/artifact/snapshot/snapshot.go | 234 +++++++ pkg/artifact/snapshot/snapshot_test.go | 282 ++++++++ pkg/artifact/snapshot/walk.go | 164 +++++ 69 files changed, 11950 insertions(+) create mode 100644 pkg/archive/deterministic.go create mode 100644 pkg/archive/deterministic_test.go create mode 100644 pkg/artifact/artifacttest/conformance.go create mode 100644 pkg/artifact/artifacttest/conformance_test.go create mode 100644 pkg/artifact/artifacttest/doc.go create mode 100644 pkg/artifact/artifacttest/lock.go create mode 100644 pkg/artifact/artifacttest/read.go create mode 100644 pkg/artifact/artifacttest/repository.go create mode 100644 pkg/artifact/artifacttest/sign.go create mode 100644 pkg/artifact/artifacttest/write.go create mode 100644 pkg/artifact/evidence.go create mode 100644 pkg/artifact/fixtures/fixtures.go create mode 100644 pkg/artifact/fixtures/fixtures_other.go create mode 100644 pkg/artifact/fixtures/fixtures_unix.go create mode 100644 pkg/artifact/materialize/archive.go create mode 100644 pkg/artifact/materialize/doc.go create mode 100644 pkg/artifact/materialize/extract.go create mode 100644 pkg/artifact/materialize/links.go create mode 100644 pkg/artifact/materialize/materialize_test.go create mode 100644 pkg/artifact/materialize/replace.go create mode 100644 pkg/artifact/materialize/tree.go create mode 100644 pkg/artifact/mock_evidence.go create mode 100644 pkg/artifact/produce/assemble.go create mode 100644 pkg/artifact/produce/doc.go create mode 100644 pkg/artifact/produce/documents.go create mode 100644 pkg/artifact/produce/files.go create mode 100644 pkg/artifact/produce/identity.go create mode 100644 pkg/artifact/produce/produce_test.go create mode 100644 pkg/artifact/produce/repack.go create mode 100644 pkg/artifact/produce/repack_test.go create mode 100644 pkg/artifact/producers.go create mode 100644 pkg/artifact/producers_test.go create mode 100644 pkg/artifact/service/boundary_test.go create mode 100644 pkg/artifact/service/commit.go create mode 100644 pkg/artifact/service/config_repositories.go create mode 100644 pkg/artifact/service/config_repositories_test.go create mode 100644 pkg/artifact/service/consume_test.go create mode 100644 pkg/artifact/service/copy.go create mode 100644 pkg/artifact/service/copy_test.go create mode 100644 pkg/artifact/service/doc.go create mode 100644 pkg/artifact/service/edge_test.go create mode 100644 pkg/artifact/service/enumerate.go create mode 100644 pkg/artifact/service/execute.go create mode 100644 pkg/artifact/service/helpers_test.go create mode 100644 pkg/artifact/service/internal_helpers_test.go create mode 100644 pkg/artifact/service/internal_test.go create mode 100644 pkg/artifact/service/interrupt_test.go create mode 100644 pkg/artifact/service/locks.go create mode 100644 pkg/artifact/service/mock_repositories.go create mode 100644 pkg/artifact/service/mutate.go create mode 100644 pkg/artifact/service/mutate_test.go create mode 100644 pkg/artifact/service/publish.go create mode 100644 pkg/artifact/service/publish_test.go create mode 100644 pkg/artifact/service/pull.go create mode 100644 pkg/artifact/service/query.go create mode 100644 pkg/artifact/service/repositories.go create mode 100644 pkg/artifact/service/service.go create mode 100644 pkg/artifact/service/sign.go create mode 100644 pkg/artifact/service/static.go create mode 100644 pkg/artifact/service/verify.go create mode 100644 pkg/artifact/service/verify_test.go create mode 100644 pkg/artifact/snapshot/copy.go create mode 100644 pkg/artifact/snapshot/doc.go create mode 100644 pkg/artifact/snapshot/finish_test.go create mode 100644 pkg/artifact/snapshot/reader.go create mode 100644 pkg/artifact/snapshot/snapshot.go create mode 100644 pkg/artifact/snapshot/snapshot_test.go create mode 100644 pkg/artifact/snapshot/walk.go diff --git a/errors/errors.go b/errors/errors.go index 91e322a1422..36a6281ebea 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2117,6 +2117,18 @@ var ( // ErrArtifactTooLarge is returned when a record, document, or evidence blob exceeds its bounded-read limit. ErrArtifactTooLarge = errors.New("artifact data exceeds the size limit") + // ErrArtifactSourceChanged is returned when source content changed while it was being captured, + // so an inventory would describe different bytes than the ones staged. + ErrArtifactSourceChanged = errors.New("artifact source changed during capture") + + // ErrArtifactVerificationFailed is returned when no permitted signature satisfies a required + // trust set. It is a policy error raised before any consumption or destination publication. + ErrArtifactVerificationFailed = errors.New("artifact signature verification failed") + + // ErrArtifactPublicationAmbiguous is returned when a mutation failed in a way that could not be + // reconciled: the completed reference could not be proven to hold, or to lack, the expected content. + ErrArtifactPublicationAmbiguous = errors.New("artifact publication outcome is ambiguous") + // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The // underlying I/O error is wrapped alongside it for diagnostics. ErrArtifactStreamFailed = errors.New("artifact stream failed") diff --git a/pkg/archive/deterministic.go b/pkg/archive/deterministic.go new file mode 100644 index 00000000000..2a2be7d3119 --- /dev/null +++ b/pkg/archive/deterministic.go @@ -0,0 +1,240 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "sort" + "strings" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MemberKind is the type of a [Member] written by [WriteDeterministic]. +type MemberKind int + +// Member kinds. Devices, sockets, and other special files are never representable. +const ( + // MemberFile is a regular file whose bytes come from [Member.Open]. + MemberFile MemberKind = iota + + // MemberDirectory is a directory entry. + MemberDirectory + + // MemberSymlink is a symbolic link to [Member.Target]. + MemberSymlink +) + +// deterministicPermMask holds the permission bits a deterministic member may carry. Privileged +// bits (setuid, setgid, sticky) are always cleared. +const deterministicPermMask fs.FileMode = 0o777 + +// Member is one explicitly described archive entry for [WriteDeterministic]. Unlike [PackOptions] +// sources, members never touch the filesystem on their own: the caller supplies the bytes, so the +// output depends only on the declared paths, modes, and content. +type Member struct { + // Path is the relative, slash-separated path inside the archive. + Path string + + // Kind is file, directory, or symlink. + Kind MemberKind + + // Mode holds the ordinary permission bits. Executable bits are preserved; other bits are + // written as given. Privileged bits are cleared. + Mode fs.FileMode + + // Size is the exact byte length of a file's content. + Size int64 + + // Target is a symlink's target; empty for other kinds. + Target string + + // Open returns a reader over a file's content. The writer closes it. Nil for other kinds. + Open func() (io.ReadCloser, error) +} + +// tarEpoch is the timestamp stamped into every deterministic tar entry. +var tarEpoch = time.Unix(0, 0).UTC() + +// zipEpoch is the timestamp stamped into every deterministic zip entry. 1980-01-01 is the earliest +// time the zip DOS timestamp field can represent. +var zipEpoch = time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC) + +// WriteDeterministic writes members to w as a zip, tar, or tgz archive whose bytes depend only on +// the members: entries are sorted by path, timestamps are pinned to a fixed epoch, owners are +// normalized to zero and empty names, gzip headers carry no name or time, and permission bits +// (including executable bits) are preserved. Identical members always produce identical bytes for +// one Go toolchain. It does not alter the behavior of [Run]; that path keeps its own defaults. +func WriteDeterministic(w io.Writer, format string, members []Member) error { + defer perf.Track(nil, "archive.WriteDeterministic")() + + sorted, err := sortMembers(members) + if err != nil { + return err + } + switch normalizeFormat(format) { + case FormatTar: + return writeDeterministicTar(w, sorted) + case FormatTGZ: + return writeDeterministicTGZ(w, sorted) + case FormatZip: + return writeDeterministicZip(w, sorted) + default: + return errUtils.Build(errUtils.ErrArchiveFormatNotImplemented). + WithExplanationf("deterministic writing supports zip, tar, and tgz, not %q", format). + WithContext("format", format). + Err() + } +} + +// sortMembers returns a path-sorted copy and rejects duplicate paths. +func sortMembers(members []Member) ([]Member, error) { + sorted := make([]Member, len(members)) + copy(sorted, members) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + for index := range sorted { + if sorted[index].Path == "" || strings.HasPrefix(sorted[index].Path, "/") { + return nil, memberError(sorted[index].Path, errUtils.ErrArchiveInvalidSubpath) + } + if index > 0 && sorted[index].Path == sorted[index-1].Path { + return nil, memberError(sorted[index].Path, errUtils.ErrArchiveWriteFailed) + } + } + return sorted, nil +} + +func memberError(path string, sentinel error) error { + return fmt.Errorf("%w: member %q is empty, absolute, or duplicated", sentinel, path) +} + +func writeDeterministicTGZ(w io.Writer, members []Member) error { + gz := gzip.NewWriter(w) + if err := writeDeterministicTar(gz, members); err != nil { + _ = gz.Close() + return err + } + if err := gz.Close(); err != nil { + return writeFailedError("gzip", err) + } + return nil +} + +func writeDeterministicTar(w io.Writer, members []Member) error { + tw := tar.NewWriter(w) + for index := range members { + if err := writeTarMember(tw, &members[index]); err != nil { + _ = tw.Close() + return err + } + } + if err := tw.Close(); err != nil { + return writeFailedError("tar", err) + } + return nil +} + +func writeTarMember(tw *tar.Writer, member *Member) error { + header := &tar.Header{ + Name: member.Path, + Mode: int64(member.Mode & deterministicPermMask), + ModTime: tarEpoch, + } + switch member.Kind { + case MemberDirectory: + header.Typeflag = tar.TypeDir + header.Name += "/" + case MemberSymlink: + header.Typeflag = tar.TypeSymlink + header.Linkname = member.Target + header.Mode = 0 + case MemberFile: + header.Typeflag = tar.TypeReg + header.Size = member.Size + default: + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + if err := tw.WriteHeader(header); err != nil { + return writeFailedError(member.Path, err) + } + if member.Kind != MemberFile { + return nil + } + return copyMemberContent(tw, member) +} + +func writeDeterministicZip(w io.Writer, members []Member) error { + zw := zip.NewWriter(w) + for index := range members { + if err := writeZipMember(zw, &members[index]); err != nil { + _ = zw.Close() + return err + } + } + if err := zw.Close(); err != nil { + return writeFailedError("zip", err) + } + return nil +} + +func writeZipMember(zw *zip.Writer, member *Member) error { + header := &zip.FileHeader{Name: member.Path, Method: zip.Deflate, Modified: zipEpoch} + perm := member.Mode & deterministicPermMask + switch member.Kind { + case MemberDirectory: + header.Name += "/" + header.Method = zip.Store + header.SetMode(fs.ModeDir | perm) + case MemberSymlink: + header.Method = zip.Store + header.SetMode(fs.ModeSymlink) + case MemberFile: + header.SetMode(perm) + default: + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + out, err := zw.CreateHeader(header) + if err != nil { + return writeFailedError(member.Path, err) + } + switch member.Kind { + case MemberSymlink: + if _, err := io.WriteString(out, member.Target); err != nil { + return writeFailedError(member.Path, err) + } + case MemberFile: + return copyMemberContent(out, member) + } + return nil +} + +// copyMemberContent copies exactly member.Size bytes and fails if the source is shorter or longer, +// so an archive never describes bytes other than the ones it carries. +func copyMemberContent(dst io.Writer, member *Member) error { + if member.Open == nil { + return memberError(member.Path, errUtils.ErrArchiveWriteFailed) + } + src, err := member.Open() + if err != nil { + return writeFailedError(member.Path, err) + } + defer src.Close() + + written, err := io.Copy(dst, io.LimitReader(src, member.Size)) + if err != nil { + return writeFailedError(member.Path, err) + } + if written != member.Size { + return writeFailedError(member.Path, io.ErrUnexpectedEOF) + } + var extra [1]byte + if n, readErr := src.Read(extra[:]); n > 0 || (readErr != nil && !errors.Is(readErr, io.EOF)) { + return writeFailedError(member.Path, fmt.Errorf("%w: content is longer than the declared %d bytes", errUtils.ErrArchiveWriteFailed, member.Size)) + } + return nil +} diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go new file mode 100644 index 00000000000..cfc5d6500a4 --- /dev/null +++ b/pkg/archive/deterministic_test.go @@ -0,0 +1,171 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "errors" + "io" + "io/fs" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func fileMember(path, content string, mode fs.FileMode) Member { + return Member{ + Path: path, Kind: MemberFile, Mode: mode, Size: int64(len(content)), + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader(content)), nil }, + } +} + +func sampleMembers() []Member { + return []Member{ + fileMember("z/last.txt", "last", 0o644), + {Path: "z", Kind: MemberDirectory, Mode: 0o755}, + fileMember("bin/run.sh", "#!/bin/sh\n", 0o755), + {Path: "bin", Kind: MemberDirectory, Mode: 0o755}, + {Path: "link", Kind: MemberSymlink, Target: "z/last.txt"}, + fileMember("unicode-é.txt", "ü", 0o640), + fileMember("empty", "", 0o644), + } +} + +func TestWriteDeterministic_IdenticalBytesRegardlessOfInputOrder(t *testing.T) { + for _, format := range []string{FormatTar, FormatTGZ, FormatZip} { + t.Run(format, func(t *testing.T) { + forward := sampleMembers() + reversed := sampleMembers() + for i, j := 0, len(reversed)-1; i < j; i, j = i+1, j-1 { + reversed[i], reversed[j] = reversed[j], reversed[i] + } + var first, second, third bytes.Buffer + require.NoError(t, WriteDeterministic(&first, format, forward)) + require.NoError(t, WriteDeterministic(&second, format, reversed)) + require.NoError(t, WriteDeterministic(&third, format, forward)) + assert.Equal(t, first.Bytes(), second.Bytes(), "member order must not change the bytes") + assert.Equal(t, first.Bytes(), third.Bytes(), "repeated writes must be identical") + }) + } +} + +func TestWriteDeterministic_TarNormalizesMetadata(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatTar, sampleMembers())) + reader := tar.NewReader(&buf) + var names []string + modes := map[string]int64{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + names = append(names, header.Name) + modes[header.Name] = header.Mode + assert.True(t, header.ModTime.Equal(tarEpoch), "%s mtime", header.Name) + assert.Zero(t, header.Uid) + assert.Zero(t, header.Gid) + assert.Empty(t, header.Uname) + assert.Empty(t, header.Gname) + } + assert.Equal(t, []string{"bin/", "bin/run.sh", "empty", "link", "unicode-é.txt", "z/", "z/last.txt"}, names) + assert.Equal(t, int64(0o755), modes["bin/run.sh"], "executable bit must survive") + assert.Equal(t, int64(0o640), modes["unicode-é.txt"]) +} + +func TestWriteDeterministic_TGZHeaderCarriesNoNameOrTime(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatTGZ, sampleMembers())) + gz, err := gzip.NewReader(&buf) + require.NoError(t, err) + assert.Empty(t, gz.Name) + assert.True(t, gz.ModTime.IsZero() || gz.ModTime.Unix() == 0) +} + +func TestWriteDeterministic_ZipEntries(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteDeterministic(&buf, FormatZip, sampleMembers())) + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + byName := map[string]*zip.File{} + for _, f := range reader.File { + byName[f.Name] = f + assert.True(t, f.Modified.UTC().Equal(zipEpoch), "%s modified", f.Name) + } + require.Contains(t, byName, "bin/run.sh") + assert.Equal(t, fs.FileMode(0o755), byName["bin/run.sh"].Mode().Perm()) + assert.True(t, byName["bin/"].Mode().IsDir()) + assert.NotZero(t, byName["link"].Mode()&fs.ModeSymlink) + rc, err := byName["link"].Open() + require.NoError(t, err) + target, err := io.ReadAll(rc) + require.NoError(t, err) + require.NoError(t, rc.Close()) + assert.Equal(t, "z/last.txt", string(target)) +} + +func TestWriteDeterministic_Rejections(t *testing.T) { + tests := []struct { + name string + format string + members []Member + want error + }{ + {"duplicate path", FormatTar, []Member{fileMember("a", "x", 0o644), fileMember("a", "y", 0o644)}, errUtils.ErrArchiveWriteFailed}, + {"absolute path", FormatTar, []Member{fileMember("/a", "x", 0o644)}, errUtils.ErrArchiveInvalidSubpath}, + {"empty path", FormatZip, []Member{fileMember("", "x", 0o644)}, errUtils.ErrArchiveInvalidSubpath}, + {"unsupported format", "tar.xz", nil, errUtils.ErrArchiveFormatNotImplemented}, + {"file without opener", FormatTar, []Member{{Path: "a", Kind: MemberFile, Size: 1}}, errUtils.ErrArchiveWriteFailed}, + {"unknown kind", FormatTar, []Member{{Path: "a", Kind: MemberKind(9)}}, errUtils.ErrArchiveWriteFailed}, + {"shorter than declared", FormatTar, []Member{{ + Path: "a", Kind: MemberFile, Size: 5, + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("ab")), nil }, + }}, errUtils.ErrArchiveWriteFailed}, + {"longer than declared", FormatZip, []Member{{ + Path: "a", Kind: MemberFile, Size: 1, + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("abc")), nil }, + }}, errUtils.ErrArchiveWriteFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := WriteDeterministic(io.Discard, tt.format, tt.members) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + }) + } +} + +func TestWriteDeterministic_ClearsPrivilegedBits(t *testing.T) { + var buf bytes.Buffer + member := fileMember("a", "x", 0o4755|fs.ModeSetuid) + require.NoError(t, WriteDeterministic(&buf, FormatTar, []Member{member})) + header, err := tar.NewReader(&buf).Next() + require.NoError(t, err) + assert.Equal(t, int64(0o755), header.Mode) +} + +type failingWriter struct{ after int } + +func (f *failingWriter) Write(p []byte) (int, error) { + if f.after <= 0 { + return 0, errors.New("disk full") + } + f.after -= len(p) + return len(p), nil +} + +func TestWriteDeterministic_PropagatesWriterFailures(t *testing.T) { + for _, format := range []string{FormatTar, FormatTGZ, FormatZip} { + t.Run(format, func(t *testing.T) { + err := WriteDeterministic(&failingWriter{}, format, sampleMembers()) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + }) + } +} diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go new file mode 100644 index 00000000000..6c0c6857625 --- /dev/null +++ b/pkg/artifact/artifacttest/conformance.go @@ -0,0 +1,461 @@ +package artifacttest + +import ( + "context" + "errors" + "io" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Names and modes the suite uses for its fixtures. +const ( + appName = "team/app" + firstVersion = "1.0.0" + secondVersion = "2.0.0" + filePerm = 0o644 + execPerm = 0o755 + pageSafety = 100 +) + +// Harness configures [RunRepositoryConformance] for one backend. +type Harness struct { + // New returns a fresh, empty repository. It is called once per subtest; the harness never + // shares state between subtests. Register cleanup with t.Cleanup. + New func(t *testing.T) artifact.Repository + + // Packaging is the payload packaging used for publications; empty uses the first packaging + // the repository advertises. + Packaging artifact.Packaging + + // Signer produces evidence the repository can store. Nil is allowed only for repositories that + // advertise no evidence formats; the suite then asserts Sign fails with a capability error. + Signer artifact.Signer + + // TrustedPolicy is satisfied by evidence the Signer produced. Required when Signer is set. + TrustedPolicy artifact.VerificationPolicy + + // UntrustedPolicy is a policy that evidence from the Signer must NOT satisfy. Required when + // Signer is set. + UntrustedPolicy artifact.VerificationPolicy +} + +// published is one artifact stored through the repository under test. +type published struct { + prepared *artifact.PreparedArtifact + result *artifact.PublishResult + ref artifact.Ref +} + +// RunRepositoryConformance runs the shared repository contract suite against h. Every backend +// calls it from its own tests; backend-specific differences are expressed through Capabilities, +// never by omitting an operation that claims success. Operations a repository does not support +// must return a capability error, which the suite asserts. +func RunRepositoryConformance(t *testing.T, h *Harness) { + defer perf.Track(nil, "artifacttest.RunRepositoryConformance")() + + t.Helper() + suite := &conformance{h: *h} + t.Run("CapabilitiesAreStatic", suite.capabilities) + t.Run("MultipleNamesAndVersions", suite.multipleNamesAndVersions) + t.Run("PublishReadRoundTrip", suite.roundTrip) + t.Run("MetadataAndDocumentsWithoutPayload", suite.metadataAndDocuments) + t.Run("DigestReferences", suite.digestReferences) + t.Run("Enumeration", suite.enumeration) + t.Run("ForceReplacementNeverServesNewBytesUnderOldDigest", suite.replacement) + t.Run("ConditionalPublish", suite.conditionalPublish) + t.Run("Delete", suite.deletion) + t.Run("Pin", suite.pin) + t.Run("SignVerify", suite.signVerify) + t.Run("Lock", suite.lock) + t.Run("ReadOnlyOperationsReturnCapabilityErrors", suite.capabilityErrors) +} + +type conformance struct { + h Harness +} + +func (c *conformance) packaging(repo artifact.Repository) artifact.Packaging { + if c.h.Packaging != "" { + return c.h.Packaging + } + caps := repo.Capabilities() + if len(caps.Packagings) > 0 { + return caps.Packagings[0] + } + return artifact.PackagingNone +} + +// spec names and fills one artifact the suite publishes. +type spec struct { + name string + version string + content string +} + +// publish prepares a file package and publishes it. The spec's content varies the payload. +func (c *conformance) publish(t *testing.T, repo artifact.Repository, sp spec, options artifact.PublishOptions) published { + t.Helper() + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", sp.content, filePerm) + fixtures.WriteFile(t, src, "bin/run.sh", fixtures.HelperScript, execPerm) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "readme for "+sp.name+"\n", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: sp.name, Version: sp.version, Packaging: c.packaging(repo), + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "conformance"}}, + OperationID: "op-" + sp.name + "-" + sp.version, PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + result, err := repo.Publish(context.Background(), prepared, options) + require.NoError(t, err) + return published{prepared: prepared, result: result, ref: artifact.Ref{Name: sp.name, Version: sp.version}} +} + +func (c *conformance) requireWritable(t *testing.T, repo artifact.Repository) { + t.Helper() + if !repo.Capabilities().Write { + t.Skip("the repository is read-only; capability errors are asserted by the dedicated subtest") + } +} + +func (c *conformance) capabilities(t *testing.T) { + repo := c.h.New(t) + first, second := repo.Capabilities(), repo.Capabilities() + assert.Equal(t, first, second, "Capabilities must be stable and need no network access") + assert.NotEmpty(t, first.LockScopes, "every repository can coordinate at least host-scope writes") +} + +func (c *conformance) multipleNamesAndVersions(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + items := []struct{ name, version string }{{appName, "1.0.0"}, {appName, "2.0.0"}, {"other/tool", "1.0.0"}} + var all []published + for _, item := range items { + all = append(all, c.publish(t, repo, spec{name: item.name, version: item.version, content: item.name + item.version}, artifact.PublishOptions{})) + } + seen := map[digest.Digest]bool{} + for _, item := range all { + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.NotNil(t, inspection.Record) + assert.Equal(t, item.ref.Name, inspection.Record.Name) + assert.Equal(t, item.ref.Version, inspection.Record.Version) + assert.False(t, seen[pinned.ResolvedDigest], "distinct artifacts resolve to distinct digests") + seen[pinned.ResolvedDigest] = true + } +} + +func (c *conformance) roundTrip(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "round trip"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NotNil(t, content.Record) + + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "pulled content matches the inventory, modes included") +} + +func (c *conformance) metadataAndDocuments(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "docs"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + data, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + require.NoError(t, stream.Reader.Close()) + assert.Equal(t, "readme for team/app\n", string(data)) + assert.Equal(t, digest.FromBytes(data), stream.Document.Digest) + + _, err = repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a missing optional document is a typed error") +} + +func (c *conformance) digestReferences(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "digest"}, artifact.PublishOptions{}) + byVersion, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + + byDigest, err := repo.Resolve(context.Background(), artifact.Ref{Name: item.ref.Name, Digest: byVersion.ResolvedDigest}) + require.NoError(t, err) + assert.Equal(t, byVersion.ResolvedDigest, byDigest.ResolvedDigest) + + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: "other/name", Digest: byVersion.ResolvedDigest}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a digest read checks the Record's embedded name") + + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: item.ref.Name, Version: "9.9.9"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func (c *conformance) enumeration(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + caps := repo.Capabilities() + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}} { + c.publish(t, repo, spec{name: item.name, version: item.version, content: item.name + item.version}, artifact.PublishOptions{}) + } + if !caps.EnumerateCollection { + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + var names []string + cursor := "" + for pages := 0; pages < pageSafety; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{Cursor: cursor}) + require.NoError(t, err) + for index := range page.Entries { + names = append(names, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + require.NotEqual(t, cursor, page.NextCursor, "the cursor must advance") + cursor = page.NextCursor + } + assert.Equal(t, []string{"a/app:1", "a/app:2", "b/app:1"}, names, "lexical order, every entry exactly once") + + if caps.EnumerateNamed { + page, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app", PageSize: pageSafety}) + require.NoError(t, err) + require.Len(t, page.Entries, 2) + assert.Equal(t, "1", page.Entries[0].Version) + assert.Equal(t, "2", page.Entries[1].Version) + } +} + +func (c *conformance) replacement(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + first := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "first"}, artifact.PublishOptions{}) + oldPinned, err := repo.Resolve(context.Background(), first.ref) + require.NoError(t, err) + + second := c.publishForce(t, repo, first.ref, "second", &oldPinned) + newPinned, err := repo.Resolve(context.Background(), first.ref) + require.NoError(t, err) + assert.NotEqual(t, oldPinned.ResolvedDigest, newPinned.ResolvedDigest) + assert.Equal(t, artifact.OutcomeReplaced, second.result.Outcome) + + content, err := repo.Read(context.Background(), oldPinned) + if err != nil { + assert.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "retired bytes report not found") + return + } + defer content.Close() + require.NotNil(t, content.Record) + recordDigest, err := content.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, oldPinned.RecordDigest, recordDigest, "an old digest never serves the replacement's record") +} + +func (c *conformance) publishForce(t *testing.T, repo artifact.Repository, ref artifact.Ref, content string, prior *artifact.PinnedReference) published { + return c.publish(t, repo, spec{name: ref.Name, version: ref.Version, content: content}, artifact.PublishOptions{Force: true, ExpectedPrior: prior}) +} + +func (c *conformance) conditionalPublish(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + if !repo.Capabilities().ConditionalUpdate { + t.Skip("the repository does not advertise conditional updates") + } + ref := artifact.Ref{Name: appName, Version: "1.0.0"} + c.publish(t, repo, spec{name: ref.Name, version: ref.Version, content: "base"}, artifact.PublishOptions{}) + + stale := artifact.PinnedReference{Ref: ref, ResolvedDigest: digest.FromString("stale")} + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "changed", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: ref.Name, Version: ref.Version, Packaging: c.packaging(repo), + OperationID: "op-stale", PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{Force: true, ExpectedPrior: &stale}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected-prior reference is a conflict, not a silent overwrite") + + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "publishing over an existing reference expecting none is a conflict") +} + +func (c *conformance) deletion(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + keep := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "keep"}, artifact.PublishOptions{}) + drop := c.publish(t, repo, spec{name: appName, version: secondVersion, content: "drop"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), drop.ref) + require.NoError(t, err) + + caps := repo.Capabilities() + if caps.Delete == artifact.DeleteNone || caps.Delete == "" { + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: caps.Delete}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: caps.Delete}) + require.NoError(t, err) + assert.NotEmpty(t, result.Deleted) + _, err = repo.Resolve(context.Background(), drop.ref) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Resolve(context.Background(), keep.ref) + require.NoError(t, err, "deletion is scoped to the selected artifact") +} + +func (c *conformance) pin(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "pin"}, artifact.PublishOptions{}) + resolved, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + pinned, err := repo.Pin(context.Background(), item.ref) + require.NoError(t, err) + assert.Equal(t, resolved.ResolvedDigest, pinned.ResolvedDigest) + assert.Equal(t, resolved.RecordDigest, pinned.RecordDigest) + + _, err = repo.Pin(context.Background(), artifact.Ref{Name: "missing/artifact", Version: "1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func (c *conformance) signVerify(t *testing.T) { + repo := c.h.New(t) + c.requireWritable(t, repo) + item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "sign"}, artifact.PublishOptions{}) + pinned, err := repo.Resolve(context.Background(), item.ref) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + + caps := repo.Capabilities() + if c.h.Signer == nil { + require.Empty(t, caps.EvidenceFormats, "a repository that stores evidence needs a Signer in the harness") + _, err := repo.Sign(context.Background(), subject, &unsupportedSigner{}, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + require.NotEmpty(t, c.h.TrustedPolicy.TrustSets, "TrustedPolicy is required with a Signer") + + unsigned, err := repo.Verify(context.Background(), subject, c.h.TrustedPolicy) + require.Error(t, err, "no evidence never verifies") + if unsigned != nil { + assert.NotEqual(t, artifact.VerificationVerified, unsigned.Status) + } + + evidence, err := repo.Sign(context.Background(), subject, c.h.Signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) + require.NotEmpty(t, evidence) + assert.Equal(t, subject.Digest, evidence[0].Subject.Digest) + + verified, err := repo.Verify(context.Background(), subject, c.h.TrustedPolicy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + assert.Equal(t, subject.Digest, verified.Subject.Digest) + assert.NotEmpty(t, verified.EvidenceDigests) + + rejected, err := repo.Verify(context.Background(), subject, c.h.UntrustedPolicy) + require.Error(t, err, "evidence from an untrusted signer never satisfies the policy") + if rejected != nil { + assert.NotEqual(t, artifact.VerificationVerified, rejected.Status) + } + + other := subject + other.Digest = digest.FromString("another subject") + if mismatched, err := repo.Verify(context.Background(), other, c.h.TrustedPolicy); err == nil && mismatched != nil { + assert.NotEqual(t, artifact.VerificationVerified, mismatched.Status, "evidence is bound to its subject digest") + } +} + +// unsupportedSigner is a signer for a format no repository under test advertises. +type unsupportedSigner struct{} + +func (*unsupportedSigner) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "artifacttest.unsupportedSigner.Formats")() + + return []artifact.EvidenceFormat{"unsupported"} +} + +func (*unsupportedSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + defer perf.Track(nil, "artifacttest.unsupportedSigner.Sign")() + + return &artifact.EvidencePayload{Format: "unsupported", Data: []byte("x")}, nil +} + +func (c *conformance) lock(t *testing.T) { + repo := c.h.New(t) + ctx := context.Background() + request := artifact.LockRequest{Resource: "team/app:1.0.0", Owner: "owner-a", OperationID: "op-a", Required: artifact.LockScopeHost} + first, err := repo.Lock(ctx, request) + require.NoError(t, err) + assert.Equal(t, request.Resource, first.Resource()) + assert.Equal(t, "owner-a", first.Owner()) + assert.NotEmpty(t, first.Token()) + assert.True(t, first.Scope().Satisfies(artifact.LockScopeHost)) + + blocked, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + second := request + second.Owner = "owner-b" + handle, err := repo.Lock(blocked, second) + require.Error(t, err, "a competing owner cannot take a held lock") + assert.Nil(t, handle) + assert.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errUtils.ErrArtifactConflict), "waiting is bounded by the context or rejected as a conflict: %v", err) + + require.NoError(t, first.Release(ctx)) + again, err := repo.Lock(ctx, second) + require.NoError(t, err, "a canceled acquisition leaked no ownership") + require.NoError(t, again.Release(ctx)) + + caps := repo.Capabilities() + if !caps.HasLockScope(artifact.LockScopeRepository) { + request.Required = artifact.LockScopeRepository + _, err := repo.Lock(ctx, request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an insufficient guarantee is an error, never a silent downgrade") + } +} + +func (c *conformance) capabilityErrors(t *testing.T) { + repo := c.h.New(t) + caps := repo.Capabilities() + if caps.Write { + t.Skip("the repository is writable; read-only behavior is not applicable") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", filePerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: "x/y", Version: "1", Packaging: c.packaging(repo), OperationID: "op", PreparedAt: time.Now(), + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/artifacttest/conformance_test.go b/pkg/artifact/artifacttest/conformance_test.go new file mode 100644 index 00000000000..ba61ae92da3 --- /dev/null +++ b/pkg/artifact/artifacttest/conformance_test.go @@ -0,0 +1,71 @@ +package artifacttest_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +func TestInMemoryRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { return artifacttest.New(artifacttest.Options{}) }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} + +func TestInMemoryRepositoryConformance_RestrictedCapabilities(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Write = false + caps.EnumerateCollection = false + caps.Delete = artifact.DeleteNone + caps.EvidenceFormats = nil + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} + +func TestInMemoryRepositoryConformance_NoEnumerationNoDelete(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.EnumerateCollection = false + caps.EnumerateNamed = false + caps.Delete = artifact.DeleteNone + caps.EvidenceFormats = nil + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} + +func TestRepository_LockRejectsLeasesAndWrongTokens(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + ctx := context.Background() + _, err := repo.Lock(ctx, artifact.LockRequest{Resource: "r", Owner: "o", Lease: 1}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: "r", Owner: "o"}) + require.NoError(t, err) + assert.Equal(t, []string{"r"}, repo.HeldLocks()) + require.ErrorIs(t, handle.Renew(ctx, 1), errUtils.ErrArtifactUnsupportedCapability) + _, expires := handle.ExpiresAt() + assert.False(t, expires) + require.NoError(t, handle.Release(ctx)) + require.ErrorIs(t, handle.Release(ctx), errUtils.ErrArtifactConflict, "a stale handle cannot release a lock it no longer owns") + assert.Empty(t, repo.HeldLocks()) +} diff --git a/pkg/artifact/artifacttest/doc.go b/pkg/artifact/artifacttest/doc.go new file mode 100644 index 00000000000..d70c45ad13c --- /dev/null +++ b/pkg/artifact/artifacttest/doc.go @@ -0,0 +1,11 @@ +// Package artifacttest provides test support for artifact repositories and the artifact service: +// a minimal in-memory [Repository] that implements the full [artifact.Repository] contract with +// failure injection, a fake signer and evidence format, and [RunRepositoryConformance], the shared +// conformance suite every backend (local directory, local archive, S3, OCI layout, OCI registry) +// runs against its own implementation. +// +// The in-memory repository is a harness, not a backend: it models the publication lifecycle +// (pending, completed, unavailable after interruption), conditional updates, digest-addressed +// reads that never serve replacement bytes, pagination, and ownership-checked locks, so service +// behavior such as reconciliation can be tested without a real store. +package artifacttest diff --git a/pkg/artifact/artifacttest/lock.go b/pkg/artifact/artifacttest/lock.go new file mode 100644 index 00000000000..7e37bfa3b6a --- /dev/null +++ b/pkg/artifact/artifacttest/lock.go @@ -0,0 +1,150 @@ +package artifacttest + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// lockState is one claimed resource. +type lockState struct { + owner string + operationID string + token string + released chan struct{} +} + +// handle is an ownership-checked lock handle. +type handle struct { + repo *Repository + resource string + owner string + token string + scope artifact.LockScope +} + +var _ artifact.LockHandle = (*handle)(nil) + +// Lock implements [artifact.LockProvider]. Acquisition waits for a competing owner and returns the +// context error, leaking nothing, when canceled. Locks never expire and are never stolen. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "artifacttest.Repository.Lock")() + + r.mu.Lock() + r.count("Lock") + r.mu.Unlock() + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "lock", "lease", "claims never expire, so a lease cannot be fenced") + } + required := request.Required + if required == "" { + required = artifact.LockScopeHost + } + if !r.caps.HasLockScope(required) { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "lock", "lock/"+string(required), "the repository cannot provide this guarantee") + } + for { + r.mu.Lock() + held, busy := r.locks[request.Resource] + if !busy { + token := uuid.NewString() + r.locks[request.Resource] = &lockState{owner: request.Owner, operationID: request.OperationID, token: token, released: make(chan struct{})} + r.mu.Unlock() + return &handle{repo: r, resource: request.Resource, owner: request.Owner, token: token, scope: r.bestScope()}, nil + } + wait := held.released + r.mu.Unlock() + select { + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for lock %s held by %s: %w", request.Resource, held.owner, ctx.Err()) + case <-wait: + } + } +} + +// bestScope is the strongest scope the repository advertises. +func (r *Repository) bestScope() artifact.LockScope { + if r.caps.HasLockScope(artifact.LockScopeRepository) { + return artifact.LockScopeRepository + } + return artifact.LockScopeHost +} + +// HeldLocks lists the currently claimed resources, sorted. +func (r *Repository) HeldLocks() []string { + defer perf.Track(nil, "artifacttest.Repository.HeldLocks")() + + r.mu.Lock() + defer r.mu.Unlock() + resources := make([]string, 0, len(r.locks)) + for resource := range r.locks { + resources = append(resources, resource) + } + sort.Strings(resources) + return resources +} + +// Resource implements [artifact.LockHandle]. +func (h *handle) Resource() string { + defer perf.Track(nil, "artifacttest.handle.Resource")() + + return h.resource +} + +// Owner implements [artifact.LockHandle]. +func (h *handle) Owner() string { + defer perf.Track(nil, "artifacttest.handle.Owner")() + + return h.owner +} + +// Scope implements [artifact.LockHandle]. +func (h *handle) Scope() artifact.LockScope { + defer perf.Track(nil, "artifacttest.handle.Scope")() + + return h.scope +} + +// Token implements [artifact.LockHandle]. +func (h *handle) Token() string { + defer perf.Track(nil, "artifacttest.handle.Token")() + + return h.token +} + +// ExpiresAt implements [artifact.LockHandle]; claims never expire. +func (h *handle) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "artifacttest.handle.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; there are no leases. +func (h *handle) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "artifacttest.handle.Renew")() + + return artifact.NewUnsupportedCapability(h.repo.opts.Alias, "lock", "lease", "claims never expire") +} + +// Release implements [artifact.LockHandle]. Releasing checks the ownership token. +func (h *handle) Release(context.Context) error { + defer perf.Track(nil, "artifacttest.handle.Release")() + + h.repo.mu.Lock() + defer h.repo.mu.Unlock() + h.repo.count("Release") + held, ok := h.repo.locks[h.resource] + if !ok || held.token != h.token { + return fmt.Errorf("%w: lock %s is not owned by token %s", errUtils.ErrArtifactConflict, h.resource, h.token) + } + delete(h.repo.locks, h.resource) + close(held.released) + return nil +} diff --git a/pkg/artifact/artifacttest/read.go b/pkg/artifact/artifacttest/read.go new file mode 100644 index 00000000000..e1c4edf57dd --- /dev/null +++ b/pkg/artifact/artifacttest/read.go @@ -0,0 +1,253 @@ +package artifacttest + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Resolve implements [artifact.Repository]. A pending reference is unavailable to readers; a +// digest reference checks the Record's embedded name. +func (r *Repository) Resolve(_ context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "artifacttest.Repository.Resolve")() + + pinned, err := r.resolve(ref) + r.mu.Lock() + hook := r.opts.OnResolved + r.mu.Unlock() + if err == nil && ref.Version != "" && hook != nil { + hook(ref) + } + return pinned, err +} + +func (r *Repository) resolve(ref artifact.Ref) (artifact.PinnedReference, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.count("Resolve") + if err := ref.Validate(); err != nil { + return artifact.PinnedReference{}, err + } + if ref.IsDigest() { + s, ok := r.records[ref.Digest] + if !ok || s.record.Name != ref.Name { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + return r.pinnedFor(ref, s), nil + } + key := refKey(ref.Name, ref.Version) + if p, isPending := r.pendings[key]; isPending { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s has a pending publication by operation %s", errUtils.ErrArtifactPublicationUnavailable, ref, p.operationID) + } + current, ok := r.refs[key] + if !ok { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + return r.pinnedFor(ref, r.records[current]), nil +} + +// Inspect implements [artifact.Repository]. It reads only the retained record and descriptors. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Inspect(_ context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "artifacttest.Repository.Inspect")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Inspect") + s, ok := r.records[pinned.RecordDigest] + if !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + record := s.record + pinnedOut := r.pinnedFor(pinned.Ref, s) + inspection := &artifact.Inspection{Pinned: pinnedOut, Record: &record, Representation: r.representation(s)} + held := r.evidence[s.recordDigest] + for index := range held { + inspection.Evidence = append(inspection.Evidence, held[index].Evidence) + } + return inspection, nil +} + +// OpenDocument implements [artifact.Repository] without touching the payload. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) OpenDocument(_ context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "artifacttest.Repository.OpenDocument")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("OpenDocument") + s, ok := r.records[pinned.RecordDigest] + if !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + for index := range s.record.Documents { + if s.record.Documents[index].Role != role { + continue + } + for blobIndex := range s.blobs { + held := &s.blobs[blobIndex] + if held.role == artifact.BlobDocument && held.documentRole == role { + return &artifact.DocumentStream{Document: s.record.Documents[index], Reader: newVerifyingReader(held.data, held.descriptor.Digest)}, nil + } + } + } + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDocumentMissing, role) +} + +// Read implements [artifact.Repository]. It serves exactly the pinned digest; once the payload of +// a replaced or deleted record is gone it reports not found rather than the current bytes. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Read(_ context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "artifacttest.Repository.Read")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Read") + s, ok := r.records[pinned.RecordDigest] + if !ok || !s.payloadAvailable { + return nil, fmt.Errorf("%w: the payload of %s is no longer available", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + record := s.record + content := &artifact.Content{Record: &record, Representation: r.representation(s)} + for index := range s.blobs { + held := &s.blobs[index] + content.Streams = append(content.Streams, artifact.ContentStream{ + Descriptor: held.descriptor, Role: held.role, Path: held.path, Reader: newVerifyingReader(held.data, held.descriptor.Digest), + }) + } + return content, nil +} + +// Pin implements [artifact.Repository]. It acquires no lock and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "artifacttest.Repository.Pin")() + + pinned, err := r.Resolve(ctx, ref) + if err != nil { + return pinned, err + } + r.mu.Lock() + defer r.mu.Unlock() + r.count("Pin") + held := r.evidence[pinned.RecordDigest] + for index := range held { + pinned.Evidence = append(pinned.Evidence, held[index].Evidence.Descriptor) + } + return pinned, nil +} + +// List implements [artifact.Repository], paginating over completed references only. +func (r *Repository) List(_ context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "artifacttest.Repository.List")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("List") + if query.Name != "" && !r.caps.EnumerateNamed || query.Name == "" && !r.caps.EnumerateCollection { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "list", "enumerate", "enumeration is not supported") + } + var entries []artifact.ListEntry + for key, recordDigest := range r.refs { + if _, isPending := r.pendings[key]; isPending { + continue + } + s := r.records[recordDigest] + if query.Name != "" && s.record.Name != query.Name { + continue + } + if !labelsMatch(s.record.Metadata.Labels, query.Labels) { + continue + } + entries = append(entries, r.listEntry(s)) + } + sort.Slice(entries, func(i, j int) bool { return listKey(&entries[i]) < listKey(&entries[j]) }) + return r.page(entries, query), nil +} + +func (r *Repository) listEntry(s *stored) artifact.ListEntry { + ref := artifact.Ref{Name: s.record.Name, Version: s.record.Version} + return artifact.ListEntry{ + Pinned: r.pinnedFor(ref, s), + Name: s.record.Name, + Version: s.record.Version, + Kind: s.record.Kind, + Description: s.record.Metadata.Description, + Labels: s.record.Metadata.Labels, + Documents: s.record.Documents, + } +} + +func listKey(entry *artifact.ListEntry) string { + return entry.Name + "\x00" + entry.Version +} + +func (r *Repository) page(entries []artifact.ListEntry, query artifact.ListQuery) *artifact.ListPage { + size := query.PageSize + if size <= 0 { + size = r.opts.PageSize + } + start := 0 + if query.Cursor != "" { + start = sort.Search(len(entries), func(i int) bool { return listKey(&entries[i]) > query.Cursor }) + } + end := start + size + page := &artifact.ListPage{} + if end < len(entries) { + page.NextCursor = listKey(&entries[end-1]) + } else { + end = len(entries) + } + page.Entries = append(page.Entries, entries[start:end]...) + return page +} + +func labelsMatch(have map[string]string, want []artifact.Label) bool { + for _, label := range want { + if value, ok := have[label.Key]; !ok || value != label.Value { + return false + } + } + return true +} + +// verifyingReader verifies the digest of the bytes it yields at EOF. +type verifyingReader struct { + reader *bytes.Reader + data []byte + expected digest.Digest + done bool +} + +func newVerifyingReader(data []byte, expected digest.Digest) io.ReadCloser { + return &verifyingReader{reader: bytes.NewReader(data), data: data, expected: expected} +} + +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "artifacttest.verifyingReader.Read")() + + n, err := v.reader.Read(p) + if err == io.EOF && !v.done { + v.done = true + if digest.FromBytes(v.data) != v.expected { + return n, fmt.Errorf("%w: blob does not match descriptor %s", errUtils.ErrArtifactIntegrityFailed, v.expected) + } + } + return n, err +} + +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "artifacttest.verifyingReader.Close")() + + return nil +} diff --git a/pkg/artifact/artifacttest/repository.go b/pkg/artifact/artifacttest/repository.go new file mode 100644 index 00000000000..b67557ba19b --- /dev/null +++ b/pkg/artifact/artifacttest/repository.go @@ -0,0 +1,284 @@ +package artifacttest + +import ( + "fmt" + "sync" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultPageSize is the fake's list page size; it is small so tests exercise pagination. +const DefaultPageSize = 2 + +// FailPoint names an injectable interruption in the publication lifecycle. +type FailPoint string + +// Injectable interruptions, in lifecycle order. +const ( + // FailBeforePayload fails before anything changed. + FailBeforePayload FailPoint = "before-payload" + + // FailDuringPayload fails while payload blobs are being read; the publication stays pending. + FailDuringPayload FailPoint = "during-payload" + + // FailAfterMetadata fails after the immutable record was written but before the reference moved. + FailAfterMetadata FailPoint = "after-metadata" + + // FailAfterCommit fails after the reference was published: the server-side commit succeeded + // but the response was lost. + FailAfterCommit FailPoint = "after-commit" +) + +// Options configure a [Repository]. +type Options struct { + // Alias names the repository in locators and errors; default "memory". + Alias string + + // Capabilities overrides the default full-featured capabilities. + Capabilities *artifact.Capabilities + + // PageSize is the list page size; default [DefaultPageSize]. + PageSize int + + // OnResolved runs after Resolve has pinned a version reference and released its internal + // lock, so a test can move the reference to simulate a tag changing mid-operation. + OnResolved func(ref artifact.Ref) +} + +// blob is one stored payload or document blob. +type blob struct { + descriptor ocispec.Descriptor + role artifact.BlobRole + documentRole artifact.DocumentRole + path string + data []byte +} + +// stored is one immutable published record with its blobs. +type stored struct { + record artifact.Record + canonical []byte + recordDigest digest.Digest + blobs []blob + packaging artifact.Packaging + payloadAvailable bool +} + +// pending is an in-flight or interrupted publication. +type pending struct { + operationID string + intent digest.Digest +} + +// failure is an armed injected failure. +type failure struct { + err error + times int +} + +// Repository is an in-memory [artifact.Repository]. It is safe for concurrent use. +type Repository struct { + mu sync.Mutex + opts Options + caps artifact.Capabilities + refs map[string]digest.Digest + records map[digest.Digest]*stored + pendings map[string]*pending + evidence map[digest.Digest][]artifact.EvidenceBlob + locks map[string]*lockState + failures map[FailPoint]*failure + calls map[string]int +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// New returns an empty in-memory repository. +func New(opts Options) *Repository { + defer perf.Track(nil, "artifacttest.New")() + + if opts.Alias == "" { + opts.Alias = "memory" + } + if opts.PageSize <= 0 { + opts.PageSize = DefaultPageSize + } + caps := DefaultCapabilities() + if opts.Capabilities != nil { + caps = *opts.Capabilities + } + return &Repository{ + opts: opts, + caps: caps, + refs: map[string]digest.Digest{}, + records: map[digest.Digest]*stored{}, + pendings: map[string]*pending{}, + evidence: map[digest.Digest][]artifact.EvidenceBlob{}, + locks: map[string]*lockState{}, + failures: map[FailPoint]*failure{}, + calls: map[string]int{}, + } +} + +// DefaultCapabilities is the full-featured capability set of the in-memory repository. +func DefaultCapabilities() artifact.Capabilities { + defer perf.Track(nil, "artifacttest.DefaultCapabilities")() + + return artifact.Capabilities{ + Read: true, + Write: true, + EnumerateNamed: true, + EnumerateCollection: true, + Delete: artifact.DeleteObject, + ConditionalUpdate: true, + Packagings: []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip}, + PreservesModes: true, + SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}, + EvidenceFormats: []artifact.EvidenceFormat{FakeEvidenceFormat}, + LockScopes: []artifact.LockScope{artifact.LockScopeHost}, + } +} + +// Capabilities implements [artifact.Repository]. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "artifacttest.Repository.Capabilities")() + + return r.caps +} + +// InjectFailure arms err to be returned from Publish at point, times times (0 means once). +func (r *Repository) InjectFailure(point FailPoint, err error, times int) { + defer perf.Track(nil, "artifacttest.Repository.InjectFailure")() + + r.mu.Lock() + defer r.mu.Unlock() + if times <= 0 { + times = 1 + } + r.failures[point] = &failure{err: err, times: times} +} + +// Calls returns how many times the named repository method was called. +func (r *Repository) Calls(method string) int { + defer perf.Track(nil, "artifacttest.Repository.Calls")() + + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[method] +} + +// TotalCalls returns the total number of repository method calls (including Capabilities-free +// methods only), which dry-run tests assert is zero. +func (r *Repository) TotalCalls() int { + defer perf.Track(nil, "artifacttest.Repository.TotalCalls")() + + r.mu.Lock() + defer r.mu.Unlock() + total := 0 + for _, n := range r.calls { + total += n + } + return total +} + +// MoveRef repoints the version reference at another stored record digest, simulating a mutable +// tag being moved. It reports false when the digest is unknown. +func (r *Repository) MoveRef(ref artifact.Ref, to digest.Digest) bool { + defer perf.Track(nil, "artifacttest.Repository.MoveRef")() + + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.records[to]; !ok { + return false + } + r.refs[refKey(ref.Name, ref.Version)] = to + return true +} + +// SetOnResolved installs (or clears) the hook that runs after a version reference was pinned. +func (r *Repository) SetOnResolved(hook func(ref artifact.Ref)) { + defer perf.Track(nil, "artifacttest.Repository.SetOnResolved")() + + r.mu.Lock() + defer r.mu.Unlock() + r.opts.OnResolved = hook +} + +// IsPending reports whether the reference has an in-flight or interrupted publication. +func (r *Repository) IsPending(ref artifact.Ref) bool { + defer perf.Track(nil, "artifacttest.Repository.IsPending")() + + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.pendings[refKey(ref.Name, ref.Version)] + return ok +} + +// RecordDigestOf returns the record digest a version reference currently points at. +func (r *Repository) RecordDigestOf(ref artifact.Ref) (digest.Digest, bool) { + defer perf.Track(nil, "artifacttest.Repository.RecordDigestOf")() + + r.mu.Lock() + defer r.mu.Unlock() + d, ok := r.refs[refKey(ref.Name, ref.Version)] + return d, ok +} + +func refKey(name, version string) string { + return name + ":" + version +} + +// count records a call; callers hold r.mu. +func (r *Repository) count(method string) { + r.calls[method]++ +} + +// trip consumes an armed failure at point; callers hold r.mu. +func (r *Repository) trip(point FailPoint) error { + armed, ok := r.failures[point] + if !ok || armed.times <= 0 { + return nil + } + armed.times-- + return armed.err +} + +func (r *Repository) locator(name, version string) string { + return fmt.Sprintf("mem://%s/%s/%s", r.opts.Alias, name, version) +} + +func (r *Repository) pinnedFor(ref artifact.Ref, s *stored) artifact.PinnedReference { + return artifact.PinnedReference{ + Ref: ref, + Repository: "mem://" + r.opts.Alias, + ResolvedDigest: s.recordDigest, + RecordDigest: s.recordDigest, + ContentDigest: s.record.ContentDigest, + } +} + +func (r *Repository) representation(s *stored) artifact.Representation { + rep := artifact.Representation{ + Packaging: s.packaging, + Locator: r.locator(s.record.Name, s.record.Version), + RecordDigest: s.recordDigest, + } + payloadDigests := "" + for index := range s.blobs { + if s.blobs[index].role == artifact.BlobPayload { + rep.Payload = append(rep.Payload, s.blobs[index].descriptor) + payloadDigests += s.blobs[index].descriptor.Digest.String() + "\n" + } else { + rep.Documents = append(rep.Documents, s.blobs[index].descriptor) + } + } + rep.RepresentationDigest = digest.FromString(payloadDigests) + return rep +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go new file mode 100644 index 00000000000..e51f7ebff12 --- /dev/null +++ b/pkg/artifact/artifacttest/sign.go @@ -0,0 +1,222 @@ +package artifacttest + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// FakeEvidenceFormat is the evidence format of [FakeSigner]; the in-memory repository stores and +// verifies it. It stands in for a real format only inside tests. +const FakeEvidenceFormat = artifact.EvidenceSigstoreBundle + +// fakeEvidenceMediaType is the media type of fake evidence. +const fakeEvidenceMediaType = "application/vnd.atmos.artifacttest.evidence.v1+json" + +// fakeEvidence is the payload [FakeSigner] produces. +type fakeEvidence struct { + Subject digest.Digest `json:"subject"` + Identity string `json:"identity"` + Issuer string `json:"issuer"` +} + +// FakeSigner signs with a fixed identity and issuer. Evidence is bound to the subject digest. +type FakeSigner struct { + // Identity is the signer identity claim. + Identity string + + // Issuer is the issuer claim. + Issuer string +} + +var _ artifact.Signer = (*FakeSigner)(nil) + +// Formats implements [artifact.Signer]. +func (f *FakeSigner) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "artifacttest.FakeSigner.Formats")() + + return []artifact.EvidenceFormat{FakeEvidenceFormat} +} + +// Sign implements [artifact.Signer]. +func (f *FakeSigner) Sign(_ context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + defer perf.Track(nil, "artifacttest.FakeSigner.Sign")() + + data, err := json.Marshal(fakeEvidence{Subject: subject.Digest, Identity: f.Identity, Issuer: f.Issuer}) + if err != nil { + return nil, err + } + return &artifact.EvidencePayload{Format: FakeEvidenceFormat, MediaType: fakeEvidenceMediaType, Data: data}, nil +} + +// TrustedPolicy returns a policy that only [FakeSigner] with the same identity and issuer satisfies. +func (f *FakeSigner) TrustedPolicy() artifact.VerificationPolicy { + defer perf.Track(nil, "artifacttest.FakeSigner.TrustedPolicy")() + + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: f.Identity, Issuer: f.Issuer}}}}, + } +} + +// UntrustedPolicy returns a policy that no evidence from [FakeSigner] satisfies. +func (f *FakeSigner) UntrustedPolicy() artifact.VerificationPolicy { + defer perf.Track(nil, "artifacttest.FakeSigner.UntrustedPolicy")() + + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: f.Identity + "-other", Issuer: f.Issuer}}}}, + } +} + +// Sign implements [artifact.Repository]: the evidence is stored keyed by the subject digest. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "artifacttest.Repository.Sign")() + + r.mu.Lock() + r.count("Sign") + if err := r.checkSubject(subject); err != nil { + r.mu.Unlock() + return nil, err + } + r.mu.Unlock() + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, err + } + if !r.caps.HasEvidenceFormat(payload.Format) { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "evidence/"+string(payload.Format), "the repository cannot store this evidence format") + } + evidence := artifact.SignatureEvidence{ + Subject: subject, + Format: payload.Format, + Descriptor: ocispec.Descriptor{ + MediaType: payload.MediaType, Digest: digest.FromBytes(payload.Data), Size: int64(len(payload.Data)), + }, + Locator: fmt.Sprintf("mem://%s/signatures/%s/%s", r.opts.Alias, subject.Digest.Encoded(), digest.FromBytes(payload.Data).Encoded()), + } + r.mu.Lock() + defer r.mu.Unlock() + r.evidence[subject.Digest] = append(r.evidence[subject.Digest], artifact.EvidenceBlob{Evidence: evidence, Data: payload.Data}) + return []artifact.SignatureEvidence{evidence}, nil +} + +// checkSubject requires the subject to be a stored record; callers hold r.mu. +func (r *Repository) checkSubject(subject artifact.SignatureSubject) error { + if !r.caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "subject/"+string(subject.Kind), "unsupported subject kind") + } + if _, ok := r.records[subject.Digest]; !ok { + return fmt.Errorf("%w: subject %s", errUtils.ErrArtifactNotFound, subject.Digest) + } + return nil +} + +// Verify implements [artifact.Repository]. It authenticates stored evidence against the policy: +// every trust set must be satisfied by at least one evidence item bound to the subject. Unrelated +// or unsupported evidence is reported and ignored; it can never override matching evidence. +func (r *Repository) Verify(_ context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Verify")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Verify") + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, err := policy.Fingerprint(); err == nil { + result.PolicyFingerprint = fingerprint + } + var claims []artifact.SignerClaim + held := r.evidence[subject.Digest] + for index := range held { + claim, ok := r.authenticate(subject, &held[index], result) + if ok { + claims = append(claims, claim) + result.EvidenceDigests = append(result.EvidenceDigests, held[index].Evidence.Descriptor.Digest) + } + } + if !satisfies(policy, claims) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-permitted-signature", Message: "no valid permitted signature satisfies the trust set"}) + return result, fmt.Errorf("%w: no valid permitted signature for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + result.Claims = claims + result.Status = artifact.VerificationVerified + return result, nil +} + +// authenticate checks one evidence item; callers hold r.mu. +func (r *Repository) authenticate(subject artifact.SignatureSubject, held *artifact.EvidenceBlob, result *artifact.VerificationResult) (artifact.SignerClaim, bool) { + skip := func(code, message string) (artifact.SignerClaim, bool) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) + return artifact.SignerClaim{}, false + } + if !r.caps.HasEvidenceFormat(held.Evidence.Format) { + return skip("unsupported-evidence", "evidence in an unsupported format was ignored") + } + var parsed fakeEvidence + if err := json.Unmarshal(held.Data, &parsed); err != nil { + return skip("malformed-evidence", "malformed evidence was ignored") + } + if parsed.Subject != subject.Digest { + return skip("subject-mismatch", "evidence bound to a different subject was ignored") + } + return artifact.SignerClaim{Identity: parsed.Identity, Issuer: parsed.Issuer}, true +} + +// satisfies reports whether the claims meet every trust set; without sets any valid claim suffices. +func satisfies(policy artifact.VerificationPolicy, claims []artifact.SignerClaim) bool { + if len(claims) == 0 { + return false + } + for _, set := range policy.TrustSets { + if !setSatisfied(set, claims) { + return false + } + } + return true +} + +func setSatisfied(set artifact.TrustSet, claims []artifact.SignerClaim) bool { + for _, claim := range claims { + for _, trusted := range set.Identities { + if trusted.Identity == claim.Identity && trusted.Issuer == claim.Issuer { + return true + } + } + } + return false +} + +// ReadEvidence implements [artifact.EvidenceReader]. +func (r *Repository) ReadEvidence(_ context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "artifacttest.Repository.ReadEvidence")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("ReadEvidence") + return append([]artifact.EvidenceBlob(nil), r.evidence[pinned.RecordDigest]...), nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]. +func (r *Repository) WriteEvidence(_ context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "artifacttest.Repository.WriteEvidence")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("WriteEvidence") + if _, ok := r.records[pinned.RecordDigest]; !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], evidence[index]) + stored = append(stored, evidence[index].Evidence) + } + return stored, nil +} diff --git a/pkg/artifact/artifacttest/write.go b/pkg/artifact/artifacttest/write.go new file mode 100644 index 00000000000..2dc3cb07837 --- /dev/null +++ b/pkg/artifact/artifacttest/write.go @@ -0,0 +1,202 @@ +package artifacttest + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Publish implements [artifact.Repository] with the publication lifecycle of the contracts: the +// pending operation is recorded before payload changes, the immutable record is written, and the +// completed reference is published last. Injected failures leave the state a real interruption +// would. A reference that is pending for a different intent stays unavailable, with the owning +// operation ID in the error, instead of being stolen. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Publish")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Publish") + if !r.caps.Write { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "publish", "write", "the repository is read-only") + } + if err := r.trip(FailBeforePayload); err != nil { + return nil, err + } + record := prepared.Record + key := refKey(record.Name, record.Version) + recordDigest, err := record.RecordDigest() + if err != nil { + return nil, err + } + outcome, existing, err := r.admit(key, prepared, options, recordDigest) + if err != nil { + return nil, err + } + if outcome == artifact.OutcomeUnchanged { + return r.publishResult(&existing.record, existing, artifact.OutcomeUnchanged), nil + } + r.pendings[key] = &pending{operationID: prepared.OperationID, intent: prepared.IntentDigest} + blobs, err := r.readBlobs(ctx, prepared) + if err != nil { + return nil, err + } + s := &stored{record: record, recordDigest: recordDigest, packaging: prepared.Packaging, blobs: blobs, payloadAvailable: true} + if s.canonical, err = record.Canonical(); err != nil { + return nil, err + } + r.records[recordDigest] = s + if err := r.trip(FailAfterMetadata); err != nil { + return nil, err + } + r.commit(key, recordDigest) + if err := r.trip(FailAfterCommit); err != nil { + return nil, err + } + return r.publishResult(&record, s, outcome), nil +} + +// admit applies conditional-update, intent, and force rules and returns the outcome to produce. +// For an unchanged outcome it also returns the stored record that already satisfies the intent. +func (r *Repository) admit(key string, prepared *artifact.PreparedArtifact, options artifact.PublishOptions, recordDigest digest.Digest) (artifact.Outcome, *stored, error) { + record := &prepared.Record + if held, isPending := r.pendings[key]; isPending { + if held.intent != prepared.IntentDigest { + return "", nil, fmt.Errorf("%w: %s:%s is pending for operation %s with a different intent; recover it before publishing", errUtils.ErrArtifactPublicationUnavailable, record.Name, record.Version, held.operationID) + } + if _, exists := r.refs[key]; exists { + return artifact.OutcomeReplaced, nil, nil + } + return artifact.OutcomeCreated, nil, nil + } + current, exists := r.refs[key] + if err := checkExpectedPrior(key, current, exists, options.ExpectedPrior); err != nil { + return "", nil, err + } + if !exists { + return artifact.OutcomeCreated, nil, nil + } + existing := r.records[current] + if current == recordDigest { + return artifact.OutcomeUnchanged, existing, nil + } + if intent, err := existing.record.IntentDigest(); err == nil && intent == prepared.IntentDigest { + return artifact.OutcomeUnchanged, existing, nil + } + if !options.Force { + return "", nil, fmt.Errorf("%w: %s already exists with a different intent", errUtils.ErrArtifactConflict, key) + } + return artifact.OutcomeReplaced, nil, nil +} + +// checkExpectedPrior is the compare-and-swap guard. +func checkExpectedPrior(key string, current digest.Digest, exists bool, expected *artifact.PinnedReference) error { + switch { + case expected == nil && exists: + return fmt.Errorf("%w: %s now exists (%s) but the writer expected it to be absent", errUtils.ErrArtifactConflict, key, current) + case expected != nil && !exists: + return fmt.Errorf("%w: %s no longer exists but the writer expected %s", errUtils.ErrArtifactConflict, key, expected.ResolvedDigest) + case expected != nil && expected.ResolvedDigest != current: + return fmt.Errorf("%w: %s is %s but the writer expected %s", errUtils.ErrArtifactConflict, key, current, expected.ResolvedDigest) + } + return nil +} + +// readBlobs reads every staged blob, verifying its descriptor. +func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedArtifact) ([]blob, error) { + blobs := make([]blob, 0, len(prepared.Blobs)) + for index := range prepared.Blobs { + staged := &prepared.Blobs[index] + reader, err := staged.Open(ctx) + if err != nil { + return nil, err + } + if index == 0 { + if err := r.trip(FailDuringPayload); err != nil { + _ = reader.Close() + return nil, err + } + } + data, readErr := io.ReadAll(reader) + closeErr := reader.Close() + if err := errors.Join(readErr, closeErr); err != nil { + return nil, err + } + if int64(len(data)) != staged.Descriptor.Size || digest.FromBytes(data) != staged.Descriptor.Digest { + return nil, fmt.Errorf("%w: staged blob %q does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, staged.Path) + } + blobs = append(blobs, blob{descriptor: staged.Descriptor, role: staged.Role, documentRole: staged.DocumentRole, path: staged.Path, data: data}) + } + return blobs, nil +} + +// commit publishes the completed reference last and retires the replaced payload. +func (r *Repository) commit(key string, recordDigest digest.Digest) { + previous, hadPrevious := r.refs[key] + r.refs[key] = recordDigest + delete(r.pendings, key) + if hadPrevious && previous != recordDigest && !r.referenced(previous) { + r.records[previous].payloadAvailable = false + } +} + +// referenced reports whether any reference still points at the record digest. +func (r *Repository) referenced(recordDigest digest.Digest) bool { + for _, d := range r.refs { + if d == recordDigest { + return true + } + } + return false +} + +func (r *Repository) publishResult(record *artifact.Record, s *stored, outcome artifact.Outcome) *artifact.PublishResult { + ref := artifact.Ref{Name: record.Name, Version: record.Version} + return &artifact.PublishResult{ + Outcome: outcome, + Pinned: r.pinnedFor(ref, s), + Representation: r.representation(s), + Location: r.locator(record.Name, record.Version), + LockScope: artifact.LockScopeHost, + } +} + +// Delete implements [artifact.Repository]. The deletion is scoped to the pinned record: the +// references pointing at it and its exclusively owned payload. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Delete(_ context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + defer perf.Track(nil, "artifacttest.Repository.Delete")() + + r.mu.Lock() + defer r.mu.Unlock() + r.count("Delete") + if r.caps.Delete == artifact.DeleteNone || r.caps.Delete == "" { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "delete", "delete", "the repository cannot delete") + } + s, ok := r.records[pinned.RecordDigest] + if !ok { + return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) + } + if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != pinned.RecordDigest { + return nil, fmt.Errorf("%w: the delete target changed", errUtils.ErrArtifactConflict) + } + result := &artifact.DeleteResult{Digest: pinned.RecordDigest} + for key, d := range r.refs { + if d != pinned.RecordDigest { + continue + } + delete(r.refs, key) + result.Deleted = append(result.Deleted, r.locator(s.record.Name, s.record.Version)) + } + delete(r.records, pinned.RecordDigest) + delete(r.evidence, pinned.RecordDigest) + return result, nil +} diff --git a/pkg/artifact/evidence.go b/pkg/artifact/evidence.go new file mode 100644 index 00000000000..78e9400ec88 --- /dev/null +++ b/pkg/artifact/evidence.go @@ -0,0 +1,20 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_evidence.go github.com/cloudposse/atmos/pkg/artifact EvidenceReader,EvidenceWriter + +// EvidenceReader is an optional Repository extension that exports the signature evidence stored +// for a pinned artifact, so a copy through the shared staging fallback can carry it along. +type EvidenceReader interface { + // ReadEvidence returns the evidence blobs bound to the pinned artifact. The evidence is + // untrusted until verified. + ReadEvidence(ctx context.Context, pinned *PinnedReference) ([]EvidenceBlob, error) +} + +// EvidenceWriter is an optional Repository extension that imports signature evidence for an +// artifact the repository already holds. It never changes the signed Record or native graph root. +type EvidenceWriter interface { + // WriteEvidence stores evidence for the pinned artifact and returns its stored descriptors. + WriteEvidence(ctx context.Context, pinned *PinnedReference, evidence []EvidenceBlob, lock LockHandle) ([]SignatureEvidence, error) +} diff --git a/pkg/artifact/fixtures/fixtures.go b/pkg/artifact/fixtures/fixtures.go new file mode 100644 index 00000000000..e9a1f4b7794 --- /dev/null +++ b/pkg/artifact/fixtures/fixtures.go @@ -0,0 +1,153 @@ +// Package fixtures builds the artifact test fixtures described by the artifact validation plan +// ("File package" and "Invalid package") with Go code inside a caller-supplied temporary +// directory. It never shells out, so it behaves identically on every platform. +package fixtures + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Permission bits used by the fixtures. +const ( + dirPerm = 0o755 + tempPerm = 0o600 + execPerm = 0o755 + filePerm = 0o644 + privPerm = 0o640 +) + +// Well-known fixture content. +const ( + // RootReadme is the content of the root README.md, which stays ordinary payload. + RootReadme = "# Root readme\n" + + // PackageReadme is the content of the distribution README in the reserved directory. + PackageReadme = "# Package readme\n" + + // PackageChangelog is the content of the distribution CHANGELOG in the reserved directory. + PackageChangelog = "# Changelog\n\n- first\n" + + // HelperScript is the content of the executable helper. + HelperScript = "#!/bin/sh\necho helper\n" + + // UnicodeName is the Unicode filename in the file package. + UnicodeName = "unicode-é-日本.txt" + + // ReservedDir is the reserved distribution directory below the source root. + ReservedDir = ".atmos/artifact" +) + +// POSIXModes reports whether the platform honors Unix permission bits. Windows does not, so tests +// that assert executable bits or mode-only changes skip there. +func POSIXModes() bool { + defer perf.Track(nil, "fixtures.POSIXModes")() + + return runtime.GOOS != "windows" +} + +// RequirePOSIXModes skips the test on platforms that ignore Unix permission bits. +func RequirePOSIXModes(t testing.TB) { + defer perf.Track(nil, "fixtures.RequirePOSIXModes")() + + t.Helper() + if !POSIXModes() { + t.Skip("this platform does not honor Unix permission bits") + } +} + +// WriteFile writes content at the slash-separated rel path below dir and sets mode explicitly so +// the result does not depend on the process umask. +func WriteFile(t testing.TB, dir, rel string, content string, mode fs.FileMode) { + defer perf.Track(nil, "fixtures.WriteFile")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), dirPerm)) + require.NoError(t, os.WriteFile(path, []byte(content), tempPerm)) + require.NoError(t, os.Chmod(path, mode)) +} + +// MkdirAll creates the slash-separated directory below dir with mode 0755. +func MkdirAll(t testing.TB, dir, rel string) { + defer perf.Track(nil, "fixtures.MkdirAll")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(path, dirPerm)) + require.NoError(t, os.Chmod(path, dirPerm)) +} + +// Symlink creates a symlink at rel pointing at target and reports false (leaving nothing behind) +// when the platform or account cannot create symlinks. +func Symlink(t testing.TB, dir, rel, target string) bool { + defer perf.Track(nil, "fixtures.Symlink")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), dirPerm)) + if err := os.Symlink(filepath.FromSlash(target), path); err != nil { + t.Logf("symlinks are unavailable here: %v", err) + return false + } + return true +} + +// FilePackage describes the files written by [WriteFilePackage]. +type FilePackage struct { + // Dir is the source directory. + Dir string + + // Symlinks reports whether the confined symlink could be created. + Symlinks bool +} + +// WriteFilePackage writes the valid "File package" fixture into dir: nested files, an executable +// helper, an empty file, an empty directory, a confined symlink, a Unicode filename, a root README +// (ordinary payload), and a distinct package README and CHANGELOG in the reserved directory. +func WriteFilePackage(t testing.TB, dir string) FilePackage { + defer perf.Track(nil, "fixtures.WriteFilePackage")() + + t.Helper() + WriteFile(t, dir, "README.md", RootReadme, filePerm) + WriteFile(t, dir, "bin/helper.sh", HelperScript, execPerm) + WriteFile(t, dir, "nested/deep/file.txt", "deep content\n", filePerm) + WriteFile(t, dir, "empty.txt", "", filePerm) + MkdirAll(t, dir, "emptydir") + WriteFile(t, dir, UnicodeName, "unicode content\n", privPerm) + WriteFile(t, dir, ReservedDir+"/README.md", PackageReadme, filePerm) + WriteFile(t, dir, ReservedDir+"/CHANGELOG.md", PackageChangelog, filePerm) + symlinks := Symlink(t, dir, "link-to-file", "nested/deep/file.txt") + return FilePackage{Dir: dir, Symlinks: symlinks} +} + +// FilePackagePaths lists the inventory paths of the file package in sorted order, excluding the +// reserved distribution directory and, when symlinks are unavailable, the symlink. +func FilePackagePaths(symlinks bool) []string { + defer perf.Track(nil, "fixtures.FilePackagePaths")() + + paths := []string{ + "README.md", + "bin", + "bin/helper.sh", + "empty.txt", + "emptydir", + "nested", + "nested/deep", + "nested/deep/file.txt", + UnicodeName, + } + if symlinks { + paths = append(paths, "link-to-file") + } + sort.Strings(paths) + return paths +} diff --git a/pkg/artifact/fixtures/fixtures_other.go b/pkg/artifact/fixtures/fixtures_other.go new file mode 100644 index 00000000000..806e24665a7 --- /dev/null +++ b/pkg/artifact/fixtures/fixtures_other.go @@ -0,0 +1,11 @@ +//go:build !unix + +package fixtures + +import "testing" + +// SpecialFile reports false: special files cannot be created portably on this platform. +func SpecialFile(t testing.TB, _, _ string) bool { + t.Helper() + return false +} diff --git a/pkg/artifact/fixtures/fixtures_unix.go b/pkg/artifact/fixtures/fixtures_unix.go new file mode 100644 index 00000000000..f31e6af1dfd --- /dev/null +++ b/pkg/artifact/fixtures/fixtures_unix.go @@ -0,0 +1,27 @@ +//go:build unix + +package fixtures + +import ( + "path/filepath" + "syscall" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// pipePerm is the mode of the named pipe fixture. +const pipePerm = 0o600 + +// SpecialFile creates a named pipe (a special file) at rel below dir and reports whether it could. +func SpecialFile(t testing.TB, dir, rel string) bool { + defer perf.Track(nil, "fixtures.SpecialFile")() + + t.Helper() + path := filepath.Join(dir, filepath.FromSlash(rel)) + if err := syscall.Mkfifo(path, pipePerm); err != nil { + t.Logf("named pipes are unavailable here: %v", err) + return false + } + return true +} diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go new file mode 100644 index 00000000000..f141ac5c0ab --- /dev/null +++ b/pkg/artifact/materialize/archive.go @@ -0,0 +1,334 @@ +package materialize + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Names of the distribution members generated inside a packaged representation. +const ( + // The distribution directory. + managedDir = ".atmos/artifact" + + // The parent of the distribution directory. + managedParent = ".atmos" + + // The generated manifest. + managedManifest = "manifest.json" + + // The longest symlink target read from a zip member. + maxLinkTargetBytes = 4096 + + // The chunk size of context-aware copies. + copyChunkSize = 32 * 1024 +) + +// fillArchive extracts the single payload stream of a tar, tgz, or zip representation. +func fillArchive(ctx context.Context, t *tree, content *artifact.Content, scratch string) error { + var payload *artifact.ContentStream + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if payload != nil { + return fmt.Errorf("%w: an archive representation must have exactly one payload", errUtils.ErrArtifactIntegrityFailed) + } + payload = stream + } + if payload == nil || payload.Reader == nil { + return fmt.Errorf("%w: the archive payload is missing", errUtils.ErrArtifactIntegrityFailed) + } + checker, err := newManagedChecker(content.Record) + if err != nil { + return err + } + if content.Representation.Packaging == artifact.PackagingZip { + return fillZip(ctx, t, payload.Reader, scratch, checker) + } + return fillTar(ctx, t, payload.Reader, content.Representation.Packaging == artifact.PackagingTgz, checker) +} + +func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, checker *managedChecker) error { + if gzipped { + gz, err := gzip.NewReader(reader) + if err != nil { + return fmt.Errorf("%w: gzip: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + defer gz.Close() + reader = gz + } + tr := tar.NewReader(reader) + for { + if err := ctx.Err(); err != nil { + return err + } + header, err := tr.Next() + if errors.Is(err, io.EOF) { + // Drain so the verifying payload reader checks its full digest at end of stream. + _, err = io.Copy(io.Discard, reader) + return err + } + if err != nil { + return fmt.Errorf("%w: tar: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + if err := applyTarHeader(t, tr, header, checker); err != nil { + return err + } + } +} + +func applyTarHeader(t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { + name := strings.TrimSuffix(path.Clean(header.Name), "/") + switch header.Typeflag { + case tar.TypeDir: + return applyMember(t, checker, &archiveMember{name: name, kind: memberDir, size: header.Size}) + case tar.TypeSymlink: + return applyMember(t, checker, &archiveMember{name: name, kind: memberLink, target: header.Linkname}) + case tar.TypeReg: + return applyMember(t, checker, &archiveMember{name: name, kind: memberFile, size: header.Size, body: tr}) + default: + return fmt.Errorf("%w: archive member %q has unsupported type %q (hard links, devices, and special files are rejected)", errUtils.ErrArtifactPackagingInvalid, header.Name, header.Typeflag) + } +} + +func fillZip(ctx context.Context, t *tree, reader io.Reader, scratch string, checker *managedChecker) error { + spool, err := os.CreateTemp(scratch, "archive-*.zip") + if err != nil { + return fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer os.Remove(spool.Name()) + defer spool.Close() + size, err := copyContext(ctx, spool, reader) + if err != nil { + return fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + zr, err := zip.NewReader(spool, size) + if err != nil { + return fmt.Errorf("%w: zip: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + for _, file := range zr.File { + if err := ctx.Err(); err != nil { + return err + } + if err := applyZipFile(t, file, checker); err != nil { + return err + } + } + return nil +} + +func applyZipFile(t *tree, file *zip.File, checker *managedChecker) error { + name := strings.TrimSuffix(path.Clean(file.Name), "/") + mode := file.Mode() + switch { + case mode.IsDir(): + return applyMember(t, checker, &archiveMember{name: name, kind: memberDir}) + case mode&fs.ModeSymlink != 0: + target, err := readZipText(file) + if err != nil { + return err + } + return applyMember(t, checker, &archiveMember{name: name, kind: memberLink, target: target}) + case mode.IsRegular(): + rc, err := file.Open() + if err != nil { + return fmt.Errorf("%w: zip member %q: %w", errUtils.ErrArtifactIntegrityFailed, file.Name, err) + } + defer rc.Close() + return applyMember(t, checker, &archiveMember{name: name, kind: memberFile, size: file.FileInfo().Size(), body: rc}) + default: + return fmt.Errorf("%w: archive member %q has an unsupported type", errUtils.ErrArtifactPackagingInvalid, file.Name) + } +} + +func readZipText(file *zip.File) (string, error) { + rc, err := file.Open() + if err != nil { + return "", fmt.Errorf("%w: zip member %q: %w", errUtils.ErrArtifactIntegrityFailed, file.Name, err) + } + defer rc.Close() + data, err := artifact.ReadBounded(rc, maxLinkTargetBytes) + if err != nil { + return "", err + } + return string(data), nil +} + +type memberKind int + +const ( + memberFile memberKind = iota + memberDir + memberLink +) + +// archiveMember is one decoded archive member. +type archiveMember struct { + name string + kind memberKind + target string + size int64 + body io.Reader +} + +// applyMember validates one archive member against the inventory (or the managed directory) and +// writes it through the tree. +func applyMember(t *tree, checker *managedChecker, m *archiveMember) error { + if err := validateMemberName(m.name); err != nil { + return err + } + if checker.owns(m.name, t) { + return checker.check(m) + } + entry, err := t.entryFor(m.name) + if err != nil { + return err + } + if !m.matches(entry) { + return fmt.Errorf("%w: archive member %q does not match its inventory entry (%s)", errUtils.ErrArtifactIntegrityFailed, m.name, entry.Type) + } + switch entry.Type { + case artifact.EntryDirectory: + return t.mkdir(entry) + case artifact.EntrySymlink: + return t.symlink(entry) + default: + return t.writeFile(entry, m.body) + } +} + +// matches reports whether the member has the kind, link target, and size its inventory entry declares. +func (m *archiveMember) matches(entry *artifact.Entry) bool { + switch entry.Type { + case artifact.EntryDirectory: + return m.kind == memberDir + case artifact.EntrySymlink: + return m.kind == memberLink && m.target == entry.Target + case artifact.EntryFile: + return m.kind == memberFile && m.size == entry.Size + default: + return false + } +} + +// validateMemberName rejects traversal, absolute, backslash, and empty names. Managed names pass +// through the same check, so even the reserved directory cannot be used to escape. +func validateMemberName(name string) error { + if name == "." || name == "" || name == ".." || strings.HasPrefix(name, "../") || filepath.IsAbs(name) { + return fmt.Errorf("%w: archive member %q escapes the destination", errUtils.ErrArtifactUnsafePath, name) + } + return artifact.ValidatePayloadPath(name) +} + +// managedChecker verifies the generated distribution members of a packaged representation +// (manifest.json plus embedded documents) without ever writing them to the user's tree. +type managedChecker struct { + manifest digest.Digest + documents map[string]artifact.Document + seen map[string]bool +} + +func newManagedChecker(record *artifact.Record) (*managedChecker, error) { + canonical, err := record.Canonical() + if err != nil { + return nil, err + } + checker := &managedChecker{manifest: digest.FromBytes(canonical), documents: map[string]artifact.Document{}, seen: map[string]bool{}} + for _, doc := range record.Documents { + checker.documents[managedDir+"/"+strings.ToUpper(string(doc.Role))+".md"] = doc + } + return checker, nil +} + +// owns reports whether name belongs to the managed directory and the inventory does not claim it. +func (m *managedChecker) owns(name string, t *tree) bool { + if _, claimed := t.inventory[name]; claimed { + return false + } + return name == managedParent || name == managedDir || strings.HasPrefix(name, managedDir+"/") +} + +// check verifies one managed member without ever writing it into the user's tree. +func (m *managedChecker) check(member *archiveMember) error { + name := member.name + if name == managedParent || name == managedDir { + if member.kind != memberDir { + return fmt.Errorf("%w: %q must be a directory", errUtils.ErrArtifactPackagingInvalid, name) + } + return nil + } + want, ok := m.expected(name) + if member.kind != memberFile || m.seen[name] || !ok { + return fmt.Errorf("%w: unexpected managed member %q", errUtils.ErrArtifactPackagingInvalid, name) + } + m.seen[name] = true + return verifyManagedFile(member, want) +} + +// verifyManagedFile reads a managed file within its size bound and compares it to the Record. +func verifyManagedFile(member *archiveMember, want digest.Digest) error { + limit := artifact.MaxDocumentSize + if member.name == managedDir+"/"+managedManifest { + limit = artifact.MaxRecordSize + } + if member.size > limit { + return fmt.Errorf("%w: managed member %q is %d bytes", errUtils.ErrArtifactTooLarge, member.name, member.size) + } + data, err := artifact.ReadBounded(member.body, limit) + if err != nil { + return err + } + if digest.FromBytes(data) != want { + return fmt.Errorf("%w: embedded %q does not match the Record", errUtils.ErrArtifactIntegrityFailed, member.name) + } + return nil +} + +func (m *managedChecker) expected(name string) (digest.Digest, bool) { + if name == managedDir+"/"+managedManifest { + return m.manifest, true + } + doc, ok := m.documents[name] + return doc.Digest, ok +} + +// copyContext copies src to dst in chunks and stops as soon as ctx is canceled. +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, copyChunkSize) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + n, readErr := src.Read(buffer) + if n > 0 { + written, writeErr := dst.Write(buffer[:n]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} diff --git a/pkg/artifact/materialize/doc.go b/pkg/artifact/materialize/doc.go new file mode 100644 index 00000000000..e59c22b41d2 --- /dev/null +++ b/pkg/artifact/materialize/doc.go @@ -0,0 +1,10 @@ +// Package materialize extracts verified artifact content into a local directory. +// +// Extraction always happens in a private staging directory next to the destination, written through +// an os.Root so no entry can land outside it. Every path is validated against the Record +// inventory: traversal, entries absent from the inventory, escaping or chained symlinks, special +// members, size or digest mismatches, and oversize content are explicit errors. A newly pulled tree +// is never merged over old files: replacing an existing destination deletes the old children +// inside the destination root only and then moves the staged tree in. Ordinary permission bits +// (including executable bits) are restored exactly; owner, group, and ACLs never are. +package materialize diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go new file mode 100644 index 00000000000..d5ba0eaeaf1 --- /dev/null +++ b/pkg/artifact/materialize/extract.go @@ -0,0 +1,151 @@ +package materialize + +import ( + "context" + "fmt" + "os" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultMaxBytes bounds the total declared size of extracted file content. +const DefaultMaxBytes int64 = 64 << 30 + +// Option customizes extraction. +type Option func(*options) + +type options struct { + maxBytes int64 +} + +// WithMaxBytes overrides the limit on total extracted file content. +func WithMaxBytes(limit int64) Option { + defer perf.Track(nil, "materialize.WithMaxBytes")() + + return func(o *options) { o.maxBytes = limit } +} + +func applyOptions(opts []Option) options { + applied := options{maxBytes: DefaultMaxBytes} + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.maxBytes <= 0 { + applied.maxBytes = DefaultMaxBytes + } + return applied +} + +// Extract writes content into the empty directory stage, verifying every member against the +// Record inventory, checking that no symlink resolves outside stage, and applying final modes +// last. The scratch directory is private and holds temporary spool files. Extract does not close +// content. +func Extract(ctx context.Context, content *artifact.Content, stage, scratch string, opts ...Option) error { + defer perf.Track(nil, "materialize.Extract")() + + applied := applyOptions(opts) + if content == nil || content.Record == nil { + return artifact.NewUnsupportedCapability("", "pull", "materialize/graph", "a native OCI graph without a portable Record cannot be materialized as a file tree") + } + record := content.Record + if err := record.Validate(); err != nil { + return err + } + if err := checkTotalSize(record, applied.maxBytes); err != nil { + return err + } + root, err := os.OpenRoot(stage) + if err != nil { + return fmt.Errorf("%w: open staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer root.Close() + + t := newTree(root, record) + if err := fill(ctx, t, content, scratch); err != nil { + return err + } + if err := checkLinkContainment(stage, record); err != nil { + return err + } + return t.complete() +} + +func checkTotalSize(record *artifact.Record, limit int64) error { + var total int64 + for index := range record.Inventory { + total += record.Inventory[index].Size + if total > limit || total < 0 { + return fmt.Errorf("%w: content exceeds the %d byte extraction limit", errUtils.ErrArtifactTooLarge, limit) + } + } + return nil +} + +// fill dispatches on the packaging mode. +func fill(ctx context.Context, t *tree, content *artifact.Content, scratch string) error { + switch content.Representation.Packaging { + case artifact.PackagingNone: + return fillFiles(ctx, t, content) + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return fillArchive(ctx, t, content, scratch) + default: + return fmt.Errorf("%w: cannot materialize packaging %q", errUtils.ErrArtifactPackagingInvalid, content.Representation.Packaging) + } +} + +// fillFiles writes a per-file payload: file entries come from their path-addressed streams; +// directories and symlinks come from the inventory. +func fillFiles(ctx context.Context, t *tree, content *artifact.Content) error { + streams := make(map[string]*artifact.ContentStream, len(content.Streams)) + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if _, duplicate := streams[stream.Path]; duplicate { + return fmt.Errorf("%w: payload %q appears more than once", errUtils.ErrArtifactIntegrityFailed, stream.Path) + } + streams[stream.Path] = stream + } + paths := make([]string, 0, len(t.inventory)) + for entryPath := range t.inventory { + paths = append(paths, entryPath) + } + sort.Strings(paths) + for _, entryPath := range paths { + if err := ctx.Err(); err != nil { + return err + } + if err := fillInventoryEntry(t, t.inventory[entryPath], streams); err != nil { + return err + } + delete(streams, entryPath) + } + for extra := range streams { + return fmt.Errorf("%w: payload %q is not in the artifact inventory", errUtils.ErrArtifactIntegrityFailed, extra) + } + return nil +} + +func fillInventoryEntry(t *tree, entry *artifact.Entry, streams map[string]*artifact.ContentStream) error { + if _, err := t.entryFor(entry.Path); err != nil { + return err + } + switch entry.Type { + case artifact.EntryDirectory: + return t.mkdir(entry) + case artifact.EntrySymlink: + return t.symlink(entry) + default: + stream, ok := streams[entry.Path] + if !ok || stream.Reader == nil { + return fmt.Errorf("%w: payload for %q is missing", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + return t.writeFile(entry, stream.Reader) + } +} diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go new file mode 100644 index 00000000000..b61ddcc80d0 --- /dev/null +++ b/pkg/artifact/materialize/links.go @@ -0,0 +1,44 @@ +package materialize + +import ( + "fmt" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// checkLinkContainment resolves every symlink physically and rejects any that leaves root. A +// lexical check of each target is not enough, because a link chain can clean to a contained path +// yet resolve outside it. Dangling links stay allowed. +func checkLinkContainment(stage string, record *artifact.Record) error { + realRoot, err := filepath.EvalSymlinks(stage) + if err != nil { + return fmt.Errorf("%w: resolve staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + for index := range record.Inventory { + entry := &record.Inventory[index] + if entry.Type != artifact.EntrySymlink { + continue + } + resolved, reachable := resolveLink(filepath.Join(stage, filepath.FromSlash(entry.Path))) + if reachable && !within(realRoot, resolved) { + return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) + } + } + return nil +} + +// resolveLink fully resolves a symlink and reports false for a dangling or looping link, which +// points at nothing reachable. +func resolveLink(linkPath string) (string, bool) { + resolved, err := filepath.EvalSymlinks(linkPath) + return resolved, err == nil +} + +// within reports whether target is root or lies beneath it. +func within(root, target string) bool { + relative, err := filepath.Rel(root, target) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go new file mode 100644 index 00000000000..7c1fb5be486 --- /dev/null +++ b/pkg/artifact/materialize/materialize_test.go @@ -0,0 +1,396 @@ +package materialize_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +var fixedTime = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +func prepared(t *testing.T, src string, packaging artifact.Packaging) *artifact.PreparedArtifact { + t.Helper() + p, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: "pkg/example", Version: "1.0.0", Packaging: packaging, OperationID: "op", PreparedAt: fixedTime, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = p.Close() }) + return p +} + +// contentOf converts staged blobs into the streams a repository would return. +func contentOf(t *testing.T, p *artifact.PreparedArtifact) *artifact.Content { + t.Helper() + record := p.Record + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: p.Packaging}} + for index := range p.Blobs { + blob := &p.Blobs[index] + reader, err := blob.Open(context.Background()) + require.NoError(t, err) + content.Streams = append(content.Streams, artifact.ContentStream{Descriptor: blob.Descriptor, Role: blob.Role, Path: blob.Path, Reader: reader}) + } + t.Cleanup(func() { _ = content.Close() }) + return content +} + +func dest(t *testing.T) string { + return filepath.Join(t.TempDir(), "out", "pulled") +} + +func assertNoScratch(t *testing.T, destination string) { + t.Helper() + items, err := os.ReadDir(filepath.Dir(destination)) + require.NoError(t, err) + for _, item := range items { + assert.NotContains(t, item.Name(), ".atmos-pull-", "staging must be removed") + } +} + +func TestMaterialize_RoundTripEveryPackaging(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + p := prepared(t, src, packaging) + out := dest(t) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + assertNoScratch(t, out) + + state, err := materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + + data, err := os.ReadFile(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, fixtures.HelperScript, string(data)) + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "executable bit survives") + } + _, err = os.Stat(filepath.Join(out, "emptydir")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(out, fixtures.UnicodeName)) + require.NoError(t, err) + _, err = os.Lstat(filepath.Join(out, ".atmos")) + assert.True(t, os.IsNotExist(err), "managed distribution members are not materialized") + if pkg.Symlinks { + target, err := os.Readlink(filepath.Join(out, "link-to-file")) + require.NoError(t, err) + assert.Equal(t, "nested/deep/file.txt", filepath.ToSlash(target)) + } + }) + } +} + +func TestMaterialize_ReplacesWithoutMergingAndStaysInsideDestination(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + + root := t.TempDir() + out := filepath.Join(root, "pulled") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(outside, 0o755)) + fixtures.WriteFile(t, outside, "keep.txt", "keep", 0o644) + fixtures.WriteFile(t, root, "sibling.txt", "sibling", 0o644) + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + fixtures.WriteFile(t, out, "nested/stale2.txt", "stale", 0o644) + linkOK := fixtures.Symlink(t, out, "link-out", "../outside") + + state, err := materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state) + + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + items, err := os.ReadDir(out) + require.NoError(t, err) + require.Len(t, items, 1, "no stale file is merged with the new tree") + assert.Equal(t, "new.txt", items[0].Name()) + assertNoScratch(t, out) + + keep, err := os.ReadFile(filepath.Join(outside, "keep.txt")) + require.NoError(t, err, "a stale link must not let deletion escape the destination") + assert.Equal(t, "keep", string(keep)) + _, err = os.Stat(filepath.Join(root, "sibling.txt")) + require.NoError(t, err) + _ = linkOK +} + +func TestInspectDestination_States(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + root := t.TempDir() + + state, err := materialize.InspectDestination(filepath.Join(root, "absent"), &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationAbsent, state) + + empty := filepath.Join(root, "empty") + require.NoError(t, os.Mkdir(empty, 0o755)) + state, err = materialize.InspectDestination(empty, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationEmpty, state) + + differs := filepath.Join(root, "differs") + fixtures.WriteFile(t, differs, "a.txt", "different", 0o644) + state, err = materialize.InspectDestination(differs, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state) + + extra := filepath.Join(root, "extra") + fixtures.WriteFile(t, extra, "a.txt", "a", 0o644) + fixtures.WriteFile(t, extra, "b.txt", "b", 0o644) + state, err = materialize.InspectDestination(extra, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "extra files are a difference") + + file := filepath.Join(root, "file") + fixtures.WriteFile(t, root, "file", "x", 0o644) + _, err = materialize.InspectDestination(file, &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + + if fixtures.Symlink(t, root, "linked", "empty") { + _, err = materialize.InspectDestination(filepath.Join(root, "linked"), &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + err = materialize.Materialize(context.Background(), contentOf(t, p), filepath.Join(root, "linked")) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + } +} + +func TestMaterialize_PerFileIntegrityFailures(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bbbb", 0o644) + tests := []struct { + name string + mutate func(content *artifact.Content) + want error + }{ + {"tampered bytes", func(c *artifact.Content) { + c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("xxxx"))) + }, errUtils.ErrArtifactIntegrityFailed}, + {"truncated", func(c *artifact.Content) { + c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("aa"))) + }, errUtils.ErrArtifactIntegrityFailed}, + {"oversize", func(c *artifact.Content) { + c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("aaaaaaaa"))) + }, errUtils.ErrArtifactTooLarge}, + {"missing payload", func(c *artifact.Content) { c.Streams = c.Streams[1:] }, errUtils.ErrArtifactIntegrityFailed}, + {"extra payload", func(c *artifact.Content) { + c.Streams = append(c.Streams, artifact.ContentStream{Role: artifact.BlobPayload, Path: "extra.txt", Reader: io.NopCloser(bytes.NewReader(nil))}) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate payload", func(c *artifact.Content) { + c.Streams = append(c.Streams, c.Streams[0]) + }, errUtils.ErrArtifactIntegrityFailed}, + {"traversal path", func(c *artifact.Content) { + c.Streams[0].Path = "../a.txt" + }, errUtils.ErrArtifactIntegrityFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + tt.mutate(content) + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr), "a failed extraction never creates the destination") + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_GraphWithoutRecordIsUnsupported(t *testing.T) { + err := materialize.Materialize(context.Background(), &artifact.Content{}, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + err = materialize.Materialize(context.Background(), nil, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + require.ErrorIs(t, materialize.Materialize(context.Background(), &artifact.Content{}, ""), errUtils.ErrArtifactConfigInvalid) +} + +func TestMaterialize_SizeLimit(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingTgz)) + err := materialize.Materialize(context.Background(), content, dest(t), materialize.WithMaxBytes(5)) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestMaterialize_CanceledContextCleansUp(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz, artifact.PackagingZip} { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := dest(t) + err := materialize.Materialize(ctx, contentOf(t, prepared(t, src, packaging)), out) + require.ErrorIs(t, err, context.Canceled, packaging) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + } +} + +func TestMaterialize_RestrictiveDirectoryModeAppliedLast(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not enforced on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "locked/inner.txt", "x", 0o644) + require.NoError(t, os.Chmod(filepath.Join(src, "locked"), 0o555)) + t.Cleanup(func() { _ = os.Chmod(filepath.Join(src, "locked"), 0o755) }) + p := prepared(t, src, artifact.PackagingTar) + out := dest(t) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + info, err := os.Stat(filepath.Join(out, "locked")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o555), info.Mode().Perm()) + require.NoError(t, os.Chmod(filepath.Join(out, "locked"), 0o755)) +} + +// sealedRecord builds a valid Record around an inventory for crafted-content tests. +func sealedRecord(t *testing.T, inventory []artifact.Entry) artifact.Record { + t.Helper() + record := artifact.Record{ + Name: "pkg/crafted", Version: "1", Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(fixedTime), Inventory: inventory, + } + require.NoError(t, record.Seal()) + return record +} + +func fileEntry(path, content string, mode uint32) artifact.Entry { + return artifact.Entry{Path: path, Type: artifact.EntryFile, Size: int64(len(content)), Mode: mode, Digest: digest.FromString(content)} +} + +type tarMember struct { + header tar.Header + body string +} + +func tarOf(t *testing.T, members []tarMember) io.ReadCloser { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for index := range members { + m := &members[index] + h := m.header + h.Size = int64(len(m.body)) + require.NoError(t, tw.WriteHeader(&h)) + _, err := tw.Write([]byte(m.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + return io.NopCloser(&buf) +} + +func tgzOf(t *testing.T, members []tarMember) io.ReadCloser { + t.Helper() + raw, err := io.ReadAll(tarOf(t, members)) + require.NoError(t, err) + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err = gz.Write(raw) + require.NoError(t, err) + require.NoError(t, gz.Close()) + return io.NopCloser(&buf) +} + +func archiveContent(record *artifact.Record, packaging artifact.Packaging, reader io.ReadCloser) *artifact.Content { + return &artifact.Content{ + Record: record, + Representation: artifact.Representation{Packaging: packaging}, + Streams: []artifact.ContentStream{{Role: artifact.BlobPayload, Reader: reader}}, + } +} + +func TestMaterialize_HostileArchives(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + good := tarMember{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "good"} + tests := []struct { + name string + members []tarMember + want error + }{ + {"traversal", []tarMember{{header: tar.Header{Name: "../evil.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"nested traversal", []tarMember{{header: tar.Header{Name: "a/../../evil.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"absolute", []tarMember{{header: tar.Header{Name: "/etc/passwd", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}, good}, errUtils.ErrArtifactUnsafePath}, + {"not in inventory", []tarMember{good, {header: tar.Header{Name: "extra.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "x"}}, errUtils.ErrArtifactIntegrityFailed}, + {"hard link", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeLink, Linkname: "other"}}}, errUtils.ErrArtifactPackagingInvalid}, + {"device", []tarMember{{header: tar.Header{Name: "dev", Typeflag: tar.TypeChar}}}, errUtils.ErrArtifactPackagingInvalid}, + {"wrong size", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "goodgood"}}, errUtils.ErrArtifactIntegrityFailed}, + {"wrong digest", []tarMember{{header: tar.Header{Name: "good.txt", Typeflag: tar.TypeReg, Mode: 0o644}, body: "evil"}}, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate", []tarMember{good, good}, errUtils.ErrArtifactIntegrityFailed}, + {"missing member", nil, errUtils.ErrArtifactIntegrityFailed}, + {"unexpected managed member", []tarMember{good, {header: tar.Header{Name: ".atmos/artifact/evil.sh", Typeflag: tar.TypeReg, Mode: 0o755}, body: "x"}}, errUtils.ErrArtifactPackagingInvalid}, + {"tampered manifest", []tarMember{good, {header: tar.Header{Name: ".atmos/artifact/manifest.json", Typeflag: tar.TypeReg, Mode: 0o644}, body: "{}"}}, errUtils.ErrArtifactIntegrityFailed}, + } + for _, tt := range tests { + for _, wrap := range []struct { + name string + packaging artifact.Packaging + build func(*testing.T, []tarMember) io.ReadCloser + }{{"tar", artifact.PackagingTar, tarOf}, {"tgz", artifact.PackagingTgz, tgzOf}} { + t.Run(tt.name+"/"+wrap.name, func(t *testing.T) { + out := dest(t) + err := materialize.Materialize(context.Background(), archiveContent(&record, wrap.packaging, wrap.build(t, tt.members)), out) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + }) + } + } +} + +func TestMaterialize_ArchiveRejectsCorruptStreams(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + for _, packaging := range []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingZip, artifact.PackagingTar} { + err := materialize.Materialize(context.Background(), archiveContent(&record, packaging, io.NopCloser(bytes.NewReader([]byte("not an archive")))), dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, packaging) + } + noPayload := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingTar}} + require.ErrorIs(t, materialize.Materialize(context.Background(), noPayload, dest(t)), errUtils.ErrArtifactIntegrityFailed) + badPackaging := archiveContent(&record, "rar", io.NopCloser(bytes.NewReader(nil))) + require.ErrorIs(t, materialize.Materialize(context.Background(), badPackaging, dest(t)), errUtils.ErrArtifactPackagingInvalid) +} + +func TestMaterialize_ChainedSymlinkEscapeIsRejected(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/b/../.."}, + }) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + if err == nil { + t.Skip("this platform cannot create the symlinks needed for the chained escape") + } + if !assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) { + t.Logf("unexpected error: %v", err) + } + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go new file mode 100644 index 00000000000..9f5e5e10eb8 --- /dev/null +++ b/pkg/artifact/materialize/replace.go @@ -0,0 +1,240 @@ +package materialize + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + scratchPattern = ".atmos-pull-*" + parentDirMode = 0o755 + treeDirName = "tree" +) + +// errMismatch stops a destination walk at the first difference. It never leaves this package. +var errMismatch = errors.New("destination differs from the artifact inventory") + +// DestinationState classifies an output directory relative to the artifact being pulled. +type DestinationState int + +// Destination states. +const ( + // DestinationAbsent means the destination does not exist. + DestinationAbsent DestinationState = iota + + // DestinationEmpty means the destination is an existing empty directory. + DestinationEmpty + + // DestinationMatches means the destination already holds exactly the artifact's inventory. + DestinationMatches + + // DestinationDiffers means the destination holds other content; replacing it requires force. + DestinationDiffers +) + +// InspectDestination classifies dest without modifying it. A symlink or non-directory +// destination is an error: a replacement must never follow a link out of the requested root. +func InspectDestination(dest string, record *artifact.Record) (DestinationState, error) { + defer perf.Track(nil, "materialize.InspectDestination")() + + info, err := os.Lstat(dest) + switch { + case os.IsNotExist(err): + return DestinationAbsent, nil + case err != nil: + return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + case info.Mode()&os.ModeSymlink != 0: + return DestinationDiffers, fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) + case !info.IsDir(): + return DestinationDiffers, fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) + } + children, err := os.ReadDir(dest) + if err != nil { + return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + if len(children) == 0 { + return DestinationEmpty, nil + } + if treeMatches(dest, record) { + return DestinationMatches, nil + } + return DestinationDiffers, nil +} + +// treeMatches reports whether dir holds exactly the Record inventory: the same paths, types, link +// targets, file sizes and digests, and (off Windows) permission bits. Any unreadable entry counts +// as a difference. +func treeMatches(dir string, record *artifact.Record) bool { + want := make(map[string]artifact.Entry, len(record.Inventory)) + for _, entry := range record.Inventory { + want[entry.Path] = entry + } + matched := 0 + walkErr := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == dir { + return nil + } + relative, err := filepath.Rel(dir, p) + if err != nil { + return err + } + entry, ok := want[filepath.ToSlash(relative)] + if !ok || !entryMatches(p, d, &entry) { + return errMismatch + } + matched++ + return nil + }) + return walkErr == nil && matched == len(want) +} + +func entryMatches(p string, d fs.DirEntry, entry *artifact.Entry) bool { + info, err := d.Info() + if err != nil { + return false + } + switch entry.Type { + case artifact.EntryDirectory: + return info.IsDir() && modeMatches(info, entry) + case artifact.EntrySymlink: + return linkMatches(p, info, entry) + default: + return fileMatches(p, info, entry) + } +} + +func linkMatches(p string, info os.FileInfo, entry *artifact.Entry) bool { + target, err := os.Readlink(p) + return err == nil && info.Mode()&os.ModeSymlink != 0 && filepath.ToSlash(target) == entry.Target +} + +func fileMatches(p string, info os.FileInfo, entry *artifact.Entry) bool { + return info.Mode().IsRegular() && info.Size() == entry.Size && modeMatches(info, entry) && fileDigest(p) == entry.Digest +} + +func modeMatches(info os.FileInfo, entry *artifact.Entry) bool { + if runtime.GOOS == "windows" { + return true + } + return uint32(info.Mode().Perm()) == entry.Mode +} + +func fileDigest(p string) digest.Digest { + file, err := os.Open(p) + if err != nil { + return "" + } + defer file.Close() + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "" + } + return digest.NewDigest(digest.SHA256, hasher) +} + +// Materialize extracts content into a private staging directory beside dest and then replaces dest +// with it. Staging is removed on every path. An existing destination keeps no stale file: its +// children are removed inside the destination root only, then the staged children move in. The +// caller decides beforehand (see [InspectDestination]) whether replacing is permitted. +func Materialize(ctx context.Context, content *artifact.Content, dest string, opts ...Option) error { + defer perf.Track(nil, "materialize.Materialize")() + + if dest == "" { + return fmt.Errorf("%w: the destination is empty", errUtils.ErrArtifactConfigInvalid) + } + parent := filepath.Dir(dest) + if err := os.MkdirAll(parent, parentDirMode); err != nil { + return fmt.Errorf("%w: create %q: %w", errUtils.ErrArtifactPackagingInvalid, parent, err) + } + scratch, err := os.MkdirTemp(parent, scratchPattern) + if err != nil { + return fmt.Errorf("%w: create staging beside %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + defer os.RemoveAll(scratch) + + stage := filepath.Join(scratch, treeDirName) + if err := os.Mkdir(stage, stagingDirMode); err != nil { + return fmt.Errorf("%w: create staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + if err := Extract(ctx, content, stage, scratch, opts...); err != nil { + return err + } + if err := os.Chmod(stage, parentDirMode); err != nil { + return fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return replaceDestination(stage, dest) +} + +// replaceDestination moves the staged tree into place without merging it over old files. +func replaceDestination(stage, dest string) error { + info, err := os.Lstat(dest) + switch { + case os.IsNotExist(err): + return rename(stage, dest) + case err != nil: + return fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + case info.Mode()&os.ModeSymlink != 0: + return fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) + case !info.IsDir(): + return fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) + } + if err := clearDirectory(dest); err != nil { + return err + } + return moveChildren(stage, dest) +} + +// clearDirectory removes every child of dir through an os.Root, so removal can never leave dir. +func clearDirectory(dir string) error { + root, err := os.OpenRoot(dir) + if err != nil { + return fmt.Errorf("%w: open destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dir, err) + } + defer root.Close() + children, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("%w: read destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dir, err) + } + for _, child := range children { + if err := root.RemoveAll(child.Name()); err != nil { + return fmt.Errorf("%w: remove stale %q: %w", errUtils.ErrArtifactPackagingInvalid, child.Name(), err) + } + } + return nil +} + +func moveChildren(stage, dest string) error { + children, err := os.ReadDir(stage) + if err != nil { + return fmt.Errorf("%w: read staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + for _, child := range children { + if err := rename(filepath.Join(stage, child.Name()), filepath.Join(dest, child.Name())); err != nil { + return err + } + } + return nil +} + +func rename(from, to string) error { + if err := os.Rename(from, to); err != nil { + return fmt.Errorf("%w: move %q into place: %w", errUtils.ErrArtifactPackagingInvalid, to, err) + } + return nil +} diff --git a/pkg/artifact/materialize/tree.go b/pkg/artifact/materialize/tree.go new file mode 100644 index 00000000000..b04b3df8cbe --- /dev/null +++ b/pkg/artifact/materialize/tree.go @@ -0,0 +1,121 @@ +package materialize + +import ( + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +const ( + stagingDirMode fs.FileMode = 0o700 + stagingFileMode fs.FileMode = 0o600 +) + +// tree writes verified entries below a root and remembers directory modes to apply last. +type tree struct { + root *os.Root + inventory map[string]*artifact.Entry + written map[string]bool +} + +func newTree(root *os.Root, record *artifact.Record) *tree { + inventory := make(map[string]*artifact.Entry, len(record.Inventory)) + for index := range record.Inventory { + inventory[record.Inventory[index].Path] = &record.Inventory[index] + } + return &tree{root: root, inventory: inventory, written: make(map[string]bool, len(inventory))} +} + +// entryFor returns the inventory entry a member must match, rejecting unsafe or unknown paths. +func (t *tree) entryFor(name string) (*artifact.Entry, error) { + if err := artifact.ValidatePayloadPath(name); err != nil { + return nil, err + } + entry, ok := t.inventory[name] + if !ok { + return nil, fmt.Errorf("%w: member %q is not in the artifact inventory", errUtils.ErrArtifactIntegrityFailed, name) + } + if t.written[name] { + return nil, fmt.Errorf("%w: member %q appears more than once", errUtils.ErrArtifactIntegrityFailed, name) + } + t.written[name] = true + return entry, nil +} + +func (t *tree) mkdir(entry *artifact.Entry) error { + if err := t.root.MkdirAll(filepath.FromSlash(entry.Path), stagingDirMode); err != nil { + return fmt.Errorf("%w: create directory %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + return nil +} + +// writeFile writes a regular file, enforcing the declared size and digest. +func (t *tree) writeFile(entry *artifact.Entry, src io.Reader) error { + if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { + return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + file, err := t.root.OpenFile(filepath.FromSlash(entry.Path), os.O_WRONLY|os.O_CREATE|os.O_EXCL, stagingFileMode) + if err != nil { + return fmt.Errorf("%w: create file %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + hasher := sha256.New() + written, copyErr := io.Copy(io.MultiWriter(file, hasher), io.LimitReader(src, entry.Size+1)) + if closeErr := file.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("%w: write %q: %w", errUtils.ErrArtifactIntegrityFailed, entry.Path, copyErr) + } + if written > entry.Size { + return fmt.Errorf("%w: %q is larger than its declared %d bytes", errUtils.ErrArtifactTooLarge, entry.Path, entry.Size) + } + if written != entry.Size || digest.NewDigest(digest.SHA256, hasher) != entry.Digest { + return fmt.Errorf("%w: %q does not match its inventory size and digest", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + return nil +} + +func (t *tree) symlink(entry *artifact.Entry) error { + if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { + return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + if err := t.root.Symlink(filepath.FromSlash(entry.Target), filepath.FromSlash(entry.Path)); err != nil { + return fmt.Errorf("%w: this platform cannot create symlink %q -> %q (it is not silently replaced): %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, entry.Target, err) + } + return nil +} + +// complete checks that every inventory entry was produced and applies final modes: files first, +// then directories deepest-first so a restrictive directory mode never blocks earlier chmods. +func (t *tree) complete() error { + paths := make([]string, 0, len(t.inventory)) + for entryPath := range t.inventory { + if !t.written[entryPath] { + return fmt.Errorf("%w: inventory entry %q is missing from the content", errUtils.ErrArtifactIntegrityFailed, entryPath) + } + paths = append(paths, entryPath) + } + sort.Slice(paths, func(i, j int) bool { + return len(paths[i]) > len(paths[j]) || (len(paths[i]) == len(paths[j]) && paths[i] > paths[j]) + }) + for _, entryPath := range paths { + entry := t.inventory[entryPath] + if entry.Type == artifact.EntrySymlink { + continue + } + if err := t.root.Chmod(filepath.FromSlash(entryPath), fs.FileMode(entry.Mode)); err != nil { + return fmt.Errorf("%w: set mode %#o on %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Mode, entryPath, err) + } + } + return nil +} diff --git a/pkg/artifact/mock_evidence.go b/pkg/artifact/mock_evidence.go new file mode 100644 index 00000000000..18e95df58cd --- /dev/null +++ b/pkg/artifact/mock_evidence.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: EvidenceReader,EvidenceWriter) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_evidence.go github.com/cloudposse/atmos/pkg/artifact EvidenceReader,EvidenceWriter +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockEvidenceReader is a mock of EvidenceReader interface. +type MockEvidenceReader struct { + ctrl *gomock.Controller + recorder *MockEvidenceReaderMockRecorder + isgomock struct{} +} + +// MockEvidenceReaderMockRecorder is the mock recorder for MockEvidenceReader. +type MockEvidenceReaderMockRecorder struct { + mock *MockEvidenceReader +} + +// NewMockEvidenceReader creates a new mock instance. +func NewMockEvidenceReader(ctrl *gomock.Controller) *MockEvidenceReader { + mock := &MockEvidenceReader{ctrl: ctrl} + mock.recorder = &MockEvidenceReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEvidenceReader) EXPECT() *MockEvidenceReaderMockRecorder { + return m.recorder +} + +// ReadEvidence mocks base method. +func (m *MockEvidenceReader) ReadEvidence(ctx context.Context, pinned *PinnedReference) ([]EvidenceBlob, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadEvidence", ctx, pinned) + ret0, _ := ret[0].([]EvidenceBlob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadEvidence indicates an expected call of ReadEvidence. +func (mr *MockEvidenceReaderMockRecorder) ReadEvidence(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadEvidence", reflect.TypeOf((*MockEvidenceReader)(nil).ReadEvidence), ctx, pinned) +} + +// MockEvidenceWriter is a mock of EvidenceWriter interface. +type MockEvidenceWriter struct { + ctrl *gomock.Controller + recorder *MockEvidenceWriterMockRecorder + isgomock struct{} +} + +// MockEvidenceWriterMockRecorder is the mock recorder for MockEvidenceWriter. +type MockEvidenceWriterMockRecorder struct { + mock *MockEvidenceWriter +} + +// NewMockEvidenceWriter creates a new mock instance. +func NewMockEvidenceWriter(ctrl *gomock.Controller) *MockEvidenceWriter { + mock := &MockEvidenceWriter{ctrl: ctrl} + mock.recorder = &MockEvidenceWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEvidenceWriter) EXPECT() *MockEvidenceWriterMockRecorder { + return m.recorder +} + +// WriteEvidence mocks base method. +func (m *MockEvidenceWriter) WriteEvidence(ctx context.Context, pinned *PinnedReference, evidence []EvidenceBlob, lock LockHandle) ([]SignatureEvidence, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteEvidence", ctx, pinned, evidence, lock) + ret0, _ := ret[0].([]SignatureEvidence) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteEvidence indicates an expected call of WriteEvidence. +func (mr *MockEvidenceWriterMockRecorder) WriteEvidence(ctx, pinned, evidence, lock any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteEvidence", reflect.TypeOf((*MockEvidenceWriter)(nil).WriteEvidence), ctx, pinned, evidence, lock) +} diff --git a/pkg/artifact/produce/assemble.go b/pkg/artifact/produce/assemble.go new file mode 100644 index 00000000000..058da1004bc --- /dev/null +++ b/pkg/artifact/produce/assemble.go @@ -0,0 +1,199 @@ +package produce + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/archive" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Constants for staged payloads. +const ( + // The media type of a per-file payload blob. + payloadMediaType = "application/octet-stream" + + // The mode of generated archive files. + generatedFilePerm = 0o644 + + // The mode of generated archive directories. + generatedDirPerm = 0o755 + + // The mode of the staged archive file itself. + archiveFilePerm = 0o600 +) + +// opener opens the bytes of one inventory file. +type opener func(entryPath string) (io.ReadCloser, error) + +// assembly turns a sealed Record, its documents, and a file opener into staged blobs. +type assembly struct { + record *artifact.Record + recordBytes []byte + docs []stagedDocument + open opener + stagingDir string +} + +// blobs stages the payload for packaging and then the separate document blobs. +func (a *assembly) blobs(ctx context.Context, packaging artifact.Packaging) ([]artifact.StagedBlob, error) { + var ( + payload []artifact.StagedBlob + err error + ) + switch packaging { + case artifact.PackagingNone: + payload = a.fileBlobs() + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + var blob artifact.StagedBlob + blob, err = a.archiveBlob(ctx, packaging) + payload = []artifact.StagedBlob{blob} + default: + err = fmt.Errorf("%w: cannot assemble packaging %q", errUtils.ErrArtifactPackagingInvalid, packaging) + } + if err != nil { + return nil, err + } + return append(payload, a.documentBlobs()...), nil +} + +func (a *assembly) fileBlobs() []artifact.StagedBlob { + var blobs []artifact.StagedBlob + for index := range a.record.Inventory { + entry := a.record.Inventory[index] + if entry.Type != artifact.EntryFile { + continue + } + blobs = append(blobs, artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: payloadMediaType, Digest: entry.Digest, Size: entry.Size}, + Role: artifact.BlobPayload, + Path: entry.Path, + Open: func(context.Context) (io.ReadCloser, error) { return a.open(entry.Path) }, + }) + } + return blobs +} + +func (a *assembly) documentBlobs() []artifact.StagedBlob { + blobs := make([]artifact.StagedBlob, 0, len(a.docs)) + for index := range a.docs { + doc := &a.docs[index] + blobs = append(blobs, artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: doc.descriptor.MediaType, Digest: doc.descriptor.Digest, Size: doc.descriptor.Size}, + Role: artifact.BlobDocument, + DocumentRole: doc.descriptor.Role, + Open: func(context.Context) (io.ReadCloser, error) { return doc.open(), nil }, + }) + } + return blobs +} + +// archiveBlob writes the deterministic archive into staging, hashing it as it is written. The +// context is checked before every member is opened, so cancellation stops a large assembly. +func (a *assembly) archiveBlob(ctx context.Context, packaging artifact.Packaging) (artifact.StagedBlob, error) { + format, mediaType := archiveFormat(packaging) + path := filepath.Join(a.stagingDir, "payload."+format) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, archiveFilePerm) + if err != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: create archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + hasher := sha256.New() + writeErr := archive.WriteDeterministic(io.MultiWriter(file, hasher), format, withContext(ctx, a.members())) + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: %w", errUtils.ErrArtifactPackagingInvalid, writeErr) + } + info, err := os.Stat(path) + if err != nil { + return artifact.StagedBlob{}, fmt.Errorf("%w: stat archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return artifact.StagedBlob{ + Descriptor: ocispec.Descriptor{MediaType: mediaType, Digest: digest.NewDigest(digest.SHA256, hasher), Size: info.Size()}, + Role: artifact.BlobPayload, + Open: func(context.Context) (io.ReadCloser, error) { return os.Open(path) }, + }, nil +} + +// withContext makes every file member fail to open once ctx is canceled. +func withContext(ctx context.Context, members []archive.Member) []archive.Member { + for index := range members { + open := members[index].Open + if open == nil { + continue + } + members[index].Open = func() (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return open() + } + } + return members +} + +func archiveFormat(packaging artifact.Packaging) (string, string) { + switch packaging { + case artifact.PackagingTar: + return archive.FormatTar, artifact.MediaTypeTar + case artifact.PackagingZip: + return archive.FormatZip, artifact.MediaTypeZip + default: + return archive.FormatTGZ, artifact.MediaTypeTarGzip + } +} + +// members lists the archive members: the inventory plus the generated reserved directory holding +// manifest.json and the embedded documents. The generated members never enter the Record. +func (a *assembly) members() []archive.Member { + members := make([]archive.Member, 0, len(a.record.Inventory)+len(a.docs)+3) + present := make(map[string]bool, len(a.record.Inventory)) + for index := range a.record.Inventory { + entry := a.record.Inventory[index] + present[entry.Path] = true + members = append(members, a.inventoryMember(&entry)) + } + if !present[reservedParent] { + members = append(members, archive.Member{Path: reservedParent, Kind: archive.MemberDirectory, Mode: generatedDirPerm}) + } + members = append(members, archive.Member{Path: reservedDir, Kind: archive.MemberDirectory, Mode: generatedDirPerm}) + members = append(members, bytesMember(reservedDir+"/"+manifestName, a.recordBytes)) + for index := range a.docs { + members = append(members, bytesMember(reservedDir+"/"+reservedDocumentNames[a.docs[index].descriptor.Role], a.docs[index].data)) + } + return members +} + +func (a *assembly) inventoryMember(entry *artifact.Entry) archive.Member { + member := archive.Member{Path: entry.Path, Mode: os.FileMode(entry.Mode)} + switch entry.Type { + case artifact.EntryDirectory: + member.Kind = archive.MemberDirectory + case artifact.EntrySymlink: + member.Kind = archive.MemberSymlink + member.Target = entry.Target + default: + member.Kind = archive.MemberFile + member.Size = entry.Size + entryPath := entry.Path + member.Open = func() (io.ReadCloser, error) { return a.open(entryPath) } + } + return member +} + +func bytesMember(path string, data []byte) archive.Member { + doc := stagedDocument{data: data} + return archive.Member{ + Path: path, Kind: archive.MemberFile, Mode: generatedFilePerm, Size: int64(len(data)), + Open: func() (io.ReadCloser, error) { return doc.open(), nil }, + } +} diff --git a/pkg/artifact/produce/doc.go b/pkg/artifact/produce/doc.go new file mode 100644 index 00000000000..b53fb20a97b --- /dev/null +++ b/pkg/artifact/produce/doc.go @@ -0,0 +1,10 @@ +// Package produce implements the generic "files" artifact producer and the seams that +// kind-specific producers (scaffold, component, and so on) reuse. +// +// Preparation snapshots the source into bounded disk staging (see package snapshot), resolves the +// artifact name and version, merges labels, recognizes the reserved .atmos/artifact distribution +// directory and explicit README/CHANGELOG overrides, seals a canonical Record, and stages the +// payload either as per-file blobs or as a deterministic tar, tgz, or zip archive. The user's +// source tree is never modified; the generated manifest.json exists only inside the packaged +// archive and never appears in the Record's inventory. +package produce diff --git a/pkg/artifact/produce/documents.go b/pkg/artifact/produce/documents.go new file mode 100644 index 00000000000..1fc14074d15 --- /dev/null +++ b/pkg/artifact/produce/documents.go @@ -0,0 +1,247 @@ +package produce + +import ( + "bytes" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +// Names of the reserved distribution paths. +const ( + // The hidden directory that holds the reserved distribution directory. + reservedParent = ".atmos" + + // The source directory reserved for README and CHANGELOG distribution documents. + reservedDir = ".atmos/artifact" + + // The generated manifest, written only inside the packaged representation. + manifestName = "manifest.json" +) + +var reservedDocumentNames = map[artifact.DocumentRole]string{ + artifact.DocumentReadme: "README.md", + artifact.DocumentChangelog: "CHANGELOG.md", +} + +// documentRoles lists the roles in canonical (sorted) order. +var documentRoles = []artifact.DocumentRole{artifact.DocumentChangelog, artifact.DocumentReadme} + +// stagedDocument is a document descriptor plus its bytes (at most [artifact.MaxDocumentSize]). +type stagedDocument struct { + descriptor artifact.Document + data []byte +} + +// checkReserved rejects reserved-directory contents other than the two distribution documents, +// including a conflicting source manifest.json, subdirectories, and links. +func checkReserved(entries []artifact.Entry) error { + for index := range entries { + entry := &entries[index] + switch { + case entry.Path == reservedDir: + if entry.Type != artifact.EntryDirectory { + return reservedError(entry.Path, "must be a directory") + } + case strings.HasPrefix(entry.Path, reservedDir+"/"): + if !isReservedDocument(entry) { + return reservedError(entry.Path, "is not README.md or CHANGELOG.md; the generated "+manifestName+" is written only inside the packaged representation") + } + } + } + return nil +} + +func isReservedDocument(entry *artifact.Entry) bool { + if entry.Type != artifact.EntryFile { + return false + } + name := strings.TrimPrefix(entry.Path, reservedDir+"/") + for _, reserved := range reservedDocumentNames { + if name == reserved { + return true + } + } + return false +} + +func reservedError(entryPath, reason string) error { + return fmt.Errorf("%w: %q in the reserved %s directory %s", errUtils.ErrArtifactPackagingInvalid, entryPath, reservedDir, reason) +} + +// payloadInventory removes the reserved distribution subtree from the snapshot inventory and drops +// the .atmos parent when nothing else lives under it. +func payloadInventory(entries []artifact.Entry) []artifact.Entry { + hasOtherAtmos := false + for index := range entries { + p := entries[index].Path + if strings.HasPrefix(p, reservedParent+"/") && p != reservedDir && !strings.HasPrefix(p, reservedDir+"/") { + hasOtherAtmos = true + break + } + } + inventory := make([]artifact.Entry, 0, len(entries)) + for index := range entries { + p := entries[index].Path + if p == reservedDir || strings.HasPrefix(p, reservedDir+"/") { + continue + } + if p == reservedParent && !hasOtherAtmos { + continue + } + inventory = append(inventory, entries[index]) + } + return inventory +} + +// documentOverrides carries the explicit --readme and --changelog paths. +type documentOverrides map[artifact.DocumentRole]string + +// collectDocuments resolves each document role: an explicit override wins over the reserved +// distribution file. Overrides resolve relative to the source directory or, for a single-file +// source, the file's parent. +func collectDocuments(snap *snapshot.Snapshot, source string, overrides documentOverrides) ([]stagedDocument, error) { + var docs []stagedDocument + for _, role := range documentRoles { + doc, found, err := collectDocument(snap, source, role, overrides[role]) + if err != nil { + return nil, err + } + if found { + docs = append(docs, doc) + } + } + return docs, nil +} + +func collectDocument(snap *snapshot.Snapshot, source string, role artifact.DocumentRole, override string) (stagedDocument, bool, error) { + if override != "" { + rel, err := cleanDocumentPath(role, override) + if err != nil { + return stagedDocument{}, false, err + } + doc, err := loadOverride(snap, source, role, rel) + return doc, err == nil, err + } + rel := reservedDir + "/" + reservedDocumentNames[role] + if snap.SourceIsFile { + return stagedDocument{}, false, nil + } + entry, ok := snap.Lookup(rel) + if !ok { + return stagedDocument{}, false, nil + } + data, err := readStagedDocument(snap, &entry) + if err != nil { + return stagedDocument{}, false, err + } + return newStagedDocument(role, rel, data), true, nil +} + +// cleanDocumentPath validates an explicit document path: relative, slash-normalized, contained. +func cleanDocumentPath(role artifact.DocumentRole, override string) (string, error) { + slashed := filepath.ToSlash(override) + cleaned := path.Clean(slashed) + if filepath.IsAbs(override) || strings.HasPrefix(slashed, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") || cleaned == "." { + return "", fmt.Errorf("%w: %s path %q must be relative to the source and stay inside it", errUtils.ErrArtifactUnsafePath, role, override) + } + if err := artifact.ValidatePayloadPath(cleaned); err != nil { + return "", fmt.Errorf("%s path %q: %w", role, override, err) + } + return cleaned, nil +} + +func loadOverride(snap *snapshot.Snapshot, source string, role artifact.DocumentRole, rel string) (stagedDocument, error) { + if snap.SourceIsFile { + data, err := readSiblingDocument(source, role, rel) + if err != nil { + return stagedDocument{}, err + } + return newStagedDocument(role, rel, data), nil + } + entry, ok := snap.Lookup(rel) + if !ok || entry.Type != artifact.EntryFile { + return stagedDocument{}, fmt.Errorf("%w: %s file %q is not a regular file inside the source", errUtils.ErrArtifactPackagingInvalid, role, rel) + } + data, err := readStagedDocument(snap, &entry) + if err != nil { + return stagedDocument{}, err + } + return newStagedDocument(role, rel, data), nil +} + +func readStagedDocument(snap *snapshot.Snapshot, entry *artifact.Entry) ([]byte, error) { + if entry.Size > artifact.MaxDocumentSize { + return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, entry.Path, entry.Size, artifact.MaxDocumentSize) + } + reader, err := snap.Open(entry.Path) + if err != nil { + return nil, err + } + defer reader.Close() + return readBounded(reader, entry.Path) +} + +// readSiblingDocument reads an explicit document next to a single-file source, rejecting anything +// that is not a regular file contained in the source's parent directory. +func readSiblingDocument(source string, role artifact.DocumentRole, rel string) ([]byte, error) { + resolvedSource, err := filepath.EvalSymlinks(source) + if err != nil { + return nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + parent := filepath.Dir(resolvedSource) + full, err := filepath.EvalSymlinks(filepath.Join(parent, filepath.FromSlash(rel))) + if err != nil { + return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + if relative, err := filepath.Rel(parent, full); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("%w: %s file %q resolves outside the source directory", errUtils.ErrArtifactUnsafePath, role, rel) + } + info, err := os.Lstat(full) + if err != nil || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%w: %s file %q is not a regular file", errUtils.ErrArtifactPackagingInvalid, role, rel) + } + if info.Size() > artifact.MaxDocumentSize { + return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, rel, info.Size(), artifact.MaxDocumentSize) + } + file, err := os.Open(full) + if err != nil { + return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + defer file.Close() + return readBounded(file, rel) +} + +func readBounded(reader io.Reader, name string) ([]byte, error) { + data, err := artifact.ReadBounded(reader, artifact.MaxDocumentSize) + if err != nil { + return nil, fmt.Errorf("document %q: %w", name, err) + } + return data, nil +} + +func newStagedDocument(role artifact.DocumentRole, sourcePath string, data []byte) stagedDocument { + return stagedDocument{ + descriptor: artifact.Document{ + Role: role, + SourcePath: sourcePath, + MediaType: artifact.DocumentMediaType, + Size: int64(len(data)), + Digest: digest.FromBytes(data), + }, + data: data, + } +} + +func (d *stagedDocument) open() io.ReadCloser { + return io.NopCloser(bytes.NewReader(d.data)) +} diff --git a/pkg/artifact/produce/files.go b/pkg/artifact/produce/files.go new file mode 100644 index 00000000000..c60960e94ea --- /dev/null +++ b/pkg/artifact/produce/files.go @@ -0,0 +1,228 @@ +package produce + +import ( + "context" + "fmt" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Producer is the generic "files" producer. It satisfies [artifact.Producer]. +type Producer struct { + stagingRoot string + limits snapshot.Limits + metadata MetadataSource + snapshotOps []snapshot.Option +} + +var _ artifact.Producer = (*Producer)(nil) + +// Option customizes a [Producer]. +type Option func(*Producer) + +// WithStagingRoot sets the directory that holds bounded disk staging. +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "produce.WithStagingRoot")() + + return func(p *Producer) { p.stagingRoot = dir } +} + +// WithLimits overrides the staging limits. +func WithLimits(limits snapshot.Limits) Option { + defer perf.Track(nil, "produce.WithLimits")() + + return func(p *Producer) { p.limits = limits } +} + +// WithMetadataSource supplies producer metadata (name, version, description, labels, kind data). +func WithMetadataSource(source MetadataSource) Option { + defer perf.Track(nil, "produce.WithMetadataSource")() + + return func(p *Producer) { p.metadata = source } +} + +// WithSnapshotOptions passes extra options to the snapshot capture, for example a capture hook in tests. +func WithSnapshotOptions(opts ...snapshot.Option) Option { + defer perf.Track(nil, "produce.WithSnapshotOptions")() + + return func(p *Producer) { p.snapshotOps = append(p.snapshotOps, opts...) } +} + +// New returns a files producer. +func New(opts ...Option) *Producer { + defer perf.Track(nil, "produce.New")() + + p := &Producer{} + for _, opt := range opts { + if opt != nil { + opt(p) + } + } + return p +} + +// Register adds a files producer to registry. +func Register(registry *artifact.ProducerRegistry, opts ...Option) error { + defer perf.Track(nil, "produce.Register")() + + return registry.Register(New(opts...)) +} + +// Kind returns [artifact.KindFiles]. +func (p *Producer) Kind() string { + defer perf.Track(nil, "produce.Producer.Kind")() + + return artifact.KindFiles +} + +// Prepare snapshots the source and returns a sealed PreparedArtifact. The caller owns it and must +// Close it; on any error all owned temporary data is already removed. +// +//nolint:gocritic // The request is passed by value because artifact.Producer fixes the signature. +func (p *Producer) Prepare(ctx context.Context, request artifact.PrepareRequest) (*artifact.PreparedArtifact, error) { + defer perf.Track(nil, "produce.Producer.Prepare")() + + if err := validatePrepareRequest(&request); err != nil { + return nil, err + } + meta, err := p.producerMetadata(ctx, request.Source) + if err != nil { + return nil, err + } + name, version, err := ResolveIdentity(request.Name, request.Version, &meta, request.Source) + if err != nil { + return nil, err + } + if err := artifact.ValidateLabels(mergeLabels(meta.Labels, request.Metadata.Labels)); err != nil { + return nil, err + } + snap, err := snapshot.Capture(ctx, request.Source, p.captureOptions()...) + if err != nil { + return nil, err + } + prepared, err := p.prepareFrom(ctx, snap, &request, &identity{name: name, version: version, meta: meta}) + if err != nil { + _ = snap.Close() + return nil, err + } + return prepared, nil +} + +func (p *Producer) captureOptions() []snapshot.Option { + opts := []snapshot.Option{snapshot.WithStagingRoot(p.stagingRoot), snapshot.WithLimits(p.limits)} + return append(opts, p.snapshotOps...) +} + +func (p *Producer) producerMetadata(ctx context.Context, source string) (ProducerMetadata, error) { + if p.metadata == nil { + return ProducerMetadata{}, nil + } + return p.metadata(ctx, source) +} + +// identity is the resolved name, version, and producer metadata of one preparation. +type identity struct { + name string + version string + meta ProducerMetadata +} + +func (p *Producer) prepareFrom(ctx context.Context, snap *snapshot.Snapshot, request *artifact.PrepareRequest, id *identity) (*artifact.PreparedArtifact, error) { + if err := checkReserved(snap.Entries); err != nil { + return nil, err + } + docs, err := collectDocuments(snap, request.Source, documentOverrides{ + artifact.DocumentReadme: request.Readme, + artifact.DocumentChangelog: request.Changelog, + }) + if err != nil { + return nil, err + } + record := artifact.Record{ + Name: id.name, + Version: id.version, + Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(request.PreparedAt), + Metadata: artifact.Metadata{ + Description: firstNonEmpty(request.Metadata.Description, id.meta.Description), + Labels: mergeLabels(id.meta.Labels, request.Metadata.Labels), + Provenance: request.Metadata.Provenance, + }, + Inventory: payloadInventory(snap.Entries), + Documents: documentDescriptors(docs), + KindData: id.meta.KindData, + } + if err := record.Seal(); err != nil { + return nil, err + } + return finalize(ctx, &finalizeInput{ + record: &record, docs: docs, open: snap.Open, stagingDir: snap.Root, packaging: request.Packaging, + operationID: request.OperationID, preparedAt: request.PreparedAt, cleanup: snap.Close, + }) +} + +// finalizeInput is everything needed to stage blobs and build the PreparedArtifact. +type finalizeInput struct { + record *artifact.Record + docs []stagedDocument + open opener + stagingDir string + packaging artifact.Packaging + operationID string + preparedAt time.Time + cleanup func() error +} + +// finalize canonicalizes the Record, stages the blobs, and builds the PreparedArtifact. +func finalize(ctx context.Context, in *finalizeInput) (*artifact.PreparedArtifact, error) { + recordBytes, err := in.record.Canonical() + if err != nil { + return nil, err + } + intent, err := in.record.IntentDigest() + if err != nil { + return nil, err + } + assembled := assembly{record: in.record, recordBytes: recordBytes, docs: in.docs, open: in.open, stagingDir: in.stagingDir} + blobs, err := assembled.blobs(ctx, in.packaging) + if err != nil { + return nil, err + } + return &artifact.PreparedArtifact{ + Record: *in.record, + IntentDigest: intent, + OperationID: in.operationID, + PreparedAt: in.preparedAt, + Packaging: in.packaging, + Blobs: blobs, + Cleanup: in.cleanup, + }, nil +} + +func documentDescriptors(docs []stagedDocument) []artifact.Document { + descriptors := make([]artifact.Document, 0, len(docs)) + for index := range docs { + descriptors = append(descriptors, docs[index].descriptor) + } + return descriptors +} + +// validatePrepareRequest checks the fields the service is required to supply. +func validatePrepareRequest(request *artifact.PrepareRequest) error { + if request.Kind != "" && request.Kind != artifact.KindFiles { + return fmt.Errorf("%w: the files producer cannot prepare kind %q", errUtils.ErrArtifactConfigInvalid, request.Kind) + } + if request.OperationID == "" || request.PreparedAt.IsZero() { + return fmt.Errorf("%w: the operation ID and preparation time are captured once by the service before preparing", errUtils.ErrArtifactConfigInvalid) + } + switch request.Packaging { + case artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return nil + default: + return fmt.Errorf("%w: unsupported packaging %q", errUtils.ErrArtifactPackagingInvalid, request.Packaging) + } +} diff --git a/pkg/artifact/produce/identity.go b/pkg/artifact/produce/identity.go new file mode 100644 index 00000000000..c7bc6e3687d --- /dev/null +++ b/pkg/artifact/produce/identity.go @@ -0,0 +1,97 @@ +package produce + +import ( + "context" + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ProducerMetadata is name, version, and descriptive metadata a producer derives from its source +// (for example a scaffold manifest). Generic file packages need none of it. +type ProducerMetadata struct { + // Name is the producer-supplied artifact name. + Name string + + // Version is the producer-supplied version. + Version string + + // Description is the producer-supplied description. + Description string + + // Labels are producer-supplied default labels; explicit labels override them. + Labels map[string]string + + // KindData is optional kind-specific data stored in the Record. + KindData artifact.KindData +} + +// MetadataSource reads producer metadata for a source. A source without metadata returns the zero +// value and no error. +type MetadataSource func(ctx context.Context, source string) (ProducerMetadata, error) + +// ResolveIdentity applies the name and version precedence rules. Name precedence is the explicit +// value, then producer metadata, then the source basename. Version precedence is the explicit +// value, then producer metadata; otherwise it fails because there is no generated version and no +// implicit latest. Backend-invalid values are rejected rather than normalized. +func ResolveIdentity(explicitName, explicitVersion string, meta *ProducerMetadata, source string) (string, string, error) { + defer perf.Track(nil, "produce.ResolveIdentity")() + + name := firstNonEmpty(explicitName, meta.Name) + if name == "" { + base, err := sourceBasename(source) + if err != nil { + return "", "", err + } + name = base + if err := artifact.ValidateName(name); err != nil { + return "", "", fmt.Errorf("%w: the source basename %q is not a valid artifact name; pass an explicit name", err, name) + } + } + if err := artifact.ValidateName(name); err != nil { + return "", "", err + } + version := firstNonEmpty(explicitVersion, meta.Version) + if version == "" { + return "", "", fmt.Errorf("%w: a version is required (there is no generated version and no implicit latest)", errUtils.ErrArtifactRefInvalid) + } + if err := artifact.ValidateVersion(version); err != nil { + return "", "", err + } + return name, version, nil +} + +func sourceBasename(source string) (string, error) { + abs, err := filepath.Abs(source) + if err != nil { + return "", fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + return filepath.Base(abs), nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +// mergeLabels returns producer labels overlaid by explicit labels, or nil when there are none. +func mergeLabels(producer, explicit map[string]string) map[string]string { + if len(producer) == 0 && len(explicit) == 0 { + return nil + } + merged := make(map[string]string, len(producer)+len(explicit)) + for key, value := range producer { + merged[key] = value + } + for key, value := range explicit { + merged[key] = value + } + return merged +} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go new file mode 100644 index 00000000000..1d91d057997 --- /dev/null +++ b/pkg/artifact/produce/produce_test.go @@ -0,0 +1,610 @@ +package produce_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +var fixedTime = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +func request(src string, packaging artifact.Packaging) *artifact.PrepareRequest { + return &artifact.PrepareRequest{ + Source: src, + Name: "tools/example", + Version: "1.0.0", + Packaging: packaging, + OperationID: "op-1", + PreparedAt: fixedTime, + } +} + +func prepare(t *testing.T, req *artifact.PrepareRequest, opts ...produce.Option) (*artifact.PreparedArtifact, string) { + t.Helper() + staging := t.TempDir() + opts = append([]produce.Option{produce.WithStagingRoot(staging)}, opts...) + prepared, err := produce.New(opts...).Prepare(context.Background(), *req) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared, staging +} + +func readBlob(t *testing.T, blob *artifact.StagedBlob) []byte { + t.Helper() + reader, err := blob.Open(context.Background()) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + return data +} + +func payloadBlobs(prepared *artifact.PreparedArtifact) []artifact.StagedBlob { + var blobs []artifact.StagedBlob + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobPayload { + blobs = append(blobs, prepared.Blobs[index]) + } + } + return blobs +} + +func inventoryPaths(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for _, entry := range record.Inventory { + paths = append(paths, entry.Path) + } + return paths +} + +func TestFilesProducer_FilePackageRecord(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + record := prepared.Record + + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), inventoryPaths(&record), "reserved directory is not payload; root README is") + assert.Equal(t, artifact.KindFiles, record.Kind) + assert.Equal(t, "tools/example", record.Name) + assert.Equal(t, artifact.NewTimestamp(fixedTime), record.CreatedAt) + require.Len(t, record.Documents, 2) + assert.Equal(t, artifact.DocumentChangelog, record.Documents[0].Role) + assert.Equal(t, ".atmos/artifact/CHANGELOG.md", record.Documents[0].SourcePath) + assert.Equal(t, artifact.DocumentReadme, record.Documents[1].Role) + assert.Equal(t, int64(len(fixtures.PackageReadme)), record.Documents[1].Size) + assert.Equal(t, "op-1", prepared.OperationID) + assert.True(t, fixedTime.Equal(prepared.PreparedAt)) + require.NoError(t, record.Validate()) + + for _, entry := range record.Inventory { + switch entry.Path { + case "bin/helper.sh": + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), entry.Mode) + } + case "empty.txt": + assert.Zero(t, entry.Size) + } + } +} + +func TestFilesProducer_PerFileBlobs(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + + var files []string + blobs := payloadBlobs(prepared) + for index := range blobs { + files = append(files, blobs[index].Path) + assert.NotEmpty(t, blobs[index].Path) + assert.Equal(t, int64(len(readBlob(t, &blobs[index]))), blobs[index].Descriptor.Size) + } + want := []string{"README.md", "bin/helper.sh", "empty.txt", "nested/deep/file.txt", fixtures.UnicodeName} + assert.ElementsMatch(t, want, files, "directories and symlinks have no blob (symlinks=%v)", pkg.Symlinks) + + documents := 0 + for _, blob := range prepared.Blobs { + if blob.Role == artifact.BlobDocument { + documents++ + assert.Equal(t, artifact.DocumentMediaType, blob.Descriptor.MediaType) + } + } + assert.Equal(t, 2, documents) +} + +func TestFilesProducer_RepeatedPreparationIsByteIdentical(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + first, _ := prepare(t, request(src, packaging)) + // Touch mtimes so only content, paths, and modes can influence the output. + future := time.Now().Add(48 * time.Hour) + require.NoError(t, os.Chtimes(filepath.Join(src, "README.md"), future, future)) + second, _ := prepare(t, request(src, packaging)) + + firstRecord, err := first.Record.Canonical() + require.NoError(t, err) + secondRecord, err := second.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstRecord, secondRecord) + assert.Equal(t, first.IntentDigest, second.IntentDigest) + require.Len(t, second.Blobs, len(first.Blobs)) + for index := range first.Blobs { + assert.Equal(t, first.Blobs[index].Descriptor, second.Blobs[index].Descriptor) + assert.Equal(t, readBlob(t, &first.Blobs[index]), readBlob(t, &second.Blobs[index])) + } + }) + } +} + +func TestFilesProducer_IntentIgnoresTimeButRecordDoesNot(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + first, _ := prepare(t, request(src, artifact.PackagingTgz)) + later := request(src, artifact.PackagingTgz) + later.PreparedAt = fixedTime.Add(26 * time.Hour) + later.OperationID = "op-2" + second, _ := prepare(t, later) + + assert.Equal(t, first.IntentDigest, second.IntentDigest) + assert.Equal(t, first.Record.ContentDigest, second.Record.ContentDigest) + firstDigest, err := first.Record.RecordDigest() + require.NoError(t, err) + secondDigest, err := second.Record.RecordDigest() + require.NoError(t, err) + assert.NotEqual(t, firstDigest, secondDigest) +} + +func TestFilesProducer_ChangedModeLabelOrDocumentChangesIntent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + base, _ := prepare(t, request(src, artifact.PackagingNone)) + + labeled := request(src, artifact.PackagingNone) + labeled.Metadata.Labels = map[string]string{"team": "platform"} + withLabel, _ := prepare(t, labeled) + assert.NotEqual(t, base.IntentDigest, withLabel.IntentDigest) + assert.Equal(t, base.Record.ContentDigest, withLabel.Record.ContentDigest, "labels do not change content identity") + + withMode := base + if fixtures.POSIXModes() { + require.NoError(t, os.Chmod(filepath.Join(src, "nested", "deep", "file.txt"), 0o755)) + withMode, _ = prepare(t, request(src, artifact.PackagingNone)) + assert.NotEqual(t, base.Record.ContentDigest, withMode.Record.ContentDigest) + } + + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "changed readme\n", 0o644) + withDoc, _ := prepare(t, request(src, artifact.PackagingNone)) + assert.NotEqual(t, withMode.Record.ContentDigest, withDoc.Record.ContentDigest) +} + +func TestFilesProducer_ArchiveCarriesManifestAndLeavesSourceUntouched(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + before := listTree(t, src) + prepared, _ := prepare(t, request(src, artifact.PackagingTgz)) + assert.Equal(t, before, listTree(t, src), "the source tree must not be modified") + + blobs := payloadBlobs(prepared) + require.Len(t, blobs, 1) + assert.Equal(t, artifact.MediaTypeTarGzip, blobs[0].Descriptor.MediaType) + data := readBlob(t, &blobs[0]) + assert.Equal(t, int64(len(data)), blobs[0].Descriptor.Size) + + gz, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + reader := tar.NewReader(gz) + contents := map[string][]byte{} + modes := map[string]int64{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + modes[header.Name] = header.Mode + body, err := io.ReadAll(reader) + require.NoError(t, err) + contents[header.Name] = body + } + recordBytes, err := prepared.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, recordBytes, contents[".atmos/artifact/manifest.json"]) + assert.Equal(t, fixtures.PackageReadme, string(contents[".atmos/artifact/README.md"])) + assert.Equal(t, fixtures.PackageChangelog, string(contents[".atmos/artifact/CHANGELOG.md"])) + assert.Equal(t, fixtures.RootReadme, string(contents["README.md"]), "root README stays ordinary payload") + if fixtures.POSIXModes() { + assert.Equal(t, int64(0o755), modes["bin/helper.sh"]) + } + assert.Contains(t, contents, "emptydir/") + if pkg.Symlinks { + assert.Contains(t, modes, "link-to-file") + } + + for _, entry := range prepared.Record.Inventory { + assert.NotContains(t, entry.Path, "manifest.json", "generated members never enter the Record") + assert.NotContains(t, entry.Path, ".atmos/artifact") + } +} + +func listTree(t *testing.T, root string) []string { + t.Helper() + var out []string + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + info, err := d.Info() + require.NoError(t, err) + out = append(out, path+"|"+info.Mode().String()+"|"+info.ModTime().String()) + return nil + })) + return out +} + +func TestFilesProducer_IdentityPrecedence(t *testing.T) { + src := filepath.Join(t.TempDir(), "my-source") + require.NoError(t, os.Mkdir(src, 0o755)) + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + metadata := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{Name: "from/producer", Version: "9.9.9", Description: "producer desc", Labels: map[string]string{"a": "producer", "b": "producer"}}, nil + } + tests := []struct { + name string + explicitName string + explicitVer string + opts []produce.Option + wantName string + wantVersion string + wantErr error + }{ + {"explicit beats producer", "explicit/name", "2.0.0", []produce.Option{produce.WithMetadataSource(metadata)}, "explicit/name", "2.0.0", nil}, + {"producer beats basename", "", "", []produce.Option{produce.WithMetadataSource(metadata)}, "from/producer", "9.9.9", nil}, + {"basename fallback", "", "1.0.0", nil, "my-source", "1.0.0", nil}, + {"version never generated", "x", "", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"invalid explicit name", "Bad Name", "1", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"invalid version", "x", "has space", nil, "", "", errUtils.ErrArtifactRefInvalid}, + {"traversal name", "../x", "1", nil, "", "", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Name, req.Version = tt.explicitName, tt.explicitVer + staging := t.TempDir() + prepared, err := produce.New(append([]produce.Option{produce.WithStagingRoot(staging)}, tt.opts...)...).Prepare(context.Background(), *req) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items, "nothing is staged for an invalid identity") + return + } + require.NoError(t, err) + defer prepared.Close() + assert.Equal(t, tt.wantName, prepared.Record.Name) + assert.Equal(t, tt.wantVersion, prepared.Record.Version) + }) + } +} + +func TestFilesProducer_InvalidBasenameNeedsExplicitName(t *testing.T) { + src := filepath.Join(t.TempDir(), "My Source") + require.NoError(t, os.Mkdir(src, 0o755)) + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + req := request(src, artifact.PackagingNone) + req.Name = "" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Contains(t, err.Error(), "explicit name") +} + +func TestFilesProducer_LabelMergeAndDescription(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + metadata := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{Description: "producer desc", Labels: map[string]string{"a": "producer", "b": "producer"}}, nil + } + req := request(src, artifact.PackagingNone) + req.Metadata = artifact.Metadata{Labels: map[string]string{"b": "explicit", "c": "explicit"}} + prepared, _ := prepare(t, req, produce.WithMetadataSource(metadata)) + assert.Equal(t, map[string]string{"a": "producer", "b": "explicit", "c": "explicit"}, prepared.Record.Metadata.Labels) + assert.Equal(t, "producer desc", prepared.Record.Metadata.Description) + + req.Metadata.Description = "explicit desc" + prepared, _ = prepare(t, req, produce.WithMetadataSource(metadata)) + assert.Equal(t, "explicit desc", prepared.Record.Metadata.Description) +} + +func TestFilesProducer_InvalidLabelsRejectedBeforeStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + req := request(src, artifact.PackagingNone) + req.Metadata.Labels = map[string]string{"bad=key": "v"} + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_RequestValidation(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + base := request(src, artifact.PackagingNone) + tests := []struct { + name string + mutate func(*artifact.PrepareRequest) + want error + }{ + {"other kind", func(r *artifact.PrepareRequest) { r.Kind = "scaffold" }, errUtils.ErrArtifactConfigInvalid}, + {"no operation id", func(r *artifact.PrepareRequest) { r.OperationID = "" }, errUtils.ErrArtifactConfigInvalid}, + {"no prepared time", func(r *artifact.PrepareRequest) { r.PreparedAt = time.Time{} }, errUtils.ErrArtifactConfigInvalid}, + {"bad packaging", func(r *artifact.PrepareRequest) { r.Packaging = "rar" }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := *base + tt.mutate(&req) + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), req) + require.ErrorIs(t, err, tt.want) + }) + } + assert.Equal(t, artifact.KindFiles, produce.New().Kind()) +} + +func TestFilesProducer_ReservedDirectoryConflicts(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + }{ + {"source manifest.json", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/manifest.json", "{}", 0o644) + return true + }}, + {"unknown reserved file", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/notes.txt", "x", 0o644) + return true + }}, + {"nested reserved directory", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/sub/README.md", "x", 0o644) + return true + }}, + {"reserved symlink document", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, "real.md", "x", 0o644) + return fixtures.Symlink(t, dir, fixtures.ReservedDir+"/README.md", "../../real.md") + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + if !tt.build(t, src) { + t.Skip("fixture unavailable") + } + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} + +func TestFilesProducer_OtherAtmosContentStaysPayload(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, ".atmos/other/config.yaml", "x: 1\n", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "doc", 0o644) + prepared, _ := prepare(t, request(src, artifact.PackagingTar)) + assert.Equal(t, []string{".atmos", ".atmos/other", ".atmos/other/config.yaml"}, inventoryPaths(&prepared.Record)) +} + +func TestFilesProducer_ReservedParentDroppedWhenOnlyDistribution(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "doc", 0o644) + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + assert.Equal(t, []string{"a.txt"}, inventoryPaths(&prepared.Record)) +} + +func TestFilesProducer_ExplicitDocuments(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "docs/GUIDE.md", "explicit guide", 0o644) + fixtures.WriteFile(t, src, "docs/HISTORY.md", "explicit history", 0o644) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "reserved readme", 0o644) + + req := request(src, artifact.PackagingNone) + req.Readme = "docs/GUIDE.md" + req.Changelog = "docs/HISTORY.md" + prepared, _ := prepare(t, req) + byRole := map[artifact.DocumentRole]artifact.Document{} + for _, doc := range prepared.Record.Documents { + byRole[doc.Role] = doc + } + assert.Equal(t, "docs/GUIDE.md", byRole[artifact.DocumentReadme].SourcePath, "explicit override wins over the reserved file") + assert.Equal(t, "docs/HISTORY.md", byRole[artifact.DocumentChangelog].SourcePath) + assert.Contains(t, inventoryPaths(&prepared.Record), "docs/GUIDE.md", "an explicit document inside the source stays payload") +} + +func TestFilesProducer_ExplicitDocumentRejections(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "big.md", string(bytes.Repeat([]byte("x"), int(artifact.MaxDocumentSize)+1)), 0o644) + fixtures.MkdirAll(t, src, "adir") + tests := []struct { + name string + override string + want error + }{ + {"traversal", "../outside.md", errUtils.ErrArtifactUnsafePath}, + {"absolute", filepath.Join(src, "a.txt"), errUtils.ErrArtifactUnsafePath}, + {"dot", ".", errUtils.ErrArtifactUnsafePath}, + {"missing", "missing.md", errUtils.ErrArtifactPackagingInvalid}, + {"directory", "adir", errUtils.ErrArtifactPackagingInvalid}, + {"oversize", "big.md", errUtils.ErrArtifactTooLarge}, + {"reserved namespace", artifact.ReservedNamespace + "/x.md", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Readme = tt.override + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, tt.want) + }) + } +} + +func TestFilesProducer_SingleFileSource(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "binary", 0o755) + fixtures.WriteFile(t, dir, "NOTES.md", "sibling notes", 0o644) + fixtures.MkdirAll(t, dir, "sub") + req := request(filepath.Join(dir, "tool.bin"), artifact.PackagingZip) + req.Name = "" + req.Readme = "NOTES.md" + prepared, _ := prepare(t, req) + assert.Equal(t, "tool.bin", prepared.Record.Name) + assert.Equal(t, []string{"tool.bin"}, inventoryPaths(&prepared.Record)) + require.Len(t, prepared.Record.Documents, 1) + assert.Equal(t, "NOTES.md", prepared.Record.Documents[0].SourcePath) + + req.Readme = "../outside.md" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + + req.Readme = "sub" + _, err = produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + + req.Readme = "missing.md" + _, err = produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestFilesProducer_SingleFileDocumentEscapingThroughSymlink(t *testing.T) { + root := t.TempDir() + parent := filepath.Join(root, "parent") + outside := filepath.Join(root, "outside") + require.NoError(t, os.MkdirAll(parent, 0o755)) + require.NoError(t, os.MkdirAll(outside, 0o755)) + fixtures.WriteFile(t, parent, "tool.bin", "x", 0o644) + fixtures.WriteFile(t, outside, "secret.md", "secret", 0o644) + if err := os.Symlink(outside, filepath.Join(parent, "link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + req := request(filepath.Join(parent, "tool.bin"), artifact.PackagingNone) + req.Readme = "link/secret.md" + _, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestFilesProducer_InvalidPackagesFailBeforeAnythingIsKept(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + want error + }{ + {"symlink escape", func(t *testing.T, dir string) bool { return fixtures.Symlink(t, dir, "escape", "../../etc/passwd") }, errUtils.ErrArtifactUnsafePath}, + {"absolute symlink", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "abs", filepath.ToSlash(os.TempDir())) + }, errUtils.ErrArtifactUnsafePath}, + {"special file", func(t *testing.T, dir string) bool { return fixtures.SpecialFile(t, dir, "pipe") }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + if !tt.build(t, src) { + t.Skip("fixture unavailable") + } + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Prepare(context.Background(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, tt.want) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} + +func TestFilesProducer_ChangeDuringCaptureFails(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bbbb", 0o644) + staging := t.TempDir() + hook := snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + fixtures.WriteFile(t, src, "a.txt", "AAAAA", 0o644) + } + }) + _, err := produce.New(produce.WithStagingRoot(staging), produce.WithSnapshotOptions(hook)).Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_CloseRemovesStagingAndIsIdempotent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + prepared, staging := prepare(t, request(src, artifact.PackagingTgz)) + items, err := os.ReadDir(staging) + require.NoError(t, err) + require.NotEmpty(t, items) + require.NoError(t, prepared.Close()) + require.NoError(t, prepared.Close()) + items, err = os.ReadDir(staging) + require.NoError(t, err) + assert.Empty(t, items) +} + +func TestFilesProducer_CancellationRemovesStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + ctx, cancel := context.WithCancel(context.Background()) + staging := t.TempDir() + hook := snapshot.WithCaptureHook(func(string) { cancel() }) + _, err := produce.New(produce.WithStagingRoot(staging), produce.WithSnapshotOptions(hook)).Prepare(ctx, *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, context.Canceled) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) +} + +func TestFilesProducer_LimitsAreExplicitErrors(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + _, err := produce.New(produce.WithStagingRoot(t.TempDir()), produce.WithLimits(snapshot.Limits{MaxBytes: 5})).Prepare(context.Background(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestRegisterAddsFilesProducer(t *testing.T) { + registry := artifact.NewProducerRegistry() + require.NoError(t, produce.Register(registry)) + producer, ok := registry.Lookup("") + require.True(t, ok) + assert.Equal(t, artifact.KindFiles, producer.Kind()) + require.ErrorIs(t, produce.Register(registry), errUtils.ErrArtifactConflict) + assert.Equal(t, []string{artifact.KindFiles}, registry.Kinds()) +} diff --git a/pkg/artifact/produce/repack.go b/pkg/artifact/produce/repack.go new file mode 100644 index 00000000000..ed199f6265b --- /dev/null +++ b/pkg/artifact/produce/repack.go @@ -0,0 +1,139 @@ +package produce + +import ( + "context" + "crypto/sha256" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "time" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// RepackRequest rebuilds an existing, sealed Record for a different packaging. The Record bytes +// and content identity are preserved exactly; only the physical representation changes. +type RepackRequest struct { + // Record is the existing Record. It is not modified. + Record artifact.Record + + // Tree is a directory holding the materialized inventory (without the reserved directory). + Tree string + + // Documents holds the document bytes by role; each must match its Record descriptor. + Documents map[artifact.DocumentRole][]byte + + // Packaging is the destination packaging. + Packaging artifact.Packaging + + // OperationID is the logical operation identifier. + OperationID string + + // PreparedAt is the preparation time of this copy operation (the Record keeps its own created_at). + PreparedAt time.Time +} + +// Repack stages an existing Record from a materialized tree for the requested packaging. Every +// file read from the tree is verified against its inventory digest. +func (p *Producer) Repack(ctx context.Context, request *RepackRequest) (*artifact.PreparedArtifact, error) { + defer perf.Track(nil, "produce.Producer.Repack")() + + if request.OperationID == "" || request.PreparedAt.IsZero() { + return nil, fmt.Errorf("%w: the operation ID and preparation time are required", errUtils.ErrArtifactConfigInvalid) + } + record := request.Record + if err := record.Validate(); err != nil { + return nil, err + } + docs, err := repackDocuments(&record, request.Documents) + if err != nil { + return nil, err + } + stagingDir, err := os.MkdirTemp(p.stagingRoot, "atmos-artifact-repack-*") + if err != nil { + return nil, fmt.Errorf("%w: create staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + prepared, err := finalize(ctx, &finalizeInput{ + record: &record, docs: docs, open: treeOpener(&record, request.Tree), stagingDir: stagingDir, packaging: request.Packaging, + operationID: request.OperationID, preparedAt: request.PreparedAt, cleanup: func() error { return os.RemoveAll(stagingDir) }, + }) + if err != nil { + _ = os.RemoveAll(stagingDir) + return nil, err + } + return prepared, nil +} + +// treeOpener opens inventory files from a materialized tree, verifying each against its digest. +func treeOpener(record *artifact.Record, tree string) opener { + inventory := make(map[string]*artifact.Entry, len(record.Inventory)) + for index := range record.Inventory { + inventory[record.Inventory[index].Path] = &record.Inventory[index] + } + return func(entryPath string) (io.ReadCloser, error) { + entry, ok := inventory[entryPath] + if !ok || entry.Type != artifact.EntryFile { + return nil, fmt.Errorf("%w: %q is not an inventory file", errUtils.ErrArtifactNotFound, entryPath) + } + file, err := os.Open(filepath.Join(tree, filepath.FromSlash(entryPath))) + if err != nil { + return nil, err + } + return newVerifyingReader(file, entry.Digest, entry.Size), nil + } +} + +// repackDocuments pairs each Record document descriptor with its bytes and verifies the digest. +func repackDocuments(record *artifact.Record, provided map[artifact.DocumentRole][]byte) ([]stagedDocument, error) { + docs := make([]stagedDocument, 0, len(record.Documents)) + for _, descriptor := range record.Documents { + data, ok := provided[descriptor.Role] + if !ok { + return nil, fmt.Errorf("%w: %s bytes are required to repack", errUtils.ErrArtifactDocumentMissing, descriptor.Role) + } + if int64(len(data)) != descriptor.Size || digest.FromBytes(data) != descriptor.Digest { + return nil, fmt.Errorf("%w: %s does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, descriptor.Role) + } + docs = append(docs, stagedDocument{descriptor: descriptor, data: data}) + } + return docs, nil +} + +// verifyingReader fails at EOF unless the bytes read match the expected digest and size. +type verifyingReader struct { + rc io.ReadCloser + hasher hash.Hash + expected digest.Digest + size int64 + read int64 +} + +func newVerifyingReader(rc io.ReadCloser, expected digest.Digest, size int64) *verifyingReader { + return &verifyingReader{rc: rc, hasher: sha256.New(), expected: expected, size: size} +} + +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "produce.verifyingReader.Read")() + + n, err := v.rc.Read(p) + v.read += int64(n) + _, _ = v.hasher.Write(p[:n]) + if err == io.EOF { + if v.read != v.size || digest.NewDigest(digest.SHA256, v.hasher) != v.expected { + return n, fmt.Errorf("%w: file content does not match its inventory digest", errUtils.ErrArtifactIntegrityFailed) + } + } + return n, err +} + +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "produce.verifyingReader.Close")() + + return v.rc.Close() +} diff --git a/pkg/artifact/produce/repack_test.go b/pkg/artifact/produce/repack_test.go new file mode 100644 index 00000000000..7f48f8e813b --- /dev/null +++ b/pkg/artifact/produce/repack_test.go @@ -0,0 +1,111 @@ +package produce_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +// treeOf materializes a prepared artifact's per-file payload into a directory. +func treeOf(t *testing.T, prepared *artifact.PreparedArtifact) string { + t.Helper() + tree := t.TempDir() + blobs := payloadBlobs(prepared) + for index := range blobs { + fixtures.WriteFile(t, tree, blobs[index].Path, string(readBlob(t, &blobs[index])), 0o644) + } + return tree +} + +func documentsOf(t *testing.T, prepared *artifact.PreparedArtifact) map[artifact.DocumentRole][]byte { + t.Helper() + docs := map[artifact.DocumentRole][]byte{} + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobDocument { + docs[prepared.Blobs[index].DocumentRole] = readBlob(t, &prepared.Blobs[index]) + } + } + return docs +} + +func TestRepack_PreservesRecordAndMatchesFreshArchive(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + fresh, _ := prepare(t, request(src, artifact.PackagingTgz)) + + req := &produce.RepackRequest{ + Record: original.Record, + Tree: treeOf(t, original), + Documents: documentsOf(t, original), + Packaging: artifact.PackagingTgz, + OperationID: "copy-op", + PreparedAt: fixedTime.AddDate(0, 0, 3), + } + repacked, err := produce.New(produce.WithStagingRoot(t.TempDir())).Repack(context.Background(), req) + require.NoError(t, err) + defer repacked.Close() + + originalBytes, err := original.Record.Canonical() + require.NoError(t, err) + repackedBytes, err := repacked.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, originalBytes, repackedBytes, "a copy preserves Record bytes") + assert.Equal(t, original.IntentDigest, repacked.IntentDigest) + assert.Equal(t, "copy-op", repacked.OperationID) + + repackedPayload := payloadBlobs(repacked) + freshPayload := payloadBlobs(fresh) + require.Len(t, repackedPayload, 1) + assert.Equal(t, readBlob(t, &freshPayload[0]), readBlob(t, &repackedPayload[0]), "deterministic assembly yields identical archive bytes") +} + +func TestRepack_Failures(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + good := func() *produce.RepackRequest { + return &produce.RepackRequest{ + Record: original.Record, Tree: treeOf(t, original), Documents: documentsOf(t, original), + Packaging: artifact.PackagingTar, OperationID: "op", PreparedAt: fixedTime, + } + } + tests := []struct { + name string + mutate func(*produce.RepackRequest) + want error + }{ + {"no operation", func(r *produce.RepackRequest) { r.OperationID = "" }, errUtils.ErrArtifactConfigInvalid}, + {"missing document", func(r *produce.RepackRequest) { delete(r.Documents, artifact.DocumentReadme) }, errUtils.ErrArtifactDocumentMissing}, + {"tampered document", func(r *produce.RepackRequest) { r.Documents[artifact.DocumentReadme] = []byte("tampered") }, errUtils.ErrArtifactIntegrityFailed}, + {"tampered tree file", func(r *produce.RepackRequest) { + require.NoError(t, os.WriteFile(filepath.Join(r.Tree, "nested", "deep", "file.txt"), []byte("tampered content"), 0o644)) + }, errUtils.ErrArtifactPackagingInvalid}, + {"missing tree file", func(r *produce.RepackRequest) { + require.NoError(t, os.Remove(filepath.Join(r.Tree, "empty.txt"))) + }, errUtils.ErrArtifactPackagingInvalid}, + {"invalid record", func(r *produce.RepackRequest) { r.Record.ContentDigest = "" }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := good() + tt.mutate(req) + staging := t.TempDir() + _, err := produce.New(produce.WithStagingRoot(staging)).Repack(context.Background(), req) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + items, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, items) + }) + } +} diff --git a/pkg/artifact/producers.go b/pkg/artifact/producers.go new file mode 100644 index 00000000000..2b0cad4f40b --- /dev/null +++ b/pkg/artifact/producers.go @@ -0,0 +1,72 @@ +package artifact + +import ( + "fmt" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ProducerRegistry maps artifact kinds to the [Producer] that prepares them. The generic files +// producer is registered by a composition root; kind-specific producers (scaffold, component, and +// so on) register the same way and add their own validation and exclusions. +type ProducerRegistry struct { + mu sync.RWMutex + producers map[string]Producer +} + +// NewProducerRegistry returns an empty registry. +func NewProducerRegistry() *ProducerRegistry { + defer perf.Track(nil, "artifact.NewProducerRegistry")() + + return &ProducerRegistry{producers: make(map[string]Producer)} +} + +// Register adds a producer under its Kind. A duplicate kind or an invalid kind is an error. +func (r *ProducerRegistry) Register(producer Producer) error { + defer perf.Track(nil, "artifact.ProducerRegistry.Register")() + + if producer == nil { + return fmt.Errorf("%w: producer must not be nil", errUtils.ErrArtifactConfigInvalid) + } + kind := producer.Kind() + if err := ValidateKind(kind); err != nil { + return fmt.Errorf("%w: producer kind %q: %w", errUtils.ErrArtifactConfigInvalid, kind, err) + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.producers[kind]; exists { + return fmt.Errorf("%w: producer for kind %q is already registered", errUtils.ErrArtifactConflict, kind) + } + r.producers[kind] = producer + return nil +} + +// Lookup returns the producer for kind; an empty kind means [KindFiles]. +func (r *ProducerRegistry) Lookup(kind string) (Producer, bool) { + defer perf.Track(nil, "artifact.ProducerRegistry.Lookup")() + + if kind == "" { + kind = KindFiles + } + r.mu.RLock() + defer r.mu.RUnlock() + producer, ok := r.producers[kind] + return producer, ok +} + +// Kinds returns the registered kinds, sorted. +func (r *ProducerRegistry) Kinds() []string { + defer perf.Track(nil, "artifact.ProducerRegistry.Kinds")() + + r.mu.RLock() + defer r.mu.RUnlock() + kinds := make([]string, 0, len(r.producers)) + for kind := range r.producers { + kinds = append(kinds, kind) + } + sort.Strings(kinds) + return kinds +} diff --git a/pkg/artifact/producers_test.go b/pkg/artifact/producers_test.go new file mode 100644 index 00000000000..1723eb49e1d --- /dev/null +++ b/pkg/artifact/producers_test.go @@ -0,0 +1,38 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestProducerRegistry(t *testing.T) { + ctrl := gomock.NewController(t) + files := NewMockProducer(ctrl) + files.EXPECT().Kind().Return(KindFiles).AnyTimes() + scaffold := NewMockProducer(ctrl) + scaffold.EXPECT().Kind().Return("scaffold").AnyTimes() + invalid := NewMockProducer(ctrl) + invalid.EXPECT().Kind().Return("Bad Kind").AnyTimes() + + registry := NewProducerRegistry() + require.NoError(t, registry.Register(scaffold)) + require.NoError(t, registry.Register(files)) + require.ErrorIs(t, registry.Register(files), errUtils.ErrArtifactConflict) + require.ErrorIs(t, registry.Register(invalid), errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, registry.Register(nil), errUtils.ErrArtifactConfigInvalid) + + found, ok := registry.Lookup("") + require.True(t, ok) + assert.Equal(t, KindFiles, found.Kind(), "an empty kind selects the generic files producer") + found, ok = registry.Lookup("scaffold") + require.True(t, ok) + assert.Equal(t, "scaffold", found.Kind()) + _, ok = registry.Lookup("component/terraform") + assert.False(t, ok) + assert.Equal(t, []string{KindFiles, "scaffold"}, registry.Kinds()) +} diff --git a/pkg/artifact/service/boundary_test.go b/pkg/artifact/service/boundary_test.go new file mode 100644 index 00000000000..61942f6a8bd --- /dev/null +++ b/pkg/artifact/service/boundary_test.go @@ -0,0 +1,45 @@ +package service_test + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestArtifactPackagesImportNoCLIRunnerHookPluginOrCI keeps the artifact service and the packages +// it composes (preparation, staging, materialization) free of CLI, runner, hook, plugin, and CI +// dependencies, as the package boundary contract requires. Test-support packages are checked too. +func TestArtifactPackagesImportNoCLIRunnerHookPluginOrCI(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + } + files, err := filepath.Glob(filepath.Join("..", "*", "*.go")) + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go new file mode 100644 index 00000000000..318c01bea09 --- /dev/null +++ b/pkg/artifact/service/commit.go @@ -0,0 +1,219 @@ +package service + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// target is the repository a mutation writes to. +type target struct { + alias string + repo artifact.Repository + info *RepositoryInfo +} + +// incoming describes the artifact being written, for comparison and reconciliation. +type incoming struct { + ref artifact.Ref + record *artifact.Record + intent digest.Digest + recordDigest digest.Digest + payload []ocispec.Descriptor + size int64 + prepared *artifact.PreparedArtifact + inspection *artifact.Inspection +} + +func newIncoming(prepared *artifact.PreparedArtifact) (*incoming, error) { + recordDigest, err := prepared.Record.RecordDigest() + if err != nil { + return nil, err + } + in := &incoming{ref: artifact.Ref{Name: prepared.Record.Name, Version: prepared.Record.Version}, record: &prepared.Record, intent: prepared.IntentDigest, recordDigest: recordDigest, prepared: prepared} + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobPayload { + in.payload = append(in.payload, prepared.Blobs[index].Descriptor) + in.size += prepared.Blobs[index].Descriptor.Size + } + } + return in, nil +} + +// current is what a reference holds right now. +type current struct { + pinned *artifact.PinnedReference + inspection *artifact.Inspection + unavailable bool +} + +type verdict int + +const ( + verdictCreate verdict = iota + verdictReplace + verdictUnchanged +) + +// writer performs one repository write attempt. +type writer func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) + +// publishState is filled by commit for the caller's outcome. +type publishState struct { + record artifact.Record + existing bool +} + +// publication bundles everything one write needs: the operation, the target repository, the +// incoming artifact, the force flag, the write closure, and the state commit fills in. +type publication struct { + op *operation + tgt *target + in *incoming + force bool + write writer + state *publishState +} + +// commit runs the shared publication sequence under the writer lock: read the current reference, +// compare intent, apply conflict and immutability rules, then write with reconciliation. +func (s *Service) commit(ctx context.Context, pub *publication) (err error) { + ref := pub.in.ref + pub.op.result.Ref = ref.String() + locks, err := s.acquireLocks(ctx, pub.op, opPublish, []lockTarget{{alias: pub.tgt.alias, repo: pub.tgt.repo, info: pub.tgt.info, resource: ref.String()}}) + if err != nil { + return err + } + s.event(ctx, pub, PhaseLocked, 0, nil) + defer func() { + _ = locks.release(ctx, pub.op) + s.event(ctx, pub, PhaseReleased, 0, nil) + if err != nil { + s.event(ctx, pub, PhaseFailed, 0, err) + } + }() + + cur, err := s.readCurrent(ctx, pub.tgt, ref) + if err != nil { + return err + } + decision, err := s.decide(pub.tgt, cur, pub.in, pub.force) + if err != nil { + return err + } + if decision == verdictUnchanged { + s.reportUnchanged(pub, cur) + return nil + } + result, err := s.execute(ctx, pub, cur, decision, locks.handleFor(pub.tgt.alias)) + if err != nil { + return err + } + s.reportWritten(pub, result, locks, decision, cur) + return nil +} + +func (s *Service) event(ctx context.Context, pub *publication, phase JournalPhase, attempt int, err error) { + s.record(ctx, &JournalEntry{ + OperationID: pub.op.id, Phase: phase, Repository: pub.tgt.alias, Ref: pub.in.ref.String(), + IntentDigest: pub.in.intent.String(), Attempt: attempt, Err: err, + }) +} + +// readCurrent resolves and inspects the reference once. A missing reference is not an error; a +// reference that exists but is unavailable (pending or interrupted) is reported so the same-intent +// retry can repair it. +func (s *Service) readCurrent(ctx context.Context, tgt *target, ref artifact.Ref) (*current, error) { + pinned, err := tgt.repo.Resolve(ctx, ref) + switch { + case errors.Is(err, errUtils.ErrArtifactNotFound): + return ¤t{}, nil + case errors.Is(err, errUtils.ErrArtifactPublicationUnavailable): + return ¤t{unavailable: true}, nil + case err != nil: + return nil, fmt.Errorf("resolve %s in repository %q: %w", ref, tgt.alias, err) + } + inspection, err := tgt.repo.Inspect(ctx, pinned) + if err != nil { + return nil, fmt.Errorf("inspect %s in repository %q: %w", ref, tgt.alias, err) + } + return ¤t{pinned: &pinned, inspection: inspection}, nil +} + +// decide applies the intent comparison, conflict, force, and immutability rules. +func (s *Service) decide(tgt *target, cur *current, in *incoming, force bool) (verdict, error) { + if cur.pinned == nil { + return verdictCreate, nil + } + existing := cur.inspection.Record + if existing != nil { + if intent, err := existing.IntentDigest(); err == nil && intent == in.intent { + return verdictUnchanged, nil + } + } + if immutable, reason := s.immutableCandidate(tgt, cur, in); immutable { + return verdictCreate, fmt.Errorf("%w: %s stays immutable even with force", errUtils.ErrArtifactConflict, reason) + } + if !force { + return verdictCreate, fmt.Errorf("%w: %s already exists in repository %q with different %s; use force to replace it", + errUtils.ErrArtifactConflict, in.ref, tgt.alias, describeDifferences(existing, in.record)) + } + return verdictReplace, nil +} + +// immutableCandidate asks the admission policy about both the stored and the incoming artifact. +func (s *Service) immutableCandidate(tgt *target, cur *current, in *incoming) (bool, string) { + if s.immutable == nil { + return false, "" + } + if immutable, why := s.immutable.Immutable(Candidate{Existing: true, Repository: tgt.alias, Inspection: cur.inspection}); immutable { + return true, "the existing release (" + why + ")" + } + if immutable, why := s.immutable.Immutable(Candidate{Repository: tgt.alias, Inspection: in.inspection, Prepared: in.prepared}); immutable { + return true, "the incoming release (" + why + ")" + } + return false, "" +} + +// describeDifferences names which parts of identity changed. +func describeDifferences(existing, incoming *artifact.Record) string { + if existing == nil || incoming == nil { + return "content (the existing artifact carries no portable record)" + } + var differences []string + if existing.ContentDigest != incoming.ContentDigest { + differences = append(differences, "content (files, modes, or documents)") + } + if existing.Metadata.Description != incoming.Metadata.Description { + differences = append(differences, "description") + } + if !labelsEqual(existing.Metadata.Labels, incoming.Metadata.Labels) { + differences = append(differences, "labels") + } + if existing.Kind != incoming.Kind || string(existing.KindData) != string(incoming.KindData) { + differences = append(differences, "kind data") + } + if len(differences) == 0 { + return "intent" + } + return strings.Join(differences, ", ") +} + +func labelsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for key, value := range a { + if other, ok := b[key]; !ok || other != value { + return false + } + } + return true +} diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go new file mode 100644 index 00000000000..99a9308bebd --- /dev/null +++ b/pkg/artifact/service/config_repositories.go @@ -0,0 +1,138 @@ +package service + +import ( + "context" + "fmt" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// ConfigRepositories implements [Repositories] from the validated artifacts configuration and a +// repository kind registry. Describe reads only configuration and kind descriptors, so it never +// authenticates; Open constructs each repository once through the registry, which is where +// identities are resolved. +type ConfigRepositories struct { + cfg *schema.ArtifactConfig + registry *artifact.RepositoryRegistry + options []artifact.RepositoryOption + mu sync.Mutex + opened map[string]artifact.Repository +} + +var _ Repositories = (*ConfigRepositories)(nil) + +// FromConfig returns a [Repositories] backed by configuration. A nil registry uses the default +// repository registry. +func FromConfig(cfg *schema.ArtifactConfig, registry *artifact.RepositoryRegistry, opts ...artifact.RepositoryOption) *ConfigRepositories { + defer perf.Track(nil, "service.FromConfig")() + + if registry == nil { + registry = artifact.DefaultRepositoryRegistry() + } + return &ConfigRepositories{cfg: cfg, registry: registry, options: opts, opened: map[string]artifact.Repository{}} +} + +// Describe implements [Repositories]. For a repository that delegates locking, the lock scopes of +// the delegate are added to its capabilities, so the lock guarantee check sees what the delegate +// can provide. +func (c *ConfigRepositories) Describe(alias string) (RepositoryInfo, error) { + defer perf.Track(nil, "service.ConfigRepositories.Describe")() + + repoCfg, ok := c.cfg.Repositories[alias] + if !ok { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, known := c.registry.Lookup(repoCfg.Kind) + if !known { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q has unknown kind %q", errUtils.ErrArtifactConfigInvalid, alias, repoCfg.Kind) + } + plan, err := artifactconfig.ResolveLockPlan(c.cfg, alias) + if err != nil { + return RepositoryInfo{}, err + } + policy, err := artifactconfig.EffectivePolicy(c.cfg, alias) + if err != nil { + return RepositoryInfo{}, err + } + info := RepositoryInfo{ + Alias: alias, + Capabilities: desc.Capabilities, + Packaging: desc.EffectivePackaging(repoCfg.Packaging), + RequiredLockScope: plan.Required, + Policy: policy, + } + if repoCfg.Defaults != nil && repoCfg.Defaults.Metadata != nil { + info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: repoCfg.Defaults.Metadata.Labels} + } + if plan.Provider != alias { + info.Capabilities.LockScopes = c.delegatedScopes(&info.Capabilities, plan.Provider) + } + return info, nil +} + +// delegatedScopes adds the delegate repository's lock scopes to the repository's own. +func (c *ConfigRepositories) delegatedScopes(caps *artifact.Capabilities, provider string) []artifact.LockScope { + scopes := append([]artifact.LockScope(nil), caps.LockScopes...) + delegate, ok := c.cfg.Repositories[provider] + if !ok { + return scopes + } + desc, known := c.registry.Lookup(delegate.Kind) + if !known { + return scopes + } + for _, scope := range desc.Capabilities.LockScopes { + if !containsScope(scopes, scope) { + scopes = append(scopes, scope) + } + } + return scopes +} + +func containsScope(scopes []artifact.LockScope, scope artifact.LockScope) bool { + for _, have := range scopes { + if have == scope { + return true + } + } + return false +} + +// Open implements [Repositories]. Each repository is constructed once. +func (c *ConfigRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { + defer perf.Track(nil, "service.ConfigRepositories.Open")() + + c.mu.Lock() + defer c.mu.Unlock() + if repo, ok := c.opened[alias]; ok { + return repo, nil + } + repoCfg, ok := c.cfg.Repositories[alias] + if !ok { + return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + repo, err := c.registry.New(ctx, alias, &repoCfg, c.options...) + if err != nil { + return nil, err + } + c.opened[alias] = repo + return repo, nil +} + +// Aliases implements [Repositories]. +func (c *ConfigRepositories) Aliases() []string { + defer perf.Track(nil, "service.ConfigRepositories.Aliases")() + + aliases := make([]string, 0, len(c.cfg.Repositories)) + for alias := range c.cfg.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/service/config_repositories_test.go b/pkg/artifact/service/config_repositories_test.go new file mode 100644 index 00000000000..9d0071883fd --- /dev/null +++ b/pkg/artifact/service/config_repositories_test.go @@ -0,0 +1,120 @@ +package service_test + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +func configRegistry(t *testing.T, constructed *atomic.Int64) *artifact.RepositoryRegistry { + t.Helper() + registry := artifact.NewRepositoryRegistry() + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindS3} { + desc, ok := artifact.BuiltinKind(kind) + require.True(t, ok) + require.NoError(t, registry.Register(desc, func(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + constructed.Add(1) + return artifacttest.New(artifacttest.Options{Alias: alias}), nil + })) + } + return registry +} + +func boolPtr(b bool) *bool { return &b } + +func TestConfigRepositories_DescribeIsOfflineAndCombinesPolicyDefaultsAndLocking(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{ + Verification: &schema.ArtifactVerificationConfig{ + RequireSignature: boolPtr(true), + TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "global@example.com", Issuer: "https://global.example.com"}}, + }, + Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": { + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": "/tmp/shared"}, + Defaults: &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{ + Description: "default description", Labels: map[string]string{"team": "platform"}, + }}, + Verification: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "repo@example.com", Issuer: "https://repo.example.com"}}}, + Locking: &schema.ArtifactLockingConfig{RequiredScope: "repository", Repository: "coordination"}, + }, + "coordination": {Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-coordination"}}, + }, + } + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + + assert.Equal(t, []string{"coordination", "shared"}, repos.Aliases()) + info, err := repos.Describe("shared") + require.NoError(t, err) + assert.Zero(t, constructed.Load(), "describing never constructs a repository, so it never authenticates") + assert.Equal(t, artifact.PackagingNone, info.Packaging) + assert.Equal(t, artifact.LockScopeRepository, info.RequiredLockScope) + assert.True(t, info.Capabilities.HasLockScope(artifact.LockScopeRepository), "the delegate's scope is advertised") + assert.Equal(t, "default description", info.Defaults.Description) + assert.Equal(t, map[string]string{"team": "platform"}, info.Defaults.Labels) + assert.True(t, info.Policy.RequireSignature) + assert.Len(t, info.Policy.TrustSets, 2, "global and repository trust sets are separate, both required") + + coordination, err := repos.Describe("coordination") + require.NoError(t, err) + assert.Equal(t, artifact.PackagingTgz, coordination.Packaging) + assert.Equal(t, artifact.LockScopeHost, coordination.RequiredLockScope) + assert.Len(t, coordination.Policy.TrustSets, 1) +} + +func TestConfigRepositories_OpenConstructsOnceAndRejectsUnknowns(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/shared"}}, + "ghost": {Kind: "unregistered/kind"}, + "s3": {Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifacts"}}, + }} + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + + first, err := repos.Open(context.Background(), "shared") + require.NoError(t, err) + second, err := repos.Open(context.Background(), "shared") + require.NoError(t, err) + assert.Same(t, first, second) + assert.Equal(t, int64(1), constructed.Load()) + + _, err = repos.Open(context.Background(), "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Describe("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Describe("ghost") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Open(context.Background(), "ghost") + require.Error(t, err) +} + +func TestConfigRepositories_DrivesTheServiceEndToEnd(t *testing.T) { + var constructed atomic.Int64 + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/shared"}}, + }} + e := newEnv(t, envConfig{}) + svc, err := service.New(service.FromConfig(cfg, configRegistry(t, &constructed)), nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + + dry := e.publishRequest("shared", e.source("hello")) + dry.DryRun = true + _, err = svc.Publish(context.Background(), dry) + require.NoError(t, err) + assert.Zero(t, constructed.Load(), "a dry run through configuration performs no repository construction or identity resolution") + + result, err := svc.Publish(context.Background(), e.publishRequest("shared", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, int64(1), constructed.Load()) +} diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go new file mode 100644 index 00000000000..e2401af20cf --- /dev/null +++ b/pkg/artifact/service/consume_test.go @@ -0,0 +1,327 @@ +package service_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func out(t *testing.T) string { + return filepath.Join(t.TempDir(), "out", "pulled") +} + +func TestPull_RoundTripEveryPackaging(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = packaging }) + src := e.source("hello") + published := e.publish("dst", src) + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, result.Outcome) + assert.Equal(t, dest, result.Location) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + + state, err := materialize.InspectDestination(dest, &published.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + }) + } +} + +func TestPull_ExistingDestinationRules(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + dest := out(t) + request := artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest} + _, err := e.svc.Pull(context.Background(), request) + require.NoError(t, err) + reads := e.repos["dst"].Calls("Read") + + again, err := e.svc.Pull(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "an identical destination is a no-op") + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "nothing is downloaded for an identical destination") + + fixtures.WriteFile(t, dest, "stale.txt", "stale", 0o644) + require.NoError(t, os.WriteFile(filepath.Join(dest, "data.txt"), []byte("locally edited"), 0o644)) + _, err = e.svc.Pull(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a different non-empty destination needs force") + data, readErr := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, readErr) + assert.Equal(t, "locally edited", string(data), "a refused pull leaves the destination untouched") + + request.Force = true + forced, err := e.svc.Pull(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, forced.Outcome) + _, statErr := os.Stat(filepath.Join(dest, "stale.txt")) + assert.True(t, os.IsNotExist(statErr), "stale files are removed, never merged") + + empty := out(t) + require.NoError(t, os.MkdirAll(empty, 0o755)) + request.Destination, request.Force = empty, false + _, err = e.svc.Pull(context.Background(), request) + require.NoError(t, err, "an empty destination needs no force") +} + +func TestPull_Failures(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + tests := []struct { + name string + request artifact.PullRequest + want error + }{ + {"missing repository", artifact.PullRequest{Ref: exampleRef, Destination: out(t)}, errUtils.ErrArtifactConfigInvalid}, + {"missing destination", artifact.PullRequest{Repository: "dst", Ref: exampleRef}, errUtils.ErrArtifactConfigInvalid}, + {"invalid ref", artifact.PullRequest{Repository: "dst", Ref: artifact.Ref{Name: "x"}, Destination: out(t)}, errUtils.ErrArtifactRefInvalid}, + {"not found", artifact.PullRequest{Repository: "dst", Ref: ref("tools/example", "9.9.9"), Destination: out(t)}, errUtils.ErrArtifactNotFound}, + {"unknown repository", artifact.PullRequest{Repository: "nope", Ref: exampleRef, Destination: out(t)}, errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := e.svc.Pull(context.Background(), tt.request) + require.ErrorIs(t, err, tt.want) + require.NotNil(t, result) + assert.NotEmpty(t, result.OperationID) + _, statErr := os.Stat(tt.request.Destination) + assert.True(t, tt.request.Destination == "" || os.IsNotExist(statErr), "a failed pull creates nothing") + }) + } +} + +func TestPull_WriteOnlyRepositoryIsACapabilityError(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Zero(t, e.repos["dst"].TotalCalls()) +} + +// moveTagOnFirstResolve repoints exampleRef at the decoy the first time it is resolved. +func moveTagOnFirstResolve(t *testing.T, e *env, decoyVersion string) { + t.Helper() + decoy, ok := e.repos["dst"].RecordDigestOf(ref("tools/example", decoyVersion)) + require.True(t, ok) + moved := false + e.repos["dst"].SetOnResolved(func(resolved artifact.Ref) { + if resolved == exampleRef && !moved { + moved = true + assert.True(t, e.repos["dst"].MoveRef(exampleRef, decoy)) + } + }) +} + +func publishDecoy(t *testing.T, e *env) { + t.Helper() + request := e.publishRequest("dst", e.source("DECOY CONTENT")) + request.Version = "2.0.0" + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) +} + +func TestPull_TagMovedMidReadStillConsumesTheOriginallyPinnedSubject(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + resolves := e.repos["dst"].Calls("Resolve") + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["dst"].Calls("Resolve")-resolves, "the mutable reference is resolved exactly once") + assert.Equal(t, original.Operation.RecordDigest, result.RecordDigest) + data, err := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, err) + assert.Equal(t, "ORIGINAL", string(data), "the moved tag never substitutes other bytes") + moved, _ := e.repos["dst"].RecordDigestOf(exampleRef) + assert.NotEqual(t, original.Operation.RecordDigest, moved, "the tag really moved") +} + +func TestInspectAndShowResolveOnce(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + before := e.repos["dst"].Calls("Resolve") + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["dst"].Calls("Resolve")-before) + assert.Equal(t, original.Operation.RecordDigest, inspected.Operation.RecordDigest) + assert.Equal(t, original.Operation.RecordDigest, inspected.Inspection.Pinned.RecordDigest) + require.NotNil(t, inspected.Inspection.Record) + assert.Equal(t, original.Record.ContentDigest, inspected.Inspection.Record.ContentDigest) + assert.Nil(t, inspected.Verification, "no trust configuration reports unchecked, never verified") +} + +func TestShowDocument(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + reads := e.repos["dst"].Calls("Read") + + result, err := e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.NoError(t, err) + assert.Equal(t, "package readme\n", string(result.Content)) + assert.Equal(t, artifact.DocumentReadme, result.Document.Role) + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "documents are read without the payload") + assert.Equal(t, int64(len(result.Content)), result.Operation.Size) + + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentChangelog}) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing) + + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: "license"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestInspectAndShow_Failures(t *testing.T) { + e := newEnv(t, envConfig{}) + _, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func seedList(t *testing.T, e *env, alias string, items map[string]string, labels map[string]string) { + t.Helper() + for nameVersion, content := range items { + name, version, _ := splitRef(nameVersion) + request := artifact.PublishRequest{Repository: alias, Source: e.source(content), Name: name, Version: version} + request.Metadata.Labels = labels + _, err := e.svc.Publish(context.Background(), request) + require.NoError(t, err) + } +} + +func splitRef(nameVersion string) (string, string, bool) { + parsed, err := artifact.ParseRef(nameVersion) + if err != nil { + return "", "", false + } + return parsed.Name, parsed.Version, true +} + +func TestList_AcrossRepositoriesWithPaginationLabelsAndOrdering(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "beta": {}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1", "app/a:2": "a2", "app/b:1": "b1"}, map[string]string{"team": "platform"}) + seedList(t, e, "beta", map[string]string{"app/a:1": "other", "app/c:1": "c1"}, map[string]string{"team": "data"}) + + all, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + var keys []string + for _, entry := range all.Artifacts { + keys = append(keys, entry.Repository+"/"+entry.Name+":"+entry.Version) + } + assert.Equal(t, []string{"alpha/app/a:1", "beta/app/a:1", "alpha/app/a:2", "alpha/app/b:1", "beta/app/c:1"}, keys, "sorted by name, version, repository; aliases stay visible; every page consumed") + require.Len(t, all.Repositories, 2) + assert.Equal(t, 3, all.Repositories[0].Count) + + named, err := e.svc.List(context.Background(), artifact.ListRequest{Name: "app/a"}) + require.NoError(t, err) + assert.Len(t, named.Artifacts, 3) + + one, err := e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"beta"}}) + require.NoError(t, err) + assert.Len(t, one.Artifacts, 2) + + platform, err := e.svc.List(context.Background(), artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}}}) + require.NoError(t, err) + assert.Len(t, platform.Artifacts, 3) + + none, err := e.svc.List(context.Background(), artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}, {Key: "team", Value: "data"}}}) + require.NoError(t, err) + assert.Empty(t, none.Artifacts, "label selectors use AND semantics") +} + +func TestSearch_MatchesNamesDescriptionsKindsAndLabels(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}}}) + request := e.publishRequest("alpha", e.source("a")) + request.Name = "net/vpc-module" + request.Metadata = artifact.Metadata{Description: "Networking Basics", Labels: map[string]string{"purpose": "backup"}} + _, err := e.svc.Publish(context.Background(), request) + require.NoError(t, err) + other := e.publishRequest("alpha", e.source("b")) + other.Name = "tools/other" + _, err = e.svc.Publish(context.Background(), other) + require.NoError(t, err) + + for query, want := range map[string]int{"vpc": 1, "networking": 1, "BACKUP": 1, "files": 2, "nomatch": 0, "": 2} { + result, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: query}) + require.NoError(t, err, query) + assert.Len(t, result.Artifacts, want, query) + } + labeled, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "files", Labels: []artifact.Label{{Key: "purpose", Value: "backup"}}}) + require.NoError(t, err) + assert.Len(t, labeled.Artifacts, 1) +} + +func TestList_CapabilityErrorsAreReportedNotHidden(t *testing.T) { + noCollection := artifacttest.DefaultCapabilities() + noCollection.EnumerateCollection = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "registry": {Capabilities: &noCollection}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1"}, nil) + + all, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err, "a default listing tolerates a repository without collection enumeration") + assert.Len(t, all.Artifacts, 1) + var diagnostic artifact.RepositoryListing + for _, listing := range all.Repositories { + if listing.Repository == "registry" { + diagnostic = listing + } + } + require.NotEmpty(t, diagnostic.Diagnostics) + assert.Contains(t, diagnostic.Diagnostics[0].Message, "direct named operations") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"registry"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an explicitly requested repository reports the capability error") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"registry"}, Name: "app/a"}) + require.NoError(t, err, "named listing works without a catalog") + + _, err = e.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"ghost"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestList_OnlyFailingRepositoriesFailTheListing(t *testing.T) { + noCollection := artifacttest.DefaultCapabilities() + noCollection.EnumerateCollection = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"registry": {Capabilities: &noCollection}}}) + _, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "when no repository can answer, the listing fails") +} + +func TestList_ConcurrencyIsInjectable(t *testing.T) { + e := newEnv(t, envConfig{ + repos: map[string]artifacttest.Options{"a": {}, "b": {}, "c": {}}, + options: []service.Option{service.WithConcurrency(1)}, + }) + for _, alias := range []string{"a", "b", "c"} { + seedList(t, e, alias, map[string]string{"app/x:1": alias}, nil) + } + result, err := e.svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + assert.Len(t, result.Artifacts, 3) +} diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go new file mode 100644 index 00000000000..3c1dd4b224c --- /dev/null +++ b/pkg/artifact/service/copy.go @@ -0,0 +1,311 @@ +package service + +import ( + "context" + "fmt" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// copyPlan is everything resolved before a copy mutates the destination. +type copyPlan struct { + ref artifact.Ref + src, dst *target + pinned artifact.PinnedReference + inspection *artifact.Inspection +} + +// copyWrite is how a copy will write the destination: the incoming description, the write +// closure, and a cleanup for any staging it owns. +type copyWrite struct { + in *incoming + write writer + cleanup func() +} + +// Copy implements [artifact.Service]. The source reference is resolved once and verified against +// BOTH the source and destination policies before the destination is touched; force never +// bypasses that. A compatible native transfer is used when available, otherwise the bounded +// staging fallback repacks the unchanged Record for the destination's packaging. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Copy(ctx context.Context, request artifact.CopyRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Copy")() + + op := s.begin(artifact.ActionCopy, request.To, request.Ref.String()) + err := s.copyArtifact(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func validateCopy(request *artifact.CopyRequest) error { + if request.From == "" || request.To == "" || request.From == request.To { + return fmt.Errorf("%w: copy needs distinct source and destination repositories", errUtils.ErrArtifactConfigInvalid) + } + return request.Ref.Validate() +} + +func (s *Service) copyArtifact(ctx context.Context, op *operation, request *artifact.CopyRequest) error { + if err := validateCopy(request); err != nil { + return err + } + plan, err := s.planCopy(ctx, op, request) + if err != nil { + return err + } + if plan.inspection.Record == nil && !canNativeCopy(plan) { + return artifact.NewUnsupportedCapability(request.To, opCopy, "transfer/graph", "a native OCI graph can only be copied between repositories that support a compatible native transfer") + } + cw, err := s.copyWriter(ctx, op, plan) + if err != nil { + return err + } + defer cw.cleanup() + + pub := &publication{op: op, tgt: plan.dst, in: cw.in, force: request.Force, write: cw.write, state: &publishState{}} + if err := s.commit(ctx, pub); err != nil { + return err + } + if op.result.Outcome == "" { + op.result.Outcome = artifact.OutcomeCreated + } + op.result.Repository = request.To + if op.result.Outcome != artifact.OutcomeUnchanged { + s.carryEvidence(ctx, op, plan) + } + return nil +} + +// planCopy checks capabilities and lock guarantees, resolves the source once, and enforces both policies. +func (s *Service) planCopy(ctx context.Context, op *operation, request *artifact.CopyRequest) (*copyPlan, error) { + srcInfo, srcRepo, err := s.openForRead(ctx, request.From, opCopy) + if err != nil { + return nil, err + } + dstInfo, _, err := s.describeForWrite(request.To, opCopy) + if err != nil { + return nil, err + } + if err := checkLockGuarantee(&dstInfo, opCopy); err != nil { + return nil, err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, srcRepo, request.From, request.Ref) + if err != nil { + return nil, err + } + if _, err := gate(ctx, request.From, srcRepo, inspection, srcInfo.Policy, dstInfo.Policy); err != nil { + return nil, err + } + dstRepo, err := s.repos.Open(ctx, request.To) + if err != nil { + return nil, err + } + return ©Plan{ + ref: request.Ref, + src: &target{alias: request.From, repo: srcRepo, info: &srcInfo}, + dst: &target{alias: request.To, repo: dstRepo, info: &dstInfo}, + pinned: pinned, + inspection: inspection, + }, nil +} + +func canNativeCopy(plan *copyPlan) bool { + copier, ok := plan.dst.repo.(artifact.Copier) + return ok && copier.CanCopyFrom(plan.src.repo) +} + +// copyWriter chooses the native transfer or the staging fallback. +func (s *Service) copyWriter(ctx context.Context, op *operation, plan *copyPlan) (*copyWrite, error) { + if canNativeCopy(plan) { + return nativeCopyWriter(plan) + } + prepared, cleanup, err := s.stageCopy(ctx, op, plan) + if err != nil { + return nil, err + } + in, err := newIncoming(prepared) + if err != nil { + cleanup() + return nil, err + } + in.inspection = plan.inspection + write := func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return plan.dst.repo.Publish(ctx, prepared, options) + } + return ©Write{in: in, write: write, cleanup: cleanup}, nil +} + +// nativeRef is the destination reference of a native transfer: the Record's name and version, or +// the requested tag for a foreign graph. A digest-only graph has no tag to publish under. +func nativeRef(plan *copyPlan) (artifact.Ref, error) { + if record := plan.inspection.Record; record != nil { + return artifact.Ref{Name: record.Name, Version: record.Version}, nil + } + if plan.ref.Version == "" { + return artifact.Ref{}, fmt.Errorf("%w: copying a native graph needs a name:version reference to publish under in the destination", errUtils.ErrArtifactRefInvalid) + } + return artifact.Ref{Name: plan.ref.Name, Version: plan.ref.Version}, nil +} + +func nativeCopyWriter(plan *copyPlan) (*copyWrite, error) { + copier, _ := plan.dst.repo.(artifact.Copier) + ref, err := nativeRef(plan) + if err != nil { + return nil, err + } + in := &incoming{ref: ref, inspection: plan.inspection} + if record := plan.inspection.Record; record != nil { + if err := in.bindRecord(record, plan.inspection); err != nil { + return nil, err + } + } + write := func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return copier.CopyFrom(ctx, plan.src.repo, plan.pinned, artifact.CopyFromOptions(options)) + } + return ©Write{in: in, write: write, cleanup: func() {}}, nil +} + +// bindRecord fills the identity of a stored Record into the incoming description. +func (in *incoming) bindRecord(record *artifact.Record, inspection *artifact.Inspection) error { + recordDigest, err := record.RecordDigest() + if err != nil { + return err + } + intent, err := record.IntentDigest() + if err != nil { + return err + } + in.record, in.recordDigest, in.intent = record, recordDigest, intent + in.payload = inspection.Representation.Payload + in.size = payloadSize(inspection) + return nil +} + +// stageCopy reads the pinned source into a bounded temporary tree and repacks the unchanged Record +// for the destination packaging. The returned cleanup removes every owned temporary directory. +func (s *Service) stageCopy(ctx context.Context, op *operation, plan *copyPlan) (*artifact.PreparedArtifact, func(), error) { + content, err := plan.src.repo.Read(ctx, plan.pinned) + if err != nil { + return nil, nil, fmt.Errorf("read %s from repository %q: %w", plan.pinned.Ref, plan.src.alias, err) + } + defer content.Close() + if err := bindContent(content, plan.inspection); err != nil { + return nil, nil, err + } + docs, err := collectDocumentBytes(content) + if err != nil { + return nil, nil, err + } + tree, err := os.MkdirTemp(s.stagingRoot, "atmos-artifact-copy-*") + if err != nil { + return nil, nil, fmt.Errorf("%w: create copy staging: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + cleanup := func() { _ = os.RemoveAll(tree) } + files := filepath.Join(tree, "files") + if err := materialize.Materialize(ctx, content, files); err != nil { + cleanup() + return nil, nil, err + } + prepared, err := s.repacker.Repack(ctx, &produce.RepackRequest{ + Record: *plan.inspection.Record, + Tree: files, + Documents: docs, + Packaging: packagingFor(plan.dst.info), + OperationID: op.id, + PreparedAt: op.preparedAt, + }) + if err != nil { + cleanup() + return nil, nil, err + } + prior := prepared.Cleanup + prepared.Cleanup = func() error { + var firstErr error + if prior != nil { + firstErr = prior() + } + cleanup() + return firstErr + } + return prepared, func() { _ = prepared.Close() }, nil +} + +func packagingFor(info *RepositoryInfo) artifact.Packaging { + if info.Packaging != "" { + return info.Packaging + } + if len(info.Capabilities.Packagings) > 0 { + return info.Capabilities.Packagings[0] + } + return artifact.PackagingNone +} + +// collectDocumentBytes reads the (bounded) document streams of the content by role. +func collectDocumentBytes(content *artifact.Content) (map[artifact.DocumentRole][]byte, error) { + docs := map[artifact.DocumentRole][]byte{} + if content.Record == nil { + return docs, nil + } + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobDocument || stream.Reader == nil { + continue + } + data, err := artifact.ReadBounded(stream.Reader, artifact.MaxDocumentSize) + if err != nil { + return nil, err + } + role, ok := documentRoleFor(content.Record, stream.Descriptor.Digest.String()) + if !ok { + return nil, fmt.Errorf("%w: a document stream matches no descriptor in the Record", errUtils.ErrArtifactIntegrityFailed) + } + docs[role] = data + } + return docs, nil +} + +func documentRoleFor(record *artifact.Record, digestValue string) (artifact.DocumentRole, bool) { + for _, doc := range record.Documents { + if doc.Digest.String() == digestValue { + return doc.Role, true + } + } + return "", false +} + +// carryEvidence moves signature evidence along with a copy when both repositories support it. +// Evidence is never invented: without both optional interfaces, a diagnostic explains that the +// destination holds no signatures. +func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPlan) { + reader, canRead := plan.src.repo.(artifact.EvidenceReader) + writer, canWrite := plan.dst.repo.(artifact.EvidenceWriter) + if !canRead || !canWrite { + if verificationRequired(plan.dst.info.Policy) { + op.warn("evidence-not-carried", "the destination policy requires verification but the repositories cannot transfer signature evidence; consumers of the destination copy must verify against the source evidence") + } + return + } + blobs, err := reader.ReadEvidence(ctx, &plan.pinned) + if err != nil || len(blobs) == 0 { + if err != nil { + op.warn("evidence-read-failed", err.Error()) + } + return + } + dstRef, err := nativeRef(plan) + if err == nil { + var dstPinned artifact.PinnedReference + if dstPinned, err = plan.dst.repo.Resolve(ctx, dstRef); err == nil { + _, err = writer.WriteEvidence(ctx, &dstPinned, blobs, nil) + } + } + if err != nil { + op.warn("evidence-write-failed", err.Error()) + } +} diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go new file mode 100644 index 00000000000..c335cfcc953 --- /dev/null +++ b/pkg/artifact/service/copy_test.go @@ -0,0 +1,273 @@ +package service_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// copyEnv has a "src" repository and a "dst" repository with chosen packagings. +func copyEnv(t *testing.T, srcPackaging, dstPackaging artifact.Packaging, options ...service.Option) *env { + t.Helper() + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"src": {}, "dst": {}}, options: options}) + e.setInfo("src", func(info *service.RepositoryInfo) { info.Packaging = srcPackaging }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = dstPackaging }) + return e +} + +func TestCopy_PreservesRecordAcrossPackagingChanges(t *testing.T) { + packagings := []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} + for _, from := range packagings { + for _, to := range packagings { + t.Run(string(from)+"->"+string(to), func(t *testing.T) { + e := copyEnv(t, from, to) + e.setInfo("dst", func(info *service.RepositoryInfo) { + info.Defaults = artifact.Metadata{Labels: map[string]string{"destination-default": "x"}} + }) + published := e.publish("src", e.source("hello")) + e.clock.Advance(72 * 3600e9) + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, "dst", result.Repository) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest, "a copy preserves Record bytes and identity") + assert.Equal(t, published.Record.ContentDigest, result.ContentDigest) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, to, inspected.Inspection.Representation.Packaging) + want, err := published.Record.Canonical() + require.NoError(t, err) + got, err := inspected.Inspection.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, want, got, "created_at and labels are not restamped; destination defaults do not apply to copies") + + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + state, err := materialize.InspectDestination(dest, &published.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + e.assertStagingClean() + }) + } + } +} + +func TestCopy_ConflictForceAndUnchanged(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz) + e.publish("src", e.source("original")) + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, first.Outcome) + + again, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "copying an identical artifact is a no-op") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) + + e.publishForce("src", e.source("different")) + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + forced, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, forced.Outcome) + e.assertStagingClean() +} + +func (e *env) publishForce(repository, src string) { + e.t.Helper() + request := e.publishRequest(repository, src) + request.Force = true + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(e.t, err) +} + +func TestCopy_PluginReleasesStayImmutableOnTheDestinationEvenWithForce(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithImmutabilityPolicy(pluginPolicy())) + pluginRequest := e.publishRequest("dst", e.source("plugin build")) + pluginRequest.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.Publish(context.Background(), pluginRequest) + require.NoError(t, err) + e.publish("src", e.source("generic")) + + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "immutable") +} + +func TestCopy_TagMovedMidCopyCopiesTheOriginallyPinnedSubject(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz) + original := e.publish("src", e.source("ORIGINAL")) + decoy := e.publishRequest("src", e.source("DECOY")) + decoy.Version = "2.0.0" + _, err := e.svc.Publish(context.Background(), decoy) + require.NoError(t, err) + decoyDigest, _ := e.repos["src"].RecordDigestOf(ref("tools/example", "2.0.0")) + moved := false + e.repos["src"].SetOnResolved(func(resolved artifact.Ref) { + if resolved == exampleRef && !moved { + moved = true + e.repos["src"].MoveRef(exampleRef, decoyDigest) + } + }) + before := e.repos["src"].Calls("Resolve") + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, e.repos["src"].Calls("Resolve")-before, "the source reference is resolved once") + assert.Equal(t, original.Operation.RecordDigest, result.RecordDigest) + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) + data, err := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, err) + assert.Equal(t, "ORIGINAL", string(data)) +} + +func TestCopy_ValidationAndCapabilitiesPrecedeAnyRead(t *testing.T) { + readOnly := artifacttest.DefaultCapabilities() + readOnly.Write = false + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"src": {}, "dst": {Capabilities: &readOnly}}}) + e.publish("src", e.source("hello")) + reads := e.repos["src"].TotalCalls() + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, reads, e.repos["src"].TotalCalls(), "the source is untouched when the destination cannot be written") + + for name, request := range map[string]artifact.CopyRequest{ + "same repository": {Ref: exampleRef, From: "src", To: "src"}, + "missing source": {Ref: exampleRef, To: "dst"}, + "missing target": {Ref: exampleRef, From: "src"}, + "invalid ref": {Ref: artifact.Ref{Name: "x"}, From: "src", To: "dst"}, + } { + _, err := e.svc.Copy(context.Background(), request) + require.Error(t, err, name) + } + writable := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + _, err = writable.svc.Copy(context.Background(), artifact.CopyRequest{Ref: ref("tools/example", "9"), From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestCopy_SourceAndDestinationPoliciesBothApplyAndForceNeverBypassesThem(t *testing.T) { + trusted := releaseSigner.TrustedPolicy() + tests := []struct { + name string + srcPolicy artifact.VerificationPolicy + dstPolicy artifact.VerificationPolicy + }{ + {"source requires", trusted, artifact.VerificationPolicy{}}, + {"destination requires", artifact.VerificationPolicy{}, trusted}, + {"both require", trusted, trusted}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + e.setInfo("src", func(info *service.RepositoryInfo) { info.Policy = tt.srcPolicy }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = tt.dstPolicy }) + dstCalls := e.repos["dst"].TotalCalls() + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "unsigned content cannot satisfy either policy, and force does not help") + assert.Equal(t, dstCalls, e.repos["dst"].TotalCalls(), "verification ran before the destination was touched") + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + if verificationConfigured(tt.dstPolicy) { + require.NoError(t, err, "evidence travels with the copy so the destination verifies independently") + assert.Equal(t, artifact.VerificationVerified, verified.Verification.Status) + } + }) + } +} + +func verificationConfigured(policy artifact.VerificationPolicy) bool { + return policy.RequireSignature || policy.HasTrustConstraints() +} + +// plainRepository hides optional extension interfaces of the wrapped repository. +type plainRepository struct{ artifact.Repository } + +func TestCopy_WarnsWhenEvidenceCannotTravel(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.static.Repositories["dst"] = plainRepository{Repository: e.repos["dst"]} + e.setInfo("src", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + var warned bool + for _, diagnostic := range result.Diagnostics { + warned = warned || diagnostic.Code == "evidence-not-carried" + } + assert.True(t, warned, "missing evidence transfer is reported, never silently dropped") +} + +// nativeRepository adds a compatible native transfer to the in-memory repository. +type nativeRepository struct { + *artifacttest.Repository + copies int + options artifact.CopyFromOptions + compat bool +} + +func (n *nativeRepository) CanCopyFrom(artifact.Repository) bool { return n.compat } + +//nolint:gocritic // The pinned reference is passed by value because artifact.Copier fixes the signature. +func (n *nativeRepository) CopyFrom(ctx context.Context, source artifact.Repository, pinned artifact.PinnedReference, options artifact.CopyFromOptions) (*artifact.PublishResult, error) { + n.copies++ + n.options = options + inspection, err := source.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: pinned, Representation: inspection.Representation, Location: "native://copy"}, nil +} + +func TestCopy_UsesNativeTransferWhenCompatible(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + published := e.publish("src", e.source("hello")) + native := &nativeRepository{Repository: e.repos["dst"], compat: true} + e.static.Repositories["dst"] = native + reads := e.repos["src"].Calls("Read") + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies) + assert.Equal(t, reads, e.repos["src"].Calls("Read"), "the staging fallback is not used") + assert.Equal(t, "native://copy", result.Location) + assert.Equal(t, published.Operation.RecordDigest, result.RecordDigest) + assert.Nil(t, native.options.ExpectedPrior) + assert.NotNil(t, native.options.Lock, "the destination lock is held across a native transfer") + assert.False(t, native.options.Force, "force applies only when replacement was decided") + + native.compat = false + _, err = e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies, "an incompatible pair falls back to staging") +} diff --git a/pkg/artifact/service/doc.go b/pkg/artifact/service/doc.go new file mode 100644 index 00000000000..4003b651c6e --- /dev/null +++ b/pkg/artifact/service/doc.go @@ -0,0 +1,18 @@ +// Package service implements [artifact.Service]: the repository-independent orchestration of +// publish, pull, copy, inspect, list, search, delete, document reads, sign, verify, and pin over +// [artifact.Repository] implementations. +// +// The service owns name and version selection, producer preparation, effective metadata, digest +// pinning, conflict policy, retries, and result aggregation; repositories own storage mechanics +// and capabilities. Three rules hold for every operation: +// +// - A mutable reference is resolved exactly once; every later read, verification, and copy uses +// the pinned subject, so a tag moved mid-operation can never substitute different bytes. +// - Capabilities and lock guarantees are checked before the first mutation, locks are acquired in +// canonical repository/resource order, and they are released on success, error, and cancellation. +// - Required verification runs before destination publication or local materialization, and +// force never bypasses it. Without any trust configuration the result is "unchecked", never +// "verified". +// +// Every method returns its typed result alongside the error so reporting can describe completed work. +package service diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go new file mode 100644 index 00000000000..f4620ad2490 --- /dev/null +++ b/pkg/artifact/service/edge_test.go @@ -0,0 +1,436 @@ +package service_test + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// mockSetup wires a service around one mock repository named "dst" (and optionally "src"). +type mockSetup struct { + ctrl *gomock.Controller + repo *artifact.MockRepository + svc *service.Service + e *env +} + +func newMockSetup(t *testing.T, caps *artifact.Capabilities, extra ...service.Option) *mockSetup { + t.Helper() + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(*caps).AnyTimes() + e := newEnv(t, envConfig{}) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"dst": repo}, + Info: map[string]service.RepositoryInfo{"dst": {Capabilities: *caps, Packaging: artifact.PackagingNone}}, + } + options := append([]service.Option{service.WithStagingRoot(e.staging), service.WithClock(e.clock.Now), service.WithRetry(2, 0), service.WithSleep(func(context.Context, time.Duration) error { return nil })}, extra...) + svc, err := service.New(repos, nil, options...) + require.NoError(t, err) + return &mockSetup{ctrl: ctrl, repo: repo, svc: svc, e: e} +} + +func sealedRecord(t *testing.T) (artifact.Record, digest.Digest) { + t.Helper() + record := artifact.Record{Name: "tools/example", Version: "1.0.0", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + return record, recordDigest +} + +// expectNoLock lets a mock repository accept lock calls with a trivial handle. +func (m *mockSetup) expectLock() { + handle := artifact.NewMockLockHandle(m.ctrl) + handle.EXPECT().Resource().Return("r").AnyTimes() + handle.EXPECT().Owner().Return("o").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + handle.EXPECT().Release(gomock.Any()).Return(nil).AnyTimes() + m.repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil).AnyTimes() +} + +func TestPublish_ReconciliationNeverClaimsSuccessWithoutProof(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + _, otherDigest := sealedRecord(t) + + tests := []struct { + name string + reconcile func(m *mockSetup) + wantErr error + }{ + {"reference cannot be read", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errTransport) + }, errUtils.ErrArtifactPublicationAmbiguous}, + {"reference now exists with unrelated content", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{ResolvedDigest: otherDigest, RecordDigest: otherDigest}, nil) + }, errUtils.ErrArtifactConflict}, + {"matching record but descriptors differ", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).DoAndReturn(func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + return artifact.PinnedReference{ResolvedDigest: "sha256:x", RecordDigest: lastPreparedDigest}, nil + }) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.Inspection{ + Representation: artifact.Representation{Payload: []ocispec.Descriptor{{Digest: digest.FromString("other-bytes"), Size: 1}}}, + }, nil) + }, errUtils.ErrArtifactIntegrityFailed}, + {"matching record but inspection fails", func(m *mockSetup) { + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).DoAndReturn(func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + return artifact.PinnedReference{ResolvedDigest: "sha256:x", RecordDigest: lastPreparedDigest}, nil + }) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(nil, errTransport) + }, errUtils.ErrArtifactPublicationAmbiguous}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + // The first Resolve is the pre-write read of the current state. + first := m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound) + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, prepared *artifact.PreparedArtifact, _ artifact.PublishOptions) (*artifact.PublishResult, error) { + d, err := prepared.Record.RecordDigest() + require.NoError(t, err) + lastPreparedDigest = d + return nil, errTransport + }, + ).After(first) + tt.reconcile(m) + result, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", src)) + require.ErrorIs(t, err, tt.wantErr) + assert.Empty(t, result.Outcome, "never reported as success") + }) + } +} + +// lastPreparedDigest is set by the publish expectation for use in later reconcile expectations. +var lastPreparedDigest digest.Digest + +func TestPublish_ExistingReferenceVanishingDuringReconcileIsAmbiguous(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + e := newEnv(t, envConfig{}) + prior := e.publish("dst", src) + changed := e.source("changed") + + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: prior.Operation.RecordDigest, RecordDigest: prior.Operation.RecordDigest} + record := prior.Record + gomock.InOrder( + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil), + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errTransport), + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound), + ) + request := m.e.publishRequest("dst", changed) + request.Force = true + _, err := m.svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) +} + +func TestPublish_NotCommittedReconciliationReturnsTheOriginalError(t *testing.T) { + src := newEnv(t, envConfig{}).source("hello") + caps := artifacttest.DefaultCapabilities() + e := newEnv(t, envConfig{}) + prior := e.publish("dst", src) + + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: prior.Operation.RecordDigest, RecordDigest: prior.Operation.RecordDigest} + record := prior.Record + gomock.InOrder( + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil), + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errTransport), + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil), + ) + request := m.e.publishRequest("dst", m.e.source("changed")) + request.Force = true + _, err := m.svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errTransport) + require.NotErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous, "the reference provably still holds the prior state") +} + +func TestPublish_ReadFailuresOfTheCurrentStateAreReported(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + t.Run("resolve fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errTransport) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errTransport) + }) + t.Run("inspect fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.expectLock() + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{ResolvedDigest: "sha256:abc"}, nil) + m.repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(nil, errTransport) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errTransport) + }) + t.Run("open fails", func(t *testing.T) { + e := newEnv(t, envConfig{}) + ctrl := gomock.NewController(t) + repos := service.NewMockRepositories(ctrl) + repos.EXPECT().Describe("dst").Return(service.RepositoryInfo{Alias: "dst", Capabilities: caps, Packaging: artifact.PackagingNone}, nil) + repos.EXPECT().Open(gomock.Any(), "dst").Return(nil, errTransport) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + _, err = svc.Publish(context.Background(), e.publishRequest("dst", e.source("x"))) + require.ErrorIs(t, err, errTransport) + e.assertStagingClean() + }) +} + +func TestPublish_ExistingForeignGraphIsNotComparableAndNeedsForce(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + m := newMockSetup(t, &caps) + m.expectLock() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: digest.FromString("graph")} + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned}, nil) + _, err := m.svc.Publish(context.Background(), m.e.publishRequest("dst", m.e.source("x"))) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "no portable record") +} + +func TestPull_Edges(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + + t.Run("native graph cannot be materialized", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned}, nil) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("content bound to another record is rejected", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + other := record + other.Version = "9.9.9" + require.NoError(t, other.Seal()) + m.repo.EXPECT().Read(gomock.Any(), pinned).Return(&artifact.Content{Record: &other}, nil) + dest := out(t) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.True(t, missing(dest)) + }) + t.Run("read fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + m.repo.EXPECT().Read(gomock.Any(), pinned).Return(nil, errTransport) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errTransport) + }) + t.Run("inspect fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(nil, errTransport) + _, err := m.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errTransport) + }) +} + +func TestShowDocument_TamperedDocumentFailsItsDescriptor(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + want := []byte("genuine readme") + descriptor := artifact.Document{Role: artifact.DocumentReadme, SourcePath: "README.md", MediaType: artifact.DocumentMediaType, Size: int64(len(want)), Digest: digest.FromBytes(want)} + _ = record + + for name, content := range map[string][]byte{"altered bytes": []byte("tampered readme"), "oversize": bytes.Repeat([]byte("x"), int(artifact.MaxDocumentSize)+1)} { + t.Run(name, func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(&artifact.DocumentStream{Document: descriptor, Reader: io.NopCloser(bytes.NewReader(content))}, nil) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.Error(t, err) + assert.True(t, errors.Is(err, errUtils.ErrArtifactIntegrityFailed) || errors.Is(err, errUtils.ErrArtifactTooLarge), "%v", err) + }) + } + t.Run("open fails", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(nil, errTransport) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errTransport) + }) +} + +func TestList_PaginationGuards(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + t.Run("non-advancing cursor", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).Return(&artifact.ListPage{NextCursor: "stuck"}, nil).Times(2) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + }) + t.Run("list error", func(t *testing.T) { + m := newMockSetup(t, &caps) + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, errTransport) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errTransport) + }) + t.Run("canceled", func(t *testing.T) { + m := newMockSetup(t, &caps) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := m.svc.List(ctx, artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, context.Canceled) + }) + t.Run("write-only repository", func(t *testing.T) { + writeOnly := caps + writeOnly.Read = false + m := newMockSetup(t, &writeOnly) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) +} + +// graphSource is a source repository whose artifact is a foreign OCI graph without a Record. +func graphSource(ctrl *gomock.Controller, pinned *artifact.PinnedReference) *artifact.MockRepository { + repo := artifact.NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(artifacttest.DefaultCapabilities()).AnyTimes() + repo.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(*pinned, nil).AnyTimes() + root := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString("graph-root"), Size: 10} + repo.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.Inspection{ + Pinned: *pinned, + Representation: artifact.Representation{Packaging: artifact.PackagingOCIGraph, OriginalRoot: &root}, + }, nil).AnyTimes() + return repo +} + +func TestCopy_ForeignGraphNeedsANativeTransfer(t *testing.T) { + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: digest.FromString("graph-root")} + build := func(t *testing.T, dst artifact.Repository) *service.Service { + ctrl := gomock.NewController(t) + e := newEnv(t, envConfig{}) + src := graphSource(ctrl, &pinned) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"src": src, "dst": dst}, + Info: map[string]service.RepositoryInfo{ + "src": {Capabilities: artifacttest.DefaultCapabilities()}, + "dst": {Capabilities: artifacttest.DefaultCapabilities(), Packaging: artifact.PackagingNone}, + }, + } + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + return svc + } + + t.Run("no native transfer is a capability error", func(t *testing.T) { + svc := build(t, artifacttest.New(artifacttest.Options{})) + _, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("native transfer carries the graph", func(t *testing.T) { + native := &nativeRepository{Repository: artifacttest.New(artifacttest.Options{}), compat: true} + svc := build(t, native) + result, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, 1, native.copies) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + }) + t.Run("a digest-only graph has no tag to publish under", func(t *testing.T) { + native := &nativeRepository{Repository: artifacttest.New(artifacttest.Options{}), compat: true} + svc := build(t, native) + digestRef := artifact.Ref{Name: "tools/example", Digest: digest.FromString("graph-root")} + _, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: digestRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) +} + +func TestCopy_EvidenceCarryFailuresAreReported(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.static.Repositories["src"] = failingEvidenceReader{Repository: e.repos["src"]} + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + var warned bool + for _, diagnostic := range result.Diagnostics { + warned = warned || diagnostic.Code == "evidence-read-failed" + } + assert.True(t, warned) +} + +type failingEvidenceReader struct{ artifact.Repository } + +func (failingEvidenceReader) ReadEvidence(context.Context, *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + return nil, errTransport +} + +func TestSign_RejectsADigestThatIsNotTheSignableSubject(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + record, recordDigest := sealedRecord(t) + // An OCI-style repository resolves the digest to a manifest that is not the Record digest. + manifest := digest.FromString("manifest") + pinned := artifact.PinnedReference{Ref: artifact.Ref{Name: "tools/example", Digest: manifest}, ResolvedDigest: manifest, RecordDigest: recordDigest} + m := newMockSetup(t, &caps, service.WithSigner(releaseSigner)) + m.repo.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + _, err := m.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: manifest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestCopy_RepackFailureCleansUpStaging(t *testing.T) { + repacker := service.NewMockRepacker(gomock.NewController(t)) + repacker.EXPECT().Repack(gomock.Any(), gomock.Any()).Return(nil, errTransport) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithRepacker(repacker)) + e.publish("src", e.source("hello")) + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.ErrorIs(t, err, errTransport) + e.assertStagingClean() + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) +} + +func TestPublish_DefaultBackoffSleepAndCancellationDuringBackoff(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 1) + svc, err := service.New(e.static, nil, service.WithRetry(2, time.Millisecond), service.WithStagingRoot(e.staging)) + require.NoError(t, err) + result, err := svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err, "the default context-aware sleep elapses and the retry succeeds") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + + other := newEnv(t, envConfig{}) + other.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 5) + ctx, cancel := context.WithCancel(context.Background()) + svc, err = service.New(other.static, nil, service.WithRetry(3, time.Hour), service.WithStagingRoot(other.staging), + service.WithSleep(func(ctx context.Context, _ time.Duration) error { cancel(); return ctx.Err() })) + require.NoError(t, err) + _, err = svc.Publish(ctx, other.publishRequest("dst", other.source("hello"))) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, other.repos["dst"].Calls("Publish")) + + real, err := service.New(other.static, nil, service.WithRetry(2, time.Hour), service.WithStagingRoot(other.staging)) + require.NoError(t, err) + other.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 5) + cancelled, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stop() + _, err = real.Publish(cancelled, other.publishRequest("dst", other.source("hello"))) + require.ErrorIs(t, err, context.DeadlineExceeded, "an hour-long default backoff is interrupted by the context") +} diff --git a/pkg/artifact/service/enumerate.go b/pkg/artifact/service/enumerate.go new file mode 100644 index 00000000000..78553c6e4f4 --- /dev/null +++ b/pkg/artifact/service/enumerate.go @@ -0,0 +1,128 @@ +package service + +import ( + "context" + "fmt" + "sort" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// maxListPages bounds pagination so a repository with a non-advancing cursor cannot loop forever. +const maxListPages = 100_000 + +// listOutcome is the result of enumerating one repository. +type listOutcome struct { + entries []artifact.ListEntry + listing artifact.RepositoryListing + err error +} + +// listRepository enumerates one repository, consuming pagination, and applies the label and text +// filters. A repository without the needed enumeration capability yields a capability error that +// explains direct named operations still work. +func (s *Service) listRepository(ctx context.Context, alias string, request *enumeration) listOutcome { + outcome := listOutcome{listing: artifact.RepositoryListing{Repository: alias}} + fail := func(err error) listOutcome { + outcome.err = fmt.Errorf("repository %q: %w", alias, err) + outcome.listing.Diagnostics = append(outcome.listing.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "list-failed", Message: err.Error()}) + return outcome + } + info, err := s.repos.Describe(alias) + if err != nil { + return fail(err) + } + if err := checkEnumeration(alias, &info.Capabilities, request.name); err != nil { + return fail(err) + } + repo, err := s.repos.Open(ctx, alias) + if err != nil { + return fail(err) + } + entries, err := consumePages(ctx, repo, request) + if err != nil { + return fail(err) + } + for index := range entries { + entries[index].Repository = alias + if matchesFilters(&entries[index], request) { + outcome.entries = append(outcome.entries, entries[index]) + } + } + outcome.listing.Count = len(outcome.entries) + return outcome +} + +func checkEnumeration(alias string, caps *artifact.Capabilities, name string) error { + if !caps.Read { + return artifact.NewUnsupportedCapability(alias, opList, "read", "the repository is write-only") + } + if name != "" && !caps.EnumerateNamed { + return artifact.NewUnsupportedCapability(alias, opList, "enumerate/named", "this repository cannot list versions of a name; direct named operations such as inspect still work") + } + if name == "" && !caps.EnumerateCollection { + return artifact.NewUnsupportedCapability(alias, opList, "enumerate/collection", "this repository has no native collection listing or catalog; direct named operations such as inspect still work") + } + return nil +} + +func consumePages(ctx context.Context, repo artifact.Repository, request *enumeration) ([]artifact.ListEntry, error) { + var entries []artifact.ListEntry + cursor := "" + for page := 0; page < maxListPages; page++ { + if err := ctx.Err(); err != nil { + return entries, err + } + result, err := repo.List(ctx, artifact.ListQuery{Name: request.name, Labels: request.labels, Cursor: cursor, Refresh: request.refresh}) + if err != nil { + return entries, err + } + entries = append(entries, result.Entries...) + if result.NextCursor == "" { + return entries, nil + } + if result.NextCursor == cursor { + return entries, fmt.Errorf("%w: the repository returned the same list cursor twice", errUtils.ErrArtifactPublicationUnavailable) + } + cursor = result.NextCursor + } + return entries, fmt.Errorf("%w: the listing exceeded %d pages", errUtils.ErrArtifactTooLarge, maxListPages) +} + +// matchesFilters applies the exact label selectors (AND semantics) and the search text. +func matchesFilters(entry *artifact.ListEntry, request *enumeration) bool { + for _, label := range request.labels { + if value, ok := entry.Labels[label.Key]; !ok || value != label.Value { + return false + } + } + if request.text == "" { + return true + } + needle := strings.ToLower(request.text) + haystack := []string{entry.Name, entry.Kind, entry.Description} + for key, value := range entry.Labels { + haystack = append(haystack, key, value) + } + for _, text := range haystack { + if strings.Contains(strings.ToLower(text), needle) { + return true + } + } + return false +} + +// sortEntries orders entries by name, then version, then repository alias. +func sortEntries(entries []artifact.ListEntry) { + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].Name != entries[j].Name { + return entries[i].Name < entries[j].Name + } + if entries[i].Version != entries[j].Version { + return entries[i].Version < entries[j].Version + } + return entries[i].Repository < entries[j].Repository + }) +} diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go new file mode 100644 index 00000000000..131559c12db --- /dev/null +++ b/pkg/artifact/service/execute.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "errors" + "fmt" + "slices" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// reconcileState is what a read of the completed reference proves. +type reconcileState int + +const ( + // The reference could not be proven to hold or lack the expected content. + reconcileUnknown reconcileState = iota + + // The completed reference holds exactly the expected content. + reconcileCommitted + + // The reference still equals the observed prior state. + reconcileNotCommitted +) + +// execute writes with reconciliation. It never blind-retries: after an ambiguous failure it reads +// the completed reference and verifies the expected digests; only a proven not-committed state +// with a temporary error is retried, reusing the prepared artifact, operation ID, and time. +func (s *Service) execute(ctx context.Context, pub *publication, cur *current, decision verdict, lock artifact.LockHandle) (*artifact.PublishResult, error) { + for attempt := 1; ; attempt++ { + s.event(ctx, pub, PhasePending, attempt, nil) + options := artifact.PublishOptions{ExpectedPrior: cur.pinned, Force: decision == verdictReplace, Lock: lock} + result, err := pub.write(ctx, options) + if err == nil { + s.event(ctx, pub, PhaseCommitted, attempt, nil) + return result, nil + } + if isDefinitive(err) { + return nil, err + } + recovered, retry, failure := s.afterFailure(ctx, pub, cur, err, attempt) + if !retry { + return recovered, failure + } + } +} + +// afterFailure decides what an ambiguous write failure means by reading the completed reference. +// It returns a recovered result when the write is proven committed, retry when it is proven +// uncommitted and worth repeating, and otherwise the error to report. +func (s *Service) afterFailure(ctx context.Context, pub *publication, cur *current, writeErr error, attempt int) (*artifact.PublishResult, bool, error) { + found := reconcile(context.WithoutCancel(ctx), pub.tgt, pub.in, cur) + switch found.state { + case reconcileCommitted: + s.event(ctx, pub, PhaseReconciled, attempt, nil) + pub.op.info("reconciled", fmt.Sprintf("the repository reported %q but the completed reference holds the expected digests; success was confirmed by reading it", writeErr)) + return syntheticResult(&found.pinned, found.inspection), false, nil + case reconcileNotCommitted: + if !isTemporary(writeErr) || attempt >= s.attempts || ctx.Err() != nil { + return nil, false, writeErr + } + if sleepErr := s.sleep(ctx, s.backoff<<(attempt-1)); sleepErr != nil { + return nil, false, errors.Join(writeErr, sleepErr) + } + return nil, true, nil + case reconcileUnknown: + fallthrough + default: + ambiguous := fmt.Errorf("%w: operation %s on %s in repository %q failed and could not be reconciled; check the reference before retrying", errUtils.ErrArtifactPublicationAmbiguous, pub.op.id, pub.in.ref, pub.tgt.alias) + return nil, false, errors.Join(ambiguous, writeErr, found.err) + } +} + +// isDefinitive reports errors that prove nothing was committed, so reconciliation is unnecessary. +func isDefinitive(err error) bool { + for _, sentinel := range []error{ + errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactRecordInvalid, + errUtils.ErrArtifactTooLarge, errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, + errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactVerificationFailed, + } { + if errors.Is(err, sentinel) { + return true + } + } + return false +} + +// isTemporary reports errors that advertise themselves as retryable. +func isTemporary(err error) bool { + var temporary interface{ Temporary() bool } + return errors.As(err, &temporary) && temporary.Temporary() +} + +// reconciliation is what reading the completed reference proved. +type reconciliation struct { + state reconcileState + pinned artifact.PinnedReference + inspection *artifact.Inspection + err error +} + +// reconcile reads the completed reference and classifies it against the expected content. +func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) reconciliation { + ref := in.ref + pinned, err := tgt.repo.Resolve(ctx, ref) + switch { + case errors.Is(err, errUtils.ErrArtifactNotFound): + if cur.pinned == nil && !cur.unavailable { + return reconciliation{state: reconcileNotCommitted} + } + return reconciliation{err: fmt.Errorf("the previously present reference %s is now absent: %w", ref, err)} + case err != nil: + return reconciliation{err: fmt.Errorf("reconcile %s: %w", ref, err)} + } + if pinned.RecordDigest == in.recordDigest { + inspection, inspectErr := tgt.repo.Inspect(ctx, pinned) + if inspectErr != nil { + return reconciliation{pinned: pinned, err: fmt.Errorf("reconcile %s: %w", ref, inspectErr)} + } + if matchErr := matchesExpected(in, inspection); matchErr != nil { + return reconciliation{pinned: pinned, inspection: inspection, err: matchErr} + } + return reconciliation{state: reconcileCommitted, pinned: pinned, inspection: inspection} + } + if cur.pinned != nil && pinned.ResolvedDigest == cur.pinned.ResolvedDigest { + return reconciliation{state: reconcileNotCommitted, pinned: pinned} + } + return reconciliation{pinned: pinned, err: fmt.Errorf("%w: %s now resolves to %s, which is neither the prior state nor the expected content", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest)} +} + +// matchesExpected verifies the stored Record and, when both sides describe them, the payload +// descriptors against what this operation prepared. +func matchesExpected(in *incoming, inspection *artifact.Inspection) error { + if inspection.Record != nil { + stored, err := inspection.Record.RecordDigest() + if err != nil || stored != in.recordDigest { + return fmt.Errorf("%w: the stored record does not match the prepared record", errUtils.ErrArtifactIntegrityFailed) + } + } + if len(in.payload) == 0 || len(inspection.Representation.Payload) == 0 { + return nil + } + want := descriptorDigests(in.payload) + got := descriptorDigests(inspection.Representation.Payload) + if !slices.Equal(want, got) { + return fmt.Errorf("%w: the stored payload descriptors do not match the prepared payload", errUtils.ErrArtifactIntegrityFailed) + } + return nil +} + +func descriptorDigests(descriptors []ocispec.Descriptor) []string { + digests := make([]string, 0, len(descriptors)) + for index := range descriptors { + digests = append(digests, descriptors[index].Digest.String()) + } + slices.Sort(digests) + return digests +} + +func syntheticResult(pinned *artifact.PinnedReference, inspection *artifact.Inspection) *artifact.PublishResult { + return &artifact.PublishResult{Pinned: *pinned, Representation: inspection.Representation, Location: inspection.Representation.Locator} +} diff --git a/pkg/artifact/service/helpers_test.go b/pkg/artifact/service/helpers_test.go new file mode 100644 index 00000000000..ae533c2b328 --- /dev/null +++ b/pkg/artifact/service/helpers_test.go @@ -0,0 +1,196 @@ +package service_test + +import ( + "context" + "fmt" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// startTime is the fixed clock origin for service tests. +var startTime = time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC) + +// fakeClock is a settable clock. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func (c *fakeClock) Set(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = t +} + +// recordingJournal records lifecycle events. +type recordingJournal struct { + mu sync.Mutex + entries []service.JournalEntry +} + +func (j *recordingJournal) Record(_ context.Context, entry *service.JournalEntry) { + j.mu.Lock() + defer j.mu.Unlock() + j.entries = append(j.entries, *entry) +} + +func (j *recordingJournal) phases() []service.JournalPhase { + j.mu.Lock() + defer j.mu.Unlock() + phases := make([]service.JournalPhase, 0, len(j.entries)) + for _, entry := range j.entries { + phases = append(phases, entry.Phase) + } + return phases +} + +func (j *recordingJournal) snapshot() []service.JournalEntry { + j.mu.Lock() + defer j.mu.Unlock() + return append([]service.JournalEntry(nil), j.entries...) +} + +// temporaryError is a retryable failure. +type temporaryError struct{ message string } + +func (e *temporaryError) Error() string { return e.message } +func (e *temporaryError) Temporary() bool { return true } + +// env is a service wired to named in-memory repositories. +type env struct { + t *testing.T + repos map[string]*artifacttest.Repository + static *service.StaticRepositories + clock *fakeClock + journal *recordingJournal + staging string + sleeps []time.Duration + sleepMu sync.Mutex + onSleep func(time.Duration) + svc *service.Service +} + +type envConfig struct { + repos map[string]artifacttest.Options + infos map[string]service.RepositoryInfo + options []service.Option +} + +// newEnv builds an environment with a "dst" repository by default. +func newEnv(t *testing.T, config envConfig) *env { + t.Helper() + if config.repos == nil { + config.repos = map[string]artifacttest.Options{"dst": {Alias: "dst"}} + } + e := &env{ + t: t, + repos: map[string]*artifacttest.Repository{}, + clock: &fakeClock{now: startTime}, + journal: &recordingJournal{}, + staging: t.TempDir(), + } + e.static = &service.StaticRepositories{Repositories: map[string]artifact.Repository{}, Info: config.infos} + if e.static.Info == nil { + e.static.Info = map[string]service.RepositoryInfo{} + } + for alias, opts := range config.repos { + opts.Alias = alias + repo := artifacttest.New(opts) + e.repos[alias] = repo + e.static.Repositories[alias] = repo + } + var counter atomic.Int64 + options := []service.Option{ + service.WithClock(e.clock.Now), + service.WithIDGenerator(func() string { return fmt.Sprintf("op-%d", counter.Add(1)) }), + service.WithJournal(e.journal), + service.WithSleep(func(_ context.Context, d time.Duration) error { + e.sleepMu.Lock() + e.sleeps = append(e.sleeps, d) + hook := e.onSleep + e.sleepMu.Unlock() + if hook != nil { + hook(d) + } + return nil + }), + service.WithRetry(3, time.Millisecond), + service.WithStagingRoot(e.staging), + service.WithOwner("test-owner"), + } + svc, err := service.New(e.static, nil, append(options, config.options...)...) + require.NoError(t, err) + e.svc = svc + return e +} + +func (e *env) setInfo(alias string, mutate func(*service.RepositoryInfo)) { + info, err := e.static.Describe(alias) + require.NoError(e.t, err) + mutate(&info) + e.static.Info[alias] = info +} + +// source writes a small file package whose data.txt holds content and returns its directory. +func (e *env) source(content string) string { + e.t.Helper() + src := e.t.TempDir() + fixtures.WriteFile(e.t, src, "data.txt", content, 0o644) + fixtures.WriteFile(e.t, src, "bin/run.sh", fixtures.HelperScript, 0o755) + fixtures.WriteFile(e.t, src, fixtures.ReservedDir+"/README.md", "package readme\n", 0o644) + return src +} + +func (e *env) publishRequest(repository, src string) artifact.PublishRequest { + return artifact.PublishRequest{Repository: repository, Source: src, Name: "tools/example", Version: "1.0.0"} +} + +func (e *env) publish(repository, src string) *service.PublishOutcome { + e.t.Helper() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest(repository, src)) + require.NoError(e.t, err) + return outcome +} + +// assertStagingClean asserts no owned temporary data remains. +func (e *env) assertStagingClean() { + e.t.Helper() + items, err := os.ReadDir(e.staging) + require.NoError(e.t, err) + assert.Empty(e.t, items, "owned temporary staging must be removed") +} + +func ref(name, version string) artifact.Ref { + return artifact.Ref{Name: name, Version: version} +} + +var exampleRef = ref("tools/example", "1.0.0") + +// missing reports whether the path does not exist, for readable "nothing was created" assertions. +func missing(path string) bool { + _, err := os.Stat(path) + return os.IsNotExist(err) +} diff --git a/pkg/artifact/service/internal_helpers_test.go b/pkg/artifact/service/internal_helpers_test.go new file mode 100644 index 00000000000..0d628dc3a2c --- /dev/null +++ b/pkg/artifact/service/internal_helpers_test.go @@ -0,0 +1,10 @@ +package service + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func ocispecDescriptor() ocispec.Descriptor { + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageIndex, Digest: digest.FromString("root"), Size: 4} +} diff --git a/pkg/artifact/service/internal_test.go b/pkg/artifact/service/internal_test.go new file mode 100644 index 00000000000..bcc4a1aa56c --- /dev/null +++ b/pkg/artifact/service/internal_test.go @@ -0,0 +1,172 @@ +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func TestCanonicalOrder_SortsByRepositoryThenResourceAndDropsDuplicates(t *testing.T) { + targets := []lockTarget{ + {alias: "b", resource: "z"}, + {alias: "a", resource: "y"}, + {alias: "a", resource: "x"}, + {alias: "b", resource: "a"}, + {alias: "a", resource: "x"}, + } + ordered := canonicalOrder(targets) + var got []string + for _, target := range ordered { + got = append(got, target.alias+"/"+target.resource) + } + assert.Equal(t, []string{"a/x", "a/y", "b/a", "b/z"}, got) + assert.Equal(t, "b", targets[0].alias, "the caller's slice is not reordered") +} + +func TestAcquireLocks_MultiResourceUsesCanonicalOrderAndReleasesInReverse(t *testing.T) { + ctrl := gomock.NewController(t) + var order []string + mkRepo := func(alias string) *artifact.MockRepository { + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + handle.EXPECT().Resource().Return("res").AnyTimes() + handle.EXPECT().Owner().Return("owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + order = append(order, "lock "+alias+"/"+request.Resource) + assert.Equal(t, "owner-x", request.Owner) + assert.Equal(t, "op-9", request.OperationID) + assert.Equal(t, artifact.LockScopeHost, request.Required) + return handle, nil + }) + handle.EXPECT().Release(gomock.Any()).DoAndReturn(func(context.Context) error { + order = append(order, "release "+alias) + return nil + }) + return repo + } + repoA := mkRepo("a") + repoB := mkRepo("b") + caps := artifact.Capabilities{LockScopes: []artifact.LockScope{artifact.LockScopeHost}} + infoA := &RepositoryInfo{Alias: "a", Capabilities: caps} + infoB := &RepositoryInfo{Alias: "b", Capabilities: caps} + + s := &Service{owner: "owner-x"} + op := &operation{id: "op-9"} + set, err := s.acquireLocks(context.Background(), op, "copy", []lockTarget{ + {alias: "b", repo: repoB, info: infoB, resource: "r2"}, + {alias: "a", repo: repoA, info: infoA, resource: "r1"}, + }) + require.NoError(t, err) + assert.NotNil(t, set.handleFor("a")) + assert.Nil(t, set.handleFor("c")) + assert.Equal(t, artifact.LockScopeHost, set.scope()) + require.NoError(t, set.release(context.Background(), op)) + assert.Equal(t, []string{"lock a/r1", "lock b/r2", "release b", "release a"}, order) +} + +func TestAcquireLocks_FailureReleasesWhatWasTakenAndInsufficientScopeFailsFirst(t *testing.T) { + ctrl := gomock.NewController(t) + first := artifact.NewMockRepository(ctrl) + second := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + handle.EXPECT().Resource().Return("res").AnyTimes() + handle.EXPECT().Owner().Return("owner").AnyTimes() + first.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + handle.EXPECT().Release(gomock.Any()).Return(nil) + second.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(nil, errUtils.ErrArtifactConflict) + caps := artifact.Capabilities{LockScopes: []artifact.LockScope{artifact.LockScopeHost}} + + s := &Service{owner: "o"} + op := &operation{id: "op"} + _, err := s.acquireLocks(context.Background(), op, "copy", []lockTarget{ + {alias: "a", repo: first, info: &RepositoryInfo{Capabilities: caps}, resource: "r"}, + {alias: "b", repo: second, info: &RepositoryInfo{Capabilities: caps}, resource: "r"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + + weak := &RepositoryInfo{Alias: "w", Capabilities: caps, RequiredLockScope: artifact.LockScopeRepository} + untouched := artifact.NewMockRepository(ctrl) // No Lock expectation: it must never be called. + _, err = s.acquireLocks(context.Background(), op, "copy", []lockTarget{{alias: "w", repo: untouched, info: weak, resource: "r"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestLockSet_NilAndEmptySetsAreSafe(t *testing.T) { + var set *lockSet + assert.Nil(t, set.handleFor("a")) + assert.Empty(t, set.scope()) + require.NoError(t, set.release(context.Background(), &operation{})) + assert.Empty(t, (&lockSet{}).scope()) +} + +func TestLockSet_ScopeIsTheWeakestHeld(t *testing.T) { + ctrl := gomock.NewController(t) + repoHandle := artifact.NewMockLockHandle(ctrl) + hostHandle := artifact.NewMockLockHandle(ctrl) + repoHandle.EXPECT().Scope().Return(artifact.LockScopeRepository).AnyTimes() + hostHandle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + set := &lockSet{held: []heldLock{{handle: repoHandle}, {handle: hostHandle}}} + assert.Equal(t, artifact.LockScopeHost, set.scope()) + set = &lockSet{held: []heldLock{{handle: hostHandle}, {handle: repoHandle}}} + assert.Equal(t, artifact.LockScopeHost, set.scope()) +} + +func TestSplitPolicyAndMerge(t *testing.T) { + global := artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: []string{"g.pub"}}}} + repo := artifact.VerificationPolicy{TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: "i", Issuer: "https://x"}}}}, Offline: true} + merged := mergePolicies(global, repo) + assert.True(t, merged.RequireSignature) + assert.True(t, merged.Offline) + require.Len(t, merged.TrustSets, 2, "sets are kept separate, never unioned") + calls := splitPolicy(merged) + require.Len(t, calls, 2) + assert.True(t, calls[0].Offline) + assert.Len(t, calls[1].TrustSets, 1) + + bare := splitPolicy(artifact.VerificationPolicy{}) + require.Len(t, bare, 1) + assert.True(t, bare[0].RequireSignature, "an empty policy that is consulted still demands a signature") + + assert.False(t, verificationRequired(artifact.VerificationPolicy{})) + assert.True(t, verificationRequired(artifact.VerificationPolicy{RequireSignature: true})) + assert.True(t, verificationRequired(artifact.VerificationPolicy{TrustSets: []artifact.TrustSet{{Keys: []string{"k"}}}})) +} + +func TestSubjectOf(t *testing.T) { + _, err := subjectOf(&artifact.Inspection{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + + root := artifact.Inspection{Representation: artifact.Representation{OriginalRoot: &ocispecRoot}} + subject, err := subjectOf(&root) + require.NoError(t, err) + assert.Equal(t, artifact.SubjectOCIRoot, subject.Kind) + assert.Equal(t, ocispecRoot.Digest, subject.Digest, "a native graph is signed by its original root, never an archive checksum") +} + +func TestMergeMetadataAndDifferences(t *testing.T) { + merged := mergeMetadata(artifact.Metadata{Description: "d", Labels: map[string]string{"a": "1"}}, artifact.Metadata{Labels: map[string]string{"a": "2", "b": "3"}}) + assert.Equal(t, "d", merged.Description) + assert.Equal(t, map[string]string{"a": "2", "b": "3"}, merged.Labels) + assert.Nil(t, mergeMetadata(artifact.Metadata{}, artifact.Metadata{}).Labels) + + a := &artifact.Record{Kind: "files"} + b := &artifact.Record{Kind: "other", KindData: artifact.KindData(`{"x":1}`)} + assert.Contains(t, describeDifferences(a, b), "kind data") + assert.Equal(t, "intent", describeDifferences(a, a)) + assert.Contains(t, describeDifferences(nil, a), "no portable record") +} + +func TestIsDefinitiveAndTemporary(t *testing.T) { + assert.True(t, isDefinitive(errUtils.ErrArtifactConflict)) + assert.True(t, isDefinitive(errUtils.ErrArtifactUnsupportedCapability)) + assert.False(t, isDefinitive(context.Canceled)) + assert.False(t, isTemporary(context.Canceled)) +} + +var ocispecRoot = ocispecDescriptor() diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go new file mode 100644 index 00000000000..6e54312ad13 --- /dev/null +++ b/pkg/artifact/service/interrupt_test.go @@ -0,0 +1,309 @@ +package service_test + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +var errTransport = errors.New("connection reset") + +func TestPublish_InterruptionBeforePayloadLeavesNothingAndRetryCreates(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.ErrorIs(t, err, errTransport) + assert.Empty(t, outcome.Operation.Outcome, "a failed write never reports success") + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + assert.False(t, e.repos["dst"].IsPending(exampleRef)) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "a non-temporary failure is never blind-retried") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + retry, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retry.Operation.Outcome) +} + +func TestPublish_InterruptionDuringPayloadOrAfterMetadataIsAmbiguousUntilSameIntentRetry(t *testing.T) { + for _, point := range []artifacttest.FailPoint{artifacttest.FailDuringPayload, artifacttest.FailAfterMetadata} { + t.Run(string(point), func(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(point, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + require.ErrorIs(t, err, errTransport, "the cause is retained for diagnostics") + assert.Contains(t, err.Error(), outcome.Operation.OperationID, "recovery diagnostics name the operation") + assert.Empty(t, outcome.Operation.Outcome, "partial work is never advertised as success") + assert.True(t, e.repos["dst"].IsPending(exampleRef), "the reference stays unavailable") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "no blind retry of an ambiguous mutation") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "readers cannot see a pending publication") + + retry, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err, "the same-intent retry repairs the interrupted publication") + assert.Equal(t, artifact.OutcomeCreated, retry.Operation.Outcome) + assert.False(t, e.repos["dst"].IsPending(exampleRef)) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + }) + } +} + +func TestPublish_DifferentIntentCannotStealAPendingPublication(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailDuringPayload, errTransport, 1) + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("first"))) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + + other, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("different"))) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + assert.Contains(t, err.Error(), "op-1", "the pending operation ID is surfaced for recovery") + assert.Empty(t, other.Operation.Outcome) +} + +func TestPublish_CommitWithLostResponseIsReconciledNotRetried(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, errTransport, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err, "success only after reconciliation read the completed reference") + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "reconciliation replaces the retry") + var reconciled bool + for _, diagnostic := range outcome.Operation.Diagnostics { + reconciled = reconciled || diagnostic.Code == "reconciled" + } + assert.True(t, reconciled, "the reconciliation is reported") + assert.Contains(t, e.journal.phases(), service.PhaseReconciled) + current, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok) + assert.Equal(t, outcome.Operation.RecordDigest, current) +} + +func TestPublish_CommitWithLostResponseOnReplacementIsReconciled(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("first")) + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, errTransport, 1) + request := e.publishRequest("dst", e.source("second")) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, outcome.Operation.Outcome) +} + +func TestPublish_TemporaryFailuresRetryOnlyAfterReconciliationProvesNothingCommitted(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 2) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 3, e.repos["dst"].Calls("Publish")) + e.sleepMu.Lock() + assert.Equal(t, []time.Duration{time.Millisecond, 2 * time.Millisecond}, e.sleeps, "exponential backoff") + e.sleepMu.Unlock() + + var attempts []int + for _, entry := range e.journal.snapshot() { + assert.Equal(t, "op-1", entry.OperationID, "every attempt reuses the operation ID") + if entry.Phase == service.PhasePending { + attempts = append(attempts, entry.Attempt) + } + } + assert.Equal(t, []int{1, 2, 3}, attempts) +} + +func TestPublish_TemporaryFailureExhaustsAttempts(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 10) + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + var temporary *temporaryError + require.ErrorAs(t, err, &temporary) + assert.Equal(t, 3, e.repos["dst"].Calls("Publish")) +} + +func TestPublish_TemporaryFailureAfterCommitIsReconciledWithoutRetry(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, &temporaryError{"timeout"}, 5) + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "a committed publication is never repeated") +} + +func TestPublish_DateRolloverAcrossRetriesReusesCapturedTime(t *testing.T) { + e := newEnv(t, envConfig{}) + e.clock.Set(time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC)) + e.onSleep = func(time.Duration) { e.clock.Advance(2 * time.Second) } // Crosses midnight between attempts. + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 1) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, "2026-03-04T23:59:59Z", outcome.Record.CreatedAt.String(), "the retry reuses the time captured before the first attempt") + assert.Equal(t, 2, e.repos["dst"].Calls("Publish")) + assert.Equal(t, "op-1", outcome.Operation.OperationID) + assert.Equal(t, []service.JournalPhase{ + service.PhasePrepared, service.PhaseLocked, service.PhasePending, service.PhasePending, service.PhaseCommitted, service.PhaseReleased, + }, e.journal.phases(), "one lifecycle, two attempts") +} + +func TestPublish_FailedForceReplacementLeavesReferenceUnavailableUntilSameIntentRetry(t *testing.T) { + e := newEnv(t, envConfig{}) + first := e.publish("dst", e.source("first")) + oldPinned, err := e.repos["dst"].Resolve(context.Background(), exampleRef) + require.NoError(t, err) + + e.repos["dst"].InjectFailure(artifacttest.FailDuringPayload, errTransport, 1) + request := e.publishRequest("dst", e.source("second")) + request.Force = true + _, err = e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + assert.True(t, e.repos["dst"].IsPending(exampleRef)) + _, err = e.repos["dst"].Resolve(context.Background(), exampleRef) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "no rollback is claimed: the reference is unavailable, not silently the old version") + + retried, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, retried.Operation.Outcome) + assert.NotEqual(t, first.Operation.RecordDigest, retried.Operation.RecordDigest) + + _, err = e.repos["dst"].Read(context.Background(), oldPinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the old digest never serves the replacement's bytes") +} + +func TestPublish_TwoWritersOneReferenceLockSerializesThem(t *testing.T) { + const writers = 8 + e := newEnv(t, envConfig{}) + sources := make([]string, writers) + for i := range sources { + sources[i] = e.source(fmt.Sprintf("content-%d", i)) + } + var wg sync.WaitGroup + var created, conflicts atomic.Int64 + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", sources[i])) + switch { + case err == nil: + created.Add(1) + case errors.Is(err, errUtils.ErrArtifactConflict): + conflicts.Add(1) + default: + t.Errorf("unexpected error: %v", err) + } + }() + } + wg.Wait() + assert.Equal(t, int64(1), created.Load(), "exactly one writer wins the reference") + assert.Equal(t, int64(writers-1), conflicts.Load()) + assert.Empty(t, e.repos["dst"].HeldLocks()) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "losers conflict before writing anything") +} + +func TestPublish_TwoWritersWithIdenticalIntentCreateOnce(t *testing.T) { + const writers = 6 + e := newEnv(t, envConfig{}) + src := e.source("same") + var wg sync.WaitGroup + var created, unchanged atomic.Int64 + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", src)) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + switch outcome.Operation.Outcome { + case artifact.OutcomeCreated: + created.Add(1) + case artifact.OutcomeUnchanged: + unchanged.Add(1) + } + }() + } + wg.Wait() + assert.Equal(t, int64(1), created.Load()) + assert.Equal(t, int64(writers-1), unchanged.Load()) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) +} + +// unlockedRepository hides the writer lock so the repository's own conditional guard is exercised. +type unlockedRepository struct { + *artifacttest.Repository +} + +func (u *unlockedRepository) Lock(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + return noopLock{resource: request.Resource}, nil +} + +type noopLock struct{ resource string } + +func (n noopLock) Resource() string { return n.resource } +func (n noopLock) Owner() string { return "noop" } +func (n noopLock) Scope() artifact.LockScope { return artifact.LockScopeHost } +func (n noopLock) Token() string { return "noop" } +func (n noopLock) ExpiresAt() (time.Time, bool) { return time.Time{}, false } +func (n noopLock) Renew(context.Context, time.Duration) error { return nil } +func (n noopLock) Release(context.Context) error { return nil } + +func TestPublish_ConditionalGuardRejectsRacingWritersEvenWithoutAWriterLock(t *testing.T) { + const writers = 6 + e := newEnv(t, envConfig{}) + bare := artifacttest.New(artifacttest.Options{Alias: "bare"}) + e.static.Repositories["bare"] = &unlockedRepository{Repository: bare} + e.static.Info["bare"] = service.RepositoryInfo{Alias: "bare", Capabilities: bare.Capabilities(), Packaging: artifact.PackagingNone} + + sources := make([]string, writers) + for i := range sources { + sources[i] = e.source(fmt.Sprintf("racing-%d", i)) + } + var wg sync.WaitGroup + var wins atomic.Int64 + var digests sync.Map + for i := 0; i < writers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("bare", sources[i])) + if err == nil { + wins.Add(1) + digests.Store(outcome.Operation.RecordDigest, true) + } + }() + } + wg.Wait() + assert.GreaterOrEqual(t, wins.Load(), int64(1)) + current, ok := bare.RecordDigestOf(exampleRef) + require.True(t, ok) + var count int + digests.Range(func(key, _ any) bool { count++; return true }) + assert.Equal(t, int64(count), wins.Load(), "each reported success corresponds to a distinct committed record") + pinned, err := bare.Resolve(context.Background(), exampleRef) + require.NoError(t, err) + assert.Equal(t, current, pinned.RecordDigest, "the reference is bound to exactly one artifact") +} diff --git a/pkg/artifact/service/locks.go b/pkg/artifact/service/locks.go new file mode 100644 index 00000000000..fdf78da3bb7 --- /dev/null +++ b/pkg/artifact/service/locks.go @@ -0,0 +1,140 @@ +package service + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// lockTarget is one resource to lock in one repository. +type lockTarget struct { + alias string + repo artifact.Repository + info *RepositoryInfo + resource string +} + +// heldLock is an acquired lock together with where it came from. +type heldLock struct { + target lockTarget + handle artifact.LockHandle +} + +// lockSet is a group of held locks released together in reverse acquisition order. +type lockSet struct { + held []heldLock +} + +// requiredScope returns the guarantee a repository requires, defaulting to host scope. +func requiredScope(info *RepositoryInfo) artifact.LockScope { + if info.RequiredLockScope == "" { + return artifact.LockScopeHost + } + return info.RequiredLockScope +} + +// checkLockGuarantee rejects an insufficient lock guarantee before anything is mutated. +func checkLockGuarantee(info *RepositoryInfo, operation string) error { + required := requiredScope(info) + if info.Capabilities.HasLockScope(required) { + return nil + } + return artifact.NewUnsupportedCapability(info.Alias, operation, "lock/"+string(required), + fmt.Sprintf("the repository cannot provide %s-scope write coordination; configure locking.repository to delegate to a repository that can, or relax locking.required_scope", required)) +} + +// acquireLocks checks every guarantee first, then acquires the locks in canonical repository alias +// then resource order. Any failure (including cancellation) releases whatever was already taken. +func (s *Service) acquireLocks(ctx context.Context, op *operation, operation string, targets []lockTarget) (*lockSet, error) { + for index := range targets { + if err := checkLockGuarantee(targets[index].info, operation); err != nil { + return nil, err + } + } + ordered := canonicalOrder(targets) + set := &lockSet{} + for index := range ordered { + target := ordered[index] + handle, err := target.repo.Lock(ctx, artifact.LockRequest{ + Resource: target.resource, + Owner: s.owner, + OperationID: op.id, + Required: requiredScope(target.info), + }) + if err != nil { + releaseErr := set.release(ctx, op) + return nil, errors.Join(fmt.Errorf("lock %s in repository %q: %w", target.resource, target.alias, err), releaseErr) + } + set.held = append(set.held, heldLock{target: target, handle: handle}) + } + return set, nil +} + +// canonicalOrder sorts targets by repository alias then resource and drops exact duplicates. +func canonicalOrder(targets []lockTarget) []lockTarget { + ordered := append([]lockTarget(nil), targets...) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].alias != ordered[j].alias { + return ordered[i].alias < ordered[j].alias + } + return ordered[i].resource < ordered[j].resource + }) + unique := ordered[:0] + for index := range ordered { + if index > 0 && ordered[index].alias == ordered[index-1].alias && ordered[index].resource == ordered[index-1].resource { + continue + } + unique = append(unique, ordered[index]) + } + return unique +} + +// handleFor returns the lock held for the repository alias, or nil. +func (l *lockSet) handleFor(alias string) artifact.LockHandle { + if l == nil { + return nil + } + for index := range l.held { + if l.held[index].target.alias == alias { + return l.held[index].handle + } + } + return nil +} + +// scope returns the weakest actual scope across held locks, which results report. +func (l *lockSet) scope() artifact.LockScope { + if l == nil || len(l.held) == 0 { + return "" + } + weakest := l.held[0].handle.Scope() + for index := range l.held { + if scope := l.held[index].handle.Scope(); !scope.Satisfies(artifact.LockScopeRepository) { + weakest = scope + } + } + return weakest +} + +// release relinquishes every held lock in reverse order. It uses a context detached from +// cancellation so a canceled operation never leaks ownership. A failed release is reported as a +// warning with the owner and resource an operator needs for recovery, and returned. +func (l *lockSet) release(ctx context.Context, op *operation) error { + if l == nil { + return nil + } + detached := context.WithoutCancel(ctx) + var errs []error + for index := len(l.held) - 1; index >= 0; index-- { + held := l.held[index] + if err := held.handle.Release(detached); err != nil { + op.warn("lock-release-failed", fmt.Sprintf("repository %q resource %q owner %q: operator recovery may be required: %v", held.target.alias, held.handle.Resource(), held.handle.Owner(), err)) + errs = append(errs, fmt.Errorf("release lock %s in repository %q: %w", held.handle.Resource(), held.target.alias, err)) + } + } + l.held = nil + return errors.Join(errs...) +} diff --git a/pkg/artifact/service/mock_repositories.go b/pkg/artifact/service/mock_repositories.go new file mode 100644 index 00000000000..d687b2bc5ae --- /dev/null +++ b/pkg/artifact/service/mock_repositories.go @@ -0,0 +1,201 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/service (interfaces: Repositories,Journal,ImmutabilityPolicy,Repacker) +// +// Generated by this command: +// +// mockgen -package service -destination mock_repositories.go github.com/cloudposse/atmos/pkg/artifact/service Repositories,Journal,ImmutabilityPolicy,Repacker +// + +// Package service is a generated GoMock package. +package service + +import ( + context "context" + reflect "reflect" + + artifact "github.com/cloudposse/atmos/pkg/artifact" + produce "github.com/cloudposse/atmos/pkg/artifact/produce" + gomock "go.uber.org/mock/gomock" +) + +// MockRepositories is a mock of Repositories interface. +type MockRepositories struct { + ctrl *gomock.Controller + recorder *MockRepositoriesMockRecorder + isgomock struct{} +} + +// MockRepositoriesMockRecorder is the mock recorder for MockRepositories. +type MockRepositoriesMockRecorder struct { + mock *MockRepositories +} + +// NewMockRepositories creates a new mock instance. +func NewMockRepositories(ctrl *gomock.Controller) *MockRepositories { + mock := &MockRepositories{ctrl: ctrl} + mock.recorder = &MockRepositoriesMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepositories) EXPECT() *MockRepositoriesMockRecorder { + return m.recorder +} + +// Aliases mocks base method. +func (m *MockRepositories) Aliases() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Aliases") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Aliases indicates an expected call of Aliases. +func (mr *MockRepositoriesMockRecorder) Aliases() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Aliases", reflect.TypeOf((*MockRepositories)(nil).Aliases)) +} + +// Describe mocks base method. +func (m *MockRepositories) Describe(alias string) (RepositoryInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Describe", alias) + ret0, _ := ret[0].(RepositoryInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Describe indicates an expected call of Describe. +func (mr *MockRepositoriesMockRecorder) Describe(alias any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Describe", reflect.TypeOf((*MockRepositories)(nil).Describe), alias) +} + +// Open mocks base method. +func (m *MockRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Open", ctx, alias) + ret0, _ := ret[0].(artifact.Repository) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Open indicates an expected call of Open. +func (mr *MockRepositoriesMockRecorder) Open(ctx, alias any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Open", reflect.TypeOf((*MockRepositories)(nil).Open), ctx, alias) +} + +// MockJournal is a mock of Journal interface. +type MockJournal struct { + ctrl *gomock.Controller + recorder *MockJournalMockRecorder + isgomock struct{} +} + +// MockJournalMockRecorder is the mock recorder for MockJournal. +type MockJournalMockRecorder struct { + mock *MockJournal +} + +// NewMockJournal creates a new mock instance. +func NewMockJournal(ctrl *gomock.Controller) *MockJournal { + mock := &MockJournal{ctrl: ctrl} + mock.recorder = &MockJournalMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockJournal) EXPECT() *MockJournalMockRecorder { + return m.recorder +} + +// Record mocks base method. +func (m *MockJournal) Record(ctx context.Context, entry *JournalEntry) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Record", ctx, entry) +} + +// Record indicates an expected call of Record. +func (mr *MockJournalMockRecorder) Record(ctx, entry any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Record", reflect.TypeOf((*MockJournal)(nil).Record), ctx, entry) +} + +// MockImmutabilityPolicy is a mock of ImmutabilityPolicy interface. +type MockImmutabilityPolicy struct { + ctrl *gomock.Controller + recorder *MockImmutabilityPolicyMockRecorder + isgomock struct{} +} + +// MockImmutabilityPolicyMockRecorder is the mock recorder for MockImmutabilityPolicy. +type MockImmutabilityPolicyMockRecorder struct { + mock *MockImmutabilityPolicy +} + +// NewMockImmutabilityPolicy creates a new mock instance. +func NewMockImmutabilityPolicy(ctrl *gomock.Controller) *MockImmutabilityPolicy { + mock := &MockImmutabilityPolicy{ctrl: ctrl} + mock.recorder = &MockImmutabilityPolicyMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockImmutabilityPolicy) EXPECT() *MockImmutabilityPolicyMockRecorder { + return m.recorder +} + +// Immutable mocks base method. +func (m *MockImmutabilityPolicy) Immutable(candidate Candidate) (bool, string) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Immutable", candidate) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + return ret0, ret1 +} + +// Immutable indicates an expected call of Immutable. +func (mr *MockImmutabilityPolicyMockRecorder) Immutable(candidate any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Immutable", reflect.TypeOf((*MockImmutabilityPolicy)(nil).Immutable), candidate) +} + +// MockRepacker is a mock of Repacker interface. +type MockRepacker struct { + ctrl *gomock.Controller + recorder *MockRepackerMockRecorder + isgomock struct{} +} + +// MockRepackerMockRecorder is the mock recorder for MockRepacker. +type MockRepackerMockRecorder struct { + mock *MockRepacker +} + +// NewMockRepacker creates a new mock instance. +func NewMockRepacker(ctrl *gomock.Controller) *MockRepacker { + mock := &MockRepacker{ctrl: ctrl} + mock.recorder = &MockRepackerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepacker) EXPECT() *MockRepackerMockRecorder { + return m.recorder +} + +// Repack mocks base method. +func (m *MockRepacker) Repack(ctx context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Repack", ctx, request) + ret0, _ := ret[0].(*artifact.PreparedArtifact) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Repack indicates an expected call of Repack. +func (mr *MockRepackerMockRecorder) Repack(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Repack", reflect.TypeOf((*MockRepacker)(nil).Repack), ctx, request) +} diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go new file mode 100644 index 00000000000..32fefa1404c --- /dev/null +++ b/pkg/artifact/service/mutate.go @@ -0,0 +1,104 @@ +package service + +import ( + "context" + "fmt" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Service]. The reference is resolved once; deletion is scoped to the +// pinned subject within the repository's declared delete capability and guarded by the writer lock. +func (s *Service) Delete(ctx context.Context, request artifact.DeleteRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Delete")() + + op := s.begin(artifact.ActionDelete, request.Repository, request.Ref.String()) + err := s.deleteArtifact(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func (s *Service) deleteArtifact(ctx context.Context, op *operation, request *artifact.DeleteRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opDelete) + if err != nil { + return err + } + if caps := info.Capabilities; caps.Delete == artifact.DeleteNone || caps.Delete == "" || !caps.Write { + return artifact.NewUnsupportedCapability(request.Repository, opDelete, opDelete, "the repository cannot delete artifacts") + } + return s.runDelete(ctx, op, request, &info, repo) +} + +// validateRepositoryRef checks the two fields every single-artifact operation needs. +func validateRepositoryRef(repository string, ref artifact.Ref) error { + if err := requireRepository(repository); err != nil { + return err + } + return ref.Validate() +} + +func (s *Service) runDelete(ctx context.Context, op *operation, request *artifact.DeleteRequest, info *RepositoryInfo, repo artifact.Repository) error { + resource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: request.Ref.String()} + locks, err := s.acquireLocks(ctx, op, opDelete, []lockTarget{resource}) + if err != nil { + return err + } + defer func() { _ = locks.release(ctx, op) }() + + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleFor(request.Repository)}) + if err != nil { + return fmt.Errorf("delete %s in repository %q: %w", request.Ref, request.Repository, err) + } + op.result.Outcome = artifact.OutcomeDeleted + op.result.LockScope = locks.scope() + op.result.Size = payloadSize(inspection) + if len(result.Deleted) > 0 { + op.result.Location = result.Deleted[0] + op.result.Aliases = result.Deleted[1:] + } + for _, affected := range result.Affected { + op.info("affected-reference", fmt.Sprintf("the deletion also affected %s", affected)) + } + return nil +} + +// Pin implements [artifact.Service]. A pin is immutable reference data for consumer lockfiles; it +// acquires no write lock and is not a substitute for signature verification. +func (s *Service) Pin(ctx context.Context, request artifact.PinRequest) (*artifact.PinResult, error) { + defer perf.Track(nil, "service.Service.Pin")() + + op := s.begin(artifact.ActionPin, request.Repository, request.Ref.String()) + result := &artifact.PinResult{} + err := s.pin(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) pin(ctx context.Context, op *operation, request *artifact.PinRequest, result *artifact.PinResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + _, repo, err := s.openForRead(ctx, request.Repository, opPin) + if err != nil { + return err + } + pinned, err := repo.Pin(ctx, request.Ref) + if err != nil { + return fmt.Errorf("pin %s in repository %q: %w", request.Ref, request.Repository, err) + } + result.Pinned = pinned + op.result.Outcome = artifact.OutcomeRead + op.result.ResolvedDigest = pinned.ResolvedDigest + op.result.RecordDigest = pinned.RecordDigest + op.result.ContentDigest = pinned.ContentDigest + op.info("pin-not-trust", "a pin fixes the immutable reference; it does not verify signatures, and trust policy is reevaluated when it is used") + return nil +} diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go new file mode 100644 index 00000000000..e081283a85c --- /dev/null +++ b/pkg/artifact/service/mutate_test.go @@ -0,0 +1,229 @@ +package service_test + +import ( + "context" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func TestDelete_ScopedToTheSelectedArtifact(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("keep")) + other := e.publishRequest("dst", e.source("drop")) + other.Version = "2.0.0" + _, err := e.svc.Publish(context.Background(), other) + require.NoError(t, err) + + result, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDeleted, result.Outcome) + assert.NotEmpty(t, result.Location) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + assert.Empty(t, e.repos["dst"].HeldLocks()) + + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err, "other versions are untouched") + + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("tools/example", "2.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Empty(t, e.repos["dst"].HeldLocks(), "locks are released on failure too") +} + +func TestDelete_CapabilityAndRequestChecks(t *testing.T) { + noDelete := artifacttest.DefaultCapabilities() + noDelete.Delete = artifact.DeleteNone + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &noDelete}}}) + e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "an unsupported delete takes no lock") + + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: "x"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestDelete_ReportsAffectedReferences(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + caps := artifacttest.DefaultCapabilities() + handle := artifact.NewMockLockHandle(ctrl) + record := artifact.Record{Name: "x/y", Version: "1", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + pinned := artifact.PinnedReference{Ref: ref("x/y", "1"), ResolvedDigest: recordDigest, RecordDigest: recordDigest} + + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + handle.EXPECT().Release(gomock.Any()).Return(nil) + handle.EXPECT().Resource().Return("x/y:1").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Resolve(gomock.Any(), ref("x/y", "1")).Return(pinned, nil) + repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + repo.EXPECT().Delete(gomock.Any(), pinned, gomock.Any()).DoAndReturn( + func(_ context.Context, _ artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + assert.Equal(t, artifact.DeleteObject, options.Scope) + require.NotNil(t, options.ExpectedCurrent) + assert.Equal(t, recordDigest, options.ExpectedCurrent.ResolvedDigest) + assert.NotNil(t, options.Lock) + return &artifact.DeleteResult{Deleted: []string{"loc-1", "loc-2"}, Affected: []artifact.Ref{ref("x/y", "latest-ish")}}, nil + }, + ) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps}}} + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + result, err := svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: ref("x/y", "1")}) + require.NoError(t, err) + assert.Equal(t, "loc-1", result.Location) + assert.Equal(t, []string{"loc-2"}, result.Aliases) + var affected bool + for _, diagnostic := range result.Diagnostics { + affected = affected || diagnostic.Code == "affected-reference" + } + assert.True(t, affected) +} + +func TestPin_IsImmutableDataTakesNoLockAndGrantsNoTrust(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + + result, err := e.svc.Pin(context.Background(), artifact.PinRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, published.Operation.RecordDigest, result.Pinned.ResolvedDigest) + assert.Equal(t, published.Operation.RecordDigest, result.Operation.RecordDigest) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "a pin is not a write mutex") + assert.Equal(t, 0, e.repos["dst"].Calls("Verify"), "a pin is not a substitute for verification") + var note bool + for _, diagnostic := range result.Operation.Diagnostics { + note = note || diagnostic.Code == "pin-not-trust" + } + assert.True(t, note) + + _, err = e.svc.Pin(context.Background(), artifact.PinRequest{Repository: "dst", Ref: ref("tools/example", "9")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.svc.Pin(context.Background(), artifact.PinRequest{Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestLocks_HeldByAnotherOwnerBlocksUntilCanceledAndLeaksNothing(t *testing.T) { + e := newEnv(t, envConfig{}) + other, err := e.repos["dst"].Lock(context.Background(), artifact.LockRequest{Resource: exampleRef.String(), Owner: "someone-else"}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) + defer cancel() + outcome, err := e.svc.PublishDetailed(ctx, e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Zero(t, e.repos["dst"].Calls("Publish"), "nothing is written without the lock") + assert.Equal(t, []string{exampleRef.String()}, e.repos["dst"].HeldLocks(), "the other owner's claim is neither stolen nor leaked") + assert.Empty(t, outcome.Operation.Outcome) + + require.NoError(t, other.Release(context.Background())) + retried, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retried.Operation.Outcome) +} + +func TestLocks_ReleasedWithADetachedContextAfterCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + handle.EXPECT().Resource().Return("tools/example:1.0.0").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound).AnyTimes() + + ctx, cancel := context.WithCancel(context.Background()) + repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *artifact.PreparedArtifact, artifact.PublishOptions) (*artifact.PublishResult, error) { + cancel() + return nil, context.Canceled + }, + ) + var released bool + handle.EXPECT().Release(gomock.Any()).DoAndReturn(func(releaseCtx context.Context) error { + released = true + assert.NoError(t, releaseCtx.Err(), "release must not inherit the canceled context") + return nil + }) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone}}} + e := newEnv(t, envConfig{}) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging), service.WithClock(e.clock.Now), service.WithRetry(1, 0)) + require.NoError(t, err) + + _, err = svc.Publish(ctx, e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, released, "the lock was released after cancellation") + e.assertStagingClean() +} + +func TestLocks_ReleaseFailureIsReportedForOperatorRecovery(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + handle := artifact.NewMockLockHandle(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + handle.EXPECT().Resource().Return("tools/example:1.0.0").AnyTimes() + handle.EXPECT().Owner().Return("test-owner").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound) + repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: artifact.PinnedReference{ResolvedDigest: digest.FromString("x")}, Representation: artifact.Representation{Payload: []ocispec.Descriptor{}}}, nil, + ) + handle.EXPECT().Release(gomock.Any()).Return(errTransport) + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone}}} + e := newEnv(t, envConfig{}) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + require.NoError(t, err) + + result, err := svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err, "the publication itself succeeded") + var warning artifact.Diagnostic + for _, diagnostic := range result.Diagnostics { + if diagnostic.Code == "lock-release-failed" { + warning = diagnostic + } + } + assert.Contains(t, warning.Message, "test-owner") + assert.Contains(t, warning.Message, "tools/example:1.0.0") +} + +func TestStaticRepositories(t *testing.T) { + repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"b": artifacttest.New(artifacttest.Options{}), "a": artifacttest.New(artifacttest.Options{})}} + assert.Equal(t, []string{"a", "b"}, repos.Aliases()) + info, err := repos.Describe("a") + require.NoError(t, err) + assert.Equal(t, "a", info.Alias) + assert.True(t, info.Capabilities.Write) + assert.Equal(t, artifact.PackagingNone, info.Packaging) + _, err = repos.Describe("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repos.Open(context.Background(), "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + repo, err := repos.Open(context.Background(), "a") + require.NoError(t, err) + assert.NotNil(t, repo) +} diff --git a/pkg/artifact/service/publish.go b/pkg/artifact/service/publish.go new file mode 100644 index 00000000000..5140c00abd8 --- /dev/null +++ b/pkg/artifact/service/publish.go @@ -0,0 +1,246 @@ +package service + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// PublishOutcome is the detailed result of [Service.PublishDetailed]. On an unchanged no-op, +// Record is the EXISTING record, byte for byte, and Operation reports the current execution +// separately (its own operation ID and duration). +type PublishOutcome struct { + // Operation is the typed operation result. + Operation artifact.OperationResult + + // Record is the Record now published (the existing one on a no-op); it is empty on failure + // before preparation and for dry runs holds the Record that would be published. + Record artifact.Record + + // Existing reports an unchanged no-op that returned the stored Record. + Existing bool +} + +// Publish implements [artifact.Service]. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Publish(ctx context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Publish")() + + outcome, err := s.PublishDetailed(ctx, request) + return &outcome.Operation, err +} + +// PublishDetailed prepares the source and stores it in one repository, returning the Record +// alongside the typed operation result. The result is always non-nil, even on error. +// +//nolint:gocritic // The request is passed by value to mirror Publish, which artifact.Service fixes. +func (s *Service) PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*PublishOutcome, error) { + defer perf.Track(nil, "service.Service.PublishDetailed")() + + op := s.begin(artifact.ActionPublish, request.Repository, requestedName(request.Name, request.Version)) + state := &publishState{} + err := s.publish(ctx, op, &request, state) + return &PublishOutcome{Operation: op.done(err), Record: state.record, Existing: state.existing}, err +} + +func requestedName(name, version string) string { + if name == "" || version == "" { + return name + } + return artifact.Ref{Name: name, Version: version}.String() +} + +func (s *Service) publish(ctx context.Context, op *operation, request *artifact.PublishRequest, state *publishState) error { + if err := validatePublish(request); err != nil { + return err + } + info, packaging, err := s.describeForWrite(request.Repository, opPublish) + if err != nil { + return err + } + if err := checkLockGuarantee(&info, opPublish); err != nil { + return err + } + producer, ok := s.producers.Lookup(request.Kind) + if !ok { + return fmt.Errorf("%w: no producer is registered for kind %q (registered: %v)", errUtils.ErrArtifactConfigInvalid, request.Kind, s.producers.Kinds()) + } + prepared, err := producer.Prepare(ctx, artifact.PrepareRequest{ + Source: request.Source, + Name: request.Name, + Version: request.Version, + Kind: request.Kind, + Metadata: mergeMetadata(info.Defaults, request.Metadata), + Readme: request.Readme, + Changelog: request.Changelog, + Packaging: packaging, + OperationID: op.id, + PreparedAt: op.preparedAt, + }) + if err != nil { + return err + } + defer prepared.Close() + + in, err := newIncoming(prepared) + if err != nil { + return err + } + state.record = prepared.Record + fillPrepared(op, in) + if request.DryRun { + op.result.Outcome = artifact.OutcomeDryRun + op.info("dry-run", fmt.Sprintf("would publish %s to repository %q as %s packaging; no repository was contacted", op.result.Ref, request.Repository, packaging)) + return nil + } + repo, err := s.repos.Open(ctx, request.Repository) + if err != nil { + return err + } + pub := &publication{ + op: op, + tgt: &target{alias: request.Repository, repo: repo, info: &info}, + in: in, + force: request.Force, + state: state, + write: func(ctx context.Context, options artifact.PublishOptions) (*artifact.PublishResult, error) { + return repo.Publish(ctx, prepared, options) + }, + } + s.event(ctx, pub, PhasePrepared, 0, nil) + return s.commit(ctx, pub) +} + +// validatePublish checks the request before anything is staged or contacted. +func validatePublish(request *artifact.PublishRequest) error { + if err := requireRepository(request.Repository); err != nil { + return err + } + if request.Source == "" { + return fmt.Errorf("%w: a source is required", errUtils.ErrArtifactConfigInvalid) + } + if request.Kind == "" { + request.Kind = artifact.KindFiles + } + if err := artifact.ValidateKind(request.Kind); err != nil { + return err + } + if request.Name != "" { + if err := artifact.ValidateName(request.Name); err != nil { + return err + } + } + if request.Version != "" { + if err := artifact.ValidateVersion(request.Version); err != nil { + return err + } + } + return artifact.ValidateLabels(request.Metadata.Labels) +} + +// describeForWrite resolves the repository description offline and checks that it can be written. +func (s *Service) describeForWrite(alias, operation string) (RepositoryInfo, artifact.Packaging, error) { + info, err := s.repos.Describe(alias) + if err != nil { + return RepositoryInfo{}, "", err + } + if !info.Capabilities.Write { + return info, "", artifact.NewUnsupportedCapability(alias, operation, "write", "the repository is read-only") + } + packaging := info.Packaging + if packaging == "" && len(info.Capabilities.Packagings) > 0 { + packaging = info.Capabilities.Packagings[0] + } + if !info.Capabilities.HasPackaging(packaging) { + return info, packaging, artifact.NewUnsupportedCapability(alias, operation, "packaging/"+string(packaging), fmt.Sprintf("supported packagings: %v", info.Capabilities.Packagings)) + } + return info, packaging, nil +} + +// mergeMetadata layers explicit metadata over repository defaults: explicit labels override +// matching default keys, and an explicit description overrides the default. +func mergeMetadata(defaults, explicit artifact.Metadata) artifact.Metadata { + merged := artifact.Metadata{ + Description: explicit.Description, + Provenance: explicit.Provenance, + } + if merged.Description == "" { + merged.Description = defaults.Description + } + if len(defaults.Labels) > 0 || len(explicit.Labels) > 0 { + merged.Labels = make(map[string]string, len(defaults.Labels)+len(explicit.Labels)) + for key, value := range defaults.Labels { + merged.Labels[key] = value + } + for key, value := range explicit.Labels { + merged.Labels[key] = value + } + } + return merged +} + +// fillPrepared records the identity of the prepared artifact in the typed result. +func fillPrepared(op *operation, in *incoming) { + op.result.Ref = artifact.Ref{Name: in.record.Name, Version: in.record.Version}.String() + op.result.Requested = op.result.Ref + op.result.RecordDigest = in.recordDigest + op.result.ContentDigest = in.record.ContentDigest + op.result.Size = in.size +} + +// reportUnchanged fills the result from the EXISTING record; the current execution keeps its own +// operation ID and duration. +func (s *Service) reportUnchanged(pub *publication, cur *current) { + op, record := pub.op, cur.inspection.Record + pub.state.existing = true + if record != nil { + pub.state.record = *record + op.result.ContentDigest = record.ContentDigest + } + op.result.Outcome = artifact.OutcomeUnchanged + op.result.Location = cur.inspection.Representation.Locator + op.result.ResolvedDigest = cur.pinned.ResolvedDigest + op.result.RecordDigest = cur.pinned.RecordDigest + op.result.RepresentationDigest = cur.inspection.Representation.RepresentationDigest + op.result.Size = payloadSize(cur.inspection) + op.info("unchanged", fmt.Sprintf("an identical artifact already exists in repository %q; the existing record was returned unchanged", pub.tgt.alias)) +} + +func payloadSize(inspection *artifact.Inspection) int64 { + var total int64 + for index := range inspection.Representation.Payload { + total += inspection.Representation.Payload[index].Size + } + return total +} + +// reportWritten fills the result after a successful write. +func (s *Service) reportWritten(pub *publication, result *artifact.PublishResult, locks *lockSet, decision verdict, cur *current) { + op, in := pub.op, pub.in + if in.record != nil { + pub.state.record = *in.record + } + outcome := result.Outcome + if outcome == "" { + outcome = artifact.OutcomeCreated + if decision == verdictReplace || cur.unavailable { + outcome = artifact.OutcomeReplaced + } + } + op.result.Outcome = outcome + op.result.Location = result.Location + op.result.Aliases = result.Aliases + op.result.ResolvedDigest = result.Pinned.ResolvedDigest + op.result.RepresentationDigest = result.Representation.RepresentationDigest + op.result.LockScope = result.LockScope + if scope := locks.scope(); scope != "" { + op.result.LockScope = scope + } + if op.result.ResolvedDigest == "" { + op.result.ResolvedDigest = in.recordDigest + } +} diff --git a/pkg/artifact/service/publish_test.go b/pkg/artifact/service/publish_test.go new file mode 100644 index 00000000000..8ec3f684177 --- /dev/null +++ b/pkg/artifact/service/publish_test.go @@ -0,0 +1,409 @@ +package service_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func TestPublish_CreatesAndReportsTypedResult(t *testing.T) { + e := newEnv(t, envConfig{}) + outcome := e.publish("dst", e.source("hello")) + + result := outcome.Operation + assert.Equal(t, artifact.ActionPublish, result.Action) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, "op-1", result.OperationID) + assert.Equal(t, "dst", result.Repository) + assert.Equal(t, "tools/example:1.0.0", result.Ref) + assert.Equal(t, artifact.LockScopeHost, result.LockScope, "the actual lock scope is reported") + assert.NotEmpty(t, result.RecordDigest) + assert.Equal(t, outcome.Record.ContentDigest, result.ContentDigest) + assert.NotEmpty(t, result.Location) + assert.False(t, outcome.Existing) + assert.Equal(t, artifact.NewTimestamp(startTime), outcome.Record.CreatedAt) + assert.Empty(t, e.repos["dst"].HeldLocks(), "locks are released") + e.assertStagingClean() + assert.Equal(t, []service.JournalPhase{service.PhasePrepared, service.PhaseLocked, service.PhasePending, service.PhaseCommitted, service.PhaseReleased}, e.journal.phases()) + + // The interface method returns the same result. + again, err := e.svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) +} + +func TestPublish_IdenticalRepublishIsUnchangedEvenWhenTimeAndRunMetadataDiffer(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + first := e.publish("dst", src) + + e.clock.Advance(30 * time.Hour) + request := e.publishRequest("dst", src) + request.Metadata.Provenance = &artifact.Provenance{CI: &artifact.CIProvenance{Provider: "github", RunID: "999"}} + second, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + + assert.True(t, second.Existing) + assert.Equal(t, artifact.OutcomeUnchanged, second.Operation.Outcome) + firstBytes, err := first.Record.Canonical() + require.NoError(t, err) + secondBytes, err := second.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstBytes, secondBytes, "the EXISTING record is returned byte for byte") + assert.Equal(t, artifact.NewTimestamp(startTime), second.Record.CreatedAt, "created_at stays the original") + assert.Equal(t, first.Operation.RecordDigest, second.Operation.RecordDigest) + assert.Equal(t, "op-2", second.Operation.OperationID, "the current execution is reported separately") + assert.Equal(t, 1, e.repos["dst"].Calls("Publish"), "an unchanged no-op writes nothing") + assert.Nil(t, second.Record.Metadata.Provenance, "run metadata of the new execution never rewrites the stored record") +} + +func TestPublish_ChangedIntentConflictsWithoutForceAndReplacesWithIt(t *testing.T) { + tests := []struct { + name string + mutate func(t *testing.T, src string, request *artifact.PublishRequest) + reason string + }{ + {"content", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.WriteFile(t, src, "data.txt", "changed", 0o644) + }, "content"}, + {"mode", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.RequirePOSIXModes(t) + require.NoError(t, os.Chmod(filepath.Join(src, "data.txt"), 0o755)) + }, "content"}, + {"document", func(t *testing.T, src string, _ *artifact.PublishRequest) { + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "new readme\n", 0o644) + }, "content"}, + {"labels", func(_ *testing.T, _ string, request *artifact.PublishRequest) { + request.Metadata.Labels = map[string]string{"team": "platform"} + }, "labels"}, + {"description", func(_ *testing.T, _ string, request *artifact.PublishRequest) { + request.Metadata.Description = "new description" + }, "description"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + first := e.publish("dst", src) + request := e.publishRequest("dst", src) + tt.mutate(t, src, &request) + + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), tt.reason) + assert.NotNil(t, outcome, "a result is returned alongside the error") + assert.Equal(t, "op-2", outcome.Operation.OperationID) + current, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok) + assert.Equal(t, first.Operation.RecordDigest, current, "the stored artifact is untouched") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + request.Force = true + replaced, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, replaced.Operation.Outcome) + current, _ = e.repos["dst"].RecordDigestOf(exampleRef) + assert.Equal(t, replaced.Operation.RecordDigest.String(), current.String()) + e.assertStagingClean() + }) + } +} + +func TestPublish_ForceOnIdenticalContentStaysUnchanged(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + e.publish("dst", src) + request := e.publishRequest("dst", src) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) +} + +// pluginPolicy treats artifacts labeled format=plugin as immutable releases. +func pluginPolicy() service.ImmutabilityPolicy { + isPlugin := func(labels map[string]string) bool { return labels["format"] == "plugin" } + return service.ImmutabilityFunc(func(c service.Candidate) (bool, string) { + switch { + case c.Inspection != nil && c.Inspection.Record != nil && isPlugin(c.Inspection.Record.Metadata.Labels): + return true, "plugin format" + case c.Prepared != nil && isPlugin(c.Prepared.Record.Metadata.Labels): + return true, "plugin format" + } + return false, "" + }) +} + +func TestPublish_PluginFormatReleasesStayImmutableEvenWithForce(t *testing.T) { + plugin := map[string]string{"format": "plugin"} + tests := []struct { + name string + existing map[string]string + incoming map[string]string + wantImmutable bool + reason string + }{ + {"existing plugin", plugin, nil, true, "existing release"}, + {"incoming plugin", nil, plugin, true, "incoming release"}, + {"both plugin", plugin, map[string]string{"format": "plugin", "v": "2"}, true, "existing release"}, + {"neither", nil, map[string]string{"x": "y"}, false, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(pluginPolicy())}}) + src := e.source("hello") + request := e.publishRequest("dst", src) + request.Metadata.Labels = tt.existing + first, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + + request.Metadata.Labels = tt.incoming + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + if !tt.wantImmutable { + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, outcome.Operation.Outcome) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), tt.reason) + assert.Contains(t, err.Error(), "immutable") + current, _ := e.repos["dst"].RecordDigestOf(exampleRef) + assert.Equal(t, first.Operation.RecordDigest, current) + }) + } +} + +func TestPublish_IdenticalPluginReleaseRepublishIsStillAnUnchangedNoOp(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(pluginPolicy())}}) + src := e.source("hello") + request := e.publishRequest("dst", src) + request.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + request.Force = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, outcome.Operation.Outcome) +} + +func TestPublish_ImmutabilityPolicyMockSeesBothSides(t *testing.T) { + ctrl := gomock.NewController(t) + policy := service.NewMockImmutabilityPolicy(ctrl) + e := newEnv(t, envConfig{options: []service.Option{service.WithImmutabilityPolicy(policy)}}) + src := e.source("hello") + e.publish("dst", src) + + var sawExisting, sawIncoming bool + policy.EXPECT().Immutable(gomock.Any()).DoAndReturn(func(c service.Candidate) (bool, string) { + if c.Existing { + sawExisting = true + assert.NotNil(t, c.Inspection) + } else { + sawIncoming = true + assert.NotNil(t, c.Prepared) + } + return false, "" + }).Times(2) + request := e.publishRequest("dst", src) + request.Force = true + request.Metadata.Description = "changed" + _, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.True(t, sawExisting && sawIncoming, "both the existing and the incoming artifact are admitted") +} + +func TestPublish_NameAndVersionResolutionHappensBeforeAnyWrite(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + tests := []struct { + name string + mutate func(*artifact.PublishRequest) + wantErr error + }{ + {"missing version", func(r *artifact.PublishRequest) { r.Version = "" }, errUtils.ErrArtifactRefInvalid}, + {"invalid version", func(r *artifact.PublishRequest) { r.Version = "bad version" }, errUtils.ErrArtifactRefInvalid}, + {"invalid name", func(r *artifact.PublishRequest) { r.Name = "Bad_Name" }, errUtils.ErrArtifactRefInvalid}, + {"traversal name", func(r *artifact.PublishRequest) { r.Name = "../x" }, errUtils.ErrArtifactUnsafePath}, + {"missing repository", func(r *artifact.PublishRequest) { r.Repository = "" }, errUtils.ErrArtifactConfigInvalid}, + {"missing source", func(r *artifact.PublishRequest) { r.Source = "" }, errUtils.ErrArtifactConfigInvalid}, + {"invalid kind", func(r *artifact.PublishRequest) { r.Kind = "Bad Kind" }, errUtils.ErrArtifactRecordInvalid}, + {"unregistered kind", func(r *artifact.PublishRequest) { r.Kind = "scaffold" }, errUtils.ErrArtifactConfigInvalid}, + {"bad labels", func(r *artifact.PublishRequest) { r.Metadata.Labels = map[string]string{"a=b": "c"} }, errUtils.ErrArtifactConfigInvalid}, + {"unknown repository", func(r *artifact.PublishRequest) { r.Repository = "nope" }, errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := e.publishRequest("dst", src) + tt.mutate(&request) + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.ErrorIs(t, err, tt.wantErr) + require.NotNil(t, outcome) + assert.NotEmpty(t, outcome.Operation.OperationID) + assert.NotEmpty(t, outcome.Operation.Diagnostics, "the failure is reported in the result") + assert.Zero(t, e.repos["dst"].TotalCalls(), "nothing reached the repository") + e.assertStagingClean() + }) + } +} + +func TestPublish_BasenameNameFallbackAndInvalidBasename(t *testing.T) { + e := newEnv(t, envConfig{}) + good := filepath.Join(t.TempDir(), "my-app") + require.NoError(t, os.Mkdir(good, 0o755)) + fixtures.WriteFile(t, good, "a.txt", "a", 0o644) + outcome, err := e.svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: good, Version: "1"}) + require.NoError(t, err) + assert.Equal(t, "my-app", outcome.Record.Name) + + bad := filepath.Join(t.TempDir(), "My App") + require.NoError(t, os.Mkdir(bad, 0o755)) + fixtures.WriteFile(t, bad, "a.txt", "a", 0o644) + _, err = e.svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: bad, Version: "1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Contains(t, err.Error(), "explicit name") +} + +func TestPublish_MergesRepositoryDefaultMetadataUnderExplicitValues(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { + info.Defaults = artifact.Metadata{Description: "repo default", Labels: map[string]string{"team": "platform", "env": "dev"}} + }) + request := e.publishRequest("dst", e.source("hello")) + request.Metadata.Labels = map[string]string{"env": "prod"} + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "platform", "env": "prod"}, outcome.Record.Metadata.Labels) + assert.Equal(t, "repo default", outcome.Record.Metadata.Description) + + request = e.publishRequest("dst", e.source("other")) + request.Version = "2.0.0" + request.Metadata.Description = "explicit" + outcome, err = e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, "explicit", outcome.Record.Metadata.Description) +} + +func TestPublish_CapabilityAndLockGuaranteesAreCheckedBeforeMutation(t *testing.T) { + readOnly := artifacttest.DefaultCapabilities() + readOnly.Write = false + badPackaging := artifacttest.DefaultCapabilities() + badPackaging.Packagings = []artifact.Packaging{artifact.PackagingZip} + tests := []struct { + name string + caps artifact.Capabilities + info service.RepositoryInfo + }{ + {"read only", readOnly, service.RepositoryInfo{Packaging: artifact.PackagingNone}}, + {"unsupported packaging", badPackaging, service.RepositoryInfo{Packaging: artifact.PackagingTgz}}, + {"insufficient lock scope", artifacttest.DefaultCapabilities(), service.RepositoryInfo{Packaging: artifact.PackagingNone, RequiredLockScope: artifact.LockScopeRepository}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caps := tt.caps + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + info := tt.info + info.Capabilities = caps + e.static.Info["dst"] = info + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Zero(t, e.repos["dst"].TotalCalls(), "rejected before any repository call, including Lock") + assert.NotEmpty(t, outcome.Operation.OperationID) + e.assertStagingClean() + }) + } +} + +func TestPublish_RepositoryScopeLockIsReportedAsActual(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.LockScopes = []artifact.LockScope{artifact.LockScopeHost, artifact.LockScopeRepository} + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.RequiredLockScope = artifact.LockScopeRepository }) + outcome := e.publish("dst", e.source("hello")) + assert.Equal(t, artifact.LockScopeRepository, outcome.Operation.LockScope) +} + +func TestPublish_RegisteredKindProducerReceivesCapturedIdentity(t *testing.T) { + ctrl := gomock.NewController(t) + producer := artifact.NewMockProducer(ctrl) + producer.EXPECT().Kind().Return("scaffold").AnyTimes() + registry := artifact.NewProducerRegistry() + require.NoError(t, registry.Register(producer)) + + e := newEnv(t, envConfig{}) + svc, err := service.New(e.static, registry, service.WithClock(e.clock.Now), service.WithIDGenerator(func() string { return "fixed-op" }), service.WithStagingRoot(e.staging)) + require.NoError(t, err) + + producer.EXPECT().Prepare(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request artifact.PrepareRequest) (*artifact.PreparedArtifact, error) { + assert.Equal(t, "fixed-op", request.OperationID) + assert.True(t, startTime.Equal(request.PreparedAt)) + assert.Equal(t, time.UTC, request.PreparedAt.Location()) + assert.Equal(t, "scaffold", request.Kind) + assert.Equal(t, artifact.PackagingNone, request.Packaging) + return nil, errUtils.ErrArtifactPackagingInvalid + }) + request := e.publishRequest("dst", e.source("hello")) + request.Kind = "scaffold" + _, err = svc.Publish(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestPublish_DryRunPerformsNoRepositoryOrIdentityCalls(t *testing.T) { + t.Run("in-memory repository", func(t *testing.T) { + e := newEnv(t, envConfig{}) + request := e.publishRequest("dst", e.source("hello")) + request.DryRun = true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDryRun, outcome.Operation.Outcome) + assert.NotEmpty(t, outcome.Operation.RecordDigest) + assert.NotEmpty(t, outcome.Record.Inventory) + assert.Zero(t, e.repos["dst"].TotalCalls()) + _, found := e.repos["dst"].RecordDigestOf(exampleRef) + assert.False(t, found) + e.assertStagingClean() + assert.Empty(t, e.journal.phases(), "a dry run records no lifecycle") + }) + + t.Run("repositories mock asserts Open is never called", func(t *testing.T) { + ctrl := gomock.NewController(t) + repos := service.NewMockRepositories(ctrl) + caps := artifacttest.DefaultCapabilities() + repos.EXPECT().Describe("dst").Return(service.RepositoryInfo{Alias: "dst", Capabilities: caps, Packaging: artifact.PackagingTgz}, nil) + repos.EXPECT().Open(gomock.Any(), gomock.Any()).Times(0) + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "dst", Source: src, Name: "x/y", Version: "1", DryRun: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDryRun, result.Outcome) + }) +} + +func TestPublish_PackagingFollowsRepository(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Packaging = packaging }) + e.publish("dst", e.source("hello")) + result, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, packaging, result.Inspection.Representation.Packaging) + }) + } +} diff --git a/pkg/artifact/service/pull.go b/pkg/artifact/service/pull.go new file mode 100644 index 00000000000..49da05323f4 --- /dev/null +++ b/pkg/artifact/service/pull.go @@ -0,0 +1,168 @@ +package service + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Pull implements [artifact.Service]. It resolves the reference once, verifies the pinned subject +// against policy, refuses to overlay a different non-empty destination without force, extracts +// into staging, and only then replaces the destination. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) Pull(ctx context.Context, request artifact.PullRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "service.Service.Pull")() + + op := s.begin(artifact.ActionPull, request.Repository, request.Ref.String()) + err := s.pull(ctx, op, &request) + result := op.done(err) + return &result, err +} + +func (s *Service) pull(ctx context.Context, op *operation, request *artifact.PullRequest) error { + if err := validatePull(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opPull) + if err != nil { + return err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + if inspection.Record == nil { + return artifact.NewUnsupportedCapability(request.Repository, opPull, "materialize/graph", "a native OCI graph without a portable Record cannot be materialized as a file tree") + } + if _, err := gate(ctx, request.Repository, repo, inspection, info.Policy); err != nil { + return err + } + unchanged, err := checkDestination(op, request, inspection.Record) + if err != nil || unchanged { + return err + } + return s.download(ctx, op, request, &pinnedSource{repo: repo, pinned: &pinned, inspection: inspection}) +} + +// pinnedSource is a repository together with the subject that was pinned and inspected in it. +type pinnedSource struct { + repo artifact.Repository + pinned *artifact.PinnedReference + inspection *artifact.Inspection +} + +// download reads the pinned content and materializes it, replacing the destination only after the +// whole tree was extracted and verified in staging. +func (s *Service) download(ctx context.Context, op *operation, request *artifact.PullRequest, source *pinnedSource) error { + content, err := source.repo.Read(ctx, *source.pinned) + if err != nil { + return fmt.Errorf("read %s from repository %q: %w", request.Ref, request.Repository, err) + } + defer content.Close() + if err := bindContent(content, source.inspection); err != nil { + return err + } + if err := materialize.Materialize(ctx, content, request.Destination); err != nil { + return err + } + op.result.Outcome = artifact.OutcomeRead + return nil +} + +func validatePull(request *artifact.PullRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if request.Destination == "" { + return fmt.Errorf("%w: a destination is required", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// checkDestination classifies the output directory. It reports unchanged when the destination +// already holds exactly the artifact and refuses a different non-empty destination without force. +func checkDestination(op *operation, request *artifact.PullRequest, record *artifact.Record) (bool, error) { + state, err := materialize.InspectDestination(request.Destination, record) + if err != nil { + return false, err + } + op.result.Location = request.Destination + switch { + case state == materialize.DestinationMatches: + op.result.Outcome = artifact.OutcomeUnchanged + op.info("unchanged", "the destination already holds exactly this artifact; nothing was downloaded") + return true, nil + case state == materialize.DestinationDiffers && !request.Force: + return false, fmt.Errorf("%w: destination %q already holds different content; use force to replace it", errUtils.ErrArtifactConflict, request.Destination) + } + return false, nil +} + +// bindContent checks the content belongs to the inspected, pinned Record and fills a missing one. +func bindContent(content *artifact.Content, inspection *artifact.Inspection) error { + if content.Record == nil { + content.Record = inspection.Record + return nil + } + got, err := content.Record.RecordDigest() + if err != nil { + return err + } + want, err := inspection.Record.RecordDigest() + if err != nil { + return err + } + if got != want { + return fmt.Errorf("%w: the content record %s differs from the pinned record %s", errUtils.ErrArtifactIntegrityFailed, got, want) + } + return nil +} + +// openForRead describes the repository offline, requires read support, and opens it. +func (s *Service) openForRead(ctx context.Context, alias, operation string) (RepositoryInfo, artifact.Repository, error) { + info, err := s.repos.Describe(alias) + if err != nil { + return RepositoryInfo{}, nil, err + } + if !info.Capabilities.Read { + return info, nil, artifact.NewUnsupportedCapability(alias, operation, "read", "the repository is write-only") + } + repo, err := s.repos.Open(ctx, alias) + if err != nil { + return info, nil, err + } + return info, repo, nil +} + +// pinAndInspect resolves the reference exactly once and inspects the pinned subject. Everything +// after this point uses the returned pin, never the mutable reference. +func (s *Service) pinAndInspect(ctx context.Context, op *operation, repo artifact.Repository, alias string, ref artifact.Ref) (artifact.PinnedReference, *artifact.Inspection, error) { + pinned, err := repo.Resolve(ctx, ref) + if err != nil { + return pinned, nil, fmt.Errorf("resolve %s in repository %q: %w", ref, alias, err) + } + inspection, err := repo.Inspect(ctx, pinned) + if err != nil { + return pinned, nil, fmt.Errorf("inspect %s in repository %q: %w", ref, alias, err) + } + fillFromInspection(op, &pinned, inspection) + return pinned, inspection, nil +} + +// fillFromInspection records the pinned identity in the typed result. +func fillFromInspection(op *operation, pinned *artifact.PinnedReference, inspection *artifact.Inspection) { + op.result.ResolvedDigest = pinned.ResolvedDigest + op.result.RecordDigest = pinned.RecordDigest + op.result.ContentDigest = pinned.ContentDigest + op.result.RepresentationDigest = inspection.Representation.RepresentationDigest + op.result.Size = payloadSize(inspection) + if inspection.Record != nil { + op.result.Ref = artifact.Ref{Name: inspection.Record.Name, Version: inspection.Record.Version}.String() + op.result.ContentDigest = inspection.Record.ContentDigest + } +} diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go new file mode 100644 index 00000000000..ba3ed533b36 --- /dev/null +++ b/pkg/artifact/service/query.go @@ -0,0 +1,208 @@ +package service + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "sync" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Inspect implements [artifact.Service]. It reads metadata only. Verification is reported when +// policy requires or constrains it; a failed or unchecked verification is information here, not a +// reason to hide metadata, because inspect consumes no payload. +func (s *Service) Inspect(ctx context.Context, request artifact.InspectRequest) (*artifact.InspectResult, error) { + defer perf.Track(nil, "service.Service.Inspect")() + + op := s.begin(artifact.ActionInspect, request.Repository, request.Ref.String()) + result := &artifact.InspectResult{} + err := s.inspect(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) inspect(ctx context.Context, op *operation, request *artifact.InspectRequest, result *artifact.InspectResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opInspect) + if err != nil { + return err + } + _, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + result.Inspection = *inspection + op.result.Outcome = artifact.OutcomeRead + op.result.Location = inspection.Representation.Locator + verification, verifyErr := gate(ctx, request.Repository, repo, inspection, info.Policy) + if verification != nil && verification.Status != artifact.VerificationUnchecked { + result.Verification = verification + } + if verifyErr != nil { + op.warn("verification-failed", verifyErr.Error()) + } + return nil +} + +// ShowDocument implements [artifact.Service]. It never downloads the payload; the document bytes +// are bounded and verified against the Record descriptor. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (s *Service) ShowDocument(ctx context.Context, request artifact.ShowDocumentRequest) (*artifact.DocumentResult, error) { + defer perf.Track(nil, "service.Service.ShowDocument")() + + op := s.begin(artifact.ActionShow, request.Repository, request.Ref.String()) + result := &artifact.DocumentResult{} + err := s.showDocument(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) showDocument(ctx context.Context, op *operation, request *artifact.ShowDocumentRequest, result *artifact.DocumentResult) error { + if err := validateShow(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opShow) + if err != nil { + return err + } + pinned, err := repo.Resolve(ctx, request.Ref) + if err != nil { + return fmt.Errorf("resolve %s in repository %q: %w", request.Ref, request.Repository, err) + } + if err := s.gatePinned(ctx, op, &target{alias: request.Repository, repo: repo, info: &info}, &pinned); err != nil { + return err + } + stream, err := repo.OpenDocument(ctx, pinned, request.Role) + if err != nil { + return fmt.Errorf("open %s of %s in repository %q: %w", request.Role, request.Ref, request.Repository, err) + } + defer stream.Reader.Close() + content, err := readDocument(stream) + if err != nil { + return err + } + result.Document, result.Content = stream.Document, content + op.result.Outcome = artifact.OutcomeRead + op.result.Size = int64(len(content)) + return nil +} + +func validateShow(request *artifact.ShowDocumentRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if request.Role != artifact.DocumentReadme && request.Role != artifact.DocumentChangelog { + return fmt.Errorf("%w: document role must be %q or %q", errUtils.ErrArtifactConfigInvalid, artifact.DocumentReadme, artifact.DocumentChangelog) + } + return nil +} + +// gatePinned enforces required verification on an already pinned subject. It inspects only when +// policy needs the subject, so unconstrained reads stay metadata-light. +func (s *Service) gatePinned(ctx context.Context, op *operation, tgt *target, pinned *artifact.PinnedReference) error { + if !verificationRequired(tgt.info.Policy) { + return nil + } + inspection, err := tgt.repo.Inspect(ctx, *pinned) + if err != nil { + return err + } + fillFromInspection(op, pinned, inspection) + _, err = gate(ctx, tgt.alias, tgt.repo, inspection, tgt.info.Policy) + return err +} + +// readDocument reads a document within the size bound and verifies it against its descriptor. +func readDocument(stream *artifact.DocumentStream) ([]byte, error) { + data, err := artifact.ReadBounded(stream.Reader, artifact.MaxDocumentSize) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != stream.Document.Size || digest.NewDigestFromBytes(digest.SHA256, sum[:]) != stream.Document.Digest { + return nil, fmt.Errorf("%w: the %s does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, stream.Document.Role) + } + return data, nil +} + +// List implements [artifact.Service]. +func (s *Service) List(ctx context.Context, request artifact.ListRequest) (*artifact.ListResult, error) { + defer perf.Track(nil, "service.Service.List")() + + return s.enumerate(ctx, artifact.ActionList, &enumeration{ + repositories: request.Repositories, name: request.Name, labels: request.Labels, refresh: request.Refresh, + }) +} + +// Search implements [artifact.Service]. The text is matched case-insensitively against names, +// kinds, descriptions, and label keys and values of the enumerated collection. +func (s *Service) Search(ctx context.Context, request artifact.SearchRequest) (*artifact.ListResult, error) { + defer perf.Track(nil, "service.Service.Search")() + + return s.enumerate(ctx, artifact.ActionSearch, &enumeration{ + repositories: request.Repositories, labels: request.Labels, refresh: request.Refresh, text: request.Query, + }) +} + +type enumeration struct { + repositories []string + name string + labels []artifact.Label + refresh bool + text string +} + +func (s *Service) enumerate(ctx context.Context, _ artifact.Action, request *enumeration) (*artifact.ListResult, error) { + result := &artifact.ListResult{Artifacts: []artifact.ListEntry{}} + err := s.runEnumeration(ctx, request, result) + return result, err +} + +// runEnumeration fans out over the selected repositories with bounded concurrency, collects +// per-repository outcomes, and fails only when an explicitly requested repository fails or none +// succeeded. Unsupported enumeration on one of several default repositories is reported, not fatal. +func (s *Service) runEnumeration(ctx context.Context, request *enumeration, result *artifact.ListResult) error { + aliases := request.repositories + explicit := len(aliases) > 0 + if !explicit { + aliases = s.repos.Aliases() + } + outcomes := make([]listOutcome, len(aliases)) + gates := make(chan struct{}, s.concurrency) + var wg sync.WaitGroup + for index := range aliases { + wg.Add(1) + go func() { + defer wg.Done() + gates <- struct{}{} + defer func() { <-gates }() + outcomes[index] = s.listRepository(ctx, aliases[index], request) + }() + } + wg.Wait() + var errs []error + succeeded := 0 + for index := range outcomes { + result.Artifacts = append(result.Artifacts, outcomes[index].entries...) + result.Repositories = append(result.Repositories, outcomes[index].listing) + if outcomes[index].err != nil { + errs = append(errs, outcomes[index].err) + continue + } + succeeded++ + } + sortEntries(result.Artifacts) + if explicit && len(errs) > 0 || succeeded == 0 && len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go new file mode 100644 index 00000000000..f345f2befb9 --- /dev/null +++ b/pkg/artifact/service/repositories.go @@ -0,0 +1,148 @@ +package service + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate mockgen -package service -destination mock_repositories.go github.com/cloudposse/atmos/pkg/artifact/service Repositories,Journal,ImmutabilityPolicy,Repacker + +// RepositoryInfo is everything the service needs to know about a configured repository without +// touching the network or resolving credentials. +type RepositoryInfo struct { + // Alias is the configured repository alias. + Alias string + + // Capabilities describes the repository without network access. For a repository that + // delegates locking, it already includes the delegate's lock scopes. + Capabilities artifact.Capabilities + + // Packaging is the effective payload packaging mode for publications. + Packaging artifact.Packaging + + // Defaults is defaults.metadata; its labels sit under explicit publication labels. + Defaults artifact.Metadata + + // RequiredLockScope is locking.required_scope; empty means host. + RequiredLockScope artifact.LockScope + + // Policy is the effective verification policy: the global and repository trust sets, both of + // which must be satisfied. + Policy artifact.VerificationPolicy +} + +// Repositories resolves repository aliases. Describe must be free of network access and +// credential resolution so dry runs never authenticate; Open may resolve identities. +type Repositories interface { + // Describe returns the repository description without authenticating. + Describe(alias string) (RepositoryInfo, error) + + // Open constructs (or returns a cached) repository, resolving credentials as needed. + Open(ctx context.Context, alias string) (artifact.Repository, error) + + // Aliases lists every configured repository alias, sorted. + Aliases() []string +} + +// JournalPhase names a step in the publication lifecycle. +type JournalPhase string + +// Journal phases, in lifecycle order. +const ( + // PhasePrepared means the source snapshot and Record were sealed. + PhasePrepared JournalPhase = "prepared" + + // PhaseLocked means the writer lock was acquired. + PhaseLocked JournalPhase = "locked" + + // PhasePending means the publication is about to be handed to the repository. + PhasePending JournalPhase = "pending" + + // PhaseCommitted means the repository confirmed completion. + PhaseCommitted JournalPhase = "committed" + + // PhaseReconciled means an ambiguous failure was resolved by reading the completed reference. + PhaseReconciled JournalPhase = "reconciled" + + // PhaseFailed means the operation failed without a committed publication. + PhaseFailed JournalPhase = "failed" + + // PhaseReleased means the writer lock was released. + PhaseReleased JournalPhase = "released" +) + +// JournalEntry is one lifecycle event. It is an implementation recovery record, not a catalog. +type JournalEntry struct { + // OperationID is the logical operation identifier, stable across retries. + OperationID string + + // Phase is the lifecycle step. + Phase JournalPhase + + // Repository is the target repository alias. + Repository string + + // Ref is the reference being mutated. + Ref string + + // IntentDigest is the intent digest of the artifact being published. + IntentDigest string + + // Attempt is the one-based publication attempt. + Attempt int + + // Err is the failure for PhaseFailed. + Err error +} + +// Journal receives publication lifecycle events. The service never blocks on or fails because of it. +type Journal interface { + // Record observes one lifecycle event. + Record(ctx context.Context, entry *JournalEntry) +} + +// Candidate is an artifact being considered for the immutability policy. +type Candidate struct { + // Existing reports that the candidate is the artifact already stored (not the incoming one). + Existing bool + + // Repository is the repository alias involved. + Repository string + + // Inspection is the metadata view, set for stored artifacts and for copy sources. + Inspection *artifact.Inspection + + // Prepared is the artifact about to be published, when one was prepared. + Prepared *artifact.PreparedArtifact +} + +// ImmutabilityPolicy is the admission seam for immutable release formats. The OCI adapter supplies +// the plugin-format detector; the service consults it for both the stored artifact and the +// incoming one, so a generic operation can never overwrite a plugin release, with or without force. +type ImmutabilityPolicy interface { + // Immutable reports whether the candidate is in an immutable release format and why. + Immutable(candidate Candidate) (bool, string) +} + +// ImmutabilityFunc adapts a function to [ImmutabilityPolicy]. +type ImmutabilityFunc func(candidate Candidate) (bool, string) + +// Immutable implements [ImmutabilityPolicy]. +func (f ImmutabilityFunc) Immutable(candidate Candidate) (bool, string) { + defer perf.Track(nil, "service.ImmutabilityFunc.Immutable")() + + return f(candidate) +} + +// StaticRepositories is a fixed alias-to-repository table. Describe reads Capabilities from each +// repository (which never authenticates) and Open returns it unchanged. It suits tests and +// composition roots that construct repositories up front. +type StaticRepositories struct { + // Repositories maps aliases to repositories. + Repositories map[string]artifact.Repository + + // Info optionally overrides the description of an alias; the zero fields fall back to defaults. + Info map[string]RepositoryInfo +} diff --git a/pkg/artifact/service/service.go b/pkg/artifact/service/service.go new file mode 100644 index 00000000000..a6cb884404f --- /dev/null +++ b/pkg/artifact/service/service.go @@ -0,0 +1,267 @@ +package service + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Operation names used in capability errors and lock requests. +const ( + opPublish = "publish" + opPull = "pull" + opCopy = "copy" + opDelete = "delete" + opSign = "sign" + opVerify = "verify" + opInspect = "inspect" + opShow = "show" + opPin = "pin" + opList = "list" +) + +// requireRepository rejects an empty repository alias: a missing required repository is an error. +func requireRepository(alias string) error { + if alias == "" { + return fmt.Errorf("%w: a repository is required", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// Defaults for retry behavior. Retries apply only to publications that failed with a temporary +// error and whose reconciliation proves nothing was committed. +const ( + // DefaultAttempts is the default number of publication attempts. + DefaultAttempts = 3 + + // DefaultBackoff is the default base delay between attempts; it doubles each attempt. + DefaultBackoff = 200 * time.Millisecond +) + +// Repacker rebuilds an existing Record for a different packaging. [produce.Producer] implements it. +type Repacker interface { + // Repack stages an existing Record from a materialized tree. + Repack(ctx context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) +} + +// Service implements [artifact.Service] over repositories. +type Service struct { + repos Repositories + producers *artifact.ProducerRegistry + repacker Repacker + signer artifact.Signer + journal Journal + immutable ImmutabilityPolicy + clock func() time.Time + newID func() string + sleep func(ctx context.Context, d time.Duration) error + owner string + stagingRoot string + concurrency int + attempts int + backoff time.Duration +} + +var _ artifact.Service = (*Service)(nil) + +// Option customizes a [Service]. +type Option func(*Service) + +// WithClock injects the clock. The service captures the time once per operation, in UTC. +func WithClock(clock func() time.Time) Option { + defer perf.Track(nil, "service.WithClock")() + + return func(s *Service) { s.clock = clock } +} + +// WithIDGenerator injects the operation ID generator. +func WithIDGenerator(newID func() string) Option { + defer perf.Track(nil, "service.WithIDGenerator")() + + return func(s *Service) { s.newID = newID } +} + +// WithConcurrency sets the transfer and discovery concurrency. The default is +// [artifact.DefaultConcurrency]. +func WithConcurrency(n int) Option { + defer perf.Track(nil, "service.WithConcurrency")() + + return func(s *Service) { s.concurrency = n } +} + +// WithRetry sets the number of publication attempts and the base backoff. +func WithRetry(attempts int, backoff time.Duration) Option { + defer perf.Track(nil, "service.WithRetry")() + + return func(s *Service) { + s.attempts = attempts + s.backoff = backoff + } +} + +// WithSleep injects the backoff sleep; tests use a no-op. +func WithSleep(sleep func(ctx context.Context, d time.Duration) error) Option { + defer perf.Track(nil, "service.WithSleep")() + + return func(s *Service) { s.sleep = sleep } +} + +// WithJournal registers a lifecycle observer. +func WithJournal(journal Journal) Option { + defer perf.Track(nil, "service.WithJournal")() + + return func(s *Service) { s.journal = journal } +} + +// WithImmutabilityPolicy registers the immutable-format admission seam. +func WithImmutabilityPolicy(policy ImmutabilityPolicy) Option { + defer perf.Track(nil, "service.WithImmutabilityPolicy")() + + return func(s *Service) { s.immutable = policy } +} + +// WithSigner supplies the signer used by Sign. +func WithSigner(signer artifact.Signer) Option { + defer perf.Track(nil, "service.WithSigner")() + + return func(s *Service) { s.signer = signer } +} + +// WithOwner sets the lock owner recorded in claims, for example host and process. +func WithOwner(owner string) Option { + defer perf.Track(nil, "service.WithOwner")() + + return func(s *Service) { s.owner = owner } +} + +// WithStagingRoot sets the directory for bounded disk staging (preparation and copy trees). +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "service.WithStagingRoot")() + + return func(s *Service) { s.stagingRoot = dir } +} + +// WithRepacker overrides the repacker used by Copy's staging fallback. +func WithRepacker(repacker Repacker) Option { + defer perf.Track(nil, "service.WithRepacker")() + + return func(s *Service) { s.repacker = repacker } +} + +// New returns a Service. With a nil producer registry, only the generic files producer is available. +func New(repos Repositories, producers *artifact.ProducerRegistry, opts ...Option) (*Service, error) { + defer perf.Track(nil, "service.New")() + + s := &Service{ + repos: repos, + producers: producers, + clock: time.Now, + newID: uuid.NewString, + sleep: sleepContext, + concurrency: artifact.DefaultConcurrency, + attempts: DefaultAttempts, + backoff: DefaultBackoff, + } + for _, opt := range opts { + if opt != nil { + opt(s) + } + } + if s.concurrency < 1 { + s.concurrency = artifact.DefaultConcurrency + } + if s.attempts < 1 { + s.attempts = 1 + } + if s.owner == "" { + s.owner = defaultOwner() + } + if s.producers == nil { + s.producers = artifact.NewProducerRegistry() + if err := produce.Register(s.producers, produce.WithStagingRoot(s.stagingRoot)); err != nil { + return nil, err + } + } + if s.repacker == nil { + s.repacker = produce.New(produce.WithStagingRoot(s.stagingRoot)) + } + return s, nil +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s:%d", host, os.Getpid()) +} + +func sleepContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// operation carries the identity of one logical operation: the ID and the UTC preparation time +// are captured exactly once, before the first attempt, and reused by every retry. +type operation struct { + id string + preparedAt time.Time + started time.Time + clock func() time.Time + result artifact.OperationResult +} + +func (s *Service) begin(action artifact.Action, repository, requested string) *operation { + now := s.clock().UTC() + return &operation{ + id: s.newID(), + preparedAt: now, + started: now, + clock: s.clock, + result: artifact.OperationResult{ + OperationID: "", + Action: action, + Repository: repository, + Requested: requested, + }, + } +} + +// done finalizes the typed result: the operation ID, the duration, and an error diagnostic on failure. +func (o *operation) done(err error) artifact.OperationResult { + o.result.OperationID = o.id + o.result.Duration = o.clock().Sub(o.started) + if err != nil { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "operation-failed", Message: err.Error()}) + } + return o.result +} + +func (o *operation) warn(code, message string) { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) +} + +func (o *operation) info(code, message string) { + o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityInfo, Code: code, Message: message}) +} + +// record forwards a lifecycle event to the journal, if any. +func (s *Service) record(ctx context.Context, entry *JournalEntry) { + if s.journal != nil { + s.journal.Record(ctx, entry) + } +} diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go new file mode 100644 index 00000000000..0405ab792cc --- /dev/null +++ b/pkg/artifact/service/sign.go @@ -0,0 +1,146 @@ +package service + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Sign implements [artifact.Service]. It accepts digest references only, binds evidence to the +// resolved immutable subject, and reports that the subject is signed, not policy-verified. +func (s *Service) Sign(ctx context.Context, request artifact.SignRequest) (*artifact.SignResult, error) { + defer perf.Track(nil, "service.Service.Sign")() + + op := s.begin(artifact.ActionSign, request.Repository, request.Ref.String()) + result := &artifact.SignResult{} + err := s.sign(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) sign(ctx context.Context, op *operation, request *artifact.SignRequest, result *artifact.SignResult) error { + if err := s.validateSign(request); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opSign) + if err != nil { + return err + } + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + subject, err := signSubject(&pinned, inspection, request.Ref) + if err != nil { + return err + } + if err := checkSigningCapability(request.Repository, &info, s.signer, subject); err != nil { + return err + } + locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: request.Ref.String()}}) + if err != nil { + return err + } + defer func() { _ = locks.release(ctx, op) }() + + evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleFor(request.Repository)}) + if err != nil { + return fmt.Errorf("sign %s in repository %q: %w", request.Ref, request.Repository, err) + } + result.Subject, result.Evidence = subject, evidence + op.result.Outcome = artifact.OutcomeCreated + op.result.LockScope = locks.scope() + op.info("signed-not-verified", "the subject is signed; signing alone does not verify it against trust policy, use verify") + return nil +} + +// validateSign requires a repository, a valid digest reference, and a configured signer. +func (s *Service) validateSign(request *artifact.SignRequest) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + if !request.Ref.IsDigest() { + return fmt.Errorf("%w: sign accepts digest references only (%s@sha256:...), not %s", errUtils.ErrArtifactRefInvalid, request.Ref.Name, request.Ref) + } + if s.signer == nil { + return artifact.NewUnsupportedCapability(request.Repository, opSign, "signer", "no signer is configured") + } + return nil +} + +// signSubject binds the digest reference to the immutable subject: the portable Record when the +// digest names the Record (or the resolved local/S3 digest), the original OCI root otherwise. +func signSubject(pinned *artifact.PinnedReference, inspection *artifact.Inspection, ref artifact.Ref) (artifact.SignatureSubject, error) { + subject, err := subjectOf(inspection) + if err != nil { + return subject, err + } + if ref.Digest != subject.Digest { + return subject, fmt.Errorf("%w: %s does not name the signable subject %s of the resolved artifact (resolved %s)", errUtils.ErrArtifactRefInvalid, ref, subject.Digest, pinned.ResolvedDigest) + } + return subject, subject.Validate() +} + +func checkSigningCapability(alias string, info *RepositoryInfo, signer artifact.Signer, subject artifact.SignatureSubject) error { + if !info.Capabilities.Write { + return artifact.NewUnsupportedCapability(alias, opSign, "write", "the repository cannot store signature evidence") + } + if !info.Capabilities.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(alias, opSign, "subject/"+string(subject.Kind), "the repository cannot bind evidence to this subject kind") + } + for _, format := range signer.Formats() { + if info.Capabilities.HasEvidenceFormat(format) { + return nil + } + } + return artifact.NewUnsupportedCapability(alias, opSign, "evidence/format", fmt.Sprintf("the signer produces %v but the repository stores %v", signer.Formats(), info.Capabilities.EvidenceFormats)) +} + +// Verify implements [artifact.Service]. An explicit verify always requires a trusted valid +// signature: with no usable trust configuration the result is unchecked and the call fails rather +// than returning a green result. +func (s *Service) Verify(ctx context.Context, request artifact.VerifyRequest) (*artifact.VerifyResult, error) { + defer perf.Track(nil, "service.Service.Verify")() + + op := s.begin(artifact.ActionVerify, request.Repository, request.Ref.String()) + result := &artifact.VerifyResult{} + err := s.verify(ctx, op, &request, result) + result.Operation = op.done(err) + return result, err +} + +func (s *Service) verify(ctx context.Context, op *operation, request *artifact.VerifyRequest, result *artifact.VerifyResult) error { + if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { + return err + } + info, repo, err := s.openForRead(ctx, request.Repository, opVerify) + if err != nil { + return err + } + _, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) + if err != nil { + return err + } + subject, err := subjectOf(inspection) + if err != nil { + return err + } + policy := info.Policy + policy.RequireSignature = true + if !policy.HasTrustConstraints() { + result.Verification = artifact.VerificationResult{Subject: subject, Status: artifact.VerificationUnchecked} + return fmt.Errorf("%w: repository %q has no trusted identities or keys configured, so nothing can be verified; configure artifacts.verification", errUtils.ErrArtifactConfigInvalid, request.Repository) + } + verification, err := verifySubject(ctx, request.Repository, repo, subject, policy) + if verification != nil { + result.Verification = *verification + } + if err != nil { + return err + } + op.result.Outcome = artifact.OutcomeRead + return nil +} diff --git a/pkg/artifact/service/static.go b/pkg/artifact/service/static.go new file mode 100644 index 00000000000..b0124df0a63 --- /dev/null +++ b/pkg/artifact/service/static.go @@ -0,0 +1,55 @@ +package service + +import ( + "context" + "fmt" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +var _ Repositories = (*StaticRepositories)(nil) + +// Describe implements [Repositories]. +func (s *StaticRepositories) Describe(alias string) (RepositoryInfo, error) { + defer perf.Track(nil, "service.StaticRepositories.Describe")() + + repo, ok := s.Repositories[alias] + if !ok { + return RepositoryInfo{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + info := s.Info[alias] + info.Alias = alias + if info.Capabilities.Packagings == nil && !info.Capabilities.Read && !info.Capabilities.Write { + info.Capabilities = repo.Capabilities() + } + if info.Packaging == "" && len(info.Capabilities.Packagings) > 0 { + info.Packaging = info.Capabilities.Packagings[0] + } + return info, nil +} + +// Open implements [Repositories]. +func (s *StaticRepositories) Open(_ context.Context, alias string) (artifact.Repository, error) { + defer perf.Track(nil, "service.StaticRepositories.Open")() + + repo, ok := s.Repositories[alias] + if !ok { + return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + return repo, nil +} + +// Aliases implements [Repositories]. +func (s *StaticRepositories) Aliases() []string { + defer perf.Track(nil, "service.StaticRepositories.Aliases")() + + aliases := make([]string, 0, len(s.Repositories)) + for alias := range s.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/service/verify.go b/pkg/artifact/service/verify.go new file mode 100644 index 00000000000..8330d4df259 --- /dev/null +++ b/pkg/artifact/service/verify.go @@ -0,0 +1,156 @@ +package service + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// verificationRequired reports whether a policy requires, or constrains, verification. Without +// either, consumption reports "unchecked" and never "verified". +func verificationRequired(policy artifact.VerificationPolicy) bool { + return policy.RequireSignature || policy.HasTrustConstraints() +} + +// mergePolicies combines policies so that every trust set of every policy must be satisfied +// (global and repository sets are never unioned into a weaker policy). +func mergePolicies(policies ...artifact.VerificationPolicy) artifact.VerificationPolicy { + var merged artifact.VerificationPolicy + for _, policy := range policies { + merged.RequireSignature = merged.RequireSignature || policy.RequireSignature + merged.Offline = merged.Offline || policy.Offline + merged.TrustSets = append(merged.TrustSets, policy.TrustSets...) + } + return merged +} + +// subjectOf derives the immutable signature subject from an inspection: the original OCI root when +// a native graph is carried, otherwise the portable Record. Metadata claims never choose it. +func subjectOf(inspection *artifact.Inspection) (artifact.SignatureSubject, error) { + if root := inspection.Representation.OriginalRoot; root != nil { + return artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: root.Digest, MediaType: root.MediaType, Size: root.Size}, nil + } + if inspection.Record == nil { + return artifact.SignatureSubject{}, fmt.Errorf("%w: the artifact has neither a Record nor an OCI root to sign or verify", errUtils.ErrArtifactRecordInvalid) + } + canonical, err := inspection.Record.Canonical() + if err != nil { + return artifact.SignatureSubject{}, err + } + return artifact.SignatureSubject{ + Kind: artifact.SubjectRecord, + Digest: digest.FromBytes(canonical), + MediaType: artifact.MediaTypeConfig, + Size: int64(len(canonical)), + }, nil +} + +// verifySubject requires every trust set of policy to be satisfied by a valid permitted signature +// on subject. Each set is checked through its own repository Verify call, so the "both global and +// repository sets" rule holds regardless of the backend. Unsupported evidence fails closed. +func verifySubject(ctx context.Context, alias string, repo artifact.Repository, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + caps := repo.Capabilities() + if !caps.HasSubjectKind(subject.Kind) { + unsupported := artifact.NewUnsupportedCapability(alias, opVerify, "subject/"+string(subject.Kind), "the repository cannot bind or verify evidence for this subject kind") + return failedResult(subject, policy, unsupported), errors.Join(errUtils.ErrArtifactVerificationFailed, unsupported) + } + fingerprint, err := policy.Fingerprint() + if err != nil { + return nil, err + } + combined := &artifact.VerificationResult{Subject: subject, PolicyFingerprint: fingerprint, Status: artifact.VerificationVerified} + for _, call := range splitPolicy(policy) { + result, verifyErr := repo.Verify(ctx, subject, call) + if failure := checkVerifyResult(subject, result, verifyErr); failure != nil { + combined.Status = artifact.VerificationFailed + if result != nil { + combined.Diagnostics = append(combined.Diagnostics, result.Diagnostics...) + } + combined.Diagnostics = append(combined.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "verification-failed", Message: failure.Error()}) + return combined, errors.Join(errUtils.ErrArtifactVerificationFailed, failure) + } + mergeResult(combined, result) + } + return combined, nil +} + +// mergeResult folds a verified per-set result into the combined one without duplicating claims, +// evidence digests, or diagnostics that several sets share. +func mergeResult(combined, result *artifact.VerificationResult) { + for _, claim := range result.Claims { + if !slices.Contains(combined.Claims, claim) { + combined.Claims = append(combined.Claims, claim) + } + } + for _, evidence := range result.EvidenceDigests { + if !slices.Contains(combined.EvidenceDigests, evidence) { + combined.EvidenceDigests = append(combined.EvidenceDigests, evidence) + } + } + for _, diagnostic := range result.Diagnostics { + if !slices.Contains(combined.Diagnostics, diagnostic) { + combined.Diagnostics = append(combined.Diagnostics, diagnostic) + } + } +} + +// splitPolicy returns one single-trust-set policy per set, or the policy itself when it has none. +func splitPolicy(policy artifact.VerificationPolicy) []artifact.VerificationPolicy { + if len(policy.TrustSets) == 0 { + policy.RequireSignature = true + return []artifact.VerificationPolicy{policy} + } + calls := make([]artifact.VerificationPolicy, 0, len(policy.TrustSets)) + for _, set := range policy.TrustSets { + calls = append(calls, artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{set}, Offline: policy.Offline}) + } + return calls +} + +// checkVerifyResult returns a failure when the repository errored, returned nothing, returned a +// result for another subject, or did not report the verified status. +func checkVerifyResult(subject artifact.SignatureSubject, result *artifact.VerificationResult, err error) error { + switch { + case err != nil: + return err + case result == nil: + return fmt.Errorf("%w: the repository returned no verification result", errUtils.ErrArtifactVerificationFailed) + case result.Subject.Digest != subject.Digest || result.Subject.Kind != subject.Kind: + return fmt.Errorf("%w: the verification result is bound to %s, not the requested subject %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest, subject.Digest) + case result.Status != artifact.VerificationVerified: + return fmt.Errorf("%w: verification status is %q", errUtils.ErrArtifactVerificationFailed, result.Status) + } + return nil +} + +func failedResult(subject artifact.SignatureSubject, policy artifact.VerificationPolicy, cause error) *artifact.VerificationResult { + fingerprint, _ := policy.Fingerprint() + return &artifact.VerificationResult{ + Subject: subject, + PolicyFingerprint: fingerprint, + Status: artifact.VerificationFailed, + Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityError, Code: "verification-failed", Message: cause.Error()}}, + } +} + +// gate enforces the consumption policy. It returns an unchecked result when no policy requires or +// constrains verification (never "verified"), a verified result when every trust set is satisfied, +// and a failed result plus ErrArtifactVerificationFailed otherwise. Force never reaches this +// function. +func gate(ctx context.Context, alias string, repo artifact.Repository, inspection *artifact.Inspection, policies ...artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + merged := mergePolicies(policies...) + if !verificationRequired(merged) { + return &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil + } + subject, err := subjectOf(inspection) + if err != nil { + return nil, err + } + return verifySubject(ctx, alias, repo, subject, merged) +} diff --git a/pkg/artifact/service/verify_test.go b/pkg/artifact/service/verify_test.go new file mode 100644 index 00000000000..76b4a248b56 --- /dev/null +++ b/pkg/artifact/service/verify_test.go @@ -0,0 +1,254 @@ +package service_test + +import ( + "context" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +var releaseSigner = &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + +// signedEnv publishes the example artifact and signs it through the service. +func signedEnv(t *testing.T, config envConfig) (*env, *service.PublishOutcome) { + t.Helper() + config.options = append(config.options, service.WithSigner(releaseSigner)) + e := newEnv(t, config) + published := e.publish("dst", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + return e, published +} + +func TestSign_BindsEvidenceToTheDigestAndReportsSignedNotVerified(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + + result, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + assert.Equal(t, artifact.SubjectRecord, result.Subject.Kind) + assert.Equal(t, published.Operation.RecordDigest, result.Subject.Digest) + require.Len(t, result.Evidence, 1) + assert.Equal(t, artifact.OutcomeCreated, result.Operation.Outcome) + assert.Equal(t, artifact.LockScopeHost, result.Operation.LockScope) + var notVerified bool + for _, diagnostic := range result.Operation.Diagnostics { + notVerified = notVerified || diagnostic.Code == "signed-not-verified" + } + assert.True(t, notVerified, "signing alone reports signed, not policy-verified") + assert.Empty(t, e.repos["dst"].HeldLocks()) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Nil(t, inspected.Verification, "with no trust configuration the artifact stays unchecked even though it is signed") + assert.Len(t, inspected.Inspection.Evidence, 1) +} + +func TestSign_Rejections(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + good := artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest} + tests := []struct { + name string + request artifact.SignRequest + want error + }{ + {"version reference", artifact.SignRequest{Repository: "dst", Ref: exampleRef}, errUtils.ErrArtifactRefInvalid}, + {"missing repository", artifact.SignRequest{Ref: good}, errUtils.ErrArtifactConfigInvalid}, + {"digest of another subject", artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: digest.FromString("other")}}, errUtils.ErrArtifactNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := e.svc.Sign(context.Background(), tt.request) + require.ErrorIs(t, err, tt.want) + require.NotNil(t, result) + }) + } + assert.Zero(t, e.repos["dst"].Calls("Sign")) +} + +func TestSign_CapabilityChecksPrecedeLocking(t *testing.T) { + noEvidence := artifacttest.DefaultCapabilities() + noEvidence.EvidenceFormats = nil + e := newEnv(t, envConfig{ + repos: map[string]artifacttest.Options{"dst": {Capabilities: &noEvidence}}, + options: []service.Option{service.WithSigner(releaseSigner)}, + }) + published := e.publish("dst", e.source("hello")) + locks := e.repos["dst"].Calls("Lock") + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, locks, e.repos["dst"].Calls("Lock"), "no lock is taken for an unsupported signing request") + + noSigner := newEnv(t, envConfig{}) + published = noSigner.publish("dst", noSigner.source("hello")) + _, err = noSigner.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestVerify_RequiresATrustedValidSignature(t *testing.T) { + e, _ := signedEnv(t, envConfig{}) + request := artifact.VerifyRequest{Repository: "dst", Ref: exampleRef} + + t.Run("no trust configuration is not a green result", func(t *testing.T) { + result, err := e.svc.Verify(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Equal(t, artifact.VerificationUnchecked, result.Verification.Status) + assert.NotEqual(t, artifact.VerificationVerified, result.Verification.Status) + }) + + t.Run("trusted signer verifies", func(t *testing.T) { + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + result, err := e.svc.Verify(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Verification.Status) + assert.Equal(t, artifact.SubjectRecord, result.Verification.Subject.Kind) + assert.NotEmpty(t, result.Verification.EvidenceDigests) + assert.NotEmpty(t, result.Verification.PolicyFingerprint) + require.NotEmpty(t, result.Verification.Claims) + assert.Equal(t, "release@example.com", result.Verification.Claims[0].Identity) + }) + + t.Run("untrusted signer fails", func(t *testing.T) { + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.UntrustedPolicy() }) + result, err := e.svc.Verify(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Verification.Status) + }) +} + +func TestVerify_GlobalAndRepositoryTrustSetsBothMustBeSatisfied(t *testing.T) { + e, published := signedEnv(t, envConfig{}) + second := &artifacttest.FakeSigner{Identity: "security@example.com", Issuer: "https://issuer.example.com"} + policy := func(signers ...*artifacttest.FakeSigner) artifact.VerificationPolicy { + var merged artifact.VerificationPolicy + merged.RequireSignature = true + for _, s := range signers { + merged.TrustSets = append(merged.TrustSets, s.TrustedPolicy().TrustSets...) + } + return merged + } + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = policy(releaseSigner, second) }) + + _, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "satisfying one of two required trust sets is not enough; they are never unioned") + + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: published.Operation.RecordDigest, MediaType: artifact.MediaTypeConfig} + _, err = e.repos["dst"].Sign(context.Background(), subject, second, artifact.SignOptions{}) + require.NoError(t, err) + result, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Verification.Status) + assert.Len(t, result.Verification.Claims, 2) +} + +func TestConsumption_RequiredVerificationRunsBeforeMaterializationAndForceDoesNotBypassIt(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + reads := e.repos["dst"].Calls("Read") + + dest := out(t) + result, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "force never bypasses verification") + assert.NotNil(t, result) + assert.Equal(t, reads, e.repos["dst"].Calls("Read"), "verification ran before the payload was read") + assert.True(t, missing(dest), "nothing was materialized") + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.NoError(t, err) +} + +func TestInspect_ReportsVerificationWhenPolicyConstrainsIt(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + unsigned, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err, "inspect consumes no payload, so a failed verification is information, not a hidden artifact") + require.NotNil(t, unsigned.Verification) + assert.Equal(t, artifact.VerificationFailed, unsigned.Verification.Status) + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + signed, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + require.NotNil(t, signed.Verification) + assert.Equal(t, artifact.VerificationVerified, signed.Verification.Status) +} + +func TestShowDocument_RequiredVerificationApplies(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + _, err := e.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) +} + +func TestVerification_UnsupportedSubjectKindFailsClosed(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.SubjectKinds = nil + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {Capabilities: &caps}}}) + e.publish("dst", e.source("hello")) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: out(t)}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "unsupported required evidence fails closed and says why") +} + +func TestVerification_RepositoryResultsAreNeverTrustedBlindly(t *testing.T) { + record := artifact.Record{Name: "tools/example", Version: "1", Kind: artifact.KindFiles, CreatedAt: artifact.NewTimestamp(startTime)} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + wrongSubject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("other")} + + tests := []struct { + name string + result *artifact.VerificationResult + err error + }{ + {"nil result", nil, nil}, + {"status unchecked", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil}, + {"result for another subject", &artifact.VerificationResult{Subject: wrongSubject, Status: artifact.VerificationVerified}, nil}, + {"verified but errored", &artifact.VerificationResult{Status: artifact.VerificationVerified}, errTransport}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + repo := artifact.NewMockRepository(ctrl) + caps := artifacttest.DefaultCapabilities() + repo.EXPECT().Capabilities().Return(caps).AnyTimes() + pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} + repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Pinned: pinned, Record: &record}, nil) + repo.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, subject artifact.SignatureSubject, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + if tt.result != nil && tt.result.Subject.Digest == "" { + tt.result.Subject = subject + } + return tt.result, tt.err + }, + ) + repos := &service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"dst": repo}, + Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone, Policy: releaseSigner.TrustedPolicy()}}, + } + svc, err := service.New(repos, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + result, err := svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.NotEqual(t, artifact.VerificationVerified, result.Verification.Status) + }) + } +} diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go new file mode 100644 index 00000000000..6255be03131 --- /dev/null +++ b/pkg/artifact/snapshot/copy.go @@ -0,0 +1,164 @@ +package snapshot + +import ( + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// captureFile copies one regular file into staging while hashing it, detecting a file that was +// swapped, grew, shrank, or was modified while it was read. +func (c *capturer) captureFile(abs, rel string, before os.FileInfo) error { + if c.total+before.Size() > c.opts.limits.MaxBytes { + return fmt.Errorf("%w: staged content exceeds %d bytes", errUtils.ErrArtifactTooLarge, c.opts.limits.MaxBytes) + } + staged, sum, err := c.stageFile(abs, rel, before) + if err != nil { + return err + } + if hook := c.opts.onCaptured; hook != nil { + hook(rel) + } + entry := artifact.Entry{Path: rel, Type: artifact.EntryFile, Size: before.Size(), Mode: permBits(before), Digest: sum} + if err := c.addEntry(&entry); err != nil { + return err + } + c.total += before.Size() + c.staged[rel] = staged + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: before, digest: sum}) + return nil +} + +// stageFile copies abs into the staging data directory and returns the staged path and digest. +func (c *capturer) stageFile(abs, rel string, before os.FileInfo) (string, digest.Digest, error) { + src, err := openSame(abs, rel, before) + if err != nil { + return "", "", err + } + defer src.Close() + + c.counter++ + stagedPath := filepath.Join(c.dataDir, strconv.Itoa(c.counter)) + dst, err := os.OpenFile(stagedPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, stagedFilePerm) + if err != nil { + return "", "", fmt.Errorf("%w: stage %q: %w", errUtils.ErrArtifactPackagingInvalid, rel, err) + } + hasher := sha256.New() + copied, copyErr := copyContext(c.ctx, io.MultiWriter(dst, hasher), io.LimitReader(src, before.Size()+1)) + if closeErr := dst.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return "", "", fmt.Errorf("%w: stage %q: %w", errUtils.ErrArtifactPackagingInvalid, rel, copyErr) + } + after, err := src.Stat() + if err != nil { + return "", "", changedError(rel, err) + } + if copied != before.Size() || !sameFileState(before, after) { + return "", "", changedError(rel, errSizeOrTimeChanged) + } + return stagedPath, digest.NewDigest(digest.SHA256, hasher), nil +} + +// errSizeOrTimeChanged explains a file that changed while it was being read. +var errSizeOrTimeChanged = errors.New("changed size or modification time while it was being read") + +// errReplaced explains a path that was swapped for another file while it was being captured. +var errReplaced = errors.New("was replaced while it was being captured") + +// openSame opens abs and confirms it is still the regular file that was lstat'd. +func openSame(abs, rel string, before os.FileInfo) (*os.File, error) { + src, err := os.Open(abs) + if err != nil { + return nil, changedError(rel, err) + } + opened, err := src.Stat() + if err != nil { + _ = src.Close() + return nil, changedError(rel, err) + } + if !opened.Mode().IsRegular() || !os.SameFile(before, opened) { + _ = src.Close() + return nil, changedError(rel, errReplaced) + } + return src, nil +} + +// sameFileState reports whether two stats describe the same unmodified file state. +func sameFileState(a, b os.FileInfo) bool { + return a.Size() == b.Size() && a.ModTime().Equal(b.ModTime()) && a.Mode() == b.Mode() +} + +// recheck verifies every captured source entry still matches what was staged: type, size, +// modification time, directory listing, link target, and file digest. +func (c *capturer) recheck() error { + for index := range c.sources { + if err := c.ctx.Err(); err != nil { + return err + } + if err := c.recheckSource(&c.sources[index]); err != nil { + return err + } + } + for abs, names := range c.dirs { + current, err := readDirNames(abs) + if err != nil { + return err + } + if !slices.Equal(current, names) { + return fmt.Errorf("%w: directory %q gained or lost entries while it was being captured", errUtils.ErrArtifactSourceChanged, abs) + } + } + return nil +} + +func (c *capturer) recheckSource(record *sourceRecord) error { + now, err := os.Lstat(record.abs) + if err != nil { + return changedError(record.rel, err) + } + if now.Mode().Type() != record.info.Mode().Type() { + return fmt.Errorf("%w: %q changed type while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + switch { + case record.digest != "": + return c.recheckFile(record, now) + case record.target != "": + current, err := os.Readlink(record.abs) + if err != nil || filepath.ToSlash(current) != record.target { + return fmt.Errorf("%w: symlink %q changed while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + } + return nil +} + +// recheckFile compares stat state and then re-hashes the source file against the staged digest. +func (c *capturer) recheckFile(record *sourceRecord, now os.FileInfo) error { + if !sameFileState(record.info, now) { + return fmt.Errorf("%w: %q changed size, mode, or modification time while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + file, err := os.Open(record.abs) + if err != nil { + return changedError(record.rel, err) + } + defer file.Close() + hasher := sha256.New() + if _, err := copyContext(c.ctx, hasher, file); err != nil { + return fmt.Errorf("%w: recheck %q: %w", errUtils.ErrArtifactSourceChanged, record.rel, err) + } + if digest.NewDigest(digest.SHA256, hasher) != record.digest { + return fmt.Errorf("%w: %q content changed while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + return nil +} diff --git a/pkg/artifact/snapshot/doc.go b/pkg/artifact/snapshot/doc.go new file mode 100644 index 00000000000..91675669892 --- /dev/null +++ b/pkg/artifact/snapshot/doc.go @@ -0,0 +1,14 @@ +// Package snapshot captures a local source file or directory into bounded disk staging and +// produces a canonical inventory of it. +// +// Capture walks the source with lstat, copies regular file bytes into a private staging directory +// while hashing them, and then rechecks every source entry (type, size, modification time, and +// digest) so that an inventory is never published for bytes other than the ones staged. Files that +// change during capture fail the operation with ErrArtifactSourceChanged. +// +// Entry rules follow the artifact contracts: directories, file bytes, ordinary permission bits and +// contained relative symlinks are preserved; absolute or escaping symlinks, devices, sockets, and +// other special files are rejected; hard links become independent regular files; privileged mode +// bits are cleared; owner, group, and ACLs are never recorded. Owned temporary data is removed on +// success (via Close), error, and cancellation. +package snapshot diff --git a/pkg/artifact/snapshot/finish_test.go b/pkg/artifact/snapshot/finish_test.go new file mode 100644 index 00000000000..3bd635e7cd0 --- /dev/null +++ b/pkg/artifact/snapshot/finish_test.go @@ -0,0 +1,29 @@ +package snapshot + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func TestFinish_RejectsDuplicatesAfterUnicodeNormalization(t *testing.T) { + const digestOfEmpty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + composed := "café.txt" + decomposed := "café.txt" + require.NotEqual(t, composed, decomposed) + + c := &capturer{entries: []artifact.Entry{ + {Path: composed, Type: artifact.EntryFile, Mode: 0o644, Digest: digestOfEmpty}, + {Path: decomposed, Type: artifact.EntryFile, Mode: 0o644, Digest: digestOfEmpty}, + }} + err := c.finish() + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.Contains(t, err.Error(), "normalization") + + c.entries = c.entries[:1] + require.NoError(t, c.finish()) +} diff --git a/pkg/artifact/snapshot/reader.go b/pkg/artifact/snapshot/reader.go new file mode 100644 index 00000000000..9e79223cfc7 --- /dev/null +++ b/pkg/artifact/snapshot/reader.go @@ -0,0 +1,35 @@ +package snapshot + +import ( + "context" + "io" +) + +// copyChunkSize is the chunk size of context-aware copies. +const copyChunkSize = 32 * 1024 + +// copyContext copies src to dst in chunks and stops as soon as ctx is canceled, so large copies +// end promptly on cancellation. +func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { + buffer := make([]byte, copyChunkSize) + var total int64 + for { + if err := ctx.Err(); err != nil { + return total, err + } + n, readErr := src.Read(buffer) + if n > 0 { + written, writeErr := dst.Write(buffer[:n]) + total += int64(written) + if writeErr != nil { + return total, writeErr + } + } + if readErr == io.EOF { + return total, nil + } + if readErr != nil { + return total, readErr + } + } +} diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go new file mode 100644 index 00000000000..4b950ec591d --- /dev/null +++ b/pkg/artifact/snapshot/snapshot.go @@ -0,0 +1,234 @@ +package snapshot + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "golang.org/x/text/unicode/norm" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Default capture limits. Both are overridable with [WithLimits]. +const ( + // DefaultMaxBytes bounds the total staged file bytes. + DefaultMaxBytes int64 = 16 << 30 + + // DefaultMaxEntries bounds the number of inventory entries. + DefaultMaxEntries = 100_000 +) + +const ( + stagingPattern = "atmos-artifact-*" + dataDirName = "data" + stagingDirPerm = 0o700 + stagedFilePerm = 0o600 +) + +// Limits bound a capture. Zero values select the defaults. +type Limits struct { + // MaxBytes is the maximum total size of staged file content. + MaxBytes int64 + + // MaxEntries is the maximum number of inventory entries. + MaxEntries int +} + +// Option customizes [Capture]. +type Option func(*options) + +type options struct { + stagingRoot string + limits Limits + onCaptured func(path string) +} + +// WithStagingRoot sets the directory that holds the private staging directory. The default is the +// operating system's temporary directory. +func WithStagingRoot(dir string) Option { + defer perf.Track(nil, "snapshot.WithStagingRoot")() + + return func(o *options) { o.stagingRoot = dir } +} + +// WithLimits overrides the capture limits. +func WithLimits(limits Limits) Option { + defer perf.Track(nil, "snapshot.WithLimits")() + + return func(o *options) { o.limits = limits } +} + +// WithCaptureHook registers a callback invoked after each regular file has been copied and before +// the final recheck. Tests use it to mutate a source file mid-capture. +func WithCaptureHook(hook func(path string)) Option { + defer perf.Track(nil, "snapshot.WithCaptureHook")() + + return func(o *options) { o.onCaptured = hook } +} + +// Snapshot is a stable, staged copy of a source plus its sorted inventory. It owns a private +// staging directory; the caller must Close it. +type Snapshot struct { + // Root is the private staging directory. + Root string + + // Entries is the inventory, sorted by path. + Entries []artifact.Entry + + // TotalBytes is the total size of staged file content. + TotalBytes int64 + + // SourceIsFile reports that the source was a single file rather than a directory. + SourceIsFile bool + + staged map[string]string +} + +// Close removes the staging directory. It is idempotent. +func (s *Snapshot) Close() error { + defer perf.Track(nil, "snapshot.Snapshot.Close")() + + if s == nil || s.Root == "" { + return nil + } + err := os.RemoveAll(s.Root) + s.Root = "" + s.staged = nil + return err +} + +// StagedPath returns the staged copy of the regular file at the inventory path. +func (s *Snapshot) StagedPath(entryPath string) (string, bool) { + defer perf.Track(nil, "snapshot.Snapshot.StagedPath")() + + p, ok := s.staged[entryPath] + return p, ok +} + +// Open opens the staged copy of the regular file at the inventory path. +func (s *Snapshot) Open(entryPath string) (io.ReadCloser, error) { + defer perf.Track(nil, "snapshot.Snapshot.Open")() + + p, ok := s.staged[entryPath] + if !ok { + return nil, fmt.Errorf("%w: %q is not a staged file", errUtils.ErrArtifactNotFound, entryPath) + } + return os.Open(p) +} + +// Lookup returns the inventory entry at path. +func (s *Snapshot) Lookup(entryPath string) (artifact.Entry, bool) { + defer perf.Track(nil, "snapshot.Snapshot.Lookup")() + + index := sort.Search(len(s.Entries), func(i int) bool { return s.Entries[i].Path >= entryPath }) + if index < len(s.Entries) && s.Entries[index].Path == entryPath { + return s.Entries[index], true + } + return artifact.Entry{}, false +} + +// Capture snapshots source into bounded disk staging. It fails, and removes all staged data, when +// the source contains unsafe entries, exceeds the limits, changes during capture, or ctx is +// canceled. +func Capture(ctx context.Context, source string, opts ...Option) (*Snapshot, error) { + defer perf.Track(nil, "snapshot.Capture")() + + var applied options + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.limits.MaxBytes <= 0 { + applied.limits.MaxBytes = DefaultMaxBytes + } + if applied.limits.MaxEntries <= 0 { + applied.limits.MaxEntries = DefaultMaxEntries + } + root, err := os.MkdirTemp(applied.stagingRoot, stagingPattern) + if err != nil { + return nil, fmt.Errorf("%w: create staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + snap, err := captureInto(ctx, source, root, &applied) + if err != nil { + _ = os.RemoveAll(root) + return nil, err + } + return snap, nil +} + +func captureInto(ctx context.Context, source, root string, opts *options) (*Snapshot, error) { + dataDir := filepath.Join(root, dataDirName) + if err := os.Mkdir(dataDir, stagingDirPerm); err != nil { + return nil, fmt.Errorf("%w: create staging data directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + resolved, info, err := resolveSource(source) + if err != nil { + return nil, err + } + c := &capturer{ctx: ctx, opts: opts, root: resolved, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} + if info.IsDir() { + err = c.walkDir(resolved, "") + } else { + err = c.captureFile(resolved, filepath.Base(resolved), info) + } + if err != nil { + return nil, err + } + if err := c.recheck(); err != nil { + return nil, err + } + if info.IsDir() { + if err := c.checkLinkContainment(); err != nil { + return nil, err + } + } + if err := c.finish(); err != nil { + return nil, err + } + sort.Slice(c.entries, func(i, j int) bool { return c.entries[i].Path < c.entries[j].Path }) + return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), staged: c.staged}, nil +} + +// resolveSource resolves a user-chosen top-level source (which may itself be a symlink) and checks +// it is a directory or regular file. Links inside the source are never followed. +func resolveSource(source string) (string, os.FileInfo, error) { + if source == "" { + return "", nil, fmt.Errorf("%w: source is empty", errUtils.ErrArtifactPackagingInvalid) + } + resolved, err := filepath.EvalSymlinks(source) + if err != nil { + return "", nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + info, err := os.Lstat(resolved) + if err != nil { + return "", nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + if !info.IsDir() && !info.Mode().IsRegular() { + return "", nil, fmt.Errorf("%w: source %q must be a regular file or directory", errUtils.ErrArtifactPackagingInvalid, source) + } + return resolved, info, nil +} + +// finish validates every entry and rejects paths that collide after Unicode normalization. +func (c *capturer) finish() error { + seen := make(map[string]string, len(c.entries)) + for index := range c.entries { + entry := &c.entries[index] + if err := entry.Validate(); err != nil { + return err + } + normalized := norm.NFC.String(entry.Path) + if previous, duplicate := seen[normalized]; duplicate { + return fmt.Errorf("%w: paths %q and %q are duplicates after Unicode normalization", errUtils.ErrArtifactPackagingInvalid, previous, entry.Path) + } + seen[normalized] = entry.Path + } + return nil +} diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go new file mode 100644 index 00000000000..30bd4f2a48b --- /dev/null +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -0,0 +1,282 @@ +package snapshot_test + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +func entryPaths(entries []artifact.Entry) []string { + paths := make([]string, 0, len(entries)) + for _, e := range entries { + paths = append(paths, e.Path) + } + return paths +} + +func TestCapture_FilePackageInventory(t *testing.T) { + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + + want := append(fixtures.FilePackagePaths(pkg.Symlinks), ".atmos", ".atmos/artifact", ".atmos/artifact/CHANGELOG.md", ".atmos/artifact/README.md") + assert.ElementsMatch(t, want, entryPaths(snap.Entries)) + assert.IsIncreasing(t, entryPaths(snap.Entries), "inventory must be sorted by path") + + helper, ok := snap.Lookup("bin/helper.sh") + require.True(t, ok) + assert.Equal(t, artifact.EntryFile, helper.Type) + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), helper.Mode, "executable bit preserved") + } + assert.Equal(t, int64(len(fixtures.HelperScript)), helper.Size) + assert.Equal(t, digest.FromString(fixtures.HelperScript), helper.Digest) + + empty, ok := snap.Lookup("empty.txt") + require.True(t, ok) + assert.Zero(t, empty.Size) + assert.Equal(t, digest.FromString(""), empty.Digest) + + dir, ok := snap.Lookup("emptydir") + require.True(t, ok) + assert.Equal(t, artifact.EntryDirectory, dir.Type) + + _, ok = snap.Lookup(fixtures.UnicodeName) + assert.True(t, ok, "Unicode filenames are preserved") + + if pkg.Symlinks { + link, ok := snap.Lookup("link-to-file") + require.True(t, ok) + assert.Equal(t, artifact.EntrySymlink, link.Type) + assert.Equal(t, "nested/deep/file.txt", link.Target) + assert.Zero(t, link.Mode) + } + + reader, err := snap.Open("nested/deep/file.txt") + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "deep content\n", string(data)) +} + +func TestCapture_SingleFileSource(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "bytes", 0o755) + snap, err := snapshot.Capture(context.Background(), filepath.Join(dir, "tool.bin"), snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + assert.True(t, snap.SourceIsFile) + require.Len(t, snap.Entries, 1) + assert.Equal(t, "tool.bin", snap.Entries[0].Path) + if fixtures.POSIXModes() { + assert.Equal(t, uint32(0o755), snap.Entries[0].Mode) + } +} + +func TestCapture_ClearsPrivilegedBitsAndIgnoresOwnership(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "x", 0o644) + path := filepath.Join(src, "a.txt") + // Setuid is not representable on every platform; failure to set it only weakens the assertion. + _ = os.Chmod(path, 0o755|os.ModeSetuid) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + entry, _ := snap.Lookup("a.txt") + assert.Zero(t, entry.Mode&^0o777, "privileged bits must be cleared") +} + +func TestCapture_HardLinksBecomeIndependentFiles(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "original.txt", "shared", 0o644) + if err := os.Link(filepath.Join(src, "original.txt"), filepath.Join(src, "hard.txt")); err != nil { + t.Skipf("hard links are unavailable here: %v", err) + } + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + original, _ := snap.Lookup("original.txt") + hard, _ := snap.Lookup("hard.txt") + assert.Equal(t, artifact.EntryFile, hard.Type) + assert.Equal(t, original.Digest, hard.Digest) + first, _ := snap.StagedPath("original.txt") + second, _ := snap.StagedPath("hard.txt") + assert.NotEqual(t, first, second, "each hard link is staged as its own file") +} + +func TestCapture_InvalidSources(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T, dir string) bool + want error + }{ + {"absolute symlink", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "abs", filepath.ToSlash(os.TempDir())) + }, errUtils.ErrArtifactUnsafePath}, + {"escaping relative symlink", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "sub") + return fixtures.Symlink(t, dir, "sub/escape", "../../outside") + }, errUtils.ErrArtifactUnsafePath}, + {"traversal target", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, "up", "..") + }, errUtils.ErrArtifactUnsafePath}, + {"chained symlink escape", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "a") + return fixtures.Symlink(t, dir, "a/b", "..") && fixtures.Symlink(t, dir, "c", "a/b/../..") + }, errUtils.ErrArtifactUnsafePath}, + {"special file", func(t *testing.T, dir string) bool { + return fixtures.SpecialFile(t, dir, "pipe") + }, errUtils.ErrArtifactPackagingInvalid}, + {"reserved namespace", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, artifact.ReservedNamespace+"/v1/refs/x.json", "{}", 0o644) + return true + }, errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + if !tt.build(t, src) { + t.Skip("fixture is unavailable on this platform") + } + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.Error(t, err) + assert.Nil(t, snap) + assert.ErrorIs(t, err, tt.want) + assertEmptyDir(t, staging) + }) + } +} + +func assertEmptyDir(t *testing.T, dir string) { + t.Helper() + items, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, items, "owned temporary data must be removed") +} + +func TestCapture_RejectsMissingAndNonRegularSource(t *testing.T) { + _, err := snapshot.Capture(context.Background(), filepath.Join(t.TempDir(), "absent"), snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + _, err = snapshot.Capture(context.Background(), "", snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestCapture_DetectsChangeDuringCapture(t *testing.T) { + tests := []struct { + name string + mutate func(t *testing.T, src string) + }{ + {"grown", func(t *testing.T, src string) { + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("original-and-more"), 0o644)) + }}, + {"same size new mtime", func(t *testing.T, src string) { + path := filepath.Join(src, "a.txt") + require.NoError(t, os.WriteFile(path, []byte("ORIGINAL"), 0o644)) + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Hour), time.Now().Add(time.Hour))) + }}, + {"same size same mtime different digest", func(t *testing.T, src string) { + path := filepath.Join(src, "a.txt") + info, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, []byte("ORIGINAL"), 0o644)) + require.NoError(t, os.Chtimes(path, info.ModTime(), info.ModTime())) + }}, + {"file added", func(t *testing.T, src string) { + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + }}, + {"file removed", func(t *testing.T, src string) { + require.NoError(t, os.Remove(filepath.Join(src, "b.txt"))) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "original", 0o644) + fixtures.WriteFile(t, src, "b.txt", "other", 0o644) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, + snapshot.WithStagingRoot(staging), + snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + tt.mutate(t, src) + } + })) + require.Error(t, err) + assert.Nil(t, snap) + assert.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assertEmptyDir(t, staging) + }) + } +} + +func TestCapture_Limits(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) + fixtures.WriteFile(t, src, "b.txt", "0123456789", 0o644) + + staging := t.TempDir() + _, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging), snapshot.WithLimits(snapshot.Limits{MaxBytes: 15})) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + assertEmptyDir(t, staging) + + _, err = snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithLimits(snapshot.Limits{MaxEntries: 1})) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithLimits(snapshot.Limits{MaxBytes: 20, MaxEntries: 2})) + require.NoError(t, err) + assert.Equal(t, int64(20), snap.TotalBytes) + require.NoError(t, snap.Close()) +} + +func TestCapture_CancellationRemovesStaging(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + ctx, cancel := context.WithCancel(context.Background()) + staging := t.TempDir() + snap, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(staging), snapshot.WithCaptureHook(func(string) { cancel() })) + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, snap) + assertEmptyDir(t, staging) +} + +func TestSnapshot_CloseRemovesStagingAndIsIdempotent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.NoError(t, err) + require.NoError(t, snap.Close()) + require.NoError(t, snap.Close()) + assertEmptyDir(t, staging) + _, ok := snap.StagedPath("a.txt") + assert.False(t, ok) +} + +func TestSnapshot_OpenUnknownPath(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + _, err = snap.Open("missing") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, ok := snap.Lookup("missing") + assert.False(t, ok) +} diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go new file mode 100644 index 00000000000..3c057601d03 --- /dev/null +++ b/pkg/artifact/snapshot/walk.go @@ -0,0 +1,164 @@ +package snapshot + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// sourceRecord remembers one captured source entry for the final recheck. +type sourceRecord struct { + abs string + rel string + info os.FileInfo + digest digest.Digest + target string +} + +// capturer holds the state of one capture. +type capturer struct { + ctx context.Context + opts *options + root string + dataDir string + entries []artifact.Entry + staged map[string]string + dirs map[string][]string + sources []sourceRecord + total int64 + counter int +} + +// walkDir captures the directory at abs, whose slash-separated inventory path is rel ("" for the root). +func (c *capturer) walkDir(abs, rel string) error { + names, err := readDirNames(abs) + if err != nil { + return err + } + c.dirs[abs] = names + for _, name := range names { + if err := c.ctx.Err(); err != nil { + return err + } + childAbs := filepath.Join(abs, name) + childRel := name + if rel != "" { + childRel = rel + "/" + name + } + if err := c.captureChild(childAbs, childRel); err != nil { + return err + } + } + return nil +} + +// captureChild classifies one directory child with lstat and captures it. +func (c *capturer) captureChild(abs, rel string) error { + info, err := os.Lstat(abs) + if err != nil { + return changedError(rel, err) + } + mode := info.Mode() + switch { + case mode.IsDir(): + if err := c.addEntry(&artifact.Entry{Path: rel, Type: artifact.EntryDirectory, Mode: permBits(info)}); err != nil { + return err + } + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: info}) + return c.walkDir(abs, rel) + case mode.IsRegular(): + return c.captureFile(abs, rel, info) + case mode&os.ModeSymlink != 0: + return c.captureSymlink(abs, rel, info) + default: + return fmt.Errorf("%w: %q is a device, socket, pipe, or other special file (%s)", errUtils.ErrArtifactPackagingInvalid, rel, mode.Type()) + } +} + +// captureSymlink records a relative symlink without following it. Absolute or escaping targets +// are rejected when the entry is validated. +func (c *capturer) captureSymlink(abs, rel string, info os.FileInfo) error { + target, err := os.Readlink(abs) + if err != nil { + return changedError(rel, err) + } + if filepath.IsAbs(target) || filepath.VolumeName(target) != "" || strings.HasPrefix(target, "/") { + return fmt.Errorf("%w: symlink %q has absolute target %q", errUtils.ErrArtifactUnsafePath, rel, target) + } + target = filepath.ToSlash(target) + entry := artifact.Entry{Path: rel, Type: artifact.EntrySymlink, Target: target} + if err := entry.Validate(); err != nil { + return err + } + if err := c.addEntry(&entry); err != nil { + return err + } + c.sources = append(c.sources, sourceRecord{abs: abs, rel: rel, info: info, target: target}) + return nil +} + +// addEntry appends an inventory entry, enforcing the entry limit. +func (c *capturer) addEntry(entry *artifact.Entry) error { + if len(c.entries) >= c.opts.limits.MaxEntries { + return fmt.Errorf("%w: more than %d entries", errUtils.ErrArtifactTooLarge, c.opts.limits.MaxEntries) + } + c.entries = append(c.entries, *entry) + return nil +} + +// permBits returns the ordinary permission bits of info with privileged bits cleared. +func permBits(info os.FileInfo) uint32 { + return uint32(info.Mode().Perm()) +} + +// readDirNames lists a directory's child names in bytewise order. +func readDirNames(dir string) ([]string, error) { + items, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("%w: read directory %q: %w", errUtils.ErrArtifactSourceChanged, dir, err) + } + names := make([]string, 0, len(items)) + for _, item := range items { + names = append(names, item.Name()) + } + slices.Sort(names) + return names, nil +} + +// checkLinkContainment resolves every captured symlink physically and rejects any that leaves the +// source. A lexical target check is not enough, because a chain of links can clean to a contained +// path yet resolve outside it. Dangling links stay allowed. +func (c *capturer) checkLinkContainment() error { + realRoot, err := filepath.EvalSymlinks(c.root) + if err != nil { + return fmt.Errorf("%w: resolve source: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + for index := range c.sources { + record := &c.sources[index] + if record.target == "" { + continue + } + resolved, err := filepath.EvalSymlinks(record.abs) + if err != nil { + continue + } + relative, err := filepath.Rel(realRoot, resolved) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("%w: symlink %q resolves outside the source", errUtils.ErrArtifactUnsafePath, record.rel) + } + } + return nil +} + +// changedError wraps a failure observed while a source entry was being captured. +func changedError(rel string, cause error) error { + return fmt.Errorf("%w: %q: %w", errUtils.ErrArtifactSourceChanged, rel, cause) +} From 97d4f88d45fcf6a65c3d0482d34abd8744e3615d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 8 Nov 2026 16:40:04 -0600 Subject: [PATCH 03/52] fix(artifacts): close replaced readers and ignore directory times in Windows tests - Close the repository-opened reader before the integrity-failure cases swap or drop a stream. The orphaned handle kept the staged file open and Windows could not remove the temp directory afterwards. - Compare file modification times but not directory times when proving the producer leaves its source tree untouched. NTFS refreshes a directory's listed time lazily on reads, so it drifted without any change; added and removed entries are still caught by the path list. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/materialize_test.go | 18 ++++++++++++++---- pkg/artifact/produce/produce_test.go | 9 ++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index 7c1fb5be486..46cbaca3124 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -50,6 +50,13 @@ func contentOf(t *testing.T, p *artifact.PreparedArtifact) *artifact.Content { return content } +// replaceStream swaps a stream's bytes after closing the file the repository opened for it, so the +// replaced handle never outlives the test (an open handle blocks temp-dir removal on Windows). +func replaceStream(c *artifact.Content, index int, data []byte) { + _ = c.Streams[index].Reader.Close() + c.Streams[index].Reader = io.NopCloser(bytes.NewReader(data)) +} + func dest(t *testing.T) string { return filepath.Join(t.TempDir(), "out", "pulled") } @@ -186,15 +193,18 @@ func TestMaterialize_PerFileIntegrityFailures(t *testing.T) { want error }{ {"tampered bytes", func(c *artifact.Content) { - c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("xxxx"))) + replaceStream(c, 0, []byte("xxxx")) }, errUtils.ErrArtifactIntegrityFailed}, {"truncated", func(c *artifact.Content) { - c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("aa"))) + replaceStream(c, 0, []byte("aa")) }, errUtils.ErrArtifactIntegrityFailed}, {"oversize", func(c *artifact.Content) { - c.Streams[0].Reader = io.NopCloser(bytes.NewReader([]byte("aaaaaaaa"))) + replaceStream(c, 0, []byte("aaaaaaaa")) }, errUtils.ErrArtifactTooLarge}, - {"missing payload", func(c *artifact.Content) { c.Streams = c.Streams[1:] }, errUtils.ErrArtifactIntegrityFailed}, + {"missing payload", func(c *artifact.Content) { + _ = c.Streams[0].Reader.Close() + c.Streams = c.Streams[1:] + }, errUtils.ErrArtifactIntegrityFailed}, {"extra payload", func(c *artifact.Content) { c.Streams = append(c.Streams, artifact.ContentStream{Role: artifact.BlobPayload, Path: "extra.txt", Reader: io.NopCloser(bytes.NewReader(nil))}) }, errUtils.ErrArtifactIntegrityFailed}, diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 1d91d057997..784f0b3e571 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -254,7 +254,14 @@ func listTree(t *testing.T, root string) []string { } info, err := d.Info() require.NoError(t, err) - out = append(out, path+"|"+info.Mode().String()+"|"+info.ModTime().String()) + entry := path + "|" + info.Mode().String() + // NTFS refreshes a directory's listed modification time lazily when its files are only read, + // so directory times drift on Windows without any change. Entries added or removed are still + // caught by the path list, and file times are compared exactly. + if !info.IsDir() { + entry += "|" + info.ModTime().String() + } + out = append(out, entry) return nil })) return out From f667bd2b791f98f076c9ccab6ea2337aa835020a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 1 Nov 2026 00:39:31 -0500 Subject: [PATCH 04/52] fix(artifacts): address CodeRabbit findings for the ORAS transport and artifact service Co-Authored-By: Claude Sonnet 5.5 --- pkg/archive/deterministic.go | 20 +- pkg/archive/deterministic_test.go | 35 ++++ pkg/artifact/artifacttest/write.go | 27 ++- pkg/artifact/linkpath/linkpath.go | 127 ++++++++++++ pkg/artifact/linkpath/linkpath_test.go | 91 +++++++++ pkg/artifact/materialize/archive.go | 28 ++- pkg/artifact/materialize/extract.go | 47 ++++- pkg/artifact/materialize/links.go | 22 +- pkg/artifact/materialize/materialize_test.go | 190 +++++++++++++++++- pkg/artifact/materialize/replace.go | 85 ++++++-- pkg/artifact/produce/documents.go | 23 ++- .../produce/documents_internal_test.go | 43 ++++ pkg/artifact/produce/produce_test.go | 7 + pkg/artifact/service/consume_test.go | 47 +++++ pkg/artifact/service/copy.go | 67 +++++- pkg/artifact/service/copy_test.go | 93 +++++++++ pkg/artifact/service/interrupt_test.go | 2 +- pkg/artifact/service/mutate_test.go | 17 ++ pkg/artifact/service/pull.go | 19 +- pkg/artifact/service/repositories.go | 7 +- pkg/artifact/service/service.go | 3 +- pkg/artifact/snapshot/copy.go | 3 + pkg/artifact/snapshot/snapshot.go | 6 +- pkg/artifact/snapshot/snapshot_test.go | 41 ++++ pkg/artifact/snapshot/walk.go | 19 +- pkg/oci/client.go | 24 ++- pkg/oci/transport.go | 12 +- pkg/oci/transport_behavior_test.go | 47 +++++ pkg/oci/transport_layout.go | 14 +- pkg/oci/transport_registry.go | 6 + pkg/oci/transport_subject.go | 86 ++++++++ pkg/oci/transport_subject_test.go | 80 ++++++++ 32 files changed, 1247 insertions(+), 91 deletions(-) create mode 100644 pkg/artifact/linkpath/linkpath.go create mode 100644 pkg/artifact/linkpath/linkpath_test.go create mode 100644 pkg/artifact/produce/documents_internal_test.go create mode 100644 pkg/oci/transport_subject.go create mode 100644 pkg/oci/transport_subject_test.go diff --git a/pkg/archive/deterministic.go b/pkg/archive/deterministic.go index 2a2be7d3119..0029b00caf2 100644 --- a/pkg/archive/deterministic.go +++ b/pkg/archive/deterministic.go @@ -232,9 +232,23 @@ func copyMemberContent(dst io.Writer, member *Member) error { if written != member.Size { return writeFailedError(member.Path, io.ErrUnexpectedEOF) } + return rejectExtraContent(src, member) +} + +// rejectExtraContent reads until a byte or the end of the source. A reader may legally return no +// bytes and no error from a single call, so one read cannot prove the source has ended. The probe +// never touches the archive writer, so extra content is not written before the failure. +func rejectExtraContent(src io.Reader, member *Member) error { var extra [1]byte - if n, readErr := src.Read(extra[:]); n > 0 || (readErr != nil && !errors.Is(readErr, io.EOF)) { - return writeFailedError(member.Path, fmt.Errorf("%w: content is longer than the declared %d bytes", errUtils.ErrArchiveWriteFailed, member.Size)) + for { + n, readErr := src.Read(extra[:]) + switch { + case n > 0: + return writeFailedError(member.Path, fmt.Errorf("%w: content is longer than the declared %d bytes", errUtils.ErrArchiveWriteFailed, member.Size)) + case errors.Is(readErr, io.EOF): + return nil + case readErr != nil: + return writeFailedError(member.Path, readErr) + } } - return nil } diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go index cfc5d6500a4..8dce801194d 100644 --- a/pkg/archive/deterministic_test.go +++ b/pkg/archive/deterministic_test.go @@ -150,6 +150,41 @@ func TestWriteDeterministic_ClearsPrivilegedBits(t *testing.T) { assert.Equal(t, int64(0o755), header.Mode) } +// stutteringReader returns one empty, error-free read before it yields its content. +type stutteringReader struct { + stuttered bool + rest io.Reader +} + +func (r *stutteringReader) Read(p []byte) (int, error) { + if !r.stuttered { + r.stuttered = true + return 0, nil + } + return r.rest.Read(p) +} + +func TestRejectExtraContent_ReadsPastEmptyReads(t *testing.T) { + tests := []struct { + name string + rest string + wantErr bool + }{ + {"byte hidden behind an empty read", "x", true}, + {"end of source behind an empty read", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := rejectExtraContent(&stutteringReader{rest: strings.NewReader(tt.rest)}, &Member{Path: "a", Size: 0}) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + return + } + require.NoError(t, err) + }) + } +} + type failingWriter struct{ after int } func (f *failingWriter) Write(p []byte) (int, error) { diff --git a/pkg/artifact/artifacttest/write.go b/pkg/artifact/artifacttest/write.go index 2dc3cb07837..a1a6f175a31 100644 --- a/pkg/artifact/artifacttest/write.go +++ b/pkg/artifact/artifacttest/write.go @@ -168,6 +168,29 @@ func (r *Repository) publishResult(record *artifact.Record, s *stored, outcome a } } +// checkExpectedCurrent is the delete staleness guard. It compares the digest the caller observed +// with the digest currently stored under the pinned reference's own key, so a reference that moved +// after pinning is a conflict. The key comes from the pinned reference rather than the Record's +// name and version, because an alias can point at a Record named differently. A digest-addressed +// pin has no mutable reference, so it is compared with the Record digest. Callers hold r.mu. +func (r *Repository) checkExpectedCurrent(pinned *artifact.PinnedReference, expected *artifact.PinnedReference) error { + if expected == nil { + return nil + } + current := pinned.RecordDigest + if pinned.Ref.Digest == "" && pinned.Ref.Version != "" { + stored, ok := r.refs[refKey(pinned.Ref.Name, pinned.Ref.Version)] + if !ok { + return fmt.Errorf("%w: %s no longer exists", errUtils.ErrArtifactConflict, pinned.Ref) + } + current = stored + } + if expected.ResolvedDigest != current { + return fmt.Errorf("%w: the delete target changed (%s is now %s, the caller observed %s)", errUtils.ErrArtifactConflict, pinned.Ref, current, expected.ResolvedDigest) + } + return nil +} + // Delete implements [artifact.Repository]. The deletion is scoped to the pinned record: the // references pointing at it and its exclusively owned payload. // @@ -185,8 +208,8 @@ func (r *Repository) Delete(_ context.Context, pinned artifact.PinnedReference, if !ok { return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) } - if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != pinned.RecordDigest { - return nil, fmt.Errorf("%w: the delete target changed", errUtils.ErrArtifactConflict) + if err := r.checkExpectedCurrent(&pinned, options.ExpectedCurrent); err != nil { + return nil, err } result := &artifact.DeleteResult{Digest: pinned.RecordDigest} for key, d := range r.refs { diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go new file mode 100644 index 00000000000..8042763d905 --- /dev/null +++ b/pkg/artifact/linkpath/linkpath.go @@ -0,0 +1,127 @@ +// Package linkpath resolves symlink chains physically so containment checks can see where a +// link really points, including links whose final components do not exist yet. +package linkpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxLinkHops bounds the symlinks followed while resolving one path, so a loop terminates. +const maxLinkHops = 255 + +// Resolve returns the physical absolute path that target refers to. It follows every symlink it +// meets, like filepath.EvalSymlinks. Unlike EvalSymlinks, it also resolves a path whose tail does +// not exist: the longest existing prefix is resolved physically, and the missing remainder is +// joined lexically. A dangling link therefore still reports where it would land once the missing +// path is created. Resolve reports false when the path loops, passes through a non-directory, or +// cannot be inspected. +func Resolve(target string) (string, bool) { + defer perf.Track(nil, "linkpath.Resolve")() + + absolute, err := filepath.Abs(target) + if err != nil { + return "", false + } + volume := filepath.VolumeName(absolute) + w := &walker{base: volume + string(filepath.Separator), pending: splitComponents(absolute[len(volume):])} + w.resolved = w.base + for len(w.pending) > 0 { + done, ok := w.step() + if done { + return w.resolved, ok + } + } + return w.resolved, true +} + +// walker resolves a path one component at a time. +type walker struct { + // base is the filesystem root, with its volume on Windows. + base string + + // resolved is the physical directory reached so far. + resolved string + + // pending holds the components still to resolve. + pending []string + + // hops counts the symlinks followed. + hops int +} + +// step consumes one component. It reports done when resolution ended, and ok when it succeeded. +func (w *walker) step() (done, ok bool) { + component := w.pending[0] + w.pending = w.pending[1:] + if component == "." || component == ".." { + w.resolved = stepDot(w.resolved, component) + return false, true + } + next := filepath.Join(w.resolved, component) + info, err := os.Lstat(next) + if errors.Is(err, fs.ErrNotExist) { + // Nothing below a missing component can be a symlink, so the rest is lexical. + w.resolved = filepath.Join(append([]string{next}, w.pending...)...) + return true, true + } + if err != nil { + return true, false + } + if info.Mode()&fs.ModeSymlink == 0 { + w.resolved = next + return false, true + } + return w.follow(next) +} + +// follow splices the target of the symlink at link in front of the pending components. +func (w *walker) follow(link string) (done, ok bool) { + w.hops++ + components, absolute, err := linkComponents(link) + if err != nil || w.hops > maxLinkHops { + return true, false + } + if absolute { + w.resolved = w.base + } + w.pending = append(components, w.pending...) + return false, true +} + +// stepDot applies a "." or ".." component to a fully resolved directory. +func stepDot(resolved, component string) string { + if component == ".." { + return filepath.Dir(resolved) + } + return resolved +} + +// linkComponents reads the symlink at link and returns the components of its target and whether the +// target is absolute, in which case resolution restarts from the root. +func linkComponents(link string) ([]string, bool, error) { + target, err := os.Readlink(link) + if err != nil { + return nil, false, err + } + absolute := filepath.IsAbs(target) || strings.HasPrefix(target, string(filepath.Separator)) + return splitComponents(target[len(filepath.VolumeName(target)):]), absolute, nil +} + +// Within reports whether target is root or lies beneath it. +func Within(root, target string) bool { + defer perf.Track(nil, "linkpath.Within")() + + relative, err := filepath.Rel(root, target) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +// splitComponents splits a path into its non-empty components. +func splitComponents(path string) []string { + return strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == filepath.Separator }) +} diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go new file mode 100644 index 00000000000..30c6fac86f4 --- /dev/null +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -0,0 +1,91 @@ +package linkpath_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" +) + +// realTemp returns a temporary directory with symlinks in its own path already resolved. +func realTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + return dir +} + +func TestResolve(t *testing.T) { + root := realTemp(t) + fixtures.WriteFile(t, root, "tree/real/file.txt", "x", 0o644) + if !fixtures.Symlink(t, root, "tree/up", "..") { + t.Skip("this platform cannot create symlinks") + } + require.True(t, fixtures.Symlink(t, root, "tree/to-real", "real")) + require.True(t, fixtures.Symlink(t, root, "tree/dangling", "real/missing/deeper")) + require.True(t, fixtures.Symlink(t, root, "tree/escapes", "up/../outside/missing")) + require.True(t, fixtures.Symlink(t, root, "tree/loop-a", "loop-b")) + require.True(t, fixtures.Symlink(t, root, "tree/loop-b", "loop-a")) + + tests := []struct { + name string + path string + want string + reachable bool + }{ + {"plain file", filepath.Join(root, "tree", "real", "file.txt"), filepath.Join(root, "tree", "real", "file.txt"), true}, + {"link to directory", filepath.Join(root, "tree", "to-real"), filepath.Join(root, "tree", "real"), true}, + {"dangling tail stays lexical under the link target", filepath.Join(root, "tree", "dangling"), filepath.Join(root, "tree", "real", "missing", "deeper"), true}, + {"dangling tail walking out through a link", filepath.Join(root, "tree", "escapes"), filepath.Join(filepath.Dir(root), "outside", "missing"), true}, + {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), true}, + {"loop", filepath.Join(root, "tree", "loop-a"), "", false}, + {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, reachable := linkpath.Resolve(tt.path) + assert.Equal(t, tt.reachable, reachable) + if tt.reachable { + assert.Equal(t, tt.want, got) + } + }) + } +} + +func TestResolve_AbsoluteLinkTarget(t *testing.T) { + root := realTemp(t) + target := filepath.Join(root, "target") + require.NoError(t, os.MkdirAll(target, 0o755)) + link := filepath.Join(root, "abs") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks are unavailable here: %v", err) + } + got, reachable := linkpath.Resolve(filepath.Join(link, "missing")) + assert.True(t, reachable) + assert.Equal(t, filepath.Join(target, "missing"), got) +} + +func TestWithin(t *testing.T) { + root := filepath.Join(string(filepath.Separator), "base", "root") + tests := []struct { + name string + target string + want bool + }{ + {"root itself", root, true}, + {"descendant", filepath.Join(root, "a", "b"), true}, + {"sibling with shared prefix", root + "-other", false}, + {"parent", filepath.Dir(root), false}, + {"unrelated", filepath.Join(string(filepath.Separator), "elsewhere"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, linkpath.Within(root, tt.target)) + }) + } +} diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index f141ac5c0ab..7366ced06f7 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/fs" + "math" "os" "path" "path/filepath" @@ -59,7 +60,7 @@ func fillArchive(ctx context.Context, t *tree, content *artifact.Content, scratc return err } if content.Representation.Packaging == artifact.PackagingZip { - return fillZip(ctx, t, payload.Reader, scratch, checker) + return fillZip(ctx, t, payload, scratch, checker) } return fillTar(ctx, t, payload.Reader, content.Representation.Packaging == artifact.PackagingTgz, checker) } @@ -107,16 +108,16 @@ func applyTarHeader(t *tree, tr *tar.Reader, header *tar.Header, checker *manage } } -func fillZip(ctx context.Context, t *tree, reader io.Reader, scratch string, checker *managedChecker) error { +func fillZip(ctx context.Context, t *tree, payload *artifact.ContentStream, scratch string, checker *managedChecker) error { spool, err := os.CreateTemp(scratch, "archive-*.zip") if err != nil { return fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactPackagingInvalid, err) } defer os.Remove(spool.Name()) defer spool.Close() - size, err := copyContext(ctx, spool, reader) + size, err := spoolBounded(ctx, spool, payload) if err != nil { - return fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactIntegrityFailed, err) + return err } zr, err := zip.NewReader(spool, size) if err != nil { @@ -133,6 +134,25 @@ func fillZip(ctx context.Context, t *tree, reader io.Reader, scratch string, che return nil } +// spoolBounded copies the payload into the spool file without exceeding the size declared by its +// descriptor, so a padded or oversized stream cannot fill the scratch disk. +func spoolBounded(ctx context.Context, spool io.Writer, payload *artifact.ContentStream) (int64, error) { + limit := max(payload.Descriptor.Size, 0) + // Read one byte past the declared size so an oversized stream is detected without spooling it all. + probe := limit + if probe < math.MaxInt64 { + probe++ + } + size, err := copyContext(ctx, spool, io.LimitReader(payload.Reader, probe)) + if err != nil { + return size, fmt.Errorf("%w: spool archive: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + if size > limit { + return size, fmt.Errorf("%w: spool archive: the payload is longer than the declared %d bytes", errUtils.ErrArtifactIntegrityFailed, limit) + } + return size, nil +} + func applyZipFile(t *tree, file *zip.File, checker *managedChecker) error { name := strings.TrimSuffix(path.Clean(file.Name), "/") mode := file.Mode() diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go index d5ba0eaeaf1..48c55f2b948 100644 --- a/pkg/artifact/materialize/extract.go +++ b/pkg/artifact/materialize/extract.go @@ -19,6 +19,19 @@ type Option func(*options) type options struct { maxBytes int64 + + // replaceGuard runs immediately before an existing destination is replaced. + replaceGuard func(dest string) error +} + +// WithReplaceGuard registers a check that runs right before an existing destination directory is +// replaced, after extraction finished and the destination was bound. A caller uses it to re-check +// the destination state it decided on earlier, because the destination may have changed during the +// download. A non-nil error aborts the replacement and leaves the destination untouched. +func WithReplaceGuard(guard func(dest string) error) Option { + defer perf.Track(nil, "materialize.WithReplaceGuard")() + + return func(o *options) { o.replaceGuard = guard } } // WithMaxBytes overrides the limit on total extracted file content. @@ -101,16 +114,9 @@ func fill(ctx context.Context, t *tree, content *artifact.Content, scratch strin // fillFiles writes a per-file payload: file entries come from their path-addressed streams; // directories and symlinks come from the inventory. func fillFiles(ctx context.Context, t *tree, content *artifact.Content) error { - streams := make(map[string]*artifact.ContentStream, len(content.Streams)) - for index := range content.Streams { - stream := &content.Streams[index] - if stream.Role != artifact.BlobPayload { - continue - } - if _, duplicate := streams[stream.Path]; duplicate { - return fmt.Errorf("%w: payload %q appears more than once", errUtils.ErrArtifactIntegrityFailed, stream.Path) - } - streams[stream.Path] = stream + streams, err := payloadStreams(t, content) + if err != nil { + return err } paths := make([]string, 0, len(t.inventory)) for entryPath := range t.inventory { @@ -132,6 +138,27 @@ func fillFiles(ctx context.Context, t *tree, content *artifact.Content) error { return nil } +// payloadStreams indexes the payload streams by inventory path. A stream may address only a file +// entry, because directories and symlinks come from the inventory and a stream for them would +// never be read or verified. +func payloadStreams(t *tree, content *artifact.Content) (map[string]*artifact.ContentStream, error) { + streams := make(map[string]*artifact.ContentStream, len(content.Streams)) + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if _, duplicate := streams[stream.Path]; duplicate { + return nil, fmt.Errorf("%w: payload %q appears more than once", errUtils.ErrArtifactIntegrityFailed, stream.Path) + } + if entry, ok := t.inventory[stream.Path]; ok && entry.Type != artifact.EntryFile { + return nil, fmt.Errorf("%w: payload %q does not address a file entry", errUtils.ErrArtifactIntegrityFailed, stream.Path) + } + streams[stream.Path] = stream + } + return streams, nil +} + func fillInventoryEntry(t *tree, entry *artifact.Entry, streams map[string]*artifact.ContentStream) error { if _, err := t.entryFor(entry.Path); err != nil { return err diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go index b61ddcc80d0..bbe99cd5f39 100644 --- a/pkg/artifact/materialize/links.go +++ b/pkg/artifact/materialize/links.go @@ -3,15 +3,16 @@ package materialize import ( "fmt" "path/filepath" - "strings" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" ) // checkLinkContainment resolves every symlink physically and rejects any that leaves root. A // lexical check of each target is not enough, because a link chain can clean to a contained path -// yet resolve outside it. Dangling links stay allowed. +// yet resolve outside it. A dangling link is checked through its longest existing ancestor plus the +// missing remainder, so it cannot escape once the missing path is created. func checkLinkContainment(stage string, record *artifact.Record) error { realRoot, err := filepath.EvalSymlinks(stage) if err != nil { @@ -22,23 +23,10 @@ func checkLinkContainment(stage string, record *artifact.Record) error { if entry.Type != artifact.EntrySymlink { continue } - resolved, reachable := resolveLink(filepath.Join(stage, filepath.FromSlash(entry.Path))) - if reachable && !within(realRoot, resolved) { + resolved, reachable := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) + if reachable && !linkpath.Within(realRoot, resolved) { return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) } } return nil } - -// resolveLink fully resolves a symlink and reports false for a dangling or looping link, which -// points at nothing reachable. -func resolveLink(linkPath string) (string, bool) { - resolved, err := filepath.EvalSymlinks(linkPath) - return resolved, err == nil -} - -// within reports whether target is root or lies beneath it. -func within(root, target string) bool { - relative, err := filepath.Rel(root, target) - return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) -} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index 46cbaca3124..c5bc6982141 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "context" + "errors" "io" "os" "path/filepath" @@ -386,7 +387,18 @@ func TestMaterialize_ArchiveRejectsCorruptStreams(t *testing.T) { require.ErrorIs(t, materialize.Materialize(context.Background(), badPackaging, dest(t)), errUtils.ErrArtifactPackagingInvalid) } +// requireSymlinks skips the test only when the platform cannot create symlinks at all. +func requireSymlinks(t *testing.T) { + t.Helper() + probe := t.TempDir() + fixtures.WriteFile(t, probe, "target.txt", "x", 0o644) + if !fixtures.Symlink(t, probe, "link", "target.txt") { + t.Skip("this platform cannot create symlinks") + } +} + func TestMaterialize_ChainedSymlinkEscapeIsRejected(t *testing.T) { + requireSymlinks(t) record := sealedRecord(t, []artifact.Entry{ {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, @@ -395,12 +407,182 @@ func TestMaterialize_ChainedSymlinkEscapeIsRejected(t *testing.T) { content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} out := dest(t) err := materialize.Materialize(context.Background(), content, out) - if err == nil { - t.Skip("this platform cannot create the symlinks needed for the chained escape") + require.Error(t, err, "a chained symlink escape must be rejected") + assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestMaterialize_DanglingChainedSymlinkEscapeIsRejected(t *testing.T) { + requireSymlinks(t) + tests := []struct { + name string + inventory []artifact.Entry + wantErr bool + }{ + { + name: "dangling tail beyond the root", + inventory: []artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/b/../../missing"}, + }, + wantErr: true, + }, + { + name: "dangling tail inside the root", + inventory: []artifact.Entry{ + {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, + {Path: "c", Type: artifact.EntrySymlink, Target: "a/missing/deeper"}, + }, + }, } - if !assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) { - t.Logf("unexpected error: %v", err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + record := sealedRecord(t, tt.inventory) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + err := materialize.Materialize(context.Background(), content, dest(t)) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + return + } + require.NoError(t, err, "a dangling link that stays inside the tree is allowed") + }) } +} + +func TestMaterialize_PayloadForNonFileEntryIsRejected(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "sub/a.txt", "aaaa", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + content.Streams = append(content.Streams, artifact.ContentStream{Role: artifact.BlobPayload, Path: "sub", Reader: io.NopCloser(bytes.NewReader([]byte("smuggled")))}) + out := dest(t) + err := materialize.Materialize(context.Background(), content, out) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) _, statErr := os.Lstat(out) assert.True(t, os.IsNotExist(statErr)) } + +// endlessReader yields zero bytes forever and counts how many it produced. +type endlessReader struct{ read int64 } + +func (r *endlessReader) Read(p []byte) (int, error) { + r.read += int64(len(p)) + return len(p), nil +} + +func (r *endlessReader) Close() error { return nil } + +func TestMaterialize_ZipSpoolIsBoundedByTheDeclaredSize(t *testing.T) { + record := sealedRecord(t, []artifact.Entry{fileEntry("good.txt", "good", 0o644)}) + const declared = 64 + endless := &endlessReader{} + content := archiveContent(&record, artifact.PackagingZip, endless) + content.Streams[0].Descriptor.Size = declared + err := materialize.Materialize(context.Background(), content, dest(t)) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.LessOrEqual(t, endless.read, int64(copyChunkBound), "the spool must stop shortly after the declared size") +} + +func TestMaterialize_ZipLongerThanDeclaredSizeIsRejected(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "aaaa", 0o644) + p := prepared(t, src, artifact.PackagingZip) + tests := []struct { + name string + adjust int64 + wantErr bool + }{ + {"declared size matches", 0, false}, + {"stream longer than declared", -1, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content := contentOf(t, p) + for index := range content.Streams { + content.Streams[index].Descriptor.Size += tt.adjust + } + err := materialize.Materialize(context.Background(), content, dest(t)) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + return + } + require.NoError(t, err) + }) + } +} + +// copyChunkBound is the largest read the bounded spool may issue: one copy chunk. +const copyChunkBound = 32 * 1024 + +func TestMaterialize_ReplaceGuard(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + errRefused := errors.New("guard refused") + tests := []struct { + name string + guard func(dest string) error + wantErr error + wantStale bool + }{ + {"guard accepts", func(string) error { return nil }, nil, false}, + {"guard refuses", func(string) error { return errRefused }, errRefused, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := dest(t) + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(tt.guard)) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + _, staleErr := os.Stat(filepath.Join(out, "stale.txt")) + assert.Equal(t, tt.wantStale, staleErr == nil) + assertNoScratch(t, out) + }) + } + + t.Run("guard is not consulted for an absent destination", func(t *testing.T) { + called := false + out := dest(t) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(func(string) error { + called = true + return errRefused + })) + require.NoError(t, err) + assert.False(t, called) + }) +} + +func TestMaterialize_DestinationSwappedForASymlinkNeverRedirectsRemoval(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("a directory with an open handle cannot be renamed on Windows") + } + requireSymlinks(t) + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + + root := t.TempDir() + out := filepath.Join(root, "pulled") + outside := filepath.Join(root, "outside") + fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) + fixtures.WriteFile(t, outside, "keep.txt", "keep", 0o644) + + swap := materialize.WithReplaceGuard(func(dest string) error { + // Simulate an attacker replacing the checked destination right after it was inspected. + require.NoError(t, os.Rename(dest, filepath.Join(root, "moved-away"))) + require.NoError(t, os.Symlink(outside, dest)) + return nil + }) + err := materialize.Materialize(context.Background(), contentOf(t, p), out, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + keep, readErr := os.ReadFile(filepath.Join(outside, "keep.txt")) + require.NoError(t, readErr, "the swapped-in link target must not lose files") + assert.Equal(t, "keep", string(keep)) + _, newErr := os.Stat(filepath.Join(outside, "new.txt")) + assert.True(t, os.IsNotExist(newErr), "staged content must not land in the link target") +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 9f5e5e10eb8..7f8caf21b2f 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -178,11 +178,13 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if err := os.Chmod(stage, parentDirMode); err != nil { return fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) } - return replaceDestination(stage, dest) + return replaceDestination(stage, dest, applyOptions(opts).replaceGuard) } -// replaceDestination moves the staged tree into place without merging it over old files. -func replaceDestination(stage, dest string) error { +// replaceDestination moves the staged tree into place without merging it over old files. An +// existing destination is opened once and every later step is checked against that same directory, +// so a symlink swapped in after the initial check cannot redirect the removal or the moves. +func replaceDestination(stage, dest string, guard func(dest string) error) error { info, err := os.Lstat(dest) switch { case os.IsNotExist(err): @@ -194,38 +196,91 @@ func replaceDestination(stage, dest string) error { case !info.IsDir(): return fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) } - if err := clearDirectory(dest); err != nil { + bound, err := bindDestination(dest, info) + if err != nil { + return err + } + defer bound.root.Close() + if guard != nil { + if err := guard(dest); err != nil { + return err + } + if err := bound.verify(); err != nil { + return err + } + } + if err := clearDirectory(bound); err != nil { return err } - return moveChildren(stage, dest) + return moveChildren(stage, bound) +} + +// boundDestination is a destination directory opened once, with the identity it must keep. +type boundDestination struct { + path string + root *os.Root + info os.FileInfo } -// clearDirectory removes every child of dir through an os.Root, so removal can never leave dir. -func clearDirectory(dir string) error { - root, err := os.OpenRoot(dir) +// bindDestination opens dest and confirms the opened directory is the one that was checked. A +// destination swapped for a symlink or another directory between the check and the open fails. +func bindDestination(dest string, checked os.FileInfo) (*boundDestination, error) { + root, err := os.OpenRoot(dest) if err != nil { - return fmt.Errorf("%w: open destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dir, err) + return nil, fmt.Errorf("%w: open destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) } - defer root.Close() - children, err := os.ReadDir(dir) + opened, err := root.Stat(".") if err != nil { - return fmt.Errorf("%w: read destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dir, err) + _ = root.Close() + return nil, fmt.Errorf("%w: inspect destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + } + if !os.SameFile(checked, opened) { + _ = root.Close() + return nil, changedDestination(dest) + } + return &boundDestination{path: dest, root: root, info: opened}, nil +} + +// verify confirms the destination path still names the bound directory and is not a symlink. +func (b *boundDestination) verify() error { + now, err := os.Lstat(b.path) + if err != nil || now.Mode()&os.ModeSymlink != 0 || !os.SameFile(now, b.info) { + return changedDestination(b.path) + } + return nil +} + +func changedDestination(dest string) error { + return fmt.Errorf("%w: destination %q changed while it was being replaced", errUtils.ErrArtifactUnsafePath, dest) +} + +// clearDirectory removes every child of the bound destination through its os.Root, so removal can +// never leave the directory that was checked. +func clearDirectory(bound *boundDestination) error { + children, err := fs.ReadDir(bound.root.FS(), ".") + if err != nil { + return fmt.Errorf("%w: read destination %q: %w", errUtils.ErrArtifactPackagingInvalid, bound.path, err) } for _, child := range children { - if err := root.RemoveAll(child.Name()); err != nil { + if err := bound.root.RemoveAll(child.Name()); err != nil { return fmt.Errorf("%w: remove stale %q: %w", errUtils.ErrArtifactPackagingInvalid, child.Name(), err) } } return nil } -func moveChildren(stage, dest string) error { +// moveChildren renames each staged child into the destination. The standard library cannot rename +// across two directory handles, so the destination identity is re-verified before every move. +func moveChildren(stage string, bound *boundDestination) error { children, err := os.ReadDir(stage) if err != nil { return fmt.Errorf("%w: read staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) } for _, child := range children { - if err := rename(filepath.Join(stage, child.Name()), filepath.Join(dest, child.Name())); err != nil { + if err := bound.verify(); err != nil { + return err + } + if err := rename(filepath.Join(stage, child.Name()), filepath.Join(bound.path, child.Name())); err != nil { return err } } diff --git a/pkg/artifact/produce/documents.go b/pkg/artifact/produce/documents.go index 1fc14074d15..bbe82795039 100644 --- a/pkg/artifact/produce/documents.go +++ b/pkg/artifact/produce/documents.go @@ -48,6 +48,12 @@ func checkReserved(entries []artifact.Entry) error { for index := range entries { entry := &entries[index] switch { + case entry.Path == reservedParent: + // A file or symlink named .atmos would be dropped from the inventory by payloadInventory + // and then be missing after a pull, so it cannot be sealed. + if entry.Type != artifact.EntryDirectory { + return reservedError(entry.Path, "must be a directory") + } case entry.Path == reservedDir: if entry.Type != artifact.EntryDirectory { return reservedError(entry.Path, "must be a directory") @@ -213,7 +219,7 @@ func readSiblingDocument(source string, role artifact.DocumentRole, rel string) if info.Size() > artifact.MaxDocumentSize { return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, rel, info.Size(), artifact.MaxDocumentSize) } - file, err := os.Open(full) + file, err := openChecked(full, info) if err != nil { return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) } @@ -221,6 +227,21 @@ func readSiblingDocument(source string, role artifact.DocumentRole, rel string) return readBounded(file, rel) } +// openChecked opens full and confirms the opened file is the regular file that was inspected, so a +// path replaced by a symlink between the check and the open is never read. +func openChecked(full string, inspected os.FileInfo) (*os.File, error) { + file, err := os.Open(full) + if err != nil { + return nil, err + } + opened, err := file.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) { + _ = file.Close() + return nil, fmt.Errorf("%w: the file changed while it was being read", errUtils.ErrArtifactSourceChanged) + } + return file, nil +} + func readBounded(reader io.Reader, name string) ([]byte, error) { data, err := artifact.ReadBounded(reader, artifact.MaxDocumentSize) if err != nil { diff --git a/pkg/artifact/produce/documents_internal_test.go b/pkg/artifact/produce/documents_internal_test.go new file mode 100644 index 00000000000..03676829072 --- /dev/null +++ b/pkg/artifact/produce/documents_internal_test.go @@ -0,0 +1,43 @@ +package produce + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" +) + +func TestOpenChecked_RejectsAFileOtherThanTheInspectedOne(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "inspected.md", "inspected", 0o644) + fixtures.WriteFile(t, dir, "replacement.md", "replacement", 0o644) + inspected, err := os.Lstat(filepath.Join(dir, "inspected.md")) + require.NoError(t, err) + + tests := []struct { + name string + path string + wantErr error + }{ + {"the inspected file", filepath.Join(dir, "inspected.md"), nil}, + {"a different file at the opened path", filepath.Join(dir, "replacement.md"), errUtils.ErrArtifactSourceChanged}, + {"a directory at the opened path", dir, errUtils.ErrArtifactSourceChanged}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + file, err := openChecked(tt.path, inspected) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + assert.Nil(t, file) + return + } + require.NoError(t, err) + require.NoError(t, file.Close()) + }) + } +} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 784f0b3e571..aae767de056 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -395,6 +395,13 @@ func TestFilesProducer_ReservedDirectoryConflicts(t *testing.T) { fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/sub/README.md", "x", 0o644) return true }}, + {"regular file named .atmos", func(t *testing.T, dir string) bool { + fixtures.WriteFile(t, dir, ".atmos", "not a directory", 0o644) + return true + }}, + {"symlink named .atmos", func(t *testing.T, dir string) bool { + return fixtures.Symlink(t, dir, ".atmos", "a.txt") + }}, {"reserved symlink document", func(t *testing.T, dir string) bool { fixtures.WriteFile(t, dir, "real.md", "x", 0o644) return fixtures.Symlink(t, dir, fixtures.ReservedDir+"/README.md", "../../real.md") diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index e2401af20cf..29b08d70306 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -325,3 +325,50 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { require.NoError(t, err) assert.Len(t, result.Artifacts, 3) } + +// readHookRepository runs a hook right before every Read, as if another process touched the +// destination while the artifact was downloading. +type readHookRepository struct { + *artifacttest.Repository + onRead func() +} + +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *readHookRepository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + r.onRead() + return r.Repository.Read(ctx, pinned) +} + +func TestPull_RechecksTheDestinationBeforeReplacingIt(t *testing.T) { + tests := []struct { + name string + force bool + wantErr error + wantKept bool + }{ + {"files appeared during the download without force", false, errUtils.ErrArtifactConflict, true}, + {"files appeared during the download with force", true, nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t, envConfig{}) + e.publish("dst", e.source("ORIGINAL")) + dest := out(t) + require.NoError(t, os.MkdirAll(dest, 0o755)) + e.static.Repositories["dst"] = &readHookRepository{Repository: e.repos["dst"], onRead: func() { + fixtures.WriteFile(t, dest, "late.txt", "written during the download", 0o644) + }} + + _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: tt.force}) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + _, lateErr := os.Stat(filepath.Join(dest, "late.txt")) + assert.Equal(t, tt.wantKept, lateErr == nil, "a refused replacement leaves the destination untouched; a forced one replaces it") + _, dataErr := os.Stat(filepath.Join(dest, "data.txt")) + assert.Equal(t, !tt.wantKept, dataErr == nil) + }) + } +} diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index 3c1dd4b224c..d37258066db 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -76,9 +76,9 @@ func (s *Service) copyArtifact(ctx context.Context, op *operation, request *arti op.result.Outcome = artifact.OutcomeCreated } op.result.Repository = request.To - if op.result.Outcome != artifact.OutcomeUnchanged { - s.carryEvidence(ctx, op, plan) - } + // Evidence is carried even when the destination already holds the Record, because signatures + // added to the source after the first copy would otherwise never reach the destination. + s.carryEvidence(ctx, op, plan) return nil } @@ -281,7 +281,8 @@ func documentRoleFor(record *artifact.Record, digestValue string) (artifact.Docu // carryEvidence moves signature evidence along with a copy when both repositories support it. // Evidence is never invented: without both optional interfaces, a diagnostic explains that the -// destination holds no signatures. +// destination holds no signatures. Evidence is written only to the exact record the copy produced +// or found, never to a record a concurrent writer repointed the destination reference to. func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPlan) { reader, canRead := plan.src.repo.(artifact.EvidenceReader) writer, canWrite := plan.dst.repo.(artifact.EvidenceWriter) @@ -292,16 +293,17 @@ func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPl return } blobs, err := reader.ReadEvidence(ctx, &plan.pinned) - if err != nil || len(blobs) == 0 { - if err != nil { - op.warn("evidence-read-failed", err.Error()) - } + if err != nil { + op.warn("evidence-read-failed", err.Error()) return } - dstRef, err := nativeRef(plan) + if len(blobs) == 0 { + return + } + dstPinned, err := resolveWritten(ctx, op, plan) if err == nil { - var dstPinned artifact.PinnedReference - if dstPinned, err = plan.dst.repo.Resolve(ctx, dstRef); err == nil { + blobs = missingEvidence(ctx, plan.dst.repo, &dstPinned, blobs) + if len(blobs) > 0 { _, err = writer.WriteEvidence(ctx, &dstPinned, blobs, nil) } } @@ -309,3 +311,46 @@ func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPl op.warn("evidence-write-failed", err.Error()) } } + +// resolveWritten resolves the destination reference and confirms it still points at the digest the +// copy wrote or found. A mismatch means another writer repointed the reference after the +// destination lock was released. +func resolveWritten(ctx context.Context, op *operation, plan *copyPlan) (artifact.PinnedReference, error) { + dstRef, err := nativeRef(plan) + if err != nil { + return artifact.PinnedReference{}, err + } + dstPinned, err := plan.dst.repo.Resolve(ctx, dstRef) + if err != nil { + return artifact.PinnedReference{}, err + } + if want := op.result.ResolvedDigest; want != "" && dstPinned.ResolvedDigest != want { + return artifact.PinnedReference{}, fmt.Errorf("%w: %s in repository %q now points at %s, not the copied %s; evidence was not written", + errUtils.ErrArtifactConflict, dstRef, plan.dst.alias, dstPinned.ResolvedDigest, want) + } + return dstPinned, nil +} + +// missingEvidence drops the blobs the destination already holds, so repeating a copy never +// duplicates evidence. A destination that cannot list its evidence receives every blob. +func missingEvidence(ctx context.Context, repo artifact.Repository, pinned *artifact.PinnedReference, blobs []artifact.EvidenceBlob) []artifact.EvidenceBlob { + existing, ok := repo.(artifact.EvidenceReader) + if !ok { + return blobs + } + held, err := existing.ReadEvidence(ctx, pinned) + if err != nil { + return blobs + } + have := make(map[string]bool, len(held)) + for index := range held { + have[held[index].Evidence.Descriptor.Digest.String()] = true + } + var missing []artifact.EvidenceBlob + for index := range blobs { + if !have[blobs[index].Evidence.Descriptor.Digest.String()] { + missing = append(missing, blobs[index]) + } + } + return missing +} diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index c335cfcc953..3529cd8ad71 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -4,8 +4,11 @@ import ( "context" "os" "path/filepath" + "slices" + "sync/atomic" "testing" + "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -271,3 +274,93 @@ func TestCopy_UsesNativeTransferWhenCompatible(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, native.copies, "an incompatible pair falls back to staging") } + +func diagnosticCodes(result *artifact.OperationResult) []string { + codes := make([]string, 0, len(result.Diagnostics)) + for _, diagnostic := range result.Diagnostics { + codes = append(codes, diagnostic.Code) + } + return codes +} + +func TestCopy_UnchangedDestinationStillReceivesLaterSignatures(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + require.Equal(t, artifact.OutcomeCreated, first.Outcome) + assert.Zero(t, e.repos["dst"].Calls("WriteEvidence"), "the source was unsigned, so nothing could travel") + + _, err = e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = releaseSigner.TrustedPolicy() }) + + second, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, second.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("WriteEvidence"), "evidence is carried even though the Record was already there") + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: exampleRef}) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Verification.Status) + + third, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, third.Outcome) + assert.Equal(t, 1, e.repos["dst"].Calls("WriteEvidence"), "repeating a copy never duplicates evidence the destination already holds") + held, err := e.repos["dst"].ReadEvidence(context.Background(), &artifact.PinnedReference{RecordDigest: published.Operation.RecordDigest}) + require.NoError(t, err) + assert.Len(t, held, 1) +} + +// repointingRepository answers Resolve with a different digest once a publish completed, as if +// another writer forced the reference elsewhere after the destination lock was released. +type repointingRepository struct { + *artifacttest.Repository + published atomic.Bool +} + +func (r *repointingRepository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + result, err := r.Repository.Publish(ctx, prepared, options) + if err == nil { + r.published.Store(true) + } + return result, err +} + +func (r *repointingRepository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + pinned, err := r.Repository.Resolve(ctx, ref) + if err == nil && r.published.Load() { + pinned.ResolvedDigest = digest.FromString("another writer") + } + return pinned, err +} + +func TestCopy_EvidenceIsNotWrittenToARepointedDestination(t *testing.T) { + tests := []struct { + name string + repoint bool + wantWrite int + wantCode bool + }{ + {"reference still points at the copied record", false, 1, false}, + {"reference repointed by another writer", true, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithSigner(releaseSigner)) + published := e.publish("src", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + dst := e.repos["dst"] + if tt.repoint { + e.static.Repositories["dst"] = &repointingRepository{Repository: dst} + } + + result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err, "a skipped evidence write is a diagnostic, not a failed copy") + assert.Equal(t, tt.wantWrite, dst.Calls("WriteEvidence")) + assert.Equal(t, tt.wantCode, slices.Contains(diagnosticCodes(result), "evidence-write-failed")) + }) + } +} diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go index 6e54312ad13..791955bbc82 100644 --- a/pkg/artifact/service/interrupt_test.go +++ b/pkg/artifact/service/interrupt_test.go @@ -297,7 +297,7 @@ func TestPublish_ConditionalGuardRejectsRacingWritersEvenWithoutAWriterLock(t *t }() } wg.Wait() - assert.GreaterOrEqual(t, wins.Load(), int64(1)) + assert.Equal(t, int64(1), wins.Load(), "exactly one writer wins the reference") current, ok := bare.RecordDigestOf(exampleRef) require.True(t, ok) var count int diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go index e081283a85c..a19b2c2eebc 100644 --- a/pkg/artifact/service/mutate_test.go +++ b/pkg/artifact/service/mutate_test.go @@ -42,6 +42,23 @@ func TestDelete_ScopedToTheSelectedArtifact(t *testing.T) { assert.Empty(t, e.repos["dst"].HeldLocks(), "locks are released on failure too") } +func TestDelete_RefusesAReferenceThatMovedAfterItWasPinned(t *testing.T) { + e := newEnv(t, envConfig{}) + original := e.publish("dst", e.source("ORIGINAL")) + publishDecoy(t, e) + moveTagOnFirstResolve(t, e, "2.0.0") + + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: exampleRef}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the reference now names another record than the one the caller observed") + + moved, ok := e.repos["dst"].RecordDigestOf(exampleRef) + require.True(t, ok, "the moved reference is left in place") + assert.NotEqual(t, original.Operation.RecordDigest, moved) + _, ok = e.repos["dst"].RecordDigestOf(ref("tools/example", "2.0.0")) + assert.True(t, ok, "the record the reference moved to is not deleted") + assert.Empty(t, e.repos["dst"].HeldLocks()) +} + func TestDelete_CapabilityAndRequestChecks(t *testing.T) { noDelete := artifacttest.DefaultCapabilities() noDelete.Delete = artifact.DeleteNone diff --git a/pkg/artifact/service/pull.go b/pkg/artifact/service/pull.go index 49da05323f4..4d9ade84709 100644 --- a/pkg/artifact/service/pull.go +++ b/pkg/artifact/service/pull.go @@ -67,7 +67,10 @@ func (s *Service) download(ctx context.Context, op *operation, request *artifact if err := bindContent(content, source.inspection); err != nil { return err } - if err := materialize.Materialize(ctx, content, request.Destination); err != nil { + guard := materialize.WithReplaceGuard(func(dest string) error { + return recheckDestination(dest, request.Force, source.inspection.Record) + }) + if err := materialize.Materialize(ctx, content, request.Destination, guard); err != nil { return err } op.result.Outcome = artifact.OutcomeRead @@ -103,6 +106,20 @@ func checkDestination(op *operation, request *artifact.PullRequest, record *arti return false, nil } +// recheckDestination repeats the conflict decision right before the destination is replaced, +// because files may have appeared in it while the artifact was downloading. An empty destination +// and a destination that already matches are always replaceable; different content needs force. +func recheckDestination(dest string, force bool, record *artifact.Record) error { + state, err := materialize.InspectDestination(dest, record) + if err != nil { + return err + } + if state == materialize.DestinationDiffers && !force { + return fmt.Errorf("%w: destination %q changed during the download and now holds different content; use force to replace it", errUtils.ErrArtifactConflict, dest) + } + return nil +} + // bindContent checks the content belongs to the inspected, pinned Record and fills a missing one. func bindContent(content *artifact.Content, inspection *artifact.Inspection) error { if content.Record == nil { diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go index f345f2befb9..897706a58fd 100644 --- a/pkg/artifact/service/repositories.go +++ b/pkg/artifact/service/repositories.go @@ -97,9 +97,12 @@ type JournalEntry struct { Err error } -// Journal receives publication lifecycle events. The service never blocks on or fails because of it. +// Journal receives publication lifecycle events. The service never fails because of it, but it +// calls Record inline and in lifecycle order, because a recovery record must not be reordered or +// dropped. A slow Record therefore delays the publication, so an implementation must return +// promptly and hand any slow work to its own buffer or goroutine. type Journal interface { - // Record observes one lifecycle event. + // Record observes one lifecycle event. It must return promptly. Record(ctx context.Context, entry *JournalEntry) } diff --git a/pkg/artifact/service/service.go b/pkg/artifact/service/service.go index a6cb884404f..1aef0758221 100644 --- a/pkg/artifact/service/service.go +++ b/pkg/artifact/service/service.go @@ -259,7 +259,8 @@ func (o *operation) info(code, message string) { o.result.Diagnostics = append(o.result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityInfo, Code: code, Message: message}) } -// record forwards a lifecycle event to the journal, if any. +// record forwards a lifecycle event to the journal, if any. The call is synchronous so events keep +// their order; the Journal contract requires implementations to return promptly. func (s *Service) record(ctx context.Context, entry *JournalEntry) { if s.journal != nil { s.journal.Record(ctx, entry) diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index 6255be03131..1dcfcd2920c 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -131,6 +131,9 @@ func (c *capturer) recheckSource(record *sourceRecord) error { if now.Mode().Type() != record.info.Mode().Type() { return fmt.Errorf("%w: %q changed type while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) } + if now.IsDir() && now.Mode() != record.info.Mode() { + return fmt.Errorf("%w: %q changed mode while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } switch { case record.digest != "": return c.recheckFile(record, now) diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index 4b950ec591d..e21ecab9b3b 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -172,7 +172,11 @@ func captureInto(ctx context.Context, source, root string, opts *options) (*Snap if err != nil { return nil, err } - c := &capturer{ctx: ctx, opts: opts, root: resolved, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} + staging, err := os.Lstat(root) + if err != nil { + return nil, fmt.Errorf("%w: inspect staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + c := &capturer{staging: staging, ctx: ctx, opts: opts, root: resolved, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} if info.IsDir() { err = c.walkDir(resolved, "") } else { diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 30bd4f2a48b..ab68e803e2f 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "runtime" "testing" "time" @@ -135,6 +136,10 @@ func TestCapture_InvalidSources(t *testing.T) { {"traversal target", func(t *testing.T, dir string) bool { return fixtures.Symlink(t, dir, "up", "..") }, errUtils.ErrArtifactUnsafePath}, + {"chained symlink escape through a dangling tail", func(t *testing.T, dir string) bool { + fixtures.MkdirAll(t, dir, "a") + return fixtures.Symlink(t, dir, "a/b", "..") && fixtures.Symlink(t, dir, "c", "a/b/../../missing") + }, errUtils.ErrArtifactUnsafePath}, {"chained symlink escape", func(t *testing.T, dir string) bool { fixtures.MkdirAll(t, dir, "a") return fixtures.Symlink(t, dir, "a/b", "..") && fixtures.Symlink(t, dir, "c", "a/b/../..") @@ -225,6 +230,42 @@ func TestCapture_DetectsChangeDuringCapture(t *testing.T) { } } +func TestCapture_DetectsDirectoryModeChangeDuringCapture(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permission bits are not tracked on Windows") + } + src := t.TempDir() + fixtures.MkdirAll(t, src, "0dir") + fixtures.WriteFile(t, src, "0dir/inner.txt", "inner", 0o644) + fixtures.WriteFile(t, src, "b.txt", "other", 0o644) + require.NoError(t, os.Chmod(filepath.Join(src, "0dir"), 0o755)) + staging := t.TempDir() + snap, err := snapshot.Capture(context.Background(), src, + snapshot.WithStagingRoot(staging), + snapshot.WithCaptureHook(func(path string) { + if path == "b.txt" { + require.NoError(t, os.Chmod(filepath.Join(src, "0dir"), 0o700)) + } + })) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) + assertEmptyDir(t, staging) +} + +func TestCapture_StagingInsideTheSourceIsNotSourceContent(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "alpha", 0o644) + fixtures.WriteFile(t, src, "b.txt", "bravo", 0o644) + staging := filepath.Join(src, ".staging") + require.NoError(t, os.Mkdir(staging, 0o755)) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(staging)) + require.NoError(t, err, "staged files must not be mistaken for a source change") + t.Cleanup(func() { _ = snap.Close() }) + assert.Equal(t, []string{".staging", "a.txt", "b.txt"}, entryPaths(snap.Entries), "the private staging directory is left out of the inventory") + assert.Equal(t, int64(10), snap.TotalBytes) +} + func TestCapture_Limits(t *testing.T) { src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "0123456789", 0o644) diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go index 3c057601d03..97caa95af89 100644 --- a/pkg/artifact/snapshot/walk.go +++ b/pkg/artifact/snapshot/walk.go @@ -12,6 +12,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/linkpath" ) // sourceRecord remembers one captured source entry for the final recheck. @@ -35,6 +36,9 @@ type capturer struct { sources []sourceRecord total int64 counter int + + // staging identifies the private staging directory, which is skipped when it lies inside the source. + staging os.FileInfo } // walkDir captures the directory at abs, whose slash-separated inventory path is rel ("" for the root). @@ -66,6 +70,10 @@ func (c *capturer) captureChild(abs, rel string) error { if err != nil { return changedError(rel, err) } + if c.staging != nil && os.SameFile(c.staging, info) { + // The private staging directory changes while files are staged, so it is never source content. + return nil + } mode := info.Mode() switch { case mode.IsDir(): @@ -135,7 +143,8 @@ func readDirNames(dir string) ([]string, error) { // checkLinkContainment resolves every captured symlink physically and rejects any that leaves the // source. A lexical target check is not enough, because a chain of links can clean to a contained -// path yet resolve outside it. Dangling links stay allowed. +// path yet resolve outside it. A dangling link is checked through its longest existing ancestor +// plus the missing remainder, so it cannot escape once the missing path is created. func (c *capturer) checkLinkContainment() error { realRoot, err := filepath.EvalSymlinks(c.root) if err != nil { @@ -146,12 +155,8 @@ func (c *capturer) checkLinkContainment() error { if record.target == "" { continue } - resolved, err := filepath.EvalSymlinks(record.abs) - if err != nil { - continue - } - relative, err := filepath.Rel(realRoot, resolved) - if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + resolved, reachable := linkpath.Resolve(record.abs) + if reachable && !linkpath.Within(realRoot, resolved) { return fmt.Errorf("%w: symlink %q resolves outside the source", errUtils.ErrArtifactUnsafePath, record.rel) } } diff --git a/pkg/oci/client.go b/pkg/oci/client.go index 574524d74a7..668e2905bfd 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -10,6 +10,8 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" @@ -174,7 +176,7 @@ func (c *Client) Delete(ctx context.Context, ref name.Digest) error { if err != nil { return err } - target, err := store.Resolve(ctx, ref.DigestStr()) + target, err := resolveDeleteTarget(ctx, store, ref) if err != nil { return err } @@ -184,6 +186,26 @@ func (c *Client) Delete(ctx context.Context, ref name.Digest) error { return nil } +// resolveDeleteTarget looks up the manifest descriptor to delete, because its media type lets the +// store also clean up referrers indexes. A credential that may delete but not read is refused the +// lookup with 401 or 403; the delete then goes out for the bare digest, which needs no read and +// matches what a delete-only credential is allowed to do. Any other lookup failure, including a +// missing digest, is returned unchanged and no DELETE is sent. +func resolveDeleteTarget(ctx context.Context, store RepositoryStore, ref name.Digest) (ocispec.Descriptor, error) { + target, err := store.Resolve(ctx, ref.DigestStr()) + if err == nil { + return target, nil + } + if status := statusOf(err); status != http.StatusUnauthorized && status != http.StatusForbidden { + return ocispec.Descriptor{}, err + } + parsed, parseErr := digest.Parse(ref.DigestStr()) + if parseErr != nil { + return ocispec.Descriptor{}, err + } + return ocispec.Descriptor{Digest: parsed}, nil +} + // reconcileDelete treats a failed delete as complete only when the digest is gone. func (c *Client) reconcileDelete(ctx context.Context, store RepositoryStore, ref name.Digest, cause error) error { if !ambiguousWrite(ctx, cause) { diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go index 1a3f746c9d5..d0b160a021c 100644 --- a/pkg/oci/transport.go +++ b/pkg/oci/transport.go @@ -4,6 +4,7 @@ package oci import ( "context" "io" + "math" "github.com/google/go-containerregistry/pkg/name" "github.com/opencontainers/go-digest" @@ -94,7 +95,8 @@ type RepositoryStore interface { ReferencePusher // Tags lists every tag without interpreting it as a version. Tags(ctx context.Context) ([]string, error) - // Delete removes a manifest by its full descriptor. It never deletes tags or shared blobs. + // Delete removes a manifest by its full descriptor. It never deletes tags or shared blobs. A + // descriptor with only a digest deletes the manifest without reading it first. Delete(ctx context.Context, target ocispec.Descriptor) error // Referrers lists attachments of the subject, optionally filtered by artifact type. // Registries without the OCI 1.1 API use the referrers tag-schema fallback. @@ -125,7 +127,13 @@ func FetchBytes(ctx context.Context, store Fetcher, target ocispec.Descriptor, l } func readVerified(reader io.Reader, target ocispec.Descriptor, limit int64) ([]byte, error) { - raw, err := io.ReadAll(io.LimitReader(reader, limit+1)) + // Read one byte past the declared size to detect oversized content. The declared size is already + // within limit, so the bound never depends on limit+1, which overflows for a MaxInt64 limit. + readLimit := target.Size + if readLimit < math.MaxInt64 { + readLimit++ + } + raw, err := io.ReadAll(io.LimitReader(reader, readLimit)) if err != nil { return nil, wrapIntegrity(err) } diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go index e64af66b385..07abb8f9355 100644 --- a/pkg/oci/transport_behavior_test.go +++ b/pkg/oci/transport_behavior_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "log" + "math" "net/http" "net/http/httptest" "path/filepath" @@ -252,6 +253,48 @@ func TestDeleteIsReconciledByResolvingTheDigest(t *testing.T) { require.Error(t, err) } +func TestDeleteDoesNotNeedReadAccessAndOnlyFallsBackOnDeniedReads(t *testing.T) { + tests := []struct { + name string + readStatus int + wantDelete int + wantErr bool + }{ + {"read forbidden still deletes", http.StatusForbidden, 1, false}, + {"read unauthorized still deletes", http.StatusUnauthorized, 1, false}, + {"missing digest sends no delete", http.StatusNotFound, 0, true}, + {"server failure sends no delete", http.StatusInternalServerError, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var deny atomic.Bool + server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { + if deny.Load() && (r.Method == http.MethodHead || r.Method == http.MethodGet) && strings.Contains(r.URL.Path, "/manifests/") { + w.WriteHeader(tt.readStatus) + return true + } + return false + }) + client := memoryClient(t, server.host, "user", "password") + ref, err := name.NewTag(server.host + "/acme/plugin:1.0.0") + require.NoError(t, err) + index := testIndex(t) + require.NoError(t, client.WriteIndex(t.Context(), ref, index)) + digest, err := index.Digest() + require.NoError(t, err) + + deny.Store(true) + err = client.Delete(t.Context(), ref.Context().Digest(digest.String())) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err, "a delete-only credential must reach the DELETE request") + } + assert.Equal(t, tt.wantDelete, server.count(http.MethodDelete, "/manifests/")) + }) + } +} + func TestReadsAreRetriedButCredentialsAreNotDowngraded(t *testing.T) { var failures atomic.Int32 server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { @@ -289,6 +332,10 @@ func TestFetchBytesIsBoundedAndDigestVerified(t *testing.T) { _, err = oci.FetchBytes(ctx, layout, descriptor, 0) require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + got, err = oci.FetchBytes(ctx, layout, descriptor, math.MaxInt64) + require.NoError(t, err, "a MaxInt64 limit must not overflow into an empty read") + assert.Equal(t, content, got) + wrongSize := descriptor wrongSize.Size++ _, err = oci.FetchBytes(ctx, fetchOverride{descriptor.Digest.String(): content}, wrongSize, 1<<10) diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index ee43ade328c..2d05488c422 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -136,19 +136,17 @@ func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) if desc.MediaType != ocispec.MediaTypeImageManifest && desc.MediaType != ocispec.MediaTypeImageIndex { return nil } - raw, err := FetchBytes(ctx, l.store, desc, maxAttachmentManifest) + subject, err := readSubject(ctx, l.store, &desc) if err != nil { return layoutError(err) } - var manifest struct { - Subject *ocispec.Descriptor `json:"subject"` - } - if err := json.Unmarshal(raw, &manifest); err != nil { - return errors.Join(errUtils.ErrOCILayout, err) - } - if manifest.Subject == nil { + if subject == nil { return nil } + // The size cap protects attachment handling only; a large manifest without a subject is valid. + if desc.Size > maxAttachmentManifest { + return layoutError(errUtils.ErrOCIIntegrity) + } l.mu.Lock() defer l.mu.Unlock() for i := range l.index { diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go index b977c970129..2c7ee0af75c 100644 --- a/pkg/oci/transport_registry.go +++ b/pkg/oci/transport_registry.go @@ -203,7 +203,13 @@ func (s *registryStore) Tags(ctx context.Context) ([]string, error) { return tags, nil } +// Delete removes a manifest. A descriptor without a media type names the manifest by digest only: +// it is deleted with a single DELETE request that needs no read access and performs no referrers +// index cleanup, because the manifest is never fetched. func (s *registryStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + if target.MediaType == "" { + return wrapRegistry(s.repo.Manifests().Delete(ctx, target)) + } return wrapRegistry(s.repo.Delete(ctx, target)) } diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go new file mode 100644 index 00000000000..ccdb256aed7 --- /dev/null +++ b/pkg/oci/transport_subject.go @@ -0,0 +1,86 @@ +package oci + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// readSubject streams a stored manifest or index and returns its subject, or nil when it has none. +// The blob was digest-verified when it was pushed, so the scan is bounded by the descriptor size +// and keeps constant memory: every other top-level field is skipped without being buffered. A +// large manifest without a subject is therefore never rejected. +func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { + if desc.Size < 0 { + return nil, errUtils.ErrOCIIntegrity + } + reader, err := store.Fetch(ctx, *desc) + if err != nil { + return nil, err + } + defer reader.Close() + decoder := json.NewDecoder(io.LimitReader(reader, desc.Size)) + if err := expectDelim(decoder, '{'); err != nil { + return nil, err + } + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return nil, err + } + name, ok := key.(string) + if !ok { + return nil, errUtils.ErrOCIIntegrity + } + if !strings.EqualFold(name, "subject") { + if err := skipValue(decoder); err != nil { + return nil, err + } + continue + } + var subject *ocispec.Descriptor + if err := decoder.Decode(&subject); err != nil { + return nil, err + } + return subject, nil + } + return nil, nil +} + +func expectDelim(decoder *json.Decoder, want json.Delim) error { + token, err := decoder.Token() + if err != nil { + return err + } + if delim, ok := token.(json.Delim); !ok || delim != want { + return fmt.Errorf("%w: the manifest is not a JSON object", errUtils.ErrOCIIntegrity) + } + return nil +} + +// skipValue consumes one JSON value, however deeply nested, without buffering it. +func skipValue(decoder *json.Decoder) error { + depth := 0 + for { + token, err := decoder.Token() + if err != nil { + return err + } + if delim, ok := token.(json.Delim); ok { + if delim == '{' || delim == '[' { + depth++ + } else { + depth-- + } + } + if depth == 0 { + return nil + } + } +} diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go new file mode 100644 index 00000000000..af8d3565e0b --- /dev/null +++ b/pkg/oci/transport_subject_test.go @@ -0,0 +1,80 @@ +package oci_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/oci" +) + +// layoutRoots reads the digests listed in a layout's index.json. +func layoutRoots(t *testing.T, directory string) []string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + var index ocispec.Index + require.NoError(t, json.Unmarshal(raw, &index)) + roots := make([]string, 0, len(index.Manifests)) + for _, manifest := range index.Manifests { + roots = append(roots, manifest.Digest.String()) + } + return roots +} + +func TestLayoutPush_RecordsOnlyManifestsThatNameASubject(t *testing.T) { + const fourMiB = 4 << 20 + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + padding := strings.Repeat("p", fourMiB+1024) + tests := []struct { + name string + manifest string + wantRecord bool + wantErr error + }{ + {"attachment is recorded", `{"schemaVersion":2,"subject":` + subject + `}`, true, nil}, + {"subject after other fields is found", `{"schemaVersion":2,"layers":[{"a":[1,{"b":null}]}],"annotations":{"k":"v"},"subject":` + subject + `}`, true, nil}, + {"manifest without a subject is not recorded", `{"schemaVersion":2,"layers":[]}`, false, nil}, + {"null subject is not recorded", `{"schemaVersion":2,"subject":null}`, false, nil}, + {"large manifest without a subject is accepted", `{"schemaVersion":2,"annotations":{"padding":"` + padding + `"}}`, false, nil}, + {"large attachment is refused", `{"schemaVersion":2,"annotations":{"padding":"` + padding + `"},"subject":` + subject + `}`, false, errUtils.ErrOCIIntegrity}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + descriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(tt.manifest)) + + err = layout.Push(ctx, descriptor, strings.NewReader(tt.manifest)) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + return + } + require.NoError(t, err) + exists, err := layout.Exists(ctx, descriptor) + require.NoError(t, err) + assert.True(t, exists) + if tt.wantRecord { + assert.Equal(t, []string{descriptor.Digest.String()}, layoutRoots(t, directory)) + } else { + assert.Empty(t, layoutRoots(t, directory)) + } + + // Pushing the same manifest again reports it exists and keeps the index free of duplicates. + before := layoutRoots(t, directory) + err = layout.Push(ctx, descriptor, strings.NewReader(tt.manifest)) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.Equal(t, before, layoutRoots(t, directory)) + }) + } +} From 6e94a0b973b0cb24922f9ed4ec46dfe599cf1341 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 10 Nov 2026 16:42:55 -0600 Subject: [PATCH 05/52] fix(artifacts): address remaining CodeRabbit findings for the OCI transport, signing, and service - Stop legacy graph preparation on cancellation and traverse a shared index once. - Require every referrer at the destination before reconciling an extended copy as complete. - Check cancellation before tagging the fallback destination. - Keep layout state consistent with index.json, persist digest-only roots, and keep root descriptor metadata when tagging. - Verify manifest bytes before PushReference publishes a tag. - Preserve the insecure HTTP scheme through repository conversion. - Validate the bound authenticator before opening a repository. - Keep both legacy sentinels for offline signature trust failures. - Restrict key-signed verification to configured keys and try every key that shares a hint. - Route delegated locks to the locking.repository provider. - Detect list cursor cycles across all pages. - Treat publication-unavailable rejections as definitive. - Compare the graph root when reconciling a native copy without a Record. - Describe the hook's resolved base in the development guide. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/commit.go | 4 + pkg/artifact/service/config_repositories.go | 1 + .../service/config_repositories_test.go | 2 + pkg/artifact/service/copy.go | 2 +- pkg/artifact/service/edge_test.go | 9 + pkg/artifact/service/enumerate.go | 7 +- pkg/artifact/service/execute.go | 20 +- pkg/artifact/service/internal_test.go | 26 +++ pkg/artifact/service/interrupt_test.go | 17 ++ pkg/artifact/service/locks.go | 24 +- pkg/artifact/service/repositories.go | 4 + pkg/oci/client.go | 4 +- pkg/oci/signature/plugin_error_test.go | 27 +++ pkg/oci/signature/verify.go | 3 +- pkg/oci/transport.go | 11 +- pkg/oci/transport_copy.go | 45 +++- pkg/oci/transport_hardening_test.go | 214 ++++++++++++++++++ pkg/oci/transport_internal_test.go | 1 + pkg/oci/transport_layout.go | 52 +++-- pkg/oci/transport_legacy.go | 45 +++- pkg/oci/transport_registry.go | 22 +- pkg/signature/key.go | 81 ++++--- pkg/signature/key_test.go | 61 +++++ 23 files changed, 617 insertions(+), 65 deletions(-) create mode 100644 pkg/oci/signature/plugin_error_test.go create mode 100644 pkg/oci/transport_hardening_test.go diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go index 318c01bea09..4dae6b1c8b3 100644 --- a/pkg/artifact/service/commit.go +++ b/pkg/artifact/service/commit.go @@ -30,6 +30,10 @@ type incoming struct { size int64 prepared *artifact.PreparedArtifact inspection *artifact.Inspection + + // rootDigest is the pinned source root of a native graph copy; reconciliation compares it when + // there is no Record digest to compare. + rootDigest digest.Digest } func newIncoming(prepared *artifact.PreparedArtifact) (*incoming, error) { diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go index 99a9308bebd..1e76b976d17 100644 --- a/pkg/artifact/service/config_repositories.go +++ b/pkg/artifact/service/config_repositories.go @@ -71,6 +71,7 @@ func (c *ConfigRepositories) Describe(alias string) (RepositoryInfo, error) { info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: repoCfg.Defaults.Metadata.Labels} } if plan.Provider != alias { + info.LockProvider = plan.Provider info.Capabilities.LockScopes = c.delegatedScopes(&info.Capabilities, plan.Provider) } return info, nil diff --git a/pkg/artifact/service/config_repositories_test.go b/pkg/artifact/service/config_repositories_test.go index 9d0071883fd..0a2ee720f35 100644 --- a/pkg/artifact/service/config_repositories_test.go +++ b/pkg/artifact/service/config_repositories_test.go @@ -60,6 +60,7 @@ func TestConfigRepositories_DescribeIsOfflineAndCombinesPolicyDefaultsAndLocking assert.Equal(t, artifact.PackagingNone, info.Packaging) assert.Equal(t, artifact.LockScopeRepository, info.RequiredLockScope) assert.True(t, info.Capabilities.HasLockScope(artifact.LockScopeRepository), "the delegate's scope is advertised") + assert.Equal(t, "coordination", info.LockProvider, "locks are routed to the delegate") assert.Equal(t, "default description", info.Defaults.Description) assert.Equal(t, map[string]string{"team": "platform"}, info.Defaults.Labels) assert.True(t, info.Policy.RequireSignature) @@ -69,6 +70,7 @@ func TestConfigRepositories_DescribeIsOfflineAndCombinesPolicyDefaultsAndLocking require.NoError(t, err) assert.Equal(t, artifact.PackagingTgz, coordination.Packaging) assert.Equal(t, artifact.LockScopeHost, coordination.RequiredLockScope) + assert.Empty(t, coordination.LockProvider, "a repository without delegation locks for itself") assert.Len(t, coordination.Policy.TrustSets, 1) } diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index d37258066db..122620a8bc6 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -159,7 +159,7 @@ func nativeCopyWriter(plan *copyPlan) (*copyWrite, error) { if err != nil { return nil, err } - in := &incoming{ref: ref, inspection: plan.inspection} + in := &incoming{ref: ref, inspection: plan.inspection, rootDigest: plan.pinned.ResolvedDigest} if record := plan.inspection.Record; record != nil { if err := in.bindRecord(record, plan.inspection); err != nil { return nil, err diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go index f4620ad2490..f73ae368494 100644 --- a/pkg/artifact/service/edge_test.go +++ b/pkg/artifact/service/edge_test.go @@ -285,6 +285,15 @@ func TestList_PaginationGuards(t *testing.T) { _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) }) + t.Run("cursor cycle across pages", func(t *testing.T) { + m := newMockSetup(t, &caps) + next := map[string]string{"": "a", "a": "b", "b": "a"} + m.repo.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + return &artifact.ListPage{NextCursor: next[query.Cursor]}, nil + }).Times(3) + _, err := m.svc.List(context.Background(), artifact.ListRequest{Repositories: []string{"dst"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + }) t.Run("list error", func(t *testing.T) { m := newMockSetup(t, &caps) m.repo.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, errTransport) diff --git a/pkg/artifact/service/enumerate.go b/pkg/artifact/service/enumerate.go index 78553c6e4f4..c62260cba9a 100644 --- a/pkg/artifact/service/enumerate.go +++ b/pkg/artifact/service/enumerate.go @@ -71,6 +71,7 @@ func checkEnumeration(alias string, caps *artifact.Capabilities, name string) er func consumePages(ctx context.Context, repo artifact.Repository, request *enumeration) ([]artifact.ListEntry, error) { var entries []artifact.ListEntry cursor := "" + seen := map[string]bool{} for page := 0; page < maxListPages; page++ { if err := ctx.Err(); err != nil { return entries, err @@ -83,8 +84,10 @@ func consumePages(ctx context.Context, repo artifact.Repository, request *enumer if result.NextCursor == "" { return entries, nil } - if result.NextCursor == cursor { - return entries, fmt.Errorf("%w: the repository returned the same list cursor twice", errUtils.ErrArtifactPublicationUnavailable) + // Any cursor already fetched means the listing cycles, not only one that repeats immediately. + seen[cursor] = true + if seen[result.NextCursor] { + return entries, fmt.Errorf("%w: the repository returned list cursor %q again", errUtils.ErrArtifactPublicationUnavailable, result.NextCursor) } cursor = result.NextCursor } diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go index 131559c12db..3f558ad2ab5 100644 --- a/pkg/artifact/service/execute.go +++ b/pkg/artifact/service/execute.go @@ -74,9 +74,12 @@ func (s *Service) afterFailure(ctx context.Context, pub *publication, cur *curre } } -// isDefinitive reports errors that prove nothing was committed, so reconciliation is unnecessary. +// isDefinitive reports errors that prove nothing was committed, so reconciliation is unnecessary. A +// publication-unavailable rejection (a reference pending for another operation) is definitive even +// when the error also carries the ambiguous sentinel. func isDefinitive(err error) bool { for _, sentinel := range []error{ + errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactRecordInvalid, errUtils.ErrArtifactTooLarge, errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactVerificationFailed, @@ -115,6 +118,9 @@ func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) rec case err != nil: return reconciliation{err: fmt.Errorf("reconcile %s: %w", ref, err)} } + if in.recordDigest == "" { + return reconcileGraph(ref, &pinned, in, cur) + } if pinned.RecordDigest == in.recordDigest { inspection, inspectErr := tgt.repo.Inspect(ctx, pinned) if inspectErr != nil { @@ -131,6 +137,18 @@ func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) rec return reconciliation{pinned: pinned, err: fmt.Errorf("%w: %s now resolves to %s, which is neither the prior state nor the expected content", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest)} } +// reconcileGraph classifies a native OCI graph without a Record. Empty record digests prove nothing, +// so success requires the destination root to be exactly the pinned source root. +func reconcileGraph(ref artifact.Ref, pinned *artifact.PinnedReference, in *incoming, cur *current) reconciliation { + if in.rootDigest != "" && pinned.ResolvedDigest == in.rootDigest { + return reconciliation{state: reconcileCommitted, pinned: *pinned, inspection: in.inspection} + } + if cur.pinned != nil && pinned.ResolvedDigest == cur.pinned.ResolvedDigest { + return reconciliation{state: reconcileNotCommitted, pinned: *pinned} + } + return reconciliation{pinned: *pinned, err: fmt.Errorf("%w: %s now resolves to %s, not the source graph root %s", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest, in.rootDigest)} +} + // matchesExpected verifies the stored Record and, when both sides describe them, the payload // descriptors against what this operation prepared. func matchesExpected(in *incoming, inspection *artifact.Inspection) error { diff --git a/pkg/artifact/service/internal_test.go b/pkg/artifact/service/internal_test.go index bcc4a1aa56c..e2704f3aae9 100644 --- a/pkg/artifact/service/internal_test.go +++ b/pkg/artifact/service/internal_test.go @@ -2,8 +2,10 @@ package service import ( "context" + "errors" "testing" + "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -167,6 +169,30 @@ func TestIsDefinitiveAndTemporary(t *testing.T) { assert.True(t, isDefinitive(errUtils.ErrArtifactUnsupportedCapability)) assert.False(t, isDefinitive(context.Canceled)) assert.False(t, isTemporary(context.Canceled)) + assert.True(t, isDefinitive(errUtils.ErrArtifactPublicationUnavailable)) + assert.True(t, isDefinitive(errors.Join(errUtils.ErrArtifactPublicationAmbiguous, errUtils.ErrArtifactPublicationUnavailable)), + "an unavailable rejection stays definitive even when it also carries the ambiguous sentinel") + assert.False(t, isDefinitive(errUtils.ErrArtifactPublicationAmbiguous)) +} + +func TestReconcileGraphComparesTheSourceRoot(t *testing.T) { + ref := artifact.Ref{Name: "pkg/graph", Version: "1"} + root := digest.FromString("source root") + prior := artifact.PinnedReference{ResolvedDigest: digest.FromString("prior root")} + other := artifact.PinnedReference{ResolvedDigest: digest.FromString("someone else")} + + found := reconcileGraph(ref, &artifact.PinnedReference{ResolvedDigest: root}, &incoming{rootDigest: root}, ¤t{}) + assert.Equal(t, reconcileCommitted, found.state, "the destination root is the pinned source root") + + found = reconcileGraph(ref, &other, &incoming{rootDigest: root}, ¤t{}) + assert.Equal(t, reconcileUnknown, found.state, "empty record digests on both sides never prove success") + require.ErrorIs(t, found.err, errUtils.ErrArtifactConflict) + + found = reconcileGraph(ref, &prior, &incoming{rootDigest: root}, ¤t{pinned: &prior}) + assert.Equal(t, reconcileNotCommitted, found.state) + + found = reconcileGraph(ref, &other, &incoming{}, ¤t{}) + assert.Equal(t, reconcileUnknown, found.state, "without a source root nothing can be proven") } var ocispecRoot = ocispecDescriptor() diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go index 791955bbc82..9b90e59f675 100644 --- a/pkg/artifact/service/interrupt_test.go +++ b/pkg/artifact/service/interrupt_test.go @@ -307,3 +307,20 @@ func TestPublish_ConditionalGuardRejectsRacingWritersEvenWithoutAWriterLock(t *t require.NoError(t, err) assert.Equal(t, current, pinned.RecordDigest, "the reference is bound to exactly one artifact") } + +func TestPublish_DelegatedLockIsTakenFromTheProvider(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"dst": {}, "coord": {}}}) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.LockProvider = "coord" }) + + outcome, err := e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("hello"))) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, outcome.Operation.Outcome) + assert.Equal(t, 1, e.repos["coord"].Calls("Lock"), "the delegate provides the lock") + assert.Zero(t, e.repos["dst"].Calls("Lock"), "the target repository is not asked to lock") + assert.Empty(t, e.repos["coord"].HeldLocks(), "the delegated lock is released") + + e.setInfo("dst", func(info *service.RepositoryInfo) { info.LockProvider = "" }) + _, err = e.svc.PublishDetailed(context.Background(), e.publishRequest("dst", e.source("other"))) + require.Error(t, err, "the second publish conflicts, but only after locking") + assert.Equal(t, 1, e.repos["dst"].Calls("Lock"), "without delegation the target repository locks for itself") +} diff --git a/pkg/artifact/service/locks.go b/pkg/artifact/service/locks.go index fdf78da3bb7..3edf79f16df 100644 --- a/pkg/artifact/service/locks.go +++ b/pkg/artifact/service/locks.go @@ -58,8 +58,13 @@ func (s *Service) acquireLocks(ctx context.Context, op *operation, operation str set := &lockSet{} for index := range ordered { target := ordered[index] - handle, err := target.repo.Lock(ctx, artifact.LockRequest{ - Resource: target.resource, + locker, resource, err := s.lockerFor(ctx, &target) + if err != nil { + releaseErr := set.release(ctx, op) + return nil, errors.Join(err, releaseErr) + } + handle, err := locker.Lock(ctx, artifact.LockRequest{ + Resource: resource, Owner: s.owner, OperationID: op.id, Required: requiredScope(target.info), @@ -73,6 +78,21 @@ func (s *Service) acquireLocks(ctx context.Context, op *operation, operation str return set, nil } +// lockerFor returns the repository that provides the target's lock and the resource to lock there. +// A repository that delegates locking (locking.repository) is locked through its delegate, whose +// resource is qualified by the target alias because one delegate can coordinate several repositories. +func (s *Service) lockerFor(ctx context.Context, target *lockTarget) (artifact.Repository, string, error) { + provider := target.info.LockProvider + if provider == "" || provider == target.alias { + return target.repo, target.resource, nil + } + repo, err := s.repos.Open(ctx, provider) + if err != nil { + return nil, "", fmt.Errorf("open lock repository %q delegated by %q: %w", provider, target.alias, err) + } + return repo, target.alias + "/" + target.resource, nil +} + // canonicalOrder sorts targets by repository alias then resource and drops exact duplicates. func canonicalOrder(targets []lockTarget) []lockTarget { ordered := append([]lockTarget(nil), targets...) diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go index 897706a58fd..6d6f8ca2f21 100644 --- a/pkg/artifact/service/repositories.go +++ b/pkg/artifact/service/repositories.go @@ -28,6 +28,10 @@ type RepositoryInfo struct { // RequiredLockScope is locking.required_scope; empty means host. RequiredLockScope artifact.LockScope + // LockProvider is the alias of the repository that locking.repository delegates locks to. Empty + // (or the repository's own alias) means the repository locks for itself. + LockProvider string + // Policy is the effective verification policy: the global and repository trust sets, both of // which must be satisfied. Policy artifact.VerificationPolicy diff --git a/pkg/oci/client.go b/pkg/oci/client.go index 668e2905bfd..f218b905e1c 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -144,7 +144,7 @@ func (c *Client) WriteImage(ctx context.Context, ref name.Reference, img v1.Imag if err != nil { return err } - graph, err := legacyImageGraph(img) + graph, err := legacyImageGraph(ctx, img) if err != nil { return err } @@ -160,7 +160,7 @@ func (c *Client) WriteIndex(ctx context.Context, ref name.Reference, index v1.Im if err != nil { return err } - graph, err := legacyIndexGraph(index) + graph, err := legacyIndexGraph(ctx, index) if err != nil { return err } diff --git a/pkg/oci/signature/plugin_error_test.go b/pkg/oci/signature/plugin_error_test.go new file mode 100644 index 00000000000..017a0181e49 --- /dev/null +++ b/pkg/oci/signature/plugin_error_test.go @@ -0,0 +1,27 @@ +package signature + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + sigcore "github.com/cloudposse/atmos/pkg/signature" +) + +func TestPluginErrorKeepsBothLegacyClassificationsForOfflineTrust(t *testing.T) { + cause := errors.New("trusted root cache is empty") + err := pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureOffline, Cause: cause}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid, "offline trust failures are still signature failures") + require.ErrorIs(t, err, cause) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureOffline}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureConfigInvalid}) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + require.NotErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) +} diff --git a/pkg/oci/signature/verify.go b/pkg/oci/signature/verify.go index 1b13a529259..dfc5eaa2688 100644 --- a/pkg/oci/signature/verify.go +++ b/pkg/oci/signature/verify.go @@ -53,7 +53,8 @@ func pluginError(err error) error { case errors.Is(shared.Kind, errUtils.ErrSignatureConfigInvalid): legacy = errUtils.ErrPluginInvalidConfig case errors.Is(shared.Kind, errUtils.ErrSignatureOffline): - legacy = errUtils.ErrPluginOfflineArtifact + // Offline trust failures were always signature failures too; keep both classifications. + legacy = errors.Join(errUtils.ErrPluginOfflineArtifact, errUtils.ErrPluginSignatureInvalid) } if shared.Cause == nil { return legacy diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go index d0b160a021c..d7a8387c586 100644 --- a/pkg/oci/transport.go +++ b/pkg/oci/transport.go @@ -21,6 +21,9 @@ import ( type Repository struct { Registry string Path string + // Insecure selects plain HTTP for a registry that name.Insecure marked, so every operation on + // the repository uses the same scheme. Loopback registries use plain HTTP without it. + Insecure bool } // ParseRepository validates "registry/path" using the same rules as the legacy client, @@ -36,7 +39,7 @@ func ParseRepository(value string) (Repository, error) { } func repositoryOf(repository name.Repository) Repository { - return Repository{Registry: repository.RegistryStr(), Path: repository.RepositoryStr()} + return Repository{Registry: repository.RegistryStr(), Path: repository.RepositoryStr(), Insecure: repository.Scheme() == "http"} } // String returns "registry/path". @@ -46,7 +49,11 @@ func (r Repository) String() string { return r.Registry + "/" + r.Path } // legacy converts back to the type used for credential resolution. func (r Repository) legacy() (name.Repository, error) { - parsed, err := name.NewRepository(r.String()) + var options []name.Option + if r.Insecure { + options = append(options, name.Insecure) + } + parsed, err := name.NewRepository(r.String(), options...) if err != nil { return name.Repository{}, wrapRegistry(err) } diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index 5d2c9b74b63..881bb68088c 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -176,10 +176,45 @@ func ExtendedCopy(ctx context.Context, src GraphSource, dst Destination, options graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store}, graph: src.Store} root, err := oras.ExtendedCopy(ctx, graph, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.graphOptions()}}) - if err != nil { - return reconcile(ctx, err, src.Store, src.Reference, dst) + if err == nil { + return root, nil } - return root, nil + want, reconcileErr := reconcile(ctx, err, src.Store, src.Reference, dst) + if reconcileErr != nil { + return want, reconcileErr + } + // The root alone does not prove an extended copy finished: every referrer must be there too. + if !referrersPresent(ctx, src.Store, dst.Store, want) { + return ocispec.Descriptor{}, classifyCopyError(err) + } + return want, nil +} + +// referrersPresent reports whether every predecessor of root in the source (signatures, SBOMs, and +// their own referrers) also exists at the destination. Any lookup failure counts as missing. +func referrersPresent(ctx context.Context, src ReadOnlyGraphStore, dst Store, root ocispec.Descriptor) bool { + visited := map[string]bool{root.Digest.String(): true} + pending := []ocispec.Descriptor{root} + for len(pending) > 0 { + node := pending[len(pending)-1] + pending = pending[:len(pending)-1] + predecessors, err := src.Predecessors(ctx, node) + if err != nil { + return false + } + for i := range predecessors { + key := predecessors[i].Digest.String() + if visited[key] { + continue + } + visited[key] = true + if exists, err := dst.Exists(ctx, predecessors[i]); err != nil || !exists { + return false + } + pending = append(pending, predecessors[i]) + } + } + return true } // reconcile decides the outcome of a failed copy by looking at the destination. @@ -286,5 +321,9 @@ func (g guardedStore) pushReference(ctx context.Context, expected ocispec.Descri if err := g.Store.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { return err } + // The push may have taken long enough for the copy to be canceled; never start the tag then. + if err := ctx.Err(); err != nil { + return err + } return g.Tag(ctx, expected, reference) } diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go new file mode 100644 index 00000000000..c08c2438735 --- /dev/null +++ b/pkg/oci/transport_hardening_test.go @@ -0,0 +1,214 @@ +package oci + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/registry/remote" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// imageIndex is embedded under a lower-case name so the embedded field does not shadow the +// ImageIndex method of the interface. +type imageIndex = v1.ImageIndex + +// countingIndex counts how often its children are listed. +type countingIndex struct { + imageIndex + listed *int +} + +func (c countingIndex) IndexManifest() (*v1.IndexManifest, error) { + *c.listed++ + return c.imageIndex.IndexManifest() +} + +// sharedParent lists one nested index twice, as a subgraph shared by two parents appears. +type sharedParent struct { + imageIndex + manifest *v1.IndexManifest + raw []byte + child v1.ImageIndex +} + +func (p sharedParent) RawManifest() ([]byte, error) { return p.raw, nil } +func (p sharedParent) IndexManifest() (*v1.IndexManifest, error) { return p.manifest, nil } +func (p sharedParent) ImageIndex(v1.Hash) (v1.ImageIndex, error) { return p.child, nil } + +func TestLegacyIndexGraphVisitsASharedIndexOnceAndStopsOnCancel(t *testing.T) { + child, err := random.Index(64, 1, 1) + require.NoError(t, err) + listed := 0 + shared := countingIndex{imageIndex: child, listed: &listed} + childDesc, err := partial.Descriptor(child) + require.NoError(t, err) + manifest := &v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex, Manifests: []v1.Descriptor{*childDesc, *childDesc}} + raw, err := json.Marshal(manifest) + require.NoError(t, err) + parent := sharedParent{imageIndex: empty.Index, manifest: manifest, raw: raw, child: shared} + + graph, err := legacyIndexGraph(t.Context(), parent) + require.NoError(t, err) + assert.Equal(t, 1, listed, "a subgraph shared by two parents is traversed once") + childDigest, err := child.Digest() + require.NoError(t, err) + exists, err := graph.Exists(t.Context(), ocispec.Descriptor{Digest: digest.Digest(childDigest.String())}) + require.NoError(t, err) + assert.True(t, exists) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = legacyIndexGraph(ctx, parent) + require.ErrorIs(t, err, context.Canceled) + image, err := random.Image(64, 1) + require.NoError(t, err) + _, err = legacyImageGraph(ctx, image) + require.ErrorIs(t, err, context.Canceled) +} + +func TestLayoutTagKeepsMetadataAndDigestRoots(t *testing.T) { + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + tagged := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"1"}}`) + tagged.ArtifactType = "application/vnd.example" + tagged.Annotations = map[string]string{"org.example.note": "kept"} + require.NoError(t, layout.Tag(t.Context(), tagged, "release")) + assert.Equal(t, map[string]string{"org.example.note": "kept"}, tagged.Annotations, "the caller's descriptor is not mutated") + + byDigest := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"2"}}`) + require.NoError(t, layout.Tag(t.Context(), byDigest, byDigest.Digest.String())) + + reopened, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + named, err := reopened.Resolve(t.Context(), "release") + require.NoError(t, err) + assert.Equal(t, "application/vnd.example", named.ArtifactType) + assert.Equal(t, "kept", named.Annotations["org.example.note"]) + root, err := reopened.Resolve(t.Context(), byDigest.Digest.String()) + require.NoError(t, err) + assert.Equal(t, ocispec.MediaTypeImageManifest, root.MediaType, "a digest-only root keeps its manifest type after reopening") + assert.Equal(t, []string{"release"}, reopened.References(), "a digest root is listed but never named") +} + +func TestLayoutStateMatchesThePersistedIndexAfterAFailedWrite(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory write permissions are not enforced on Windows") + } + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + desc := pushManifest(t, layout, `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[]}`) + + require.NoError(t, os.Chmod(dir, 0o500)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + require.Error(t, layout.Tag(t.Context(), desc, "release")) + assert.Empty(t, layout.References(), "a failed index write leaves the in-memory index unchanged") + + require.NoError(t, os.Chmod(dir, 0o755)) + require.NoError(t, layout.Tag(t.Context(), desc, "release"), "the retry repeats the index update") + assert.Equal(t, []string{"release"}, layout.References()) + raw, err := os.ReadFile(filepath.Join(dir, ocispec.ImageIndexFile)) + require.NoError(t, err) + var index ocispec.Index + require.NoError(t, json.Unmarshal(raw, &index)) + require.Len(t, index.Manifests, 1) + assert.Equal(t, "release", index.Manifests[0].Annotations[ocispec.AnnotationRefName]) +} + +func pushManifest(t *testing.T, layout *Layout, manifest string) ocispec.Descriptor { + t.Helper() + config := []byte("{}") + configDesc := Descriptor("application/vnd.oci.empty.v1+json", config) + err := layout.Push(t.Context(), configDesc, bytes.NewReader(config)) + if err != nil { + require.ErrorContains(t, err, "already exists") + } + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(manifest)) + require.NoError(t, layout.Push(t.Context(), desc, bytes.NewReader([]byte(manifest)))) + return desc +} + +func TestPushReferenceRejectsContentThatDoesNotMatchBeforeSending(t *testing.T) { + // The repository points at a closed port: a request would fail with a network error, not integrity. + repo, err := remote.NewRepository("127.0.0.1:1/acme/plugin") + require.NoError(t, err) + repo.PlainHTTP = true + store := ®istryStore{repo: repo} + expected := Descriptor(ocispec.MediaTypeImageManifest, []byte(`{"expected":true}`)) + + for name, content := range map[string][]byte{ + "different bytes": []byte(`{"expected":false}`), + "longer": []byte(`{"expected":true} `), + "shorter": []byte(`{}`), + } { + t.Run(name, func(t *testing.T) { + err := store.PushReference(t.Context(), expected, bytes.NewReader(content), "1.0.0") + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + }) + } + oversized := expected + oversized.Size = maxPushedManifest + 1 + require.ErrorIs(t, store.PushReference(t.Context(), oversized, bytes.NewReader(nil), "1.0.0"), errUtils.ErrOCIIntegrity) +} + +func TestRepositoryConversionKeepsTheInsecureScheme(t *testing.T) { + insecure, err := name.NewRepository("registry.example.com/acme/plugin", name.Insecure) + require.NoError(t, err) + converted := repositoryOf(insecure) + assert.True(t, converted.Insecure) + back, err := converted.legacy() + require.NoError(t, err) + assert.Equal(t, "http", back.Scheme(), "every operation on the repository keeps plain HTTP") + + secure, err := name.NewRepository("registry.example.com/acme/plugin") + require.NoError(t, err) + converted = repositoryOf(secure) + assert.False(t, converted.Insecure) + back, err = converted.legacy() + require.NoError(t, err) + assert.Equal(t, "https", back.Scheme()) +} + +func TestReferrersPresentRequiresEveryReferrerAtTheDestination(t *testing.T) { + src := memory.New() + dst := memory.New() + ctx := t.Context() + config := []byte("{}") + configDesc := Descriptor("application/vnd.oci.empty.v1+json", config) + rootRaw := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"` + configDesc.Digest.String() + `","size":2},"layers":[]}`) + root := Descriptor(ocispec.MediaTypeImageManifest, rootRaw) + sigRaw, err := json.Marshal(ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: ocispec.MediaTypeImageManifest, Config: configDesc, + Layers: []ocispec.Descriptor{}, Subject: &root, + }) + require.NoError(t, err) + signature := Descriptor(ocispec.MediaTypeImageManifest, sigRaw) + for _, store := range []*memory.Store{src, dst} { + require.NoError(t, store.Push(ctx, configDesc, bytes.NewReader(config))) + require.NoError(t, store.Push(ctx, root, bytes.NewReader(rootRaw))) + } + require.NoError(t, src.Push(ctx, signature, bytes.NewReader(sigRaw))) + + assert.False(t, referrersPresent(ctx, src, dst, root), "the root alone does not complete an extended copy") + require.NoError(t, dst.Push(ctx, signature, bytes.NewReader(sigRaw))) + assert.True(t, referrersPresent(ctx, src, dst, root)) +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index bdab1c7f4cb..f8497cd26e1 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -102,6 +102,7 @@ func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { {"resolver error", nil, true, os.ErrPermission}, {"matched identity without credentials", nil, true, nil}, {"matched identity resolves to anonymous", authn.Anonymous, true, nil}, + {"bound authenticator cannot authorize", failingAuthenticator{}, true, nil}, } { t.Run(tc.name, func(t *testing.T) { controller := gomock.NewController(t) diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 2d05488c422..410e0e06c34 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -6,8 +6,10 @@ import ( "encoding/json" "errors" "io" + "maps" "os" "path/filepath" + "slices" "sync" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -89,8 +91,19 @@ func (l *Layout) loadIndex() error { return nil } -func (l *Layout) writeIndex() error { - index := ocispec.Index{MediaType: ocispec.MediaTypeImageIndex, Manifests: l.index} +// commitIndex persists entries as index.json and only then adopts them as the in-memory state, so a +// failed write leaves the layout exactly as persisted and a retry repeats the whole update. Callers +// hold l.mu. +func (l *Layout) commitIndex(entries []ocispec.Descriptor) error { + if err := writeIndexFile(l.indexPath(), entries); err != nil { + return err + } + l.index = entries + return nil +} + +func writeIndexFile(indexPath string, entries []ocispec.Descriptor) error { + index := ocispec.Index{MediaType: ocispec.MediaTypeImageIndex, Manifests: entries} index.SchemaVersion = 2 if index.Manifests == nil { index.Manifests = []ocispec.Descriptor{} @@ -99,7 +112,7 @@ func (l *Layout) writeIndex() error { if err != nil { return errors.Join(errUtils.ErrOCILayout, err) } - if err := filesystem.NewOSFileSystem().WriteFileAtomic(l.indexPath(), raw, layoutIndexMode); err != nil { + if err := filesystem.NewOSFileSystem().WriteFileAtomic(indexPath, raw, layoutIndexMode); err != nil { return errors.Join(errUtils.ErrOCILayout, err) } return nil @@ -147,6 +160,12 @@ func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) if desc.Size > maxAttachmentManifest { return layoutError(errUtils.ErrOCIIntegrity) } + return l.recordRoot(desc) +} + +// recordRoot lists desc as an unnamed root in index.json unless its digest is already listed, so the +// manifest and its graph edges still resolve after the layout is reopened. +func (l *Layout) recordRoot(desc ocispec.Descriptor) error { l.mu.Lock() defer l.mu.Unlock() for i := range l.index { @@ -154,8 +173,8 @@ func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) return nil } } - l.index = append(l.index, ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size}) - return l.writeIndex() + entries := append(slices.Clone(l.index), ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size}) + return l.commitIndex(entries) } // Resolve resolves a reference name or digest to a descriptor. @@ -172,9 +191,9 @@ func (l *Layout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]o return found, layoutError(err) } -// Tag records reference as a named root in index.json. Digest references are implicit -// and never listed. An unnamed root with the same digest is named in place, so a legacy -// single-release layout keeps exactly one entry. +// Tag records reference as a named root in index.json. A digest reference is recorded as an +// unnamed root unless the digest is already listed. An unnamed root with the same digest is named +// in place, so a legacy single-release layout keeps exactly one entry. func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { defer perf.Track(nil, "oci.Layout.Tag")() @@ -182,16 +201,19 @@ func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference str return layoutError(err) } if reference == desc.Digest.String() { - return nil + // A digest reference is never named, but the root must still be listed or reopening the + // layout loses its media type and children. + return l.recordRoot(desc) } l.mu.Lock() defer l.mu.Unlock() - named := ocispec.Descriptor{ - MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size, - Annotations: map[string]string{ocispec.AnnotationRefName: reference}, - } - l.index = withReference(l.index, named) - return l.writeIndex() + // Copy the descriptor and its annotations so root metadata (artifact type, platform, other + // annotations) survives reopening without mutating the caller's descriptor. + named := desc + named.Annotations = make(map[string]string, len(desc.Annotations)+1) + maps.Copy(named.Annotations, desc.Annotations) + named.Annotations[ocispec.AnnotationRefName] = reference + return l.commitIndex(withReference(slices.Clone(l.index), named)) } func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []ocispec.Descriptor { diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go index 9f5b2160821..363c874e580 100644 --- a/pkg/oci/transport_legacy.go +++ b/pkg/oci/transport_legacy.go @@ -28,15 +28,25 @@ const legacyRoot = "root" type legacyGraph struct { root ocispec.Descriptor blobs map[digest.Digest]func() (io.ReadCloser, error) + + // indexes holds the descriptors of indexes whose children were already added, so a subgraph + // shared by several parents is traversed once. + indexes map[digest.Digest]ocispec.Descriptor } var _ ReadOnlyStore = (*legacyGraph)(nil) func newLegacyGraph() *legacyGraph { - return &legacyGraph{blobs: make(map[digest.Digest]func() (io.ReadCloser, error))} + return &legacyGraph{ + blobs: make(map[digest.Digest]func() (io.ReadCloser, error)), + indexes: make(map[digest.Digest]ocispec.Descriptor), + } } -func legacyImageGraph(image v1.Image) (*legacyGraph, error) { +func legacyImageGraph(ctx context.Context, image v1.Image) (*legacyGraph, error) { + if err := ctx.Err(); err != nil { + return nil, err + } graph := newLegacyGraph() root, err := graph.addImage(image) if err != nil { @@ -46,9 +56,9 @@ func legacyImageGraph(image v1.Image) (*legacyGraph, error) { return graph, nil } -func legacyIndexGraph(index v1.ImageIndex) (*legacyGraph, error) { +func legacyIndexGraph(ctx context.Context, index v1.ImageIndex) (*legacyGraph, error) { graph := newLegacyGraph() - root, err := graph.addIndex(index) + root, err := graph.addIndex(ctx, index) if err != nil { return nil, err } @@ -95,7 +105,12 @@ func (g *legacyGraph) addImage(image v1.Image) (ocispec.Descriptor, error) { return g.addBytes(string(mediaType), raw), nil } -func (g *legacyGraph) addIndex(index v1.ImageIndex) (ocispec.Descriptor, error) { +// addIndex records an index and, once per distinct index digest, every child below it. Cancellation +// is checked before each accessor call and child, so a canceled write stops preparing the graph. +func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispec.Descriptor, error) { + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } raw, err := index.RawManifest() if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) @@ -104,27 +119,39 @@ func (g *legacyGraph) addIndex(index v1.ImageIndex) (ocispec.Descriptor, error) if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) } + if done, found := g.indexes[digest.FromBytes(raw)]; found { + return done, nil + } + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } manifest, err := index.IndexManifest() if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) } for i := range manifest.Manifests { - if err := g.addChild(index, &manifest.Manifests[i]); err != nil { + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if err := g.addChild(ctx, index, &manifest.Manifests[i]); err != nil { return ocispec.Descriptor{}, err } } - return g.addBytes(string(mediaType), raw), nil + // The index is marked complete only after every child was added successfully. + desc := g.addBytes(string(mediaType), raw) + g.indexes[desc.Digest] = desc + return desc, nil } // addChild records one child manifest or nested index of a legacy index. -func (g *legacyGraph) addChild(index v1.ImageIndex, child *v1.Descriptor) error { +func (g *legacyGraph) addChild(ctx context.Context, index v1.ImageIndex, child *v1.Descriptor) error { switch { case child.MediaType.IsIndex(): nested, err := index.ImageIndex(child.Digest) if err != nil { return errors.Join(errUtils.ErrOCIIntegrity, err) } - _, err = g.addIndex(nested) + _, err = g.addIndex(ctx, nested) return err case child.MediaType.IsImage(): nested, err := index.Image(child.Digest) diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go index 2c7ee0af75c..6e6771a9192 100644 --- a/pkg/oci/transport_registry.go +++ b/pkg/oci/transport_registry.go @@ -2,6 +2,7 @@ package oci import ( + "bytes" "context" "encoding/base64" "io" @@ -21,6 +22,9 @@ import ( ) const ( + // The largest manifest pushed under a reference: the 4 MiB limit registries commonly enforce. + maxPushedManifest = 4 << 20 + userAgent = "atmos" dockerHubLegacy = "index.docker.io" dockerHubRegistry = "docker.io" @@ -45,6 +49,12 @@ func (c *Client) Repository(ctx context.Context, repository Repository) (Reposit if err != nil { return nil, err } + // Resolve the bound credential once before any request, so a failing identity stops here + // instead of surfacing mid-operation or degrading to anonymous access. The callback below still + // re-invokes the authenticator on every challenge, so refreshing tokens keep refreshing. + if _, err := credentialFrom(authenticator); err != nil { + return nil, err + } repo, err := c.remoteRepository(resource, authenticator) if err != nil { return nil, err @@ -169,8 +179,18 @@ func (s *registryStore) Push(ctx context.Context, expected ocispec.Descriptor, c return wrapRegistry(s.repo.Push(ctx, expected, content)) } +// PushReference verifies the manifest bytes against expected before anything is sent, then pushes +// them under reference. A registry that omits Docker-Content-Digest therefore can never tag content +// other than the expected manifest. func (s *registryStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { - return wrapRegistry(s.repo.PushReference(ctx, expected, content, reference)) + if expected.Size < 0 || expected.Size > maxPushedManifest { + return errUtils.ErrOCIIntegrity + } + raw, err := readVerified(content, expected, maxPushedManifest) + if err != nil { + return err + } + return wrapRegistry(s.repo.PushReference(ctx, expected, bytes.NewReader(raw), reference)) } func (s *registryStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { diff --git a/pkg/signature/key.go b/pkg/signature/key.go index 462bfb8fb4a..a8747ab524d 100644 --- a/pkg/signature/key.go +++ b/pkg/signature/key.go @@ -3,6 +3,7 @@ package signature import ( "context" "crypto" + "fmt" "time" "github.com/sigstore/sigstore-go/pkg/root" @@ -37,7 +38,8 @@ type KeyVerifier struct { LoadRoot func(context.Context, bool) (root.TrustedMaterial, error) } -// Verify checks raw against the independently supplied subject digest. +// Verify checks raw against the independently supplied subject digest. When several configured keys +// share a hint, each of them is tried; the first that verifies the bundle wins. func (v KeyVerifier) Verify(ctx context.Context, raw []byte, digest string, offline bool) (*KeyResult, error) { defer perf.Track(nil, "signature.KeyVerifier.Verify")() @@ -51,29 +53,40 @@ func (v KeyVerifier) Verify(ctx context.Context, raw []byte, digest string, offl if err != nil { return nil, err } - verifier, err := v.trustedVerifier(ctx, offline) + rounds, err := v.keyRounds() if err != nil { return nil, err } - result, err := verifier.Verify(signed, verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), verify.WithKey())) + logs, err := v.loadLogs(ctx, offline) if err != nil { - return nil, fail(errUtils.ErrSignatureInvalid, err) - } - if err := ctx.Err(); err != nil { return nil, err } - if result.Signature == nil || result.Signature.PublicKeyID == nil { - return nil, fail(errUtils.ErrSignatureInvalid, nil) + policy := verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), verify.WithKey()) + var last error + for _, keys := range rounds { + verifier, err := verify.NewVerifier(root.TrustedMaterialCollection{keys, logs}, verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) + if err != nil { + return nil, fail(errUtils.ErrSignatureInvalid, err) + } + result, err := verifier.Verify(signed, policy) + if err != nil { + last = err + continue + } + if err := ctx.Err(); err != nil { + return nil, err + } + if result.Signature == nil || result.Signature.PublicKeyID == nil { + return nil, fail(errUtils.ErrSignatureInvalid, nil) + } + return &KeyResult{Hint: string(*result.Signature.PublicKeyID)}, nil } - return &KeyResult{Hint: string(*result.Signature.PublicKeyID)}, nil + return nil, fail(errUtils.ErrSignatureInvalid, last) } -// trustedVerifier combines the configured keys with transparency-log trust. -func (v KeyVerifier) trustedVerifier(ctx context.Context, offline bool) (*verify.Verifier, error) { - keys, err := v.keyMaterial() - if err != nil { - return nil, err - } +// loadLogs loads transparency-log trust and strips any public keys it carries, so only the +// configured keys can authenticate a key-signed bundle. +func (v KeyVerifier) loadLogs(ctx context.Context, offline bool) (root.TrustedMaterial, error) { loader := v.LoadRoot if loader == nil { loader = LoadPublicRoot @@ -82,17 +95,24 @@ func (v KeyVerifier) trustedVerifier(ctx context.Context, offline bool) (*verify if err != nil { return nil, fail(errUtils.ErrSignatureInvalid, err) } - verifier, err := verify.NewVerifier(root.TrustedMaterialCollection{keys, logs}, verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) - if err != nil { - return nil, fail(errUtils.ErrSignatureInvalid, err) - } - return verifier, nil + return transparencyOnly{TrustedMaterial: logs}, nil } -// keyMaterial exposes only the configured keys. Certificate authorities from other trust -// material cannot validate a key-signed bundle. -func (v KeyVerifier) keyMaterial() (root.TrustedMaterial, error) { - verifiers := make(map[string]*root.ExpiringKey, len(v.Keys)) +// transparencyOnly exposes transparency-log and timestamp trust but never resolves a signer key. +type transparencyOnly struct{ root.TrustedMaterial } + +//nolint:lintroller // Constant rejection with no work to track. +func (transparencyOnly) PublicKeyVerifier(hint string) (root.TimeConstrainedVerifier, error) { + return nil, fmt.Errorf("%w: key %q is not a configured trusted key", errUtils.ErrSignatureInvalid, hint) +} + +// keyRounds groups the configured keys so each round holds at most one key per hint. A bundle names +// a single hint, so trying every round tries every configured key for that hint, and a key never +// overwrites another that shares its hint. Certificate authorities from other trust material cannot +// validate a key-signed bundle. +func (v KeyVerifier) keyRounds() ([]root.TrustedMaterial, error) { + var rounds []map[string]*root.ExpiringKey + seen := make(map[string]int, len(v.Keys)) for _, key := range v.Keys { if key.Hint == "" || key.PublicKey == nil { return nil, fail(errUtils.ErrSignatureConfigInvalid, nil) @@ -101,7 +121,16 @@ func (v KeyVerifier) keyMaterial() (root.TrustedMaterial, error) { if err != nil { return nil, fail(errUtils.ErrSignatureConfigInvalid, err) } - verifiers[key.Hint] = root.NewExpiringKey(loaded, time.Time{}, time.Time{}) + round := seen[key.Hint] + seen[key.Hint]++ + if round == len(rounds) { + rounds = append(rounds, make(map[string]*root.ExpiringKey)) + } + rounds[round][key.Hint] = root.NewExpiringKey(loaded, time.Time{}, time.Time{}) + } + materials := make([]root.TrustedMaterial, 0, len(rounds)) + for _, keys := range rounds { + materials = append(materials, root.NewTrustedPublicKeyMaterialFromMapping(keys)) } - return root.NewTrustedPublicKeyMaterialFromMapping(verifiers), nil + return materials, nil } diff --git a/pkg/signature/key_test.go b/pkg/signature/key_test.go index 4e796c3ac35..21eb5b87345 100644 --- a/pkg/signature/key_test.go +++ b/pkg/signature/key_test.go @@ -2,13 +2,16 @@ package signature import ( "context" + "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "strings" "testing" + "time" "github.com/sigstore/sigstore-go/pkg/root" + sigstore "github.com/sigstore/sigstore/pkg/signature" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -87,3 +90,61 @@ func TestKeyAndIdentityPathsNeverSubstituteForEachOther(t *testing.T) { _, err = keys.Verify(t.Context(), certificateBundle, digestOf(artifact), true) require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) } + +// leakingRoot is transparency-log trust that also resolves a signer key, as a loader might. +type leakingRoot struct { + *signaturetest.Authority + key root.TimeConstrainedVerifier +} + +func (l leakingRoot) PublicKeyVerifier(string) (root.TimeConstrainedVerifier, error) { + return l.key, nil +} + +func TestKeyVerifierIgnoresSignerKeysFromTheLoader(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + loaded, err := sigstore.LoadVerifier(signed.PublicKey, crypto.SHA256) + require.NoError(t, err) + leaking := func(context.Context, bool) (root.TrustedMaterial, error) { + return leakingRoot{Authority: authority, key: root.NewExpiringKey(loaded, time.Time{}, time.Time{})}, nil + } + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + // The loader's key would verify the bundle, but it is not configured, so verification fails. + verifier := KeyVerifier{Keys: []TrustedKey{{Hint: "other", PublicKey: &stranger.PublicKey}}, LoadRoot: leaking} + _, err = verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + + // The loader's transparency-log trust is still used when the signer key is configured. + verifier.Keys = []TrustedKey{{Hint: signed.Hint, PublicKey: signed.PublicKey}} + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, signaturetest.KeyHint, result.Hint) +} + +func TestKeyVerifierTriesEveryKeyThatSharesAHint(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + stranger, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + right := TrustedKey{Hint: signed.Hint, PublicKey: signed.PublicKey} + wrong := TrustedKey{Hint: signed.Hint, PublicKey: &stranger.PublicKey} + + for name, keys := range map[string][]TrustedKey{"wrong key first": {wrong, right}, "right key first": {right, wrong}} { + t.Run(name, func(t *testing.T) { + result, err := KeyVerifier{Keys: keys, LoadRoot: logsOnly(authority)}.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.NoError(t, err) + assert.Equal(t, signaturetest.KeyHint, result.Hint) + }) + } + + // Two non-matching keys under the hint still fail. + other, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + _, err = KeyVerifier{Keys: []TrustedKey{wrong, {Hint: signed.Hint, PublicKey: &other.PublicKey}}, LoadRoot: logsOnly(authority)}.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) +} From 930596ffecb15a72a3e22a8b7a93ca0298e856bb Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 10 Nov 2026 19:49:26 -0600 Subject: [PATCH 06/52] refactor(artifacts): split reconcile and key verification to satisfy cyclomatic limits Extract the Record-digest match path from reconcile into reconcileRecord and the per-round loop from KeyVerifier.Verify into verifyRounds. Behavior is unchanged; revive flagged both functions above the configured complexity limit of 10. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/execute.go | 22 ++++++++++++++-------- pkg/signature/key.go | 7 +++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go index 3f558ad2ab5..220caaa18ff 100644 --- a/pkg/artifact/service/execute.go +++ b/pkg/artifact/service/execute.go @@ -122,14 +122,7 @@ func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) rec return reconcileGraph(ref, &pinned, in, cur) } if pinned.RecordDigest == in.recordDigest { - inspection, inspectErr := tgt.repo.Inspect(ctx, pinned) - if inspectErr != nil { - return reconciliation{pinned: pinned, err: fmt.Errorf("reconcile %s: %w", ref, inspectErr)} - } - if matchErr := matchesExpected(in, inspection); matchErr != nil { - return reconciliation{pinned: pinned, inspection: inspection, err: matchErr} - } - return reconciliation{state: reconcileCommitted, pinned: pinned, inspection: inspection} + return reconcileRecord(ctx, tgt, &pinned, in) } if cur.pinned != nil && pinned.ResolvedDigest == cur.pinned.ResolvedDigest { return reconciliation{state: reconcileNotCommitted, pinned: pinned} @@ -137,6 +130,19 @@ func reconcile(ctx context.Context, tgt *target, in *incoming, cur *current) rec return reconciliation{pinned: pinned, err: fmt.Errorf("%w: %s now resolves to %s, which is neither the prior state nor the expected content", errUtils.ErrArtifactConflict, ref, pinned.ResolvedDigest)} } +// reconcileRecord inspects a reference whose Record digest matches the expected one and confirms +// that the stored content is the expected content. +func reconcileRecord(ctx context.Context, tgt *target, pinned *artifact.PinnedReference, in *incoming) reconciliation { + inspection, err := tgt.repo.Inspect(ctx, *pinned) + if err != nil { + return reconciliation{pinned: *pinned, err: fmt.Errorf("reconcile %s: %w", in.ref, err)} + } + if matchErr := matchesExpected(in, inspection); matchErr != nil { + return reconciliation{pinned: *pinned, inspection: inspection, err: matchErr} + } + return reconciliation{state: reconcileCommitted, pinned: *pinned, inspection: inspection} +} + // reconcileGraph classifies a native OCI graph without a Record. Empty record digests prove nothing, // so success requires the destination root to be exactly the pinned source root. func reconcileGraph(ref artifact.Ref, pinned *artifact.PinnedReference, in *incoming, cur *current) reconciliation { diff --git a/pkg/signature/key.go b/pkg/signature/key.go index a8747ab524d..89c1cf5a589 100644 --- a/pkg/signature/key.go +++ b/pkg/signature/key.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/sigstore/sigstore-go/pkg/bundle" "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore-go/pkg/verify" sigstore "github.com/sigstore/sigstore/pkg/signature" @@ -62,6 +63,12 @@ func (v KeyVerifier) Verify(ctx context.Context, raw []byte, digest string, offl return nil, err } policy := verify.NewPolicy(verify.WithArtifactDigest("sha256", hash), verify.WithKey()) + return verifyRounds(ctx, rounds, logs, signed, policy) +} + +// verifyRounds tries each key round in order and returns the hint of the first key that verifies +// the bundle. When no round verifies it, the last verification error is reported. +func verifyRounds(ctx context.Context, rounds []root.TrustedMaterial, logs root.TrustedMaterial, signed *bundle.Bundle, policy verify.PolicyBuilder) (*KeyResult, error) { var last error for _, keys := range rounds { verifier, err := verify.NewVerifier(root.TrustedMaterialCollection{keys, logs}, verify.WithTransparencyLog(1), verify.WithObserverTimestamps(1)) From 5e6e594a5019af882eeb5180b5aa0aad2b130151 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 07:57:31 -0600 Subject: [PATCH 07/52] fix(artifacts): reject paths through regular files --- pkg/artifact/linkpath/linkpath.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index 8042763d905..cd4227f8c35 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -74,6 +74,9 @@ func (w *walker) step() (done, ok bool) { return true, false } if info.Mode()&fs.ModeSymlink == 0 { + if !info.IsDir() && len(w.pending) > 0 { + return true, false + } w.resolved = next return false, true } From 6d8491775ad75c25b78acdb5e36b78756fa46d23 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 15:54:33 -0600 Subject: [PATCH 08/52] fix(artifacts): harden service and transport review findings --- pkg/artifact/artifacttest/conformance.go | 4 ++ pkg/artifact/artifacttest/read.go | 7 +-- pkg/artifact/linkpath/linkpath.go | 22 ++++++-- pkg/artifact/linkpath/linkpath_test.go | 2 + pkg/artifact/produce/assemble.go | 25 +++++++-- pkg/artifact/produce/context_internal_test.go | 27 ++++++++++ pkg/artifact/produce/produce_test.go | 5 +- pkg/artifact/service/consume_test.go | 53 +++++++++++++++++-- pkg/artifact/service/copy.go | 31 ++++++++++- pkg/artifact/service/edge_test.go | 19 ++++++- pkg/artifact/service/execute.go | 10 +++- pkg/artifact/service/internal_test.go | 14 +++++ pkg/artifact/service/locks.go | 21 +++++++- pkg/artifact/service/mutate.go | 5 +- pkg/artifact/service/mutate_test.go | 4 +- pkg/artifact/service/query.go | 18 +++++-- pkg/artifact/service/sign.go | 12 ++++- pkg/artifact/snapshot/snapshot_test.go | 10 +++- pkg/oci/transport.go | 2 +- pkg/oci/transport_behavior_test.go | 25 ++++++++- pkg/oci/transport_copy.go | 31 +++++++---- pkg/oci/transport_hardening_test.go | 4 +- pkg/oci/transport_layout.go | 2 + pkg/signature/trust_test.go | 13 +++++ 24 files changed, 318 insertions(+), 48 deletions(-) create mode 100644 pkg/artifact/produce/context_internal_test.go diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index 6c0c6857625..43801131d54 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -222,6 +222,10 @@ func (c *conformance) enumeration(t *testing.T) { for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}} { c.publish(t, repo, spec{name: item.name, version: item.version, content: item.name + item.version}, artifact.PublishOptions{}) } + if !caps.EnumerateNamed { + _, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + } if !caps.EnumerateCollection { _, err := repo.List(context.Background(), artifact.ListQuery{}) require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) diff --git a/pkg/artifact/artifacttest/read.go b/pkg/artifact/artifacttest/read.go index e1c4edf57dd..5e641099976 100644 --- a/pkg/artifact/artifacttest/read.go +++ b/pkg/artifact/artifacttest/read.go @@ -201,12 +201,13 @@ func (r *Repository) page(entries []artifact.ListEntry, query artifact.ListQuery if query.Cursor != "" { start = sort.Search(len(entries), func(i int) bool { return listKey(&entries[i]) > query.Cursor }) } - end := start + size + end := len(entries) + if size < len(entries)-start { + end = start + size + } page := &artifact.ListPage{} if end < len(entries) { page.NextCursor = listKey(&entries[end-1]) - } else { - end = len(entries) } page.Entries = append(page.Entries, entries[start:end]...) return page diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index cd4227f8c35..549917a518f 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -34,10 +34,10 @@ func Resolve(target string) (string, bool) { for len(w.pending) > 0 { done, ok := w.step() if done { - return w.resolved, ok + return filepath.Join(append([]string{w.resolved}, w.missing...)...), ok } } - return w.resolved, true + return filepath.Join(append([]string{w.resolved}, w.missing...)...), true } // walker resolves a path one component at a time. @@ -51,6 +51,9 @@ type walker struct { // pending holds the components still to resolve. pending []string + // missing holds the lexical tail below the first missing component. + missing []string + // hops counts the symlinks followed. hops int } @@ -59,6 +62,16 @@ type walker struct { func (w *walker) step() (done, ok bool) { component := w.pending[0] w.pending = w.pending[1:] + if len(w.missing) > 0 { + switch component { + case ".": + case "..": + w.missing = w.missing[:len(w.missing)-1] + default: + w.missing = append(w.missing, component) + } + return false, true + } if component == "." || component == ".." { w.resolved = stepDot(w.resolved, component) return false, true @@ -66,9 +79,8 @@ func (w *walker) step() (done, ok bool) { next := filepath.Join(w.resolved, component) info, err := os.Lstat(next) if errors.Is(err, fs.ErrNotExist) { - // Nothing below a missing component can be a symlink, so the rest is lexical. - w.resolved = filepath.Join(append([]string{next}, w.pending...)...) - return true, true + w.missing = append(w.missing, component) + return false, true } if err != nil { return true, false diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go index 30c6fac86f4..687820dc279 100644 --- a/pkg/artifact/linkpath/linkpath_test.go +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -29,6 +29,7 @@ func TestResolve(t *testing.T) { require.True(t, fixtures.Symlink(t, root, "tree/to-real", "real")) require.True(t, fixtures.Symlink(t, root, "tree/dangling", "real/missing/deeper")) require.True(t, fixtures.Symlink(t, root, "tree/escapes", "up/../outside/missing")) + require.True(t, fixtures.Symlink(t, root, "tree/missing-return", "absent/../up/../outside")) require.True(t, fixtures.Symlink(t, root, "tree/loop-a", "loop-b")) require.True(t, fixtures.Symlink(t, root, "tree/loop-b", "loop-a")) @@ -43,6 +44,7 @@ func TestResolve(t *testing.T) { {"dangling tail stays lexical under the link target", filepath.Join(root, "tree", "dangling"), filepath.Join(root, "tree", "real", "missing", "deeper"), true}, {"dangling tail walking out through a link", filepath.Join(root, "tree", "escapes"), filepath.Join(filepath.Dir(root), "outside", "missing"), true}, {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), true}, + {"missing tail returns to physical resolution", filepath.Join(root, "tree", "missing-return"), filepath.Join(filepath.Dir(root), "outside"), true}, {"loop", filepath.Join(root, "tree", "loop-a"), "", false}, {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", false}, } diff --git a/pkg/artifact/produce/assemble.go b/pkg/artifact/produce/assemble.go index 058da1004bc..d95eaef1452 100644 --- a/pkg/artifact/produce/assemble.go +++ b/pkg/artifact/produce/assemble.go @@ -14,6 +14,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/archive" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" ) // Constants for staged payloads. @@ -97,7 +98,7 @@ func (a *assembly) documentBlobs() []artifact.StagedBlob { } // archiveBlob writes the deterministic archive into staging, hashing it as it is written. The -// context is checked before every member is opened, so cancellation stops a large assembly. +// context is checked while members are opened and read, so cancellation stops a large assembly. func (a *assembly) archiveBlob(ctx context.Context, packaging artifact.Packaging) (artifact.StagedBlob, error) { format, mediaType := archiveFormat(packaging) path := filepath.Join(a.stagingDir, "payload."+format) @@ -124,7 +125,7 @@ func (a *assembly) archiveBlob(ctx context.Context, packaging artifact.Packaging }, nil } -// withContext makes every file member fail to open once ctx is canceled. +// withContext checks cancellation when opening and reading every file member. func withContext(ctx context.Context, members []archive.Member) []archive.Member { for index := range members { open := members[index].Open @@ -135,12 +136,30 @@ func withContext(ctx context.Context, members []archive.Member) []archive.Member if err := ctx.Err(); err != nil { return nil, err } - return open() + reader, err := open() + if err != nil { + return nil, err + } + return &contextReader{ctx: ctx, ReadCloser: reader}, nil } } return members } +type contextReader struct { + ctx context.Context + io.ReadCloser +} + +func (r *contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "produce.contextReader.Read")() + + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.ReadCloser.Read(p) +} + func archiveFormat(packaging artifact.Packaging) (string, string) { switch packaging { case artifact.PackagingTar: diff --git a/pkg/artifact/produce/context_internal_test.go b/pkg/artifact/produce/context_internal_test.go new file mode 100644 index 00000000000..8b457ad7782 --- /dev/null +++ b/pkg/artifact/produce/context_internal_test.go @@ -0,0 +1,27 @@ +package produce + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/archive" +) + +func TestArchiveMemberStopsReadingAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + members := withContext(ctx, []archive.Member{{Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("payload")), nil }}}) + reader, err := members[0].Open() + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1) + _, err = reader.Read(buf) + require.NoError(t, err) + cancel() + _, err = reader.Read(buf) + require.True(t, errors.Is(err, context.Canceled)) +} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index aae767de056..0be791d5f26 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -114,7 +115,9 @@ func TestFilesProducer_PerFileBlobs(t *testing.T) { for index := range blobs { files = append(files, blobs[index].Path) assert.NotEmpty(t, blobs[index].Path) - assert.Equal(t, int64(len(readBlob(t, &blobs[index]))), blobs[index].Descriptor.Size) + data := readBlob(t, &blobs[index]) + assert.Equal(t, int64(len(data)), blobs[index].Descriptor.Size) + assert.Equal(t, digest.FromBytes(data), blobs[index].Descriptor.Digest) } want := []string{"README.md", "bin/helper.sh", "empty.txt", "nested/deep/file.txt", fixtures.UnicodeName} assert.ElementsMatch(t, want, files, "directories and symlinks have no blob (symlinks=%v)", pkg.Symlinks) diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index 29b08d70306..861ef9a1317 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -2,9 +2,12 @@ package service_test import ( "context" + "fmt" "os" "path/filepath" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -313,6 +316,28 @@ func TestList_OnlyFailingRepositoriesFailTheListing(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "when no repository can answer, the listing fails") } +type listGate struct { + inflight atomic.Int32 + peak atomic.Int32 + entered chan struct{} + release chan struct{} +} + +type gatedListRepository struct { + artifact.Repository + gate *listGate +} + +func (r *gatedListRepository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + current := r.gate.inflight.Add(1) + defer r.gate.inflight.Add(-1) + for old := r.gate.peak.Load(); current > old && !r.gate.peak.CompareAndSwap(old, current); old = r.gate.peak.Load() { + } + r.gate.entered <- struct{}{} + <-r.gate.release + return r.Repository.List(ctx, query) +} + func TestList_ConcurrencyIsInjectable(t *testing.T) { e := newEnv(t, envConfig{ repos: map[string]artifacttest.Options{"a": {}, "b": {}, "c": {}}, @@ -321,9 +346,31 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { for _, alias := range []string{"a", "b", "c"} { seedList(t, e, alias, map[string]string{"app/x:1": alias}, nil) } - result, err := e.svc.List(context.Background(), artifact.ListRequest{}) - require.NoError(t, err) - assert.Len(t, result.Artifacts, 3) + gate := &listGate{entered: make(chan struct{}, 3), release: make(chan struct{})} + for _, alias := range []string{"a", "b", "c"} { + e.static.Repositories[alias] = &gatedListRepository{Repository: e.repos[alias], gate: gate} + } + done := make(chan error, 1) + go func() { + result, err := e.svc.List(context.Background(), artifact.ListRequest{}) + if err == nil && len(result.Artifacts) != 3 { + err = fmt.Errorf("got %d artifacts", len(result.Artifacts)) + } + done <- err + }() + select { + case <-gate.entered: + case <-time.After(2 * time.Second): + close(gate.release) + t.Fatal("listing did not start") + } + select { + case <-gate.entered: + case <-time.After(200 * time.Millisecond): + } + close(gate.release) + require.NoError(t, <-done) + assert.Equal(t, int32(1), gate.peak.Load()) } // readHookRepository runs a hook right before every Read, as if another process touched the diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index 122620a8bc6..e36f64c69c5 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -2,7 +2,9 @@ package service import ( "context" + "errors" "fmt" + "io/fs" "os" "path/filepath" @@ -206,7 +208,11 @@ func (s *Service) stageCopy(ctx context.Context, op *operation, plan *copyPlan) if err != nil { return nil, nil, fmt.Errorf("%w: create copy staging: %w", errUtils.ErrArtifactPackagingInvalid, err) } - cleanup := func() { _ = os.RemoveAll(tree) } + cleanup := func() { + if err := removeStagedTree(tree); err != nil { + op.warn("staging-cleanup-failed", fmt.Sprintf("remove copy staging %q: %v", tree, err)) + } + } files := filepath.Join(tree, "files") if err := materialize.Materialize(ctx, content, files); err != nil { cleanup() @@ -354,3 +360,26 @@ func missingEvidence(ctx context.Context, repo artifact.Repository, pinned *arti } return missing } + +// removeStagedTree restores owner access as it walks directories, without following symlinks. +func removeStagedTree(tree string) error { + root, err := os.OpenRoot(tree) + if err != nil { + return err + } + walkErr := fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + return root.Chmod(path, info.Mode().Perm()|0o700) + } + return nil + }) + closeErr := root.Close() + return errors.Join(walkErr, closeErr, os.RemoveAll(tree)) +} diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go index f73ae368494..33862b02eaa 100644 --- a/pkg/artifact/service/edge_test.go +++ b/pkg/artifact/service/edge_test.go @@ -256,21 +256,38 @@ func TestShowDocument_TamperedDocumentFailsItsDescriptor(t *testing.T) { pinned := artifact.PinnedReference{Ref: exampleRef, ResolvedDigest: recordDigest, RecordDigest: recordDigest} want := []byte("genuine readme") descriptor := artifact.Document{Role: artifact.DocumentReadme, SourcePath: "README.md", MediaType: artifact.DocumentMediaType, Size: int64(len(want)), Digest: digest.FromBytes(want)} - _ = record + record.Documents = []artifact.Document{descriptor} + require.NoError(t, record.Seal()) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + pinned.ResolvedDigest, pinned.RecordDigest = recordDigest, recordDigest for name, content := range map[string][]byte{"altered bytes": []byte("tampered readme"), "oversize": bytes.Repeat([]byte("x"), int(artifact.MaxDocumentSize)+1)} { t.Run(name, func(t *testing.T) { m := newMockSetup(t, &caps) m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(&artifact.DocumentStream{Document: descriptor, Reader: io.NopCloser(bytes.NewReader(content))}, nil) _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) require.Error(t, err) assert.True(t, errors.Is(err, errUtils.ErrArtifactIntegrityFailed) || errors.Is(err, errUtils.ErrArtifactTooLarge), "%v", err) }) } + t.Run("forged descriptor", func(t *testing.T) { + m := newMockSetup(t, &caps) + forged := []byte("forged readme") + other := descriptor + other.Size, other.Digest = int64(len(forged)), digest.FromBytes(forged) + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) + m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(&artifact.DocumentStream{Document: other, Reader: io.NopCloser(bytes.NewReader(forged))}, nil) + _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) t.Run("open fails", func(t *testing.T) { m := newMockSetup(t, &caps) m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(pinned, nil) + m.repo.EXPECT().Inspect(gomock.Any(), pinned).Return(&artifact.Inspection{Record: &record}, nil) m.repo.EXPECT().OpenDocument(gomock.Any(), pinned, artifact.DocumentReadme).Return(nil, errTransport) _, err := m.svc.ShowDocument(context.Background(), artifact.ShowDocumentRequest{Repository: "dst", Ref: exampleRef, Role: artifact.DocumentReadme}) require.ErrorIs(t, err, errTransport) diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go index 220caaa18ff..171f49fb47d 100644 --- a/pkg/artifact/service/execute.go +++ b/pkg/artifact/service/execute.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "slices" + "time" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -52,14 +53,19 @@ func (s *Service) execute(ctx context.Context, pub *publication, cur *current, d // It returns a recovered result when the write is proven committed, retry when it is proven // uncommitted and worth repeating, and otherwise the error to report. func (s *Service) afterFailure(ctx context.Context, pub *publication, cur *current, writeErr error, attempt int) (*artifact.PublishResult, bool, error) { - found := reconcile(context.WithoutCancel(ctx), pub.tgt, pub.in, cur) + reconcileCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + found := reconcile(reconcileCtx, pub.tgt, pub.in, cur) + cancel() switch found.state { case reconcileCommitted: s.event(ctx, pub, PhaseReconciled, attempt, nil) pub.op.info("reconciled", fmt.Sprintf("the repository reported %q but the completed reference holds the expected digests; success was confirmed by reading it", writeErr)) return syntheticResult(&found.pinned, found.inspection), false, nil case reconcileNotCommitted: - if !isTemporary(writeErr) || attempt >= s.attempts || ctx.Err() != nil { + if ctx.Err() != nil { + return nil, false, errors.Join(writeErr, ctx.Err()) + } + if !isTemporary(writeErr) || attempt >= s.attempts { return nil, false, writeErr } if sleepErr := s.sleep(ctx, s.backoff<<(attempt-1)); sleepErr != nil { diff --git a/pkg/artifact/service/internal_test.go b/pkg/artifact/service/internal_test.go index e2704f3aae9..79faaef74a7 100644 --- a/pkg/artifact/service/internal_test.go +++ b/pkg/artifact/service/internal_test.go @@ -3,6 +3,8 @@ package service import ( "context" "errors" + "os" + "path/filepath" "testing" "github.com/opencontainers/go-digest" @@ -196,3 +198,15 @@ func TestReconcileGraphComparesTheSourceRoot(t *testing.T) { } var ocispecRoot = ocispecDescriptor() + +func TestRemoveStagedTreeRestoresDirectoryAccess(t *testing.T) { + tree := filepath.Join(t.TempDir(), "tree") + dir := filepath.Join(tree, "restricted") + require.NoError(t, os.MkdirAll(dir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "file"), []byte("x"), 0o600)) + require.NoError(t, os.Chmod(dir, 0o000)) + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + require.NoError(t, removeStagedTree(tree)) + _, err := os.Lstat(tree) + require.True(t, os.IsNotExist(err)) +} diff --git a/pkg/artifact/service/locks.go b/pkg/artifact/service/locks.go index 3edf79f16df..7149c1f40d7 100644 --- a/pkg/artifact/service/locks.go +++ b/pkg/artifact/service/locks.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sort" + "time" "github.com/cloudposse/atmos/pkg/artifact" ) @@ -125,6 +126,20 @@ func (l *lockSet) handleFor(alias string) artifact.LockHandle { return nil } +// handleForResource returns the handle for a specific lock target. +func (l *lockSet) handleForResource(alias, resource string) artifact.LockHandle { + if l != nil { + for i := range l.held { + if l.held[i].target.alias == alias && l.held[i].target.resource == resource { + return l.held[i].handle + } + } + } + return nil +} + +func subjectLockResource(name string) string { return "subject:" + name } + // scope returns the weakest actual scope across held locks, which results report. func (l *lockSet) scope() artifact.LockScope { if l == nil || len(l.held) == 0 { @@ -146,11 +161,13 @@ func (l *lockSet) release(ctx context.Context, op *operation) error { if l == nil { return nil } - detached := context.WithoutCancel(ctx) var errs []error for index := len(l.held) - 1; index >= 0; index-- { held := l.held[index] - if err := held.handle.Release(detached); err != nil { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + err := held.handle.Release(releaseCtx) + cancel() + if err != nil { op.warn("lock-release-failed", fmt.Sprintf("repository %q resource %q owner %q: operator recovery may be required: %v", held.target.alias, held.handle.Resource(), held.handle.Owner(), err)) errs = append(errs, fmt.Errorf("release lock %s in repository %q: %w", held.handle.Resource(), held.target.alias, err)) } diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go index 32fefa1404c..4189bf49dd6 100644 --- a/pkg/artifact/service/mutate.go +++ b/pkg/artifact/service/mutate.go @@ -43,7 +43,8 @@ func validateRepositoryRef(repository string, ref artifact.Ref) error { func (s *Service) runDelete(ctx context.Context, op *operation, request *artifact.DeleteRequest, info *RepositoryInfo, repo artifact.Repository) error { resource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: request.Ref.String()} - locks, err := s.acquireLocks(ctx, op, opDelete, []lockTarget{resource}) + subjectResource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: subjectLockResource(request.Ref.Name)} + locks, err := s.acquireLocks(ctx, op, opDelete, []lockTarget{resource, subjectResource}) if err != nil { return err } @@ -53,7 +54,7 @@ func (s *Service) runDelete(ctx context.Context, op *operation, request *artifac if err != nil { return err } - result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleFor(request.Repository)}) + result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleForResource(request.Repository, resource.resource)}) if err != nil { return fmt.Errorf("delete %s in repository %q: %w", request.Ref, request.Repository, err) } diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go index a19b2c2eebc..359b77a2082 100644 --- a/pkg/artifact/service/mutate_test.go +++ b/pkg/artifact/service/mutate_test.go @@ -87,8 +87,8 @@ func TestDelete_ReportsAffectedReferences(t *testing.T) { pinned := artifact.PinnedReference{Ref: ref("x/y", "1"), ResolvedDigest: recordDigest, RecordDigest: recordDigest} repo.EXPECT().Capabilities().Return(caps).AnyTimes() - repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil) - handle.EXPECT().Release(gomock.Any()).Return(nil) + repo.EXPECT().Lock(gomock.Any(), gomock.Any()).Return(handle, nil).Times(2) + handle.EXPECT().Release(gomock.Any()).Return(nil).Times(2) handle.EXPECT().Resource().Return("x/y:1").AnyTimes() handle.EXPECT().Owner().Return("test-owner").AnyTimes() handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go index ba3ed533b36..aaa640bb824 100644 --- a/pkg/artifact/service/query.go +++ b/pkg/artifact/service/query.go @@ -74,9 +74,9 @@ func (s *Service) showDocument(ctx context.Context, op *operation, request *arti if err != nil { return err } - pinned, err := repo.Resolve(ctx, request.Ref) + pinned, inspection, err := s.pinAndInspect(ctx, op, repo, request.Repository, request.Ref) if err != nil { - return fmt.Errorf("resolve %s in repository %q: %w", request.Ref, request.Repository, err) + return err } if err := s.gatePinned(ctx, op, &target{alias: request.Repository, repo: repo, info: &info}, &pinned); err != nil { return err @@ -86,6 +86,18 @@ func (s *Service) showDocument(ctx context.Context, op *operation, request *arti return fmt.Errorf("open %s of %s in repository %q: %w", request.Role, request.Ref, request.Repository, err) } defer stream.Reader.Close() + matched := false + if inspection.Record != nil { + for _, document := range inspection.Record.Documents { + if document.Role == request.Role && document == stream.Document { + matched = true + break + } + } + } + if !matched { + return fmt.Errorf("%w: document does not match the pinned record", errUtils.ErrArtifactIntegrityFailed) + } content, err := readDocument(stream) if err != nil { return err @@ -180,10 +192,10 @@ func (s *Service) runEnumeration(ctx context.Context, request *enumeration, resu gates := make(chan struct{}, s.concurrency) var wg sync.WaitGroup for index := range aliases { + gates <- struct{}{} wg.Add(1) go func() { defer wg.Done() - gates <- struct{}{} defer func() { <-gates }() outcomes[index] = s.listRepository(ctx, aliases[index], request) }() diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go index 0405ab792cc..f929b4e1f1f 100644 --- a/pkg/artifact/service/sign.go +++ b/pkg/artifact/service/sign.go @@ -40,13 +40,21 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig if err := checkSigningCapability(request.Repository, &info, s.signer, subject); err != nil { return err } - locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: request.Ref.String()}}) + resource := subjectLockResource(request.Ref.Name) + locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: resource}}) if err != nil { return err } defer func() { _ = locks.release(ctx, op) }() - evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleFor(request.Repository)}) + current, err := repo.Resolve(ctx, request.Ref) + if err != nil { + return fmt.Errorf("resolve %s after locking: %w", request.Ref, err) + } + if current.ResolvedDigest != pinned.ResolvedDigest || current.RecordDigest != pinned.RecordDigest { + return fmt.Errorf("%w: subject %s changed while acquiring its lock", errUtils.ErrArtifactConflict, request.Ref) + } + evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, resource)}) if err != nil { return fmt.Errorf("sign %s in repository %q: %w", request.Ref, request.Repository, err) } diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index ab68e803e2f..5db3becdcf5 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -93,8 +93,14 @@ func TestCapture_ClearsPrivilegedBitsAndIgnoresOwnership(t *testing.T) { src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "x", 0o644) path := filepath.Join(src, "a.txt") - // Setuid is not representable on every platform; failure to set it only weakens the assertion. - _ = os.Chmod(path, 0o755|os.ModeSetuid) + if err := os.Chmod(path, 0o755|os.ModeSetuid); err != nil { + t.Skipf("setuid unavailable: %v", err) + } + info, err := os.Stat(path) + require.NoError(t, err) + if info.Mode()&os.ModeSetuid == 0 { + t.Skip("filesystem did not apply setuid") + } snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) require.NoError(t, err) defer snap.Close() diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go index d7a8387c586..2e631ad6988 100644 --- a/pkg/oci/transport.go +++ b/pkg/oci/transport.go @@ -102,7 +102,7 @@ type RepositoryStore interface { ReferencePusher // Tags lists every tag without interpreting it as a version. Tags(ctx context.Context) ([]string, error) - // Delete removes a manifest by its full descriptor. It never deletes tags or shared blobs. A + // Delete removes a manifest by its full descriptor. Tags pointing to that digest may stop resolving; shared blobs are not deleted. A // descriptor with only a digest deletes the manifest without reading it first. Delete(ctx context.Context, target ocispec.Descriptor) error // Referrers lists attachments of the subject, optionally filtered by artifact type. diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go index 07abb8f9355..2a31b2f7fa0 100644 --- a/pkg/oci/transport_behavior_test.go +++ b/pkg/oci/transport_behavior_test.go @@ -204,15 +204,20 @@ func TestAmbiguousWriteThatDidNotLandIsAnError(t *testing.T) { } func TestRejectedWriteIsNotReconciledOrRetriedAnonymously(t *testing.T) { - var anonymous atomic.Int32 + var anonymous, anonymousAfterRejection atomic.Int32 + var rejected atomic.Bool server := newFlakyRegistry(t, func(w http.ResponseWriter, r *http.Request, _ http.Handler) bool { if _, _, ok := r.BasicAuth(); !ok { anonymous.Add(1) + if rejected.Load() { + anonymousAfterRejection.Add(1) + } w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) w.WriteHeader(http.StatusUnauthorized) return true } if r.Method == http.MethodPut { + rejected.Store(true) w.WriteHeader(http.StatusForbidden) return true } @@ -228,6 +233,8 @@ func TestRejectedWriteIsNotReconciledOrRetriedAnonymously(t *testing.T) { "a definite rejection must not trigger destination reconciliation") assert.Empty(t, server.duplicates(http.MethodPut), "a rejected mutation is not retried") assert.Positive(t, anonymous.Load(), "the registry challenged the first request") + assert.True(t, rejected.Load(), "the authenticated write was rejected") + assert.Zero(t, anonymousAfterRejection.Load(), "no anonymous fallback after rejection") } func TestDeleteIsReconciledByResolvingTheDigest(t *testing.T) { @@ -428,6 +435,18 @@ func (l *landingDestination) PushReference(ctx context.Context, expected ocispec return l.err } +type movingSource struct { + oci.ReadOnlyStore + calls atomic.Int32 +} + +func (s *movingSource) Resolve(ctx context.Context, ref string) (ocispec.Descriptor, error) { + if s.calls.Add(1) > 1 { + return ocispec.Descriptor{}, errors.New("source tag moved") + } + return s.ReadOnlyStore.Resolve(ctx, ref) +} + func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { directory := filepath.Join(t.TempDir(), "source") index := testIndex(t) @@ -452,8 +471,10 @@ func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) require.NoError(t, err) destination := &landingDestination{Store: layout, land: tc.land, err: tc.err} - root, err := oci.Copy(t.Context(), oci.Source{Store: source, Reference: digest.String()}, oci.Destination{Store: destination, Reference: "1.0.0"}) + moving := &movingSource{ReadOnlyStore: source} + root, err := oci.Copy(t.Context(), oci.Source{Store: moving, Reference: digest.String()}, oci.Destination{Store: destination, Reference: "1.0.0"}) assert.Equal(t, int32(1), destination.pushes.Load(), "the mutation is attempted exactly once") + assert.Equal(t, int32(1), moving.calls.Load(), "reconciliation must use the selected root") if tc.wantErr { require.Error(t, err) return diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index 881bb68088c..d75bcc38938 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -153,10 +153,11 @@ func Copy(ctx context.Context, src Source, dst Destination, options ...CopyOptio dst.Reference = src.Reference } config := newCopyConfig(options) - root, err := oras.Copy(ctx, guardedSource{ReadOnlyStore: src.Store}, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, + var selected ocispec.Descriptor + root, err := oras.Copy(ctx, guardedSource{ReadOnlyStore: src.Store, selected: &selected}, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, oras.CopyOptions{CopyGraphOptions: config.graphOptions()}) if err != nil { - return reconcile(ctx, err, src.Store, src.Reference, dst) + return reconcile(ctx, err, selected, dst) } return root, nil } @@ -173,13 +174,14 @@ func ExtendedCopy(ctx context.Context, src GraphSource, dst Destination, options dst.Reference = src.Reference } config := newCopyConfig(options) - graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store}, graph: src.Store} + var selected ocispec.Descriptor + graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store, selected: &selected}, graph: src.Store} root, err := oras.ExtendedCopy(ctx, graph, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.graphOptions()}}) if err == nil { return root, nil } - want, reconcileErr := reconcile(ctx, err, src.Store, src.Reference, dst) + want, reconcileErr := reconcile(ctx, err, selected, dst) if reconcileErr != nil { return want, reconcileErr } @@ -218,13 +220,9 @@ func referrersPresent(ctx context.Context, src ReadOnlyGraphStore, dst Store, ro } // reconcile decides the outcome of a failed copy by looking at the destination. -func reconcile(ctx context.Context, cause error, src ReadOnlyStore, srcRef string, dst Destination) (ocispec.Descriptor, error) { +func reconcile(ctx context.Context, cause error, want ocispec.Descriptor, dst Destination) (ocispec.Descriptor, error) { failure := classifyCopyError(cause) - if !ambiguousWrite(ctx, failure) { - return ocispec.Descriptor{}, failure - } - want, err := src.Resolve(ctx, srcRef) - if err != nil { + if !ambiguousWrite(ctx, failure) || want.Digest == "" { return ocispec.Descriptor{}, failure } have, err := dst.Store.Resolve(ctx, dst.Reference) @@ -242,7 +240,18 @@ func classifyCopyError(err error) error { } // guardedSource aborts as soon as the context ends, even for stores that ignore it. -type guardedSource struct{ ReadOnlyStore } +type guardedSource struct { + ReadOnlyStore + selected *ocispec.Descriptor +} + +func (g guardedSource) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + desc, err := g.ReadOnlyStore.Resolve(ctx, reference) + if err == nil && g.selected != nil { + *g.selected = desc + } + return desc, err +} func (g guardedSource) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { if err := ctx.Err(); err != nil { diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index c08c2438735..30ad2873031 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -110,8 +110,8 @@ func TestLayoutTagKeepsMetadataAndDigestRoots(t *testing.T) { } func TestLayoutStateMatchesThePersistedIndexAfterAFailedWrite(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory write permissions are not enforced on Windows") + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("directory write permissions are not enforced for this process") } dir := t.TempDir() layout, err := OpenLayout(t.Context(), dir) diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 410e0e06c34..3009c84600c 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -135,6 +135,8 @@ func (l *Layout) Exists(ctx context.Context, target ocispec.Descriptor) (bool, e // recorded as an unnamed root, because that is the only way its relationship to the // subject survives reopening the layout. func (l *Layout) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + defer perf.Track(nil, "oci.Layout.Push")() + err := l.store.Push(ctx, expected, content) if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { return layoutError(err) diff --git a/pkg/signature/trust_test.go b/pkg/signature/trust_test.go index 81c790867d0..d340c9158cc 100644 --- a/pkg/signature/trust_test.go +++ b/pkg/signature/trust_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -52,8 +53,20 @@ func TestTrustHTTPClientUsesVerificationContext(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrSignatureOffline) } +type countingTransport struct{ requests atomic.Int32 } + +func (t *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + t.requests.Add(1) + return nil, errUtils.ErrSignatureOffline +} + func TestOfflineTrustCannotFetch(t *testing.T) { + transport := &countingTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) t.Setenv("ATMOS_XDG_DATA_HOME", t.TempDir()) _, err := LoadPublicRoot(t.Context(), true) require.Error(t, err) + require.Zero(t, transport.requests.Load()) } From e0851d429fb0a9e585f12241a0520a4c139c4293 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:08:19 -0600 Subject: [PATCH 09/52] fix(artifacts): replace destinations atomically --- pkg/artifact/materialize/exchange_darwin.go | 21 ++ pkg/artifact/materialize/exchange_linux.go | 21 ++ pkg/artifact/materialize/exchange_other.go | 18 ++ pkg/artifact/materialize/exchange_windows.go | 28 +++ pkg/artifact/materialize/materialize_test.go | 32 ++- pkg/artifact/materialize/replace.go | 208 ++++++++++++----- .../materialize/replace_internal_test.go | 213 ++++++++++++++++++ pkg/artifact/service/consume_test.go | 20 ++ 8 files changed, 506 insertions(+), 55 deletions(-) create mode 100644 pkg/artifact/materialize/exchange_darwin.go create mode 100644 pkg/artifact/materialize/exchange_linux.go create mode 100644 pkg/artifact/materialize/exchange_other.go create mode 100644 pkg/artifact/materialize/exchange_windows.go create mode 100644 pkg/artifact/materialize/replace_internal_test.go diff --git a/pkg/artifact/materialize/exchange_darwin.go b/pkg/artifact/materialize/exchange_darwin.go new file mode 100644 index 00000000000..2aa8dbe5f3e --- /dev/null +++ b/pkg/artifact/materialize/exchange_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package materialize + +import ( + "os" + "runtime" + + "golang.org/x/sys/unix" +) + +func exchangeDirectories(stageParent *os.File, stage string, destParent *os.File, dest string) error { + err := unix.RenameatxNp(int(stageParent.Fd()), stage, int(destParent.Fd()), dest, unix.RENAME_SWAP) + runtime.KeepAlive(stageParent) + runtime.KeepAlive(destParent) + return err +} + +func renameExclusive(from, to string) error { + return unix.RenameatxNp(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_EXCL) +} diff --git a/pkg/artifact/materialize/exchange_linux.go b/pkg/artifact/materialize/exchange_linux.go new file mode 100644 index 00000000000..deb64c6bc26 --- /dev/null +++ b/pkg/artifact/materialize/exchange_linux.go @@ -0,0 +1,21 @@ +//go:build linux + +package materialize + +import ( + "os" + "runtime" + + "golang.org/x/sys/unix" +) + +func exchangeDirectories(stageParent *os.File, stage string, destParent *os.File, dest string) error { + err := unix.Renameat2(int(stageParent.Fd()), stage, int(destParent.Fd()), dest, unix.RENAME_EXCHANGE) + runtime.KeepAlive(stageParent) + runtime.KeepAlive(destParent) + return err +} + +func renameExclusive(from, to string) error { + return unix.Renameat2(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_NOREPLACE) +} diff --git a/pkg/artifact/materialize/exchange_other.go b/pkg/artifact/materialize/exchange_other.go new file mode 100644 index 00000000000..59a29432eb7 --- /dev/null +++ b/pkg/artifact/materialize/exchange_other.go @@ -0,0 +1,18 @@ +//go:build !linux && !darwin && !windows + +package materialize + +import ( + "fmt" + "os" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func exchangeDirectories(_ *os.File, _ string, _ *os.File, _ string) error { + return fmt.Errorf("%w: atomic directory replacement is unavailable on this platform", errUtils.ErrArtifactUnsupportedCapability) +} + +func renameExclusive(_, _ string) error { + return fmt.Errorf("%w: exclusive directory install is unavailable on this platform", errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/materialize/exchange_windows.go b/pkg/artifact/materialize/exchange_windows.go new file mode 100644 index 00000000000..8bdde7e1fbb --- /dev/null +++ b/pkg/artifact/materialize/exchange_windows.go @@ -0,0 +1,28 @@ +//go:build windows + +package materialize + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func exchangeDirectories(_ *os.File, _ string, _ *os.File, _ string) error { + return fmt.Errorf("%w: atomic directory replacement is unavailable on Windows", errUtils.ErrArtifactUnsupportedCapability) +} + +func renameExclusive(from, to string) error { + source, err := windows.UTF16PtrFromString(from) + if err != nil { + return err + } + target, err := windows.UTF16PtrFromString(to) + if err != nil { + return err + } + return windows.MoveFile(source, target) +} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index c5bc6982141..ccfde2b2651 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -127,7 +127,15 @@ func TestMaterialize_ReplacesWithoutMergingAndStaysInsideDestination(t *testing. require.NoError(t, err) assert.Equal(t, materialize.DestinationDiffers, state) - require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + err = materialize.Materialize(context.Background(), contentOf(t, p), out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, statErr := os.Stat(filepath.Join(out, "stale.txt")) + require.NoError(t, statErr, "unsupported replacement preserves the existing destination") + assertNoScratch(t, out) + return + } + require.NoError(t, err) items, err := os.ReadDir(out) require.NoError(t, err) require.Len(t, items, 1, "no stale file is merged with the new tree") @@ -142,6 +150,25 @@ func TestMaterialize_ReplacesWithoutMergingAndStaysInsideDestination(t *testing. _ = linkOK } +func TestMaterialize_ReplacesRestrictiveDestination(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("atomic directory replacement is unavailable on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + out := dest(t) + fixtures.WriteFile(t, out, "blocked/old.txt", "old", 0o600) + restricted := filepath.Join(out, "blocked") + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + data, err := os.ReadFile(filepath.Join(out, "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, out) +} + func TestInspectDestination_States(t *testing.T) { src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "a", 0o644) @@ -536,6 +563,9 @@ func TestMaterialize_ReplaceGuard(t *testing.T) { err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(tt.guard)) if tt.wantErr != nil { require.ErrorIs(t, err, tt.wantErr) + } else if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + tt.wantStale = true } else { require.NoError(t, err) } diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 7f8caf21b2f..d798a663ed2 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -15,6 +15,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" ) @@ -22,6 +23,7 @@ const ( scratchPattern = ".atmos-pull-*" parentDirMode = 0o755 treeDirName = "tree" + rootDirName = "." ) // errMismatch stops a destination walk at the first difference. It never leaves this package. @@ -149,9 +151,9 @@ func fileDigest(p string) digest.Digest { } // Materialize extracts content into a private staging directory beside dest and then replaces dest -// with it. Staging is removed on every path. An existing destination keeps no stale file: its -// children are removed inside the destination root only, then the staged children move in. The -// caller decides beforehand (see [InspectDestination]) whether replacing is permitted. +// with it. An existing destination is exchanged atomically with the staged tree. If atomic +// exchange is unavailable (including on Windows), replacing an existing directory fails without +// modifying it. The caller decides beforehand (see [InspectDestination]) whether replacement is permitted. func Materialize(ctx context.Context, content *artifact.Content, dest string, opts ...Option) error { defer perf.Track(nil, "materialize.Materialize")() @@ -166,7 +168,20 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if err != nil { return fmt.Errorf("%w: create staging beside %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) } - defer os.RemoveAll(scratch) + scratchRoot, err := os.OpenRoot(scratch) + if err != nil { + _ = os.Remove(scratch) + return fmt.Errorf("%w: open staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + preserveScratch := false + defer func() { + if !preserveScratch { + if cleanupErr := removeScratch(scratchRoot, scratch); cleanupErr != nil { + log.Warn("Failed to remove artifact staging directory", "path", scratch, "error", cleanupErr) + } + } + _ = scratchRoot.Close() + }() stage := filepath.Join(scratch, treeDirName) if err := os.Mkdir(stage, stagingDirMode); err != nil { @@ -178,58 +193,141 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if err := os.Chmod(stage, parentDirMode); err != nil { return fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) } - return replaceDestination(stage, dest, applyOptions(opts).replaceGuard) + preserveScratch, err = replaceDestination(stage, dest, applyOptions(opts).replaceGuard) + return err } -// replaceDestination moves the staged tree into place without merging it over old files. An -// existing destination is opened once and every later step is checked against that same directory, -// so a symlink swapped in after the initial check cannot redirect the removal or the moves. -func replaceDestination(stage, dest string, guard func(dest string) error) error { +// replaceDestination atomically exchanges the staged and existing trees. No children are +// removed or installed through the destination path, and a failed exchange leaves both intact. +// The boolean preserves scratch when a concurrent replacement cannot be safely rolled back. +func replaceDestination(stage, dest string, guard func(dest string) error) (bool, error) { info, err := os.Lstat(dest) switch { case os.IsNotExist(err): - return rename(stage, dest) + return false, rename(stage, dest) case err != nil: - return fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + return false, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) case info.Mode()&os.ModeSymlink != 0: - return fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) + return false, fmt.Errorf("%w: destination %q is a symlink; refusing to replace content through it", errUtils.ErrArtifactUnsafePath, dest) case !info.IsDir(): - return fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) + return false, fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) } bound, err := bindDestination(dest, info) if err != nil { - return err + return false, err } defer bound.root.Close() if guard != nil { if err := guard(dest); err != nil { - return err - } - if err := bound.verify(); err != nil { - return err + return false, err } } - if err := clearDirectory(bound); err != nil { - return err + if err := bound.verify(); err != nil { + return false, err + } + return exchangeStagedTree(stage, dest, bound.info) +} + +func exchangeStagedTree(stage, dest string, destInfo os.FileInfo) (bool, error) { + stageInfo, err := os.Lstat(stage) + if err != nil { + return false, fmt.Errorf("%w: inspect staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + stageParent, err := os.Open(filepath.Dir(stage)) + if err != nil { + return false, fmt.Errorf("%w: open staging parent: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + defer stageParent.Close() + destParent, err := os.Open(filepath.Dir(dest)) + if err != nil { + return false, fmt.Errorf("%w: open destination parent: %w", errUtils.ErrArtifactPackagingInvalid, err) } - return moveChildren(stage, bound) + defer destParent.Close() + pair := exchangePair{stageParent: stageParent, stage: filepath.Base(stage), destParent: destParent, dest: filepath.Base(dest), stageInfo: stageInfo, destInfo: destInfo} + return exchangeDestination(&pair, exchangeDirectories) +} + +type exchangePair struct { + stageParent *os.File + stage string + destParent *os.File + dest string + stageInfo os.FileInfo + destInfo os.FileInfo +} + +type exchangeFunc func(stageParent *os.File, stage string, destParent *os.File, dest string) error + +// exchangeDestination checks the identity displaced by the atomic exchange. If an attacker +// swapped the entry after verification, reverse the exchange; never remove an unrecognized tree. +func exchangeDestination(pair *exchangePair, exchange exchangeFunc) (bool, error) { + stageParent, stage, destParent, dest := pair.stageParent, pair.stage, pair.destParent, pair.dest + stageInfo, destInfo := pair.stageInfo, pair.destInfo + stagingRoot, err := checkedParent(stageParent) + if err != nil { + return false, err + } + defer stagingRoot.Close() + destinationRoot, err := checkedParent(destParent) + if err != nil { + return false, err + } + defer destinationRoot.Close() + if !sameRootEntry(destinationRoot, dest, destInfo) { + return false, changedDestination(filepath.Join(destParent.Name(), dest)) + } + if err := exchange(stageParent, stage, destParent, dest); err != nil { + return false, fmt.Errorf("%w: atomically replace destination: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + if pair.matchesExchange(stagingRoot, destinationRoot) { + return false, nil + } + rollbackErr := exchange(stageParent, stage, destParent, dest) + if rollbackErr == nil && sameRootEntry(stagingRoot, stage, stageInfo) { + return false, changedDestination(filepath.Join(destParent.Name(), dest)) + } + return true, errors.Join(fmt.Errorf("%w: destination changed during replacement; recovery data retained in %q", errUtils.ErrArtifactUnsafePath, stageParent.Name()), rollbackErr) +} + +func (pair *exchangePair) matchesExchange(stageRoot, destRoot *os.Root) bool { + return sameRootEntry(stageRoot, pair.stage, pair.destInfo) && sameRootEntry(destRoot, pair.dest, pair.stageInfo) +} + +func sameRootEntry(root *os.Root, name string, expected os.FileInfo) bool { + info, err := root.Lstat(name) + return err == nil && os.SameFile(info, expected) +} + +// checkedParent binds path-based inspection to the same directory as the syscall descriptor. +func checkedParent(parent *os.File) (*os.Root, error) { + root, err := os.OpenRoot(parent.Name()) + if err != nil { + return nil, changedDestination(parent.Name()) + } + opened, err := parent.Stat() + if err == nil { + var current os.FileInfo + current, err = root.Stat(rootDirName) + if err == nil && os.SameFile(opened, current) { + return root, nil + } + } + _ = root.Close() + return nil, changedDestination(parent.Name()) } -// boundDestination is a destination directory opened once, with the identity it must keep. type boundDestination struct { path string root *os.Root info os.FileInfo } -// bindDestination opens dest and confirms the opened directory is the one that was checked. A -// destination swapped for a symlink or another directory between the check and the open fails. func bindDestination(dest string, checked os.FileInfo) (*boundDestination, error) { root, err := os.OpenRoot(dest) if err != nil { return nil, fmt.Errorf("%w: open destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) } - opened, err := root.Stat(".") + opened, err := root.Stat(rootDirName) if err != nil { _ = root.Close() return nil, fmt.Errorf("%w: inspect destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) @@ -241,7 +339,6 @@ func bindDestination(dest string, checked os.FileInfo) (*boundDestination, error return &boundDestination{path: dest, root: root, info: opened}, nil } -// verify confirms the destination path still names the bound directory and is not a symlink. func (b *boundDestination) verify() error { now, err := os.Lstat(b.path) if err != nil || now.Mode()&os.ModeSymlink != 0 || !os.SameFile(now, b.info) { @@ -254,41 +351,44 @@ func changedDestination(dest string) error { return fmt.Errorf("%w: destination %q changed while it was being replaced", errUtils.ErrArtifactUnsafePath, dest) } -// clearDirectory removes every child of the bound destination through its os.Root, so removal can -// never leave the directory that was checked. -func clearDirectory(bound *boundDestination) error { - children, err := fs.ReadDir(bound.root.FS(), ".") - if err != nil { - return fmt.Errorf("%w: read destination %q: %w", errUtils.ErrArtifactPackagingInvalid, bound.path, err) - } - for _, child := range children { - if err := bound.root.RemoveAll(child.Name()); err != nil { - return fmt.Errorf("%w: remove stale %q: %w", errUtils.ErrArtifactPackagingInvalid, child.Name(), err) - } - } - return nil -} - -// moveChildren renames each staged child into the destination. The standard library cannot rename -// across two directory handles, so the destination identity is re-verified before every move. -func moveChildren(stage string, bound *boundDestination) error { - children, err := os.ReadDir(stage) - if err != nil { - return fmt.Errorf("%w: read staging tree: %w", errUtils.ErrArtifactPackagingInvalid, err) - } - for _, child := range children { - if err := bound.verify(); err != nil { +// removeScratch restores owner access before removing a displaced restrictive tree. WalkDir +// never follows symlinks and Root bounds permission changes to the private staging tree. +func removeScratch(root *os.Root, scratch string) error { + walkErr := fs.WalkDir(root.FS(), rootDirName, func(path string, entry fs.DirEntry, err error) error { + if err != nil { return err } - if err := rename(filepath.Join(stage, child.Name()), filepath.Join(bound.path, child.Name())); err != nil { - return err + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + return root.Chmod(path, info.Mode().Perm()|0o700) } + return nil + }) + children, readErr := fs.ReadDir(root.FS(), rootDirName) + var removeErr error + for _, child := range children { + removeErr = errors.Join(removeErr, root.RemoveAll(child.Name())) } - return nil + opened, statErr := root.Stat(rootDirName) + if statErr == nil { + var current os.FileInfo + current, statErr = os.Lstat(scratch) + if statErr == nil && !os.SameFile(opened, current) { + statErr = changedDestination(scratch) + } + } + closeErr := root.Close() + if statErr == nil && closeErr == nil { + statErr = os.Remove(scratch) + } + return errors.Join(walkErr, readErr, removeErr, statErr, closeErr) } func rename(from, to string) error { - if err := os.Rename(from, to); err != nil { + if err := renameExclusive(from, to); err != nil { return fmt.Errorf("%w: move %q into place: %w", errUtils.ErrArtifactPackagingInvalid, to, err) } return nil diff --git a/pkg/artifact/materialize/replace_internal_test.go b/pkg/artifact/materialize/replace_internal_test.go new file mode 100644 index 00000000000..d0f35087770 --- /dev/null +++ b/pkg/artifact/materialize/replace_internal_test.go @@ -0,0 +1,213 @@ +//go:build linux || darwin + +package materialize + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestExchangeDestinationFailurePreservesOriginal(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + restricted := filepath.Join(dest, "restricted") + require.NoError(t, os.Mkdir(restricted, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(restricted, "keep"), []byte("keep"), 0o600)) + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + sentinel := errors.New("exchange failed") + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, func(*os.File, string, *os.File, string) error { return sentinel }) + require.ErrorIs(t, err, sentinel) + require.False(t, preserve) + data, err := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, err) + require.Equal(t, "original", string(data)) + require.NoError(t, os.Chmod(restricted, 0o700)) + data, err = os.ReadFile(filepath.Join(restricted, "keep")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestExchangeDestinationRejectsLastMomentSwap(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(dest, filepath.Join(root, "original"))) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "intruder"), []byte("intact"), 0o600)) + } + return exchangeDirectories(a, b, c, d) + } + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.False(t, preserve) + require.Equal(t, 2, calls) + data, err := os.ReadFile(filepath.Join(dest, "intruder")) + require.NoError(t, err) + require.Equal(t, "intact", string(data)) + data, err = os.ReadFile(filepath.Join(root, "original", "original")) + require.NoError(t, err) + require.Equal(t, "original", string(data)) +} + +func TestRemoveScratchWithRestrictiveDirectories(t *testing.T) { + scratch := filepath.Join(t.TempDir(), "scratch") + restricted := filepath.Join(scratch, "tree", "restricted") + require.NoError(t, os.MkdirAll(restricted, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(restricted, "data"), []byte("x"), 0o600)) + require.NoError(t, os.Chmod(restricted, 0o000)) + t.Cleanup(func() { _ = os.Chmod(restricted, 0o700) }) + root, err := os.OpenRoot(scratch) + require.NoError(t, err) + defer root.Close() + require.NoError(t, removeScratch(root, scratch)) + _, err = os.Lstat(scratch) + require.True(t, os.IsNotExist(err)) +} + +func TestExchangeDestinationRetainsDisplacedTreeWhenRollbackFails(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(dest, filepath.Join(root, "original"))) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "intruder"), []byte("keep"), 0o600)) + return exchangeDirectories(a, b, c, d) + } + return errors.New("rollback failed") + } + preserve, err := exchangeDestination(&exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo}, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.True(t, preserve) + require.ErrorContains(t, err, scratch) + data, err := os.ReadFile(filepath.Join(scratch, "tree", "intruder")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestRenameExclusiveNeverOverwritesExistingDestination(t *testing.T) { + root := t.TempDir() + stage := filepath.Join(root, "stage") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(stage, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.Error(t, renameExclusive(stage, dest)) + _, err := os.Stat(stage) + require.NoError(t, err) + _, err = os.Stat(dest) + require.NoError(t, err) +} + +func TestRemoveScratchDoesNotFollowSwappedPath(t *testing.T) { + parent := t.TempDir() + scratch := filepath.Join(parent, "scratch") + outside := filepath.Join(parent, "outside") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(outside, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "keep"), []byte("keep"), 0o600)) + root, err := os.OpenRoot(scratch) + require.NoError(t, err) + defer root.Close() + require.NoError(t, os.Rename(scratch, filepath.Join(parent, "moved"))) + require.NoError(t, os.Symlink(outside, scratch)) + require.ErrorIs(t, removeScratch(root, scratch), errUtils.ErrArtifactUnsafePath) + data, err := os.ReadFile(filepath.Join(outside, "keep")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestExchangeDestinationRejectsSwappedStage(t *testing.T) { + root := t.TempDir() + scratch := filepath.Join(root, "scratch") + dest := filepath.Join(root, "dest") + require.NoError(t, os.Mkdir(scratch, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(scratch, "tree"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("keep"), 0o600)) + stageInfo, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + destInfo, err := os.Lstat(dest) + require.NoError(t, err) + stageParent, err := os.Open(scratch) + require.NoError(t, err) + defer stageParent.Close() + destParent, err := os.Open(root) + require.NoError(t, err) + defer destParent.Close() + calls := 0 + swap := func(a *os.File, b string, c *os.File, d string) error { + calls++ + if calls == 1 { + require.NoError(t, os.Rename(filepath.Join(scratch, "tree"), filepath.Join(scratch, "real"))) + require.NoError(t, os.Symlink(filepath.Join(root, "elsewhere"), filepath.Join(scratch, "tree"))) + } + return exchangeDirectories(a, b, c, d) + } + pair := &exchangePair{stageParent: stageParent, stage: "tree", destParent: destParent, dest: "dest", stageInfo: stageInfo, destInfo: destInfo} + preserve, err := exchangeDestination(pair, swap) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + require.True(t, preserve) + require.Equal(t, 2, calls) + data, err := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) + info, err := os.Lstat(filepath.Join(scratch, "tree")) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeSymlink) +} diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index 861ef9a1317..c9582aa2229 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sync/atomic" "testing" "time" @@ -71,6 +72,21 @@ func TestPull_ExistingDestinationRules(t *testing.T) { request.Force = true forced, err := e.svc.Pull(context.Background(), request) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + data, readErr := os.ReadFile(filepath.Join(dest, "data.txt")) + require.NoError(t, readErr) + assert.Equal(t, "locally edited", string(data)) + empty := out(t) + require.NoError(t, os.MkdirAll(empty, 0o755)) + request.Destination, request.Force = empty, false + _, err = e.svc.Pull(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + entries, readErr := os.ReadDir(empty) + require.NoError(t, readErr) + assert.Empty(t, entries) + return + } require.NoError(t, err) assert.Equal(t, artifact.OutcomeRead, forced.Outcome) _, statErr := os.Stat(filepath.Join(dest, "stale.txt")) @@ -407,6 +423,10 @@ func TestPull_RechecksTheDestinationBeforeReplacingIt(t *testing.T) { }} _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: tt.force}) + if runtime.GOOS == "windows" && tt.force { + tt.wantErr = errUtils.ErrArtifactUnsupportedCapability + tt.wantKept = true + } if tt.wantErr != nil { require.ErrorIs(t, err, tt.wantErr) } else { From 22f8e65139c36445b7b79c5b818fdcda9af6344a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 17:57:09 -0600 Subject: [PATCH 10/52] test(artifact): allow reported ambiguous lock creates in conformance The lock conformance case cancels a competing acquisition after 50ms and then expects the released lock to be free. When the deadline cuts the competing claim create short, a backend can report the outcome as ambiguous because that create may still commit. The S3 backend does this, and leaves such a claim for deliberate operator recovery. Under the race detector the late create landed after the release, and the reacquire failed. The leak-free check now applies only when the backend did not report an ambiguous outcome. The capability check runs first, independent of that path. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/artifacttest/conformance.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index 43801131d54..76f22c8cde9 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -425,6 +425,14 @@ func (c *conformance) lock(t *testing.T) { assert.NotEmpty(t, first.Token()) assert.True(t, first.Scope().Satisfies(artifact.LockScopeHost)) + caps := repo.Capabilities() + if !caps.HasLockScope(artifact.LockScopeRepository) { + stronger := request + stronger.Required = artifact.LockScopeRepository + _, err := repo.Lock(ctx, stronger) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an insufficient guarantee is an error, never a silent downgrade") + } + blocked, cancel := context.WithTimeout(ctx, 50*time.Millisecond) defer cancel() second := request @@ -435,16 +443,14 @@ func (c *conformance) lock(t *testing.T) { assert.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errUtils.ErrArtifactConflict), "waiting is bounded by the context or rejected as a conflict: %v", err) require.NoError(t, first.Release(ctx)) + if errors.Is(err, errUtils.ErrArtifactPublicationAmbiguous) { + // The deadline cut a claim create short and the backend reported that it may still commit. + // Such a claim is left for deliberate operator recovery, so leak-freedom is not promised. + return + } again, err := repo.Lock(ctx, second) require.NoError(t, err, "a canceled acquisition leaked no ownership") require.NoError(t, again.Release(ctx)) - - caps := repo.Capabilities() - if !caps.HasLockScope(artifact.LockScopeRepository) { - request.Required = artifact.LockScopeRepository - _, err := repo.Lock(ctx, request) - require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an insufficient guarantee is an error, never a silent downgrade") - } } func (c *conformance) capabilityErrors(t *testing.T) { From ab1d7432ba4ea81af2585fb5029fdc32ccec19d4 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 07:59:05 -0600 Subject: [PATCH 11/52] fix(oci): classify manifest scan failures as integrity errors and bound string skipping Join ErrOCIIntegrity with JSON scan failures while preserving the cause, and replace the decoder.Token based scan with a byte scanner that skips strings without buffering them. Co-Authored-By: Claude Opus 5.5 --- pkg/oci/transport_internal_test.go | 59 +++++++++ pkg/oci/transport_subject.go | 203 ++++++++++++++++++++++++----- 2 files changed, 226 insertions(+), 36 deletions(-) diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index f8497cd26e1..d1b776f9050 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -3,6 +3,7 @@ package oci import ( "encoding/base64" "errors" + "io" "net/http" "net/http/httptest" "os" @@ -220,3 +221,61 @@ var ( errorResponse403 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusForbidden} errorResponse503 = errcode.ErrorResponse{Method: http.MethodPut, StatusCode: http.StatusServiceUnavailable} ) + +func TestReadSubjectClassifiesScanFailuresAsIntegrity(t *testing.T) { + for _, tc := range []struct { + name string + manifest string + }{ + {"truncated manifest", `{"schemaVersion":2,"layers":[`}, + {"malformed subject", `{"schemaVersion":2,"subject":{"size":"x"}}`}, + {"not an object", `[1,2]`}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tc.manifest)), nil) + + subject, err := readSubject(t.Context(), store, &desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, subject) + }) + } +} + +func TestReadSubjectSkipsLongStringsAndFindsSubject(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + longString := strings.Repeat(`x\"y`, 1<<18) + for _, tc := range []struct { + name string + manifest string + want bool + }{ + {"long string value then subject", `{"annotations":{"k":"` + longString + `"},"subject":` + subject + `}`, true}, + {"long key is skipped", `{"` + strings.Repeat("k", 4096) + `":1,"subject":` + subject + `}`, true}, + {"escaped subject key", `{"subject":` + subject + `}`, true}, + {"scalars and nesting before subject", `{"a":1.5e3,"b":true,"c":null,"d":[{"e":"]}"}],"subject":` + subject + `}`, true}, + {"long string without subject", `{"annotations":{"k":"` + longString + `"},"n":7}`, false}, + {"empty object", `{}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tc.manifest)), nil) + + got, err := readSubject(t.Context(), store, &desc) + + require.NoError(t, err) + if !tc.want { + assert.Nil(t, got) + return + } + require.NotNil(t, got) + assert.Equal(t, int64(1), got.Size) + assert.Equal(t, ocispec.MediaTypeImageManifest, got.MediaType) + }) + } +} diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go index ccdb256aed7..b29d8eef0a0 100644 --- a/pkg/oci/transport_subject.go +++ b/pkg/oci/transport_subject.go @@ -1,8 +1,11 @@ package oci import ( + "bufio" + "bytes" "context" "encoding/json" + "errors" "fmt" "io" "strings" @@ -12,10 +15,15 @@ import ( errUtils "github.com/cloudposse/atmos/errors" ) +// maxManifestKeyBytes bounds how much of an object key the subject scan retains. A longer key can +// never be the "subject" field, so the remainder is skipped without being kept. +const maxManifestKeyBytes = 64 + // readSubject streams a stored manifest or index and returns its subject, or nil when it has none. // The blob was digest-verified when it was pushed, so the scan is bounded by the descriptor size -// and keeps constant memory: every other top-level field is skipped without being buffered. A -// large manifest without a subject is therefore never rejected. +// and keeps constant memory: every other top-level field, including arbitrarily long strings, is +// skipped byte by byte without being buffered. A large manifest without a subject is therefore +// never rejected. func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { if desc.Size < 0 { return nil, errUtils.ErrOCIIntegrity @@ -25,62 +33,185 @@ func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) ( return nil, err } defer reader.Close() - decoder := json.NewDecoder(io.LimitReader(reader, desc.Size)) - if err := expectDelim(decoder, '{'); err != nil { - return nil, err + scanner := bufio.NewReader(io.LimitReader(reader, desc.Size)) + if err := expectByte(scanner, '{'); err != nil { + return nil, integrityError(err) } - for decoder.More() { - key, err := decoder.Token() + for { + next, err := nextNonSpace(scanner) if err != nil { - return nil, err - } - name, ok := key.(string) - if !ok { - return nil, errUtils.ErrOCIIntegrity + return nil, integrityError(err) } - if !strings.EqualFold(name, "subject") { - if err := skipValue(decoder); err != nil { - return nil, err - } + switch next { + case '}': + return nil, nil + case ',': continue + case '"': + default: + return nil, fmt.Errorf("%w: unexpected %q in the manifest object", errUtils.ErrOCIIntegrity, next) + } + subject, found, err := scanField(scanner) + if err != nil { + return nil, integrityError(err) + } + if found { + return subject, nil + } + } +} + +// scanField consumes one object member whose opening quote was already read. It decodes the member +// only when it is the subject; every other value is skipped. +func scanField(scanner *bufio.Reader) (*ocispec.Descriptor, bool, error) { + name, err := readKey(scanner) + if err != nil { + return nil, false, err + } + if err := expectByte(scanner, ':'); err != nil { + return nil, false, err + } + if !strings.EqualFold(name, "subject") { + return nil, false, skipValue(scanner) + } + var subject *ocispec.Descriptor + if err := json.NewDecoder(scanner).Decode(&subject); err != nil { + return nil, false, err + } + return subject, true, nil +} + +// integrityError classifies a manifest scan failure with ErrOCIIntegrity and keeps the cause. An +// error that already carries the sentinel is returned unchanged. +func integrityError(err error) error { + if errors.Is(err, errUtils.ErrOCIIntegrity) { + return err + } + return fmt.Errorf("%w: reading the manifest subject: %w", errUtils.ErrOCIIntegrity, err) +} + +// nextNonSpace returns the next byte that is not JSON whitespace. +func nextNonSpace(scanner *bufio.Reader) (byte, error) { + for { + next, err := scanner.ReadByte() + if err != nil { + return 0, err } - var subject *ocispec.Descriptor - if err := decoder.Decode(&subject); err != nil { - return nil, err + switch next { + case ' ', '\t', '\r', '\n': + default: + return next, nil } - return subject, nil } - return nil, nil } -func expectDelim(decoder *json.Decoder, want json.Delim) error { - token, err := decoder.Token() +func expectByte(scanner *bufio.Reader, want byte) error { + next, err := nextNonSpace(scanner) if err != nil { return err } - if delim, ok := token.(json.Delim); !ok || delim != want { - return fmt.Errorf("%w: the manifest is not a JSON object", errUtils.ErrOCIIntegrity) + if next != want { + return fmt.Errorf("%w: expected %q in the manifest but found %q", errUtils.ErrOCIIntegrity, want, next) } return nil } -// skipValue consumes one JSON value, however deeply nested, without buffering it. -func skipValue(decoder *json.Decoder) error { - depth := 0 +// readKey reads an object key whose opening quote was consumed. It retains at most +// maxManifestKeyBytes of the key; an overlong key comes back empty because it cannot be "subject". +func readKey(scanner *bufio.Reader) (string, error) { + var raw bytes.Buffer + overflow, err := scanString(scanner, &raw, maxManifestKeyBytes) + if err != nil || overflow { + return "", err + } + if !bytes.ContainsRune(raw.Bytes(), '\\') { + return raw.String(), nil + } + var name string + if err := json.Unmarshal([]byte(`"`+raw.String()+`"`), &name); err != nil { + return "", err + } + return name, nil +} + +// scanString consumes a JSON string whose opening quote was already read. It copies at most limit +// bytes into keep (when keep is not nil) and reports whether the string was longer. Memory stays +// constant however long the string is. +func scanString(scanner *bufio.Reader, keep *bytes.Buffer, limit int) (bool, error) { + overflow := false + escaped := false for { - token, err := decoder.Token() + next, err := scanner.ReadByte() + if err != nil { + return overflow, err + } + if !escaped && next == '"' { + return overflow, nil + } + escaped = !escaped && next == '\\' + if keep == nil { + continue + } + if keep.Len() >= limit { + overflow = true + continue + } + keep.WriteByte(next) + } +} + +// skipValue consumes one JSON value, however deeply nested or long, without buffering it. +func skipValue(scanner *bufio.Reader) error { + first, err := nextNonSpace(scanner) + if err != nil { + return err + } + switch first { + case '"': + _, err := scanString(scanner, nil, 0) + return err + case '{', '[': + return skipContainer(scanner) + default: + return skipScalar(scanner) + } +} + +// skipContainer consumes the rest of an object or array whose opening bracket was already read. +func skipContainer(scanner *bufio.Reader) error { + depth := 1 + for depth > 0 { + next, err := scanner.ReadByte() if err != nil { return err } - if delim, ok := token.(json.Delim); ok { - if delim == '{' || delim == '[' { - depth++ - } else { - depth-- + switch next { + case '"': + if _, err := scanString(scanner, nil, 0); err != nil { + return err + } + case '{', '[': + depth++ + case '}', ']': + depth-- + } + } + return nil +} + +// skipScalar consumes the rest of a number, boolean, or null, leaving the delimiter unread. +func skipScalar(scanner *bufio.Reader) error { + for { + next, err := scanner.ReadByte() + if err != nil { + if errors.Is(err, io.EOF) { + return nil } + return err } - if depth == 0 { - return nil + switch next { + case ',', '}', ']', ' ', '\t', '\r', '\n': + return scanner.UnreadByte() } } } From d220e77595760010b575107cdb22941439e7b426 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:00:18 -0600 Subject: [PATCH 12/52] fix(artifact): wrap staged-file errors and bound the final source recheck Wrap Snapshot.Open and Snapshot.Close failures with ErrArtifactPackagingInvalid, and validate the opened descriptor and bound the read in recheckFile. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/snapshot/copy.go | 19 +++++++++-- pkg/artifact/snapshot/snapshot.go | 12 +++++-- pkg/artifact/snapshot/snapshot_test.go | 46 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index 1dcfcd2920c..95637ba02b6 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -151,15 +151,28 @@ func (c *capturer) recheckFile(record *sourceRecord, now os.FileInfo) error { if !sameFileState(record.info, now) { return fmt.Errorf("%w: %q changed size, mode, or modification time while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) } - file, err := os.Open(record.abs) + file, err := openSame(record.abs, record.rel, record.info) if err != nil { - return changedError(record.rel, err) + return err } defer file.Close() + opened, err := file.Stat() + if err != nil { + return changedError(record.rel, err) + } + if !sameFileState(record.info, opened) { + return changedError(record.rel, errSizeOrTimeChanged) + } + // Read at most one byte past the recorded size so a file that grew after the stat is rejected + // without being read in full. hasher := sha256.New() - if _, err := copyContext(c.ctx, hasher, file); err != nil { + copied, err := copyContext(c.ctx, hasher, io.LimitReader(file, record.info.Size()+1)) + if err != nil { return fmt.Errorf("%w: recheck %q: %w", errUtils.ErrArtifactSourceChanged, record.rel, err) } + if copied != record.info.Size() { + return changedError(record.rel, errSizeOrTimeChanged) + } if digest.NewDigest(digest.SHA256, hasher) != record.digest { return fmt.Errorf("%w: %q content changed while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) } diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index e21ecab9b3b..dd6d78eeb75 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -98,9 +98,13 @@ func (s *Snapshot) Close() error { return nil } err := os.RemoveAll(s.Root) + root := s.Root s.Root = "" s.staged = nil - return err + if err != nil { + return fmt.Errorf("%w: remove staging directory %q: %w", errUtils.ErrArtifactPackagingInvalid, root, err) + } + return nil } // StagedPath returns the staged copy of the regular file at the inventory path. @@ -119,7 +123,11 @@ func (s *Snapshot) Open(entryPath string) (io.ReadCloser, error) { if !ok { return nil, fmt.Errorf("%w: %q is not a staged file", errUtils.ErrArtifactNotFound, entryPath) } - return os.Open(p) + file, err := os.Open(p) + if err != nil { + return nil, fmt.Errorf("%w: open staged file %q: %w", errUtils.ErrArtifactPackagingInvalid, entryPath, err) + } + return file, nil } // Lookup returns the inventory entry at path. diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 5db3becdcf5..3300df6478e 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -208,6 +208,18 @@ func TestCapture_DetectsChangeDuringCapture(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("ORIGINAL"), 0o644)) require.NoError(t, os.Chtimes(path, info.ModTime(), info.ModTime())) }}, + {"replaced by another file with identical state", func(t *testing.T, src string) { + if runtime.GOOS == "windows" { + t.Skip("replacing an open path is not portable on Windows") + } + path := filepath.Join(src, "a.txt") + info, err := os.Stat(path) + require.NoError(t, err) + replacement := filepath.Join(src, "replacement.tmp") + require.NoError(t, os.WriteFile(replacement, []byte("original"), 0o644)) + require.NoError(t, os.Chtimes(replacement, info.ModTime(), info.ModTime())) + require.NoError(t, os.Rename(replacement, path)) + }}, {"file added", func(t *testing.T, src string) { fixtures.WriteFile(t, src, "new.txt", "new", 0o644) }}, @@ -327,3 +339,37 @@ func TestSnapshot_OpenUnknownPath(t *testing.T) { _, ok := snap.Lookup("missing") assert.False(t, ok) } + +func TestSnapshot_OpenWrapsStagedFileFailure(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + defer snap.Close() + staged, ok := snap.StagedPath("a.txt") + require.True(t, ok) + require.NoError(t, os.Remove(staged)) + + _, err = snap.Open("a.txt") + + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestSnapshot_CloseWrapsRemovalFailure(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("directory permissions do not block removal here") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + root := snap.Root + require.NoError(t, os.Chmod(root, 0o500)) + t.Cleanup(func() { _ = os.Chmod(root, 0o700) }) + + err = snap.Close() + + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrPermission) +} From 6d547cc5d9fb50c369411efde682ab80da0496b1 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:01:42 -0600 Subject: [PATCH 13/52] fix(signature): classify signing backend failures with a static sentinel Co-Authored-By: Claude Opus 5.5 --- errors/signature.go | 3 +++ pkg/signature/signer.go | 3 ++- pkg/signature/signer_test.go | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/errors/signature.go b/errors/signature.go index ac3e4eef484..1f74c5b9e95 100644 --- a/errors/signature.go +++ b/errors/signature.go @@ -13,4 +13,7 @@ var ( // ErrSignatureOffline indicates trust material cannot be fetched while verifying offline. ErrSignatureOffline = errors.New("signature trust material is unavailable offline") + + // ErrSignatureSigningFailed indicates a signing backend failed to produce signature evidence. + ErrSignatureSigningFailed = errors.New("signing backend failed to produce a signature") ) diff --git a/pkg/signature/signer.go b/pkg/signature/signer.go index a72bfc449c0..acbdaa70f7a 100644 --- a/pkg/signature/signer.go +++ b/pkg/signature/signer.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "github.com/sigstore/sigstore-go/pkg/bundle" @@ -27,7 +28,7 @@ func SignBlob(ctx context.Context, signer BlobSigner, artifact []byte) ([]byte, signed, err := signer.Sign(ctx, artifact) if err != nil { - return nil, err + return nil, errors.Join(errUtils.ErrSignatureSigningFailed, err) } if err := CheckBundleSubject(signed, artifact); err != nil { return nil, err diff --git a/pkg/signature/signer_test.go b/pkg/signature/signer_test.go index 97014348737..74fb47c96c1 100644 --- a/pkg/signature/signer_test.go +++ b/pkg/signature/signer_test.go @@ -30,7 +30,7 @@ func TestSignBlobAcceptsOnlyEvidenceBoundToTheSignedBytes(t *testing.T) { {"malformed bundle", []byte("not json"), nil, errUtils.ErrSignatureInvalid}, {"empty bundle", nil, nil, errUtils.ErrSignatureInvalid}, {"oversized bundle", make([]byte, MaxBundleSize+1), nil, errUtils.ErrSignatureInvalid}, - {"signer failure", nil, failure, failure}, + {"signer failure", nil, failure, errUtils.ErrSignatureSigningFailed}, } { t.Run(tc.name, func(t *testing.T) { signer := NewMockBlobSigner(gomock.NewController(t)) @@ -38,6 +38,9 @@ func TestSignBlobAcceptsOnlyEvidenceBoundToTheSignedBytes(t *testing.T) { got, err := SignBlob(t.Context(), signer, artifact) if tc.want != nil { require.ErrorIs(t, err, tc.want) + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + } assert.Nil(t, got) return } From 66c4d977f74c0e84e91e208f115343ecae028011 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:02:00 -0600 Subject: [PATCH 14/52] test(artifact): use the generated signer mock in conformance The fixture repository now refuses a signer whose formats it cannot store before invoking it. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/artifacttest/conformance.go | 23 +++++++++-------------- pkg/artifact/artifacttest/sign.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index 76f22c8cde9..95a3aabf778 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -11,6 +11,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -363,7 +364,7 @@ func (c *conformance) signVerify(t *testing.T) { caps := repo.Capabilities() if c.h.Signer == nil { require.Empty(t, caps.EvidenceFormats, "a repository that stores evidence needs a Signer in the harness") - _, err := repo.Sign(context.Background(), subject, &unsupportedSigner{}, artifact.SignOptions{}) + _, err := repo.Sign(context.Background(), subject, unsupportedSigner(t), artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) return } @@ -399,19 +400,13 @@ func (c *conformance) signVerify(t *testing.T) { } } -// unsupportedSigner is a signer for a format no repository under test advertises. -type unsupportedSigner struct{} - -func (*unsupportedSigner) Formats() []artifact.EvidenceFormat { - defer perf.Track(nil, "artifacttest.unsupportedSigner.Formats")() - - return []artifact.EvidenceFormat{"unsupported"} -} - -func (*unsupportedSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { - defer perf.Track(nil, "artifacttest.unsupportedSigner.Sign")() - - return &artifact.EvidencePayload{Format: "unsupported", Data: []byte("x")}, nil +// unsupportedSigner returns a generated signer for a format no repository under test advertises. +// Sign has no expectation, so the mock fails the test if a repository ever invokes it. +func unsupportedSigner(t *testing.T) artifact.Signer { + t.Helper() + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return([]artifact.EvidenceFormat{"unsupported"}).AnyTimes() + return signer } func (c *conformance) lock(t *testing.T) { diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index e51f7ebff12..5418b926688 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -87,6 +88,9 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return nil, err } r.mu.Unlock() + if err := r.checkSignerFormats(signer); err != nil { + return nil, err + } payload, err := signer.Sign(ctx, subject) if err != nil { return nil, err @@ -108,6 +112,25 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return []artifact.SignatureEvidence{evidence}, nil } +// checkSignerFormats refuses a signer before it is invoked when none of its formats can be stored. +func (r *Repository) checkSignerFormats(signer artifact.Signer) error { + formats := signer.Formats() + for _, format := range formats { + if r.caps.HasEvidenceFormat(format) { + return nil + } + } + return artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "evidence/"+joinFormats(formats), "the repository cannot store this evidence format") +} + +func joinFormats(formats []artifact.EvidenceFormat) string { + names := make([]string, 0, len(formats)) + for _, format := range formats { + names = append(names, string(format)) + } + return strings.Join(names, ",") +} + // checkSubject requires the subject to be a stored record; callers hold r.mu. func (r *Repository) checkSubject(subject artifact.SignatureSubject) error { if !r.caps.HasSubjectKind(subject.Kind) { From 5716ddd0f47355769063466d316f95598f5687b5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:05:15 -0600 Subject: [PATCH 15/52] fix(artifact): report a repeated native graph copy as unchanged Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/commit.go | 21 +++++++-- pkg/artifact/service/copy_test.go | 57 +++++++++++++++++++++++ pkg/artifact/service/delegate_test.go | 65 +++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 pkg/artifact/service/delegate_test.go diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go index 4dae6b1c8b3..29f99e54b35 100644 --- a/pkg/artifact/service/commit.go +++ b/pkg/artifact/service/commit.go @@ -157,10 +157,8 @@ func (s *Service) decide(tgt *target, cur *current, in *incoming, force bool) (v return verdictCreate, nil } existing := cur.inspection.Record - if existing != nil { - if intent, err := existing.IntentDigest(); err == nil && intent == in.intent { - return verdictUnchanged, nil - } + if sameIntent(existing, in) || sameGraphRoot(existing, cur, in) { + return verdictUnchanged, nil } if immutable, reason := s.immutableCandidate(tgt, cur, in); immutable { return verdictCreate, fmt.Errorf("%w: %s stays immutable even with force", errUtils.ErrArtifactConflict, reason) @@ -172,6 +170,21 @@ func (s *Service) decide(tgt *target, cur *current, in *incoming, force bool) (v return verdictReplace, nil } +// sameIntent reports whether the stored Record carries the incoming intent. +func sameIntent(existing *artifact.Record, in *incoming) bool { + if existing == nil { + return false + } + intent, err := existing.IntentDigest() + return err == nil && intent == in.intent +} + +// sameGraphRoot reports whether a Record-free destination already points at the pinned source root +// of a native graph copy. It uses the same root identity as reconcileGraph. +func sameGraphRoot(existing *artifact.Record, cur *current, in *incoming) bool { + return existing == nil && in.rootDigest != "" && cur.pinned.ResolvedDigest == in.rootDigest +} + // immutableCandidate asks the admission policy about both the stored and the incoming artifact. func (s *Service) immutableCandidate(tgt *target, cur *current, in *incoming) (bool, string) { if s.immutable == nil { diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index 3529cd8ad71..1e9e40d536d 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -11,6 +11,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -275,6 +276,62 @@ func TestCopy_UsesNativeTransferWhenCompatible(t *testing.T) { assert.Equal(t, 1, native.copies, "an incompatible pair falls back to staging") } +func TestCopy_RepeatedNativeGraphCopyWithoutRecordIsUnchanged(t *testing.T) { + for _, force := range []bool{false, true} { + t.Run(map[bool]string{false: "without force", true: "with force"}[force], func(t *testing.T) { + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) + e.publish("src", e.source("hello")) + source := delegatingRepository(t, e.repos["src"], repoHooks{inspect: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := e.repos["src"].Inspect(ctx, pinned) + if err != nil { + return nil, err + } + return withoutRecord(inspection), nil + }}) + pinned, err := e.repos["src"].Resolve(context.Background(), exampleRef) + require.NoError(t, err) + var copied atomic.Int32 + destination := delegatingRepository(t, e.repos["dst"], repoHooks{ + resolve: func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { + if copied.Load() == 0 { + return artifact.PinnedReference{}, errUtils.ErrArtifactNotFound + } + return pinned, nil + }, + inspect: func(ctx context.Context, p artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := e.repos["src"].Inspect(ctx, p) + if err != nil { + return nil, err + } + return withoutRecord(inspection), nil + }, + }) + copier := artifact.NewMockCopier(gomock.NewController(t)) + copier.EXPECT().CanCopyFrom(gomock.Any()).Return(true).AnyTimes() + copier.EXPECT().CopyFrom(gomock.Any(), gomock.Any(), pinned, gomock.Any()).DoAndReturn( + func(ctx context.Context, _ artifact.Repository, p artifact.PinnedReference, _ artifact.CopyFromOptions) (*artifact.PublishResult, error) { + copied.Add(1) + inspection, err := e.repos["src"].Inspect(ctx, p) + if err != nil { + return nil, err + } + return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: p, Representation: inspection.Representation, Location: "native://copy"}, nil + }, + ).Times(1) + e.static.Repositories["src"] = source + e.static.Repositories["dst"] = &nativeDestination{Repository: destination, Copier: copier} + + first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, first.Outcome) + + again, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: force}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "the destination already points at the source root") + }) + } +} + func diagnosticCodes(result *artifact.OperationResult) []string { codes := make([]string, 0, len(result.Diagnostics)) for _, diagnostic := range result.Diagnostics { diff --git a/pkg/artifact/service/delegate_test.go b/pkg/artifact/service/delegate_test.go new file mode 100644 index 00000000000..e5450e7e169 --- /dev/null +++ b/pkg/artifact/service/delegate_test.go @@ -0,0 +1,65 @@ +package service_test + +import ( + "context" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// repoHooks overrides individual repository methods of a delegating mock. A nil hook delegates to +// the base repository. +type repoHooks struct { + resolve func(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) + inspect func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) + read func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) + list func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) +} + +// delegatingRepository returns a generated MockRepository whose methods forward to base, except +// for the ones a hook replaces. +func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHooks) *artifact.MockRepository { + t.Helper() + if hooks.resolve == nil { + hooks.resolve = base.Resolve + } + if hooks.inspect == nil { + hooks.inspect = base.Inspect + } + if hooks.read == nil { + hooks.read = base.Read + } + if hooks.list == nil { + hooks.list = base.List + } + mock := artifact.NewMockRepository(gomock.NewController(t)) + mock.EXPECT().Capabilities().DoAndReturn(base.Capabilities).AnyTimes() + mock.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(base.Lock).AnyTimes() + mock.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(hooks.resolve).AnyTimes() + mock.EXPECT().Inspect(gomock.Any(), gomock.Any()).DoAndReturn(hooks.inspect).AnyTimes() + mock.EXPECT().OpenDocument(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.OpenDocument).AnyTimes() + mock.EXPECT().Read(gomock.Any(), gomock.Any()).DoAndReturn(hooks.read).AnyTimes() + mock.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Publish).AnyTimes() + mock.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(hooks.list).AnyTimes() + mock.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Delete).AnyTimes() + mock.EXPECT().Pin(gomock.Any(), gomock.Any()).DoAndReturn(base.Pin).AnyTimes() + mock.EXPECT().Sign(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Sign).AnyTimes() + mock.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Verify).AnyTimes() + return mock +} + +// nativeDestination composes a repository with a generated Copier so a test can add a native +// transfer to a destination. +type nativeDestination struct { + artifact.Repository + artifact.Copier +} + +// withoutRecord returns a copy of the inspection that carries no portable Record, like a foreign graph. +func withoutRecord(inspection *artifact.Inspection) *artifact.Inspection { + stripped := *inspection + stripped.Record = nil + return &stripped +} From 344da54665e0e538b21f44202746de34e108b605 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:09:37 -0600 Subject: [PATCH 16/52] test(artifact): use generated mocks for optional repository interfaces Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/copy_test.go | 44 ++++----------------------- pkg/artifact/service/delegate_test.go | 29 ++++++++++++++++-- pkg/artifact/service/edge_test.go | 17 +++++------ 3 files changed, 40 insertions(+), 50 deletions(-) diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index 1e9e40d536d..77e29f1a584 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -11,7 +11,6 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -232,31 +231,10 @@ func TestCopy_WarnsWhenEvidenceCannotTravel(t *testing.T) { assert.True(t, warned, "missing evidence transfer is reported, never silently dropped") } -// nativeRepository adds a compatible native transfer to the in-memory repository. -type nativeRepository struct { - *artifacttest.Repository - copies int - options artifact.CopyFromOptions - compat bool -} - -func (n *nativeRepository) CanCopyFrom(artifact.Repository) bool { return n.compat } - -//nolint:gocritic // The pinned reference is passed by value because artifact.Copier fixes the signature. -func (n *nativeRepository) CopyFrom(ctx context.Context, source artifact.Repository, pinned artifact.PinnedReference, options artifact.CopyFromOptions) (*artifact.PublishResult, error) { - n.copies++ - n.options = options - inspection, err := source.Inspect(ctx, pinned) - if err != nil { - return nil, err - } - return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: pinned, Representation: inspection.Representation, Location: "native://copy"}, nil -} - func TestCopy_UsesNativeTransferWhenCompatible(t *testing.T) { e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone) published := e.publish("src", e.source("hello")) - native := &nativeRepository{Repository: e.repos["dst"], compat: true} + native := newNativeRepository(t, e.repos["dst"]) e.static.Repositories["dst"] = native reads := e.repos["src"].Calls("Read") @@ -290,10 +268,10 @@ func TestCopy_RepeatedNativeGraphCopyWithoutRecordIsUnchanged(t *testing.T) { }}) pinned, err := e.repos["src"].Resolve(context.Background(), exampleRef) require.NoError(t, err) - var copied atomic.Int32 + var native *nativeRepository destination := delegatingRepository(t, e.repos["dst"], repoHooks{ resolve: func(context.Context, artifact.Ref) (artifact.PinnedReference, error) { - if copied.Load() == 0 { + if native.copies == 0 { return artifact.PinnedReference{}, errUtils.ErrArtifactNotFound } return pinned, nil @@ -306,20 +284,9 @@ func TestCopy_RepeatedNativeGraphCopyWithoutRecordIsUnchanged(t *testing.T) { return withoutRecord(inspection), nil }, }) - copier := artifact.NewMockCopier(gomock.NewController(t)) - copier.EXPECT().CanCopyFrom(gomock.Any()).Return(true).AnyTimes() - copier.EXPECT().CopyFrom(gomock.Any(), gomock.Any(), pinned, gomock.Any()).DoAndReturn( - func(ctx context.Context, _ artifact.Repository, p artifact.PinnedReference, _ artifact.CopyFromOptions) (*artifact.PublishResult, error) { - copied.Add(1) - inspection, err := e.repos["src"].Inspect(ctx, p) - if err != nil { - return nil, err - } - return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: p, Representation: inspection.Representation, Location: "native://copy"}, nil - }, - ).Times(1) + native = newNativeRepository(t, destination) e.static.Repositories["src"] = source - e.static.Repositories["dst"] = &nativeDestination{Repository: destination, Copier: copier} + e.static.Repositories["dst"] = native first, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) require.NoError(t, err) @@ -328,6 +295,7 @@ func TestCopy_RepeatedNativeGraphCopyWithoutRecordIsUnchanged(t *testing.T) { again, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: force}) require.NoError(t, err) assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "the destination already points at the source root") + assert.Equal(t, 1, native.copies, "an unchanged destination is not copied again") }) } } diff --git a/pkg/artifact/service/delegate_test.go b/pkg/artifact/service/delegate_test.go index e5450e7e169..5b0f8f7ae17 100644 --- a/pkg/artifact/service/delegate_test.go +++ b/pkg/artifact/service/delegate_test.go @@ -50,11 +50,34 @@ func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHook return mock } -// nativeDestination composes a repository with a generated Copier so a test can add a native -// transfer to a destination. -type nativeDestination struct { +// nativeRepository adds a native transfer, backed by a generated Copier, to a repository. It +// records how often and with which options the transfer ran. +type nativeRepository struct { artifact.Repository artifact.Copier + copies int + options artifact.CopyFromOptions + compat bool +} + +func newNativeRepository(t *testing.T, base artifact.Repository) *nativeRepository { + t.Helper() + native := &nativeRepository{Repository: base, compat: true} + copier := artifact.NewMockCopier(gomock.NewController(t)) + copier.EXPECT().CanCopyFrom(gomock.Any()).DoAndReturn(func(artifact.Repository) bool { return native.compat }).AnyTimes() + copier.EXPECT().CopyFrom(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, source artifact.Repository, pinned artifact.PinnedReference, options artifact.CopyFromOptions) (*artifact.PublishResult, error) { + native.copies++ + native.options = options + inspection, err := source.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + return &artifact.PublishResult{Outcome: artifact.OutcomeCreated, Pinned: pinned, Representation: inspection.Representation, Location: "native://copy"}, nil + }, + ).AnyTimes() + native.Copier = copier + return native } // withoutRecord returns a copy of the inspection that carries no portable Record, like a foreign graph. diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go index 33862b02eaa..91bf1d996cb 100644 --- a/pkg/artifact/service/edge_test.go +++ b/pkg/artifact/service/edge_test.go @@ -370,7 +370,7 @@ func TestCopy_ForeignGraphNeedsANativeTransfer(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) }) t.Run("native transfer carries the graph", func(t *testing.T) { - native := &nativeRepository{Repository: artifacttest.New(artifacttest.Options{}), compat: true} + native := newNativeRepository(t, artifacttest.New(artifacttest.Options{})) svc := build(t, native) result, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) require.NoError(t, err) @@ -378,7 +378,7 @@ func TestCopy_ForeignGraphNeedsANativeTransfer(t *testing.T) { assert.Equal(t, "tools/example:1.0.0", result.Ref) }) t.Run("a digest-only graph has no tag to publish under", func(t *testing.T) { - native := &nativeRepository{Repository: artifacttest.New(artifacttest.Options{}), compat: true} + native := newNativeRepository(t, artifacttest.New(artifacttest.Options{})) svc := build(t, native) digestRef := artifact.Ref{Name: "tools/example", Digest: digest.FromString("graph-root")} _, err := svc.Copy(context.Background(), artifact.CopyRequest{Ref: digestRef, From: "src", To: "dst"}) @@ -391,7 +391,12 @@ func TestCopy_EvidenceCarryFailuresAreReported(t *testing.T) { published := e.publish("src", e.source("hello")) _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "src", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) require.NoError(t, err) - e.static.Repositories["src"] = failingEvidenceReader{Repository: e.repos["src"]} + reader := artifact.NewMockEvidenceReader(gomock.NewController(t)) + reader.EXPECT().ReadEvidence(gomock.Any(), gomock.Any()).Return(nil, errTransport).AnyTimes() + e.static.Repositories["src"] = struct { + artifact.Repository + artifact.EvidenceReader + }{e.repos["src"], reader} result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) require.NoError(t, err) @@ -402,12 +407,6 @@ func TestCopy_EvidenceCarryFailuresAreReported(t *testing.T) { assert.True(t, warned) } -type failingEvidenceReader struct{ artifact.Repository } - -func (failingEvidenceReader) ReadEvidence(context.Context, *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { - return nil, errTransport -} - func TestSign_RejectsADigestThatIsNotTheSignableSubject(t *testing.T) { caps := artifacttest.DefaultCapabilities() record, recordDigest := sealedRecord(t) From b9a577c760429becab9020be1c03499da5082658 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:10:51 -0600 Subject: [PATCH 17/52] test(artifact): use generated repository mocks for the consume test hooks Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/consume_test.go | 45 ++++++++++------------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index c9582aa2229..d9c6c50361c 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -339,19 +339,17 @@ type listGate struct { release chan struct{} } -type gatedListRepository struct { - artifact.Repository - gate *listGate -} - -func (r *gatedListRepository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { - current := r.gate.inflight.Add(1) - defer r.gate.inflight.Add(-1) - for old := r.gate.peak.Load(); current > old && !r.gate.peak.CompareAndSwap(old, current); old = r.gate.peak.Load() { - } - r.gate.entered <- struct{}{} - <-r.gate.release - return r.Repository.List(ctx, query) +// gatedList returns a generated repository that blocks every List on the gate before delegating. +func gatedList(t *testing.T, base *artifacttest.Repository, gate *listGate) artifact.Repository { + return delegatingRepository(t, base, repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + current := gate.inflight.Add(1) + defer gate.inflight.Add(-1) + for old := gate.peak.Load(); current > old && !gate.peak.CompareAndSwap(old, current); old = gate.peak.Load() { + } + gate.entered <- struct{}{} + <-gate.release + return base.List(ctx, query) + }}) } func TestList_ConcurrencyIsInjectable(t *testing.T) { @@ -364,7 +362,7 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { } gate := &listGate{entered: make(chan struct{}, 3), release: make(chan struct{})} for _, alias := range []string{"a", "b", "c"} { - e.static.Repositories[alias] = &gatedListRepository{Repository: e.repos[alias], gate: gate} + e.static.Repositories[alias] = gatedList(t, e.repos[alias], gate) } done := make(chan error, 1) go func() { @@ -389,19 +387,6 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { assert.Equal(t, int32(1), gate.peak.Load()) } -// readHookRepository runs a hook right before every Read, as if another process touched the -// destination while the artifact was downloading. -type readHookRepository struct { - *artifacttest.Repository - onRead func() -} - -//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. -func (r *readHookRepository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { - r.onRead() - return r.Repository.Read(ctx, pinned) -} - func TestPull_RechecksTheDestinationBeforeReplacingIt(t *testing.T) { tests := []struct { name string @@ -418,9 +403,11 @@ func TestPull_RechecksTheDestinationBeforeReplacingIt(t *testing.T) { e.publish("dst", e.source("ORIGINAL")) dest := out(t) require.NoError(t, os.MkdirAll(dest, 0o755)) - e.static.Repositories["dst"] = &readHookRepository{Repository: e.repos["dst"], onRead: func() { + // Another process touches the destination right before the artifact is read. + e.static.Repositories["dst"] = delegatingRepository(t, e.repos["dst"], repoHooks{read: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { fixtures.WriteFile(t, dest, "late.txt", "written during the download", 0o644) - }} + return e.repos["dst"].Read(ctx, pinned) + }}) _, err := e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest, Force: tt.force}) if runtime.GOOS == "windows" && tt.force { From 5ced6de8d6efd36e87d442f296ee560900611d66 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:12:53 -0600 Subject: [PATCH 18/52] test(oci): use generated mocks for the transport test doubles Generate a ReadOnlyStore mock alongside the existing store mocks. Co-Authored-By: Claude Opus 5.5 --- pkg/oci/mock_transport.go | 73 +++++++++++++++- pkg/oci/transport.go | 2 +- pkg/oci/transport_behavior_test.go | 128 ++++++++++++++++------------- 3 files changed, 145 insertions(+), 58 deletions(-) diff --git a/pkg/oci/mock_transport.go b/pkg/oci/mock_transport.go index d4b3799d805..482a62479ba 100644 --- a/pkg/oci/mock_transport.go +++ b/pkg/oci/mock_transport.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/cloudposse/atmos/pkg/oci (interfaces: Store,RepositoryStore,Transport) +// Source: github.com/cloudposse/atmos/pkg/oci (interfaces: ReadOnlyStore,Store,RepositoryStore,Transport) // // Generated by this command: // -// mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci Store,RepositoryStore,Transport +// mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci ReadOnlyStore,Store,RepositoryStore,Transport // // Package oci is a generated GoMock package. @@ -18,6 +18,75 @@ import ( gomock "go.uber.org/mock/gomock" ) +// MockReadOnlyStore is a mock of ReadOnlyStore interface. +type MockReadOnlyStore struct { + ctrl *gomock.Controller + recorder *MockReadOnlyStoreMockRecorder + isgomock struct{} +} + +// MockReadOnlyStoreMockRecorder is the mock recorder for MockReadOnlyStore. +type MockReadOnlyStoreMockRecorder struct { + mock *MockReadOnlyStore +} + +// NewMockReadOnlyStore creates a new mock instance. +func NewMockReadOnlyStore(ctrl *gomock.Controller) *MockReadOnlyStore { + mock := &MockReadOnlyStore{ctrl: ctrl} + mock.recorder = &MockReadOnlyStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockReadOnlyStore) EXPECT() *MockReadOnlyStoreMockRecorder { + return m.recorder +} + +// Exists mocks base method. +func (m *MockReadOnlyStore) Exists(ctx context.Context, target v1.Descriptor) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockReadOnlyStoreMockRecorder) Exists(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockReadOnlyStore)(nil).Exists), ctx, target) +} + +// Fetch mocks base method. +func (m *MockReadOnlyStore) Fetch(ctx context.Context, target v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, target) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockReadOnlyStoreMockRecorder) Fetch(ctx, target any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockReadOnlyStore)(nil).Fetch), ctx, target) +} + +// Resolve mocks base method. +func (m *MockReadOnlyStore) Resolve(ctx context.Context, reference string) (v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, reference) + ret0, _ := ret[0].(v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockReadOnlyStoreMockRecorder) Resolve(ctx, reference any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockReadOnlyStore)(nil).Resolve), ctx, reference) +} + // MockStore is a mock of Store interface. type MockStore struct { ctrl *gomock.Controller diff --git a/pkg/oci/transport.go b/pkg/oci/transport.go index 2e631ad6988..138020541f8 100644 --- a/pkg/oci/transport.go +++ b/pkg/oci/transport.go @@ -14,7 +14,7 @@ import ( "github.com/cloudposse/atmos/pkg/perf" ) -//go:generate go run go.uber.org/mock/mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci Store,RepositoryStore,Transport +//go:generate go run go.uber.org/mock/mockgen -destination mock_transport.go -package oci github.com/cloudposse/atmos/pkg/oci ReadOnlyStore,Store,RepositoryStore,Transport // Repository identifies one OCI repository independently of any client library. // Registry is the host (with optional port) and Path the repository name. diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go index 2a31b2f7fa0..53bfa7ee0c0 100644 --- a/pkg/oci/transport_behavior_test.go +++ b/pkg/oci/transport_behavior_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/go-containerregistry/pkg/registry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry/remote/errcode" @@ -345,10 +346,10 @@ func TestFetchBytesIsBoundedAndDigestVerified(t *testing.T) { wrongSize := descriptor wrongSize.Size++ - _, err = oci.FetchBytes(ctx, fetchOverride{descriptor.Digest.String(): content}, wrongSize, 1<<10) + _, err = oci.FetchBytes(ctx, fetchOverride(t, map[string][]byte{descriptor.Digest.String(): content}), wrongSize, 1<<10) require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) - tampered := fetchOverride{descriptor.Digest.String(): []byte(`{"hello":"WORLD"}`)} + tampered := fetchOverride(t, map[string][]byte{descriptor.Digest.String(): []byte(`{"hello":"WORLD"}`)}) _, err = oci.FetchBytes(ctx, tampered, descriptor, 1<<10) require.ErrorIs(t, err, errUtils.ErrOCIIntegrity, "same size, different bytes") @@ -357,25 +358,31 @@ func TestFetchBytesIsBoundedAndDigestVerified(t *testing.T) { require.ErrorIs(t, err, errdef.ErrNotFound) } -type fetchOverride map[string][]byte - -func (f fetchOverride) Fetch(_ context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader(string(f[target.Digest.String()]))), nil -} - -// cancellingSource cancels the copy after a number of fetches and counts how many nodes were read. -type cancellingSource struct { - oci.ReadOnlyStore - fetches atomic.Int32 - cancelAfter int32 - cancel context.CancelFunc +// fetchOverride returns a generated source whose Fetch serves the supplied bytes by digest. +func fetchOverride(t *testing.T, contents map[string][]byte) *oci.MockReadOnlyStore { + t.Helper() + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(string(contents[target.Digest.String()]))), nil + }).AnyTimes() + return store } -func (c *cancellingSource) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { - if c.fetches.Add(1) == c.cancelAfter { - c.cancel() - } - return c.ReadOnlyStore.Fetch(ctx, target) +// cancellingSource returns a generated source that delegates to base, cancels the copy after +// cancelAfter fetches, and counts how many nodes were read. +func cancellingSource(t *testing.T, base oci.ReadOnlyStore, cancelAfter int32, cancel context.CancelFunc) (*oci.MockReadOnlyStore, *atomic.Int32) { + t.Helper() + var fetches atomic.Int32 + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + if fetches.Add(1) == cancelAfter { + cancel() + } + return base.Fetch(ctx, target) + }).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(base.Exists).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(base.Resolve).AnyTimes() + return store, &fetches } func TestCancellationStopsGraphTraversalAndLeavesNoReference(t *testing.T) { @@ -388,22 +395,22 @@ func TestCancellationStopsGraphTraversalAndLeavesNoReference(t *testing.T) { require.NoError(t, err) // Establish how many nodes an uninterrupted copy fetches. - complete := &cancellingSource{ReadOnlyStore: base, cancelAfter: -1} + complete, completeFetches := cancellingSource(t, base, -1, nil) full, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "full")) require.NoError(t, err) _, err = oci.Copy(t.Context(), oci.Source{Store: complete, Reference: digest.String()}, oci.Destination{Store: full, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) require.NoError(t, err) - total := complete.fetches.Load() + total := completeFetches.Load() require.Greater(t, total, int32(3)) ctx, cancel := context.WithCancel(t.Context()) t.Cleanup(cancel) - interrupted := &cancellingSource{ReadOnlyStore: base, cancelAfter: 2, cancel: cancel} + interrupted, interruptedFetches := cancellingSource(t, base, 2, cancel) target, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) require.NoError(t, err) _, err = oci.Copy(ctx, oci.Source{Store: interrupted, Reference: digest.String()}, oci.Destination{Store: target, Reference: "1.0.0"}, oci.WithCopyConcurrency(1)) require.ErrorIs(t, err, context.Canceled) - assert.Less(t, interrupted.fetches.Load(), total, "traversal must stop once the context ends") + assert.Less(t, interruptedFetches.Load(), total, "traversal must stop once the context ends") assert.Empty(t, target.References(), "an interrupted copy must not publish the reference") _, err = target.Resolve(t.Context(), "1.0.0") require.Error(t, err) @@ -414,37 +421,48 @@ func TestCancellationStopsGraphTraversalAndLeavesNoReference(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -// landingDestination reports a failed reference push after (optionally) completing it. -type landingDestination struct { - oci.Store - land bool - err error - pushes atomic.Int32 +// landingDestination returns a generated destination over layout whose reference push reports err +// after (optionally) completing it, and counts the push attempts. +func landingDestination(t *testing.T, layout *oci.Layout, land bool, pushErr error) (*oci.MockRepositoryStore, *atomic.Int32) { + t.Helper() + var pushes atomic.Int32 + store := oci.NewMockRepositoryStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(layout.Fetch).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(layout.Exists).AnyTimes() + store.EXPECT().Push(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(layout.Push).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(layout.Resolve).AnyTimes() + store.EXPECT().Tag(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(layout.Tag).AnyTimes() + store.EXPECT().PushReference(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + pushes.Add(1) + if land { + if err := layout.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { + return err + } + if err := layout.Tag(ctx, expected, reference); err != nil { + return err + } + } + return pushErr + }, + ).AnyTimes() + return store, &pushes } -func (l *landingDestination) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { - l.pushes.Add(1) - if l.land { - if err := l.Push(ctx, expected, content); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { - return err - } - if err := l.Tag(ctx, expected, reference); err != nil { - return err +// movingSource returns a generated source whose tag resolves once and then fails, as if it moved. +func movingSource(t *testing.T, base oci.ReadOnlyStore) (*oci.MockReadOnlyStore, *atomic.Int32) { + t.Helper() + var calls atomic.Int32 + store := oci.NewMockReadOnlyStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn(base.Fetch).AnyTimes() + store.EXPECT().Exists(gomock.Any(), gomock.Any()).DoAndReturn(base.Exists).AnyTimes() + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, ref string) (ocispec.Descriptor, error) { + if calls.Add(1) > 1 { + return ocispec.Descriptor{}, errors.New("source tag moved") } - } - return l.err -} - -type movingSource struct { - oci.ReadOnlyStore - calls atomic.Int32 -} - -func (s *movingSource) Resolve(ctx context.Context, ref string) (ocispec.Descriptor, error) { - if s.calls.Add(1) > 1 { - return ocispec.Descriptor{}, errors.New("source tag moved") - } - return s.ReadOnlyStore.Resolve(ctx, ref) + return base.Resolve(ctx, ref) + }).AnyTimes() + return store, &calls } func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { @@ -470,11 +488,11 @@ func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { t.Run(tc.name, func(t *testing.T) { layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "target")) require.NoError(t, err) - destination := &landingDestination{Store: layout, land: tc.land, err: tc.err} - moving := &movingSource{ReadOnlyStore: source} + destination, pushes := landingDestination(t, layout, tc.land, tc.err) + moving, resolves := movingSource(t, source) root, err := oci.Copy(t.Context(), oci.Source{Store: moving, Reference: digest.String()}, oci.Destination{Store: destination, Reference: "1.0.0"}) - assert.Equal(t, int32(1), destination.pushes.Load(), "the mutation is attempted exactly once") - assert.Equal(t, int32(1), moving.calls.Load(), "reconciliation must use the selected root") + assert.Equal(t, int32(1), pushes.Load(), "the mutation is attempted exactly once") + assert.Equal(t, int32(1), resolves.Load(), "reconciliation must use the selected root") if tc.wantErr { require.Error(t, err) return From b781e85198dc759a4c266a029a377aaa326eae15 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:14:51 -0600 Subject: [PATCH 19/52] fix(oci): serialize layout tag updates with index commits Co-Authored-By: Claude Opus 5.5 --- pkg/oci/transport_hardening_test.go | 32 +++++++++++++++++++++++++++++ pkg/oci/transport_layout.go | 13 +++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index 30ad2873031..b836cecc0ef 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "encoding/json" + "fmt" "os" "path/filepath" "runtime" + "sync" "testing" "github.com/google/go-containerregistry/pkg/name" @@ -212,3 +214,33 @@ func TestReferrersPresentRequiresEveryReferrerAtTheDestination(t *testing.T) { require.NoError(t, dst.Push(ctx, signature, bytes.NewReader(sigRaw))) assert.True(t, referrersPresent(ctx, src, dst, root)) } + +func TestLayoutConcurrentTagsKeepResolverAndIndexInStep(t *testing.T) { + const writers = 16 + dir := t.TempDir() + layout, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + descriptors := make([]ocispec.Descriptor, writers) + for i := range descriptors { + descriptors[i] = pushManifest(t, layout, fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[],"annotations":{"n":"%d"}}`, i)) + } + + var wg sync.WaitGroup + for i := range descriptors { + wg.Add(1) + go func() { + defer wg.Done() + assert.NoError(t, layout.Tag(t.Context(), descriptors[i], "release")) + }() + } + wg.Wait() + + resolved, err := layout.Resolve(t.Context(), "release") + require.NoError(t, err) + reopened, err := OpenLayout(t.Context(), dir) + require.NoError(t, err) + persisted, err := reopened.Resolve(t.Context(), "release") + require.NoError(t, err) + assert.Equal(t, resolved.Digest, persisted.Digest, "the persisted index names the same manifest the resolver returns") + assert.Equal(t, []string{"release"}, reopened.References()) +} diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 3009c84600c..7b580689c21 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -170,6 +170,11 @@ func (l *Layout) recordAttachment(ctx context.Context, desc ocispec.Descriptor) func (l *Layout) recordRoot(desc ocispec.Descriptor) error { l.mu.Lock() defer l.mu.Unlock() + return l.recordRootLocked(desc) +} + +// recordRootLocked is recordRoot for a caller that already holds l.mu. +func (l *Layout) recordRootLocked(desc ocispec.Descriptor) error { for i := range l.index { if l.index[i].Digest == desc.Digest { return nil @@ -199,16 +204,18 @@ func (l *Layout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]o func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { defer perf.Track(nil, "oci.Layout.Tag")() + // The mutex covers the resolver update and the index commit so concurrent updates to the same + // reference are ordered: the last resolver write is also the last index write. + l.mu.Lock() + defer l.mu.Unlock() if err := l.store.Tag(ctx, desc, reference); err != nil { return layoutError(err) } if reference == desc.Digest.String() { // A digest reference is never named, but the root must still be listed or reopening the // layout loses its media type and children. - return l.recordRoot(desc) + return l.recordRootLocked(desc) } - l.mu.Lock() - defer l.mu.Unlock() // Copy the descriptor and its annotations so root metadata (artifact type, platform, other // annotations) survives reopening without mutating the caller's descriptor. named := desc From c27145a7cf1dbd9ac930559e78006dd4827949a6 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:16:11 -0600 Subject: [PATCH 20/52] fix(artifact): anchor sibling document reads to the checked parent directory Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/produce/documents.go | 34 ++++++--------- .../produce/documents_internal_test.go | 43 ------------------- pkg/artifact/produce/produce_test.go | 16 +++++++ 3 files changed, 29 insertions(+), 64 deletions(-) delete mode 100644 pkg/artifact/produce/documents_internal_test.go diff --git a/pkg/artifact/produce/documents.go b/pkg/artifact/produce/documents.go index bbe82795039..0cd99393f58 100644 --- a/pkg/artifact/produce/documents.go +++ b/pkg/artifact/produce/documents.go @@ -212,34 +212,26 @@ func readSiblingDocument(source string, role artifact.DocumentRole, rel string) if relative, err := filepath.Rel(parent, full); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { return nil, fmt.Errorf("%w: %s file %q resolves outside the source directory", errUtils.ErrArtifactUnsafePath, role, rel) } - info, err := os.Lstat(full) - if err != nil || !info.Mode().IsRegular() { - return nil, fmt.Errorf("%w: %s file %q is not a regular file", errUtils.ErrArtifactPackagingInvalid, role, rel) - } - if info.Size() > artifact.MaxDocumentSize { - return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, rel, info.Size(), artifact.MaxDocumentSize) + // Open the document through a handle on the checked parent directory so a parent that is + // replaced after the containment check above cannot redirect the read outside it. + root, err := os.OpenRoot(parent) + if err != nil { + return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) } - file, err := openChecked(full, info) + defer root.Close() + file, err := root.Open(filepath.FromSlash(rel)) if err != nil { return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) } defer file.Close() - return readBounded(file, rel) -} - -// openChecked opens full and confirms the opened file is the regular file that was inspected, so a -// path replaced by a symlink between the check and the open is never read. -func openChecked(full string, inspected os.FileInfo) (*os.File, error) { - file, err := os.Open(full) - if err != nil { - return nil, err + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%w: %s file %q is not a regular file", errUtils.ErrArtifactPackagingInvalid, role, rel) } - opened, err := file.Stat() - if err != nil || !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) { - _ = file.Close() - return nil, fmt.Errorf("%w: the file changed while it was being read", errUtils.ErrArtifactSourceChanged) + if info.Size() > artifact.MaxDocumentSize { + return nil, fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, rel, info.Size(), artifact.MaxDocumentSize) } - return file, nil + return readBounded(file, rel) } func readBounded(reader io.Reader, name string) ([]byte, error) { diff --git a/pkg/artifact/produce/documents_internal_test.go b/pkg/artifact/produce/documents_internal_test.go deleted file mode 100644 index 03676829072..00000000000 --- a/pkg/artifact/produce/documents_internal_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package produce - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - errUtils "github.com/cloudposse/atmos/errors" - "github.com/cloudposse/atmos/pkg/artifact/fixtures" -) - -func TestOpenChecked_RejectsAFileOtherThanTheInspectedOne(t *testing.T) { - dir := t.TempDir() - fixtures.WriteFile(t, dir, "inspected.md", "inspected", 0o644) - fixtures.WriteFile(t, dir, "replacement.md", "replacement", 0o644) - inspected, err := os.Lstat(filepath.Join(dir, "inspected.md")) - require.NoError(t, err) - - tests := []struct { - name string - path string - wantErr error - }{ - {"the inspected file", filepath.Join(dir, "inspected.md"), nil}, - {"a different file at the opened path", filepath.Join(dir, "replacement.md"), errUtils.ErrArtifactSourceChanged}, - {"a directory at the opened path", dir, errUtils.ErrArtifactSourceChanged}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - file, err := openChecked(tt.path, inspected) - if tt.wantErr != nil { - require.ErrorIs(t, err, tt.wantErr) - assert.Nil(t, file) - return - } - require.NoError(t, err) - require.NoError(t, file.Close()) - }) - } -} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 0be791d5f26..3d87445cf94 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -535,6 +535,22 @@ func TestFilesProducer_SingleFileDocumentEscapingThroughSymlink(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) } +func TestFilesProducer_SingleFileDocumentThroughSymlinkInsideTheSourceDirectory(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "tool.bin", "x", 0o644) + fixtures.MkdirAll(t, dir, "docs") + fixtures.WriteFile(t, dir, "docs/NOTES.md", "inside notes", 0o644) + if err := os.Symlink("docs", filepath.Join(dir, "alias")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + req := request(filepath.Join(dir, "tool.bin"), artifact.PackagingNone) + req.Readme = "alias/NOTES.md" + prepared, _ := prepare(t, req) + require.Len(t, prepared.Record.Documents, 1) + assert.Equal(t, "alias/NOTES.md", prepared.Record.Documents[0].SourcePath) + assert.Equal(t, int64(len("inside notes")), prepared.Record.Documents[0].Size) +} + func TestFilesProducer_InvalidPackagesFailBeforeAnythingIsKept(t *testing.T) { tests := []struct { name string From 2de4d418cbe1b2e7d867eac18169288d56854fe0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:19:08 -0600 Subject: [PATCH 21/52] fix(oci): keep streaming layers uploadable through the legacy graph adapter Spool unconsumed stream layers into replayable files before building the descriptor graph and remove the spool when the write finishes. Co-Authored-By: Claude Opus 5.5 --- pkg/oci/client.go | 2 + pkg/oci/client_test.go | 29 ++++++++++ pkg/oci/transport_hardening_test.go | 35 ++++++++++++ pkg/oci/transport_legacy.go | 86 ++++++++++++++++++++++++++++- 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/pkg/oci/client.go b/pkg/oci/client.go index f218b905e1c..063fe854882 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -148,6 +148,7 @@ func (c *Client) WriteImage(ctx context.Context, ref name.Reference, img v1.Imag if err != nil { return err } + defer graph.close() _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) return err } @@ -164,6 +165,7 @@ func (c *Client) WriteIndex(ctx context.Context, ref name.Reference, index v1.Im if err != nil { return err } + defer graph.close() _, err = Copy(ctx, Source{Store: graph, Reference: legacyRoot}, Destination{Store: store, Reference: ref.Identifier()}, WithoutSubjectTraversal()) return err } diff --git a/pkg/oci/client_test.go b/pkg/oci/client_test.go index 18afda53e18..96e70e030cd 100644 --- a/pkg/oci/client_test.go +++ b/pkg/oci/client_test.go @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/validate" "github.com/stretchr/testify/require" @@ -292,3 +293,31 @@ func TestExportIndexRejectsRegistryFailureAndOrdinaryImage(t *testing.T) { require.NoDirExists(t, destination, "failed export must not publish a local layout") } } + +func TestWriteImageUploadsAStreamingLayer(t *testing.T) { + client, ref, _ := credentialRegistry(t, "stream-test") + payload := strings.Repeat("streamed layer content ", 4096) + streamed := stream.NewLayer(io.NopCloser(strings.NewReader(payload))) + image, err := mutate.AppendLayers(empty.Image, streamed) + require.NoError(t, err) + + require.NoError(t, client.WriteImage(t.Context(), ref, image)) + + pulled, err := client.Get(t.Context(), ref) + require.NoError(t, err) + img, err := pulled.Image() + require.NoError(t, err) + layers, err := img.Layers() + require.NoError(t, err) + require.Len(t, layers, 1) + reader, err := layers[0].Compressed() + require.NoError(t, err) + defer reader.Close() + stored, err := io.ReadAll(reader) + require.NoError(t, err) + want, err := streamed.Digest() + require.NoError(t, err) + got, _, err := v1.SHA256(strings.NewReader(string(stored))) + require.NoError(t, err) + require.Equal(t, want, got, "the uploaded layer is the streamed content") +} diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index b836cecc0ef..daf1398965d 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -5,17 +5,21 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path/filepath" "runtime" + "strings" "sync" "testing" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" @@ -244,3 +248,34 @@ func TestLayoutConcurrentTagsKeepResolverAndIndexInStep(t *testing.T) { assert.Equal(t, resolved.Digest, persisted.Digest, "the persisted index names the same manifest the resolver returns") assert.Equal(t, []string{"release"}, reopened.References()) } + +func TestLegacyGraphSpoolsStreamingLayersAndCleansUp(t *testing.T) { + payload := "streamed layer content" + image, err := mutate.AppendLayers(empty.Image, stream.NewLayer(io.NopCloser(strings.NewReader(payload)))) + require.NoError(t, err) + + graph, err := legacyImageGraph(t.Context(), image) + require.NoError(t, err) + manifest, err := image.Manifest() + require.NoError(t, err) + require.Len(t, manifest.Layers, 1) + layer := ocispec.Descriptor{Digest: digest.Digest(manifest.Layers[0].Digest.String())} + var reads [][]byte + for range 2 { + reader, err := graph.Fetch(t.Context(), layer) + require.NoError(t, err) + content, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + reads = append(reads, content) + } + assert.NotEmpty(t, reads[0], "the spooled layer holds the streamed bytes") + assert.Equal(t, reads[0], reads[1], "a streamed layer can be read again after it was spooled") + assert.Equal(t, layer.Digest, digest.FromBytes(reads[0]), "the spooled bytes match the manifest digest") + + spoolDir := graph.spoolDir + require.DirExists(t, spoolDir) + graph.close() + assert.NoDirExists(t, spoolDir) + graph.close() +} diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go index 363c874e580..84e1cb5ed29 100644 --- a/pkg/oci/transport_legacy.go +++ b/pkg/oci/transport_legacy.go @@ -8,8 +8,10 @@ import ( "errors" "fmt" "io" + "os" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -32,6 +34,11 @@ type legacyGraph struct { // indexes holds the descriptors of indexes whose children were already added, so a subgraph // shared by several parents is traversed once. indexes map[digest.Digest]ocispec.Descriptor + + // spooled maps a streaming layer's digest to the replayable file that holds its bytes, and + // spoolDir is the private directory that owns those files until close. + spooled map[digest.Digest]string + spoolDir string } var _ ReadOnlyStore = (*legacyGraph)(nil) @@ -40,7 +47,71 @@ func newLegacyGraph() *legacyGraph { return &legacyGraph{ blobs: make(map[digest.Digest]func() (io.ReadCloser, error)), indexes: make(map[digest.Digest]ocispec.Descriptor), + spooled: make(map[digest.Digest]string), + } +} + +// close removes the spooled streaming layers. It is safe to call more than once. +func (g *legacyGraph) close() { + if g == nil || g.spoolDir == "" { + return + } + _ = os.RemoveAll(g.spoolDir) + g.spoolDir = "" + g.spooled = map[digest.Digest]string{} +} + +// spoolStreamingLayers consumes every layer that is a stream not yet read (stream.NewLayer) into a +// replayable file. A stream's digest, size, and the image's manifest exist only after it was read +// once, and it can be read only once, so the bytes must be kept for the upload. +func (g *legacyGraph) spoolStreamingLayers(ctx context.Context, image v1.Image) error { + layers, err := image.Layers() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + for _, layer := range layers { + if _, err := layer.Digest(); !errors.Is(err, stream.ErrNotComputed) { + continue + } + if err := g.spoolLayer(ctx, layer); err != nil { + return err + } + } + return nil +} + +func (g *legacyGraph) spoolLayer(ctx context.Context, layer v1.Layer) error { + if err := ctx.Err(); err != nil { + return err + } + if g.spoolDir == "" { + dir, err := os.MkdirTemp("", "atmos-oci-spool-") + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.spoolDir = dir + } + reader, err := layer.Compressed() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) } + file, err := os.CreateTemp(g.spoolDir, "layer-*") + if err != nil { + _ = reader.Close() + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + _, copyErr := io.Copy(file, reader) + // Closing the stream is what computes its digest and size. + closeErr := errors.Join(reader.Close(), file.Close()) + if err := errors.Join(copyErr, closeErr); err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + hash, err := layer.Digest() + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + g.spooled[digest.Digest(hash.String())] = file.Name() + return nil } func legacyImageGraph(ctx context.Context, image v1.Image) (*legacyGraph, error) { @@ -48,8 +119,9 @@ func legacyImageGraph(ctx context.Context, image v1.Image) (*legacyGraph, error) return nil, err } graph := newLegacyGraph() - root, err := graph.addImage(image) + root, err := graph.addImage(ctx, image) if err != nil { + graph.close() return nil, err } graph.root = root @@ -60,6 +132,7 @@ func legacyIndexGraph(ctx context.Context, index v1.ImageIndex) (*legacyGraph, e graph := newLegacyGraph() root, err := graph.addIndex(ctx, index) if err != nil { + graph.close() return nil, err } graph.root = root @@ -72,7 +145,10 @@ func (g *legacyGraph) addBytes(mediaType string, raw []byte) ocispec.Descriptor return desc } -func (g *legacyGraph) addImage(image v1.Image) (ocispec.Descriptor, error) { +func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Descriptor, error) { + if err := g.spoolStreamingLayers(ctx, image); err != nil { + return ocispec.Descriptor{}, err + } raw, err := image.RawManifest() if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) @@ -94,6 +170,10 @@ func (g *legacyGraph) addImage(image v1.Image) (ocispec.Descriptor, error) { } for i := range manifest.Layers { hash := manifest.Layers[i].Digest + if path, found := g.spooled[digest.Digest(hash.String())]; found { + g.blobs[digest.Digest(hash.String())] = func() (io.ReadCloser, error) { return os.Open(path) } + continue + } g.blobs[digest.Digest(hash.String())] = func() (io.ReadCloser, error) { layer, err := image.LayerByDigest(hash) if err != nil { @@ -158,7 +238,7 @@ func (g *legacyGraph) addChild(ctx context.Context, index v1.ImageIndex, child * if err != nil { return errors.Join(errUtils.ErrOCIIntegrity, err) } - _, err = g.addImage(nested) + _, err = g.addImage(ctx, nested) return err default: return fmt.Errorf("%w: unsupported child media type %q", errUtils.ErrOCIIntegrity, child.MediaType) From f2a4cc7f2ccaf1f2f4df0b27e8adc9125c49ef98 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:22:56 -0600 Subject: [PATCH 22/52] fix(oci): require referrer registration before accepting an attachment write A manifest that exists at the destination no longer converts a failed attachment write to success unless it is listed under its subject. Co-Authored-By: Claude Opus 5.5 --- pkg/oci/transport_behavior_test.go | 59 ++++++++++++++++++++++++++++++ pkg/oci/transport_copy.go | 36 +++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go index 53bfa7ee0c0..d50eee7c85e 100644 --- a/pkg/oci/transport_behavior_test.go +++ b/pkg/oci/transport_behavior_test.go @@ -503,6 +503,65 @@ func TestCopyReconcilesAmbiguousFailuresAgainstTheDestination(t *testing.T) { } } +func TestCopyReconcilesAnAttachmentOnlyOnceItIsRegisteredUnderItsSubject(t *testing.T) { + ctx := t.Context() + source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) + require.NoError(t, err) + config := oci.Descriptor(ocispec.MediaTypeEmptyJSON, []byte("{}")) + require.NoError(t, source.Push(ctx, config, strings.NewReader("{}"))) + subject := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("subject manifest")) + raw, err := json.Marshal(ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: "application/vnd.example.signature", + Config: config, + Layers: []ocispec.Descriptor{}, + Subject: &subject, + }) + require.NoError(t, err) + attachment := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, source.Push(ctx, attachment, strings.NewReader(string(raw)))) + require.NoError(t, source.Tag(ctx, attachment, "sig")) + + for _, tc := range []struct { + name string + referrers []ocispec.Descriptor + wantErr bool + }{ + {"registered under the subject", []ocispec.Descriptor{attachment}, false}, + {"manifest exists but is not registered", nil, true}, + {"only another attachment is registered", []ocispec.Descriptor{oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("other"))}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + destination := oci.NewMockRepositoryStore(gomock.NewController(t)) + destination.EXPECT().Exists(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() + destination.EXPECT().Push(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ ocispec.Descriptor, content io.Reader) error { + _, err := io.Copy(io.Discard, content) + return err + }, + ).AnyTimes() + destination.EXPECT().PushReference(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("connection reset by peer")).AnyTimes() + destination.EXPECT().Resolve(gomock.Any(), "1.0.0").Return(attachment, nil).AnyTimes() + destination.EXPECT().Fetch(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, ocispec.Descriptor) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(string(raw))), nil + }, + ).AnyTimes() + destination.EXPECT().Predecessors(gomock.Any(), subject).Return(tc.referrers, nil).AnyTimes() + + got, err := oci.Copy(ctx, oci.Source{Store: source, Reference: "sig"}, oci.Destination{Store: destination, Reference: "1.0.0"}, oci.WithoutSubjectTraversal()) + + if tc.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + return + } + require.NoError(t, err) + assert.Equal(t, attachment.Digest, got.Digest) + }) + } +} + func errcodeResponse(status int) error { return &errcode.ErrorResponse{Method: http.MethodPut, StatusCode: status} } diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index d75bcc38938..9be153be0eb 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "slices" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -229,9 +230,40 @@ func reconcile(ctx context.Context, cause error, want ocispec.Descriptor, dst De if err != nil || have.Digest != want.Digest { return ocispec.Descriptor{}, failure } + // An attachment is complete only once it is registered under its subject; the manifest alone + // existing at the destination is not enough, so the original failure stands. + if !attachmentRegistered(ctx, dst.Store, want) { + return ocispec.Descriptor{}, failure + } return want, nil } +// attachmentRegistered reports whether the manifest or index at desc is complete at the store: a +// manifest without a subject always is, and one with a subject must be listed among the subject's +// referrers. Any lookup failure, or a store that cannot list referrers, counts as not registered so +// a failed write is never reported as a success on weaker evidence. +func attachmentRegistered(ctx context.Context, store Store, desc ocispec.Descriptor) bool { + if desc.MediaType != ocispec.MediaTypeImageManifest && desc.MediaType != ocispec.MediaTypeImageIndex { + return true + } + subject, err := readSubject(ctx, store, &desc) + if err != nil { + return false + } + if subject == nil { + return true + } + graph, ok := store.(GraphStore) + if !ok { + return false + } + referrers, err := graph.Predecessors(ctx, *subject) + if err != nil { + return false + } + return slices.ContainsFunc(referrers, func(found ocispec.Descriptor) bool { return found.Digest == desc.Digest }) +} + func classifyCopyError(err error) error { if errors.Is(err, errUtils.ErrOCIRegistry) || errors.Is(err, errUtils.ErrOCILayout) { return err @@ -300,7 +332,7 @@ func (g guardedStore) Push(ctx context.Context, expected ocispec.Descriptor, con // The write may have landed even though it was reported as failed. Look at the // destination before deciding; never write again. if ambiguousWrite(ctx, err) { - if exists, existsErr := g.Store.Exists(ctx, expected); existsErr == nil && exists { + if exists, existsErr := g.Store.Exists(ctx, expected); existsErr == nil && exists && attachmentRegistered(ctx, g.Store, expected) { return nil } } @@ -317,7 +349,7 @@ func (g guardedStore) PushReference(ctx context.Context, expected ocispec.Descri if err == nil || errors.Is(err, errdef.ErrAlreadyExists) || !ambiguousWrite(ctx, err) { return err } - if resolved, resolveErr := g.Resolve(ctx, reference); resolveErr == nil && resolved.Digest == expected.Digest { + if resolved, resolveErr := g.Resolve(ctx, reference); resolveErr == nil && resolved.Digest == expected.Digest && attachmentRegistered(ctx, g.Store, expected) { return nil } return err From 57d9fd0ee4f1a01e164d55ffbd0afb4ec5090afd Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:48:35 -0600 Subject: [PATCH 23/52] refactor(artifact): share the signer format check across repositories Move the check that refuses a signer before it runs, when the repository can store none of its formats, from the conformance fixture to Capabilities.CheckSignerFormats. Every repository backend can then enforce the contract that the conformance suite now asserts with a generated signer mock. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/artifacttest/sign.go | 22 +--------------------- pkg/artifact/capabilities.go | 18 ++++++++++++++++++ pkg/artifact/producers_test.go | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index 5418b926688..28ee94cda27 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -88,7 +87,7 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return nil, err } r.mu.Unlock() - if err := r.checkSignerFormats(signer); err != nil { + if err := r.caps.CheckSignerFormats(r.opts.Alias, signer); err != nil { return nil, err } payload, err := signer.Sign(ctx, subject) @@ -112,25 +111,6 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return []artifact.SignatureEvidence{evidence}, nil } -// checkSignerFormats refuses a signer before it is invoked when none of its formats can be stored. -func (r *Repository) checkSignerFormats(signer artifact.Signer) error { - formats := signer.Formats() - for _, format := range formats { - if r.caps.HasEvidenceFormat(format) { - return nil - } - } - return artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "evidence/"+joinFormats(formats), "the repository cannot store this evidence format") -} - -func joinFormats(formats []artifact.EvidenceFormat) string { - names := make([]string, 0, len(formats)) - for _, format := range formats { - names = append(names, string(format)) - } - return strings.Join(names, ",") -} - // checkSubject requires the subject to be a stored record; callers hold r.mu. func (r *Repository) checkSubject(subject artifact.SignatureSubject) error { if !r.caps.HasSubjectKind(subject.Kind) { diff --git a/pkg/artifact/capabilities.go b/pkg/artifact/capabilities.go index ddd00f13639..98260ff6b30 100644 --- a/pkg/artifact/capabilities.go +++ b/pkg/artifact/capabilities.go @@ -2,6 +2,7 @@ package artifact import ( "slices" + "strings" "github.com/cloudposse/atmos/pkg/perf" ) @@ -74,6 +75,23 @@ func (c *Capabilities) HasEvidenceFormat(f EvidenceFormat) bool { return slices.Contains(c.EvidenceFormats, f) } +// CheckSignerFormats refuses a signer before it is invoked when the repository can store none of +// its evidence formats, so an unusable signer never signs anything. The repository argument +// names the repository in the capability error. +func (c *Capabilities) CheckSignerFormats(repository string, signer Signer) error { + defer perf.Track(nil, "artifact.Capabilities.CheckSignerFormats")() + + formats := signer.Formats() + names := make([]string, 0, len(formats)) + for _, format := range formats { + if c.HasEvidenceFormat(format) { + return nil + } + names = append(names, string(format)) + } + return NewUnsupportedCapability(repository, "sign", "evidence/"+strings.Join(names, ","), "the repository cannot store this evidence format") +} + // HasSubjectKind reports whether the repository supports the signature subject kind. func (c *Capabilities) HasSubjectKind(k SubjectKind) bool { defer perf.Track(nil, "artifact.Capabilities.HasSubjectKind")() diff --git a/pkg/artifact/producers_test.go b/pkg/artifact/producers_test.go index 1723eb49e1d..fef643c18cc 100644 --- a/pkg/artifact/producers_test.go +++ b/pkg/artifact/producers_test.go @@ -36,3 +36,18 @@ func TestProducerRegistry(t *testing.T) { assert.False(t, ok) assert.Equal(t, []string{KindFiles, "scaffold"}, registry.Kinds()) } + +// A signer is refused before it runs when the repository can store none of its formats. +func TestCapabilitiesCheckSignerFormats(t *testing.T) { + caps := &Capabilities{EvidenceFormats: []EvidenceFormat{EvidenceSigstoreBundle}} + + usable := NewMockSigner(gomock.NewController(t)) + usable.EXPECT().Formats().Return([]EvidenceFormat{"other", EvidenceSigstoreBundle}) + require.NoError(t, caps.CheckSignerFormats("repo", usable), "one storable format is enough") + + unusable := NewMockSigner(gomock.NewController(t)) + unusable.EXPECT().Formats().Return([]EvidenceFormat{"a", "b"}) + err := caps.CheckSignerFormats("repo", unusable) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Contains(t, err.Error(), "evidence/a,b") +} From e4e1936f00287126c9f458efa53c0c81efe2cde0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 12:01:27 -0600 Subject: [PATCH 24/52] refactor(artifact): keep document and subject scanning within lint limits Code scanning flagged readSiblingDocument (cyclomatic 11) and readSubject (cyclomatic 12), plus an unused gocritic nolint directive. The parent containment check moves into checkedSiblingParent, and the manifest member loop moves into scanForSubject. The directive is removed. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/produce/documents.go | 32 ++++++++++++++++++++---------- pkg/oci/transport_behavior_test.go | 1 - pkg/oci/transport_subject.go | 6 ++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/artifact/produce/documents.go b/pkg/artifact/produce/documents.go index 0cd99393f58..a1dc200d877 100644 --- a/pkg/artifact/produce/documents.go +++ b/pkg/artifact/produce/documents.go @@ -200,20 +200,12 @@ func readStagedDocument(snap *snapshot.Snapshot, entry *artifact.Entry) ([]byte, // readSiblingDocument reads an explicit document next to a single-file source, rejecting anything // that is not a regular file contained in the source's parent directory. func readSiblingDocument(source string, role artifact.DocumentRole, rel string) ([]byte, error) { - resolvedSource, err := filepath.EvalSymlinks(source) - if err != nil { - return nil, fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) - } - parent := filepath.Dir(resolvedSource) - full, err := filepath.EvalSymlinks(filepath.Join(parent, filepath.FromSlash(rel))) + parent, err := checkedSiblingParent(source, role, rel) if err != nil { - return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) - } - if relative, err := filepath.Rel(parent, full); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return nil, fmt.Errorf("%w: %s file %q resolves outside the source directory", errUtils.ErrArtifactUnsafePath, role, rel) + return nil, err } // Open the document through a handle on the checked parent directory so a parent that is - // replaced after the containment check above cannot redirect the read outside it. + // replaced after the containment check cannot redirect the read outside it. root, err := os.OpenRoot(parent) if err != nil { return nil, fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) @@ -234,6 +226,24 @@ func readSiblingDocument(source string, role artifact.DocumentRole, rel string) return readBounded(file, rel) } +// checkedSiblingParent resolves the source's parent directory and verifies that the document +// rel resolves inside it. +func checkedSiblingParent(source string, role artifact.DocumentRole, rel string) (string, error) { + resolvedSource, err := filepath.EvalSymlinks(source) + if err != nil { + return "", fmt.Errorf("%w: source %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + parent := filepath.Dir(resolvedSource) + full, err := filepath.EvalSymlinks(filepath.Join(parent, filepath.FromSlash(rel))) + if err != nil { + return "", fmt.Errorf("%w: %s file %q: %w", errUtils.ErrArtifactPackagingInvalid, role, rel, err) + } + if relative, err := filepath.Rel(parent, full); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%w: %s file %q resolves outside the source directory", errUtils.ErrArtifactUnsafePath, role, rel) + } + return parent, nil +} + func readBounded(reader io.Reader, name string) ([]byte, error) { data, err := artifact.ReadBounded(reader, artifact.MaxDocumentSize) if err != nil { diff --git a/pkg/oci/transport_behavior_test.go b/pkg/oci/transport_behavior_test.go index d50eee7c85e..32f1614094c 100644 --- a/pkg/oci/transport_behavior_test.go +++ b/pkg/oci/transport_behavior_test.go @@ -1,4 +1,3 @@ -//nolint:gocritic // Descriptor-by-value parameters mirror the ORAS content contracts. package oci_test import ( diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go index b29d8eef0a0..b7926854390 100644 --- a/pkg/oci/transport_subject.go +++ b/pkg/oci/transport_subject.go @@ -37,6 +37,12 @@ func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) ( if err := expectByte(scanner, '{'); err != nil { return nil, integrityError(err) } + return scanForSubject(scanner) +} + +// scanForSubject reads the members of a manifest object, after its opening brace, until it finds +// the subject descriptor or the object ends. +func scanForSubject(scanner *bufio.Reader) (*ocispec.Descriptor, error) { for { next, err := nextNonSpace(scanner) if err != nil { From 09e499be57839e88b4006e5c54d9ecba2c76a46b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 07:00:44 -0600 Subject: [PATCH 25/52] fix(artifacts): address transport service review findings Normalize the materialize destination, honor cancellation during file copies, make the snapshot containment root absolute, reject required signatures with no trust configured, classify offline trust failures from the cause chain, copy evidence bytes at repository boundaries, wrap staged and tree open errors with artifact sentinels, and use generated mocks and static errors in tests. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/artifacttest/sign.go | 12 ++- pkg/artifact/artifacttest/write.go | 4 +- pkg/artifact/fixtures/fixtures.go | 2 +- pkg/artifact/materialize/extract.go | 22 +++++- pkg/artifact/materialize/materialize_test.go | 79 ++++++++++++++++++++ pkg/artifact/materialize/replace.go | 2 + pkg/artifact/produce/repack.go | 2 +- pkg/artifact/service/consume_test.go | 18 +++-- pkg/artifact/service/delegate_test.go | 6 +- pkg/artifact/service/interrupt_test.go | 33 ++++---- pkg/artifact/service/verify.go | 3 + pkg/artifact/service/verify_test.go | 15 ++++ pkg/artifact/snapshot/snapshot_test.go | 18 +++++ pkg/artifact/snapshot/walk.go | 8 +- pkg/oci/signature/plugin_error_test.go | 8 ++ pkg/oci/signature/verify.go | 2 +- pkg/oci/transport_hardening_test.go | 30 ++++++++ 17 files changed, 228 insertions(+), 36 deletions(-) diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index 28ee94cda27..05620f109d6 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -1,6 +1,7 @@ package artifacttest import ( + "bytes" "context" "encoding/json" "fmt" @@ -107,7 +108,7 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject } r.mu.Lock() defer r.mu.Unlock() - r.evidence[subject.Digest] = append(r.evidence[subject.Digest], artifact.EvidenceBlob{Evidence: evidence, Data: payload.Data}) + r.evidence[subject.Digest] = append(r.evidence[subject.Digest], artifact.EvidenceBlob{Evidence: evidence, Data: bytes.Clone(payload.Data)}) return []artifact.SignatureEvidence{evidence}, nil } @@ -203,7 +204,12 @@ func (r *Repository) ReadEvidence(_ context.Context, pinned *artifact.PinnedRefe r.mu.Lock() defer r.mu.Unlock() r.count("ReadEvidence") - return append([]artifact.EvidenceBlob(nil), r.evidence[pinned.RecordDigest]...), nil + held := r.evidence[pinned.RecordDigest] + blobs := make([]artifact.EvidenceBlob, len(held)) + for index := range held { + blobs[index] = artifact.EvidenceBlob{Evidence: held[index].Evidence, Data: bytes.Clone(held[index].Data)} + } + return blobs, nil } // WriteEvidence implements [artifact.EvidenceWriter]. @@ -218,7 +224,7 @@ func (r *Repository) WriteEvidence(_ context.Context, pinned *artifact.PinnedRef } stored := make([]artifact.SignatureEvidence, 0, len(evidence)) for index := range evidence { - r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], evidence[index]) + r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], artifact.EvidenceBlob{Evidence: evidence[index].Evidence, Data: bytes.Clone(evidence[index].Data)}) stored = append(stored, evidence[index].Evidence) } return stored, nil diff --git a/pkg/artifact/artifacttest/write.go b/pkg/artifact/artifacttest/write.go index a1a6f175a31..96094f43dff 100644 --- a/pkg/artifact/artifacttest/write.go +++ b/pkg/artifact/artifacttest/write.go @@ -116,7 +116,7 @@ func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedA staged := &prepared.Blobs[index] reader, err := staged.Open(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: open staged blob %q: %w", errUtils.ErrArtifactPackagingInvalid, staged.Path, err) } if index == 0 { if err := r.trip(FailDuringPayload); err != nil { @@ -127,7 +127,7 @@ func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedA data, readErr := io.ReadAll(reader) closeErr := reader.Close() if err := errors.Join(readErr, closeErr); err != nil { - return nil, err + return nil, fmt.Errorf("%w: read staged blob %q: %w", errUtils.ErrArtifactPackagingInvalid, staged.Path, err) } if int64(len(data)) != staged.Descriptor.Size || digest.FromBytes(data) != staged.Descriptor.Digest { return nil, fmt.Errorf("%w: staged blob %q does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, staged.Path) diff --git a/pkg/artifact/fixtures/fixtures.go b/pkg/artifact/fixtures/fixtures.go index e9a1f4b7794..9b960e4973c 100644 --- a/pkg/artifact/fixtures/fixtures.go +++ b/pkg/artifact/fixtures/fixtures.go @@ -60,7 +60,7 @@ func RequirePOSIXModes(t testing.TB) { t.Helper() if !POSIXModes() { - t.Skip("this platform does not honor Unix permission bits") + t.Skipf("this platform does not honor Unix permission bits") } } diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go index 48c55f2b948..6b99b4a976a 100644 --- a/pkg/artifact/materialize/extract.go +++ b/pkg/artifact/materialize/extract.go @@ -3,6 +3,7 @@ package materialize import ( "context" "fmt" + "io" "os" "sort" @@ -127,7 +128,7 @@ func fillFiles(ctx context.Context, t *tree, content *artifact.Content) error { if err := ctx.Err(); err != nil { return err } - if err := fillInventoryEntry(t, t.inventory[entryPath], streams); err != nil { + if err := fillInventoryEntry(ctx, t, t.inventory[entryPath], streams); err != nil { return err } delete(streams, entryPath) @@ -159,7 +160,22 @@ func payloadStreams(t *tree, content *artifact.Content) (map[string]*artifact.Co return streams, nil } -func fillInventoryEntry(t *tree, entry *artifact.Entry, streams map[string]*artifact.ContentStream) error { +// contextReader stops an in-progress copy as soon as the context is canceled. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (c contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "materialize.contextReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.reader.Read(p) +} + +func fillInventoryEntry(ctx context.Context, t *tree, entry *artifact.Entry, streams map[string]*artifact.ContentStream) error { if _, err := t.entryFor(entry.Path); err != nil { return err } @@ -173,6 +189,6 @@ func fillInventoryEntry(t *tree, entry *artifact.Entry, streams map[string]*arti if !ok || stream.Reader == nil { return fmt.Errorf("%w: payload for %q is missing", errUtils.ErrArtifactIntegrityFailed, entry.Path) } - return t.writeFile(entry, stream.Reader) + return t.writeFile(entry, contextReader{ctx: ctx, reader: stream.Reader}) } } diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index ccfde2b2651..d7bbc85077a 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -616,3 +616,82 @@ func TestMaterialize_DestinationSwappedForASymlinkNeverRedirectsRemoval(t *testi _, newErr := os.Stat(filepath.Join(outside, "new.txt")) assert.True(t, os.IsNotExist(newErr), "staged content must not land in the link target") } + +// cancelOnReadReader cancels the context on its first read and then yields one byte per read. +type cancelOnReadReader struct { + cancel context.CancelFunc + data []byte +} + +func (c *cancelOnReadReader) Read(p []byte) (int, error) { + c.cancel() + if len(c.data) == 0 { + return 0, io.EOF + } + p[0] = c.data[0] + c.data = c.data[1:] + return 1, nil +} + +func (c *cancelOnReadReader) Close() error { return nil } + +func TestMaterialize_CancellationStopsInProgressFileCopy(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "big.txt", "0123456789", 0o644) + content := contentOf(t, prepared(t, src, artifact.PackagingNone)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + replaced := false + for index := range content.Streams { + if content.Streams[index].Role != artifact.BlobPayload { + continue + } + _ = content.Streams[index].Reader.Close() + content.Streams[index].Reader = &cancelOnReadReader{cancel: cancel, data: []byte("0123456789")} + replaced = true + } + require.True(t, replaced, "the fixture must have a payload stream") + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) +} + +func TestMaterialize_TrailingSeparatorDestination(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + + t.Run("absent destination", func(t *testing.T) { + root := t.TempDir() + out := filepath.Join(root, "out") + string(filepath.Separator) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, prepared(t, src, artifact.PackagingTgz)), out)) + data, err := os.ReadFile(filepath.Join(root, "out", "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, filepath.Join(root, "out")) + items, err := os.ReadDir(root) + require.NoError(t, err) + require.Len(t, items, 1, "staging must be created beside the destination, not inside it") + }) + + t.Run("existing destination", func(t *testing.T) { + root := t.TempDir() + existing := filepath.Join(root, "out") + fixtures.WriteFile(t, existing, "stale.txt", "stale", 0o644) + out := existing + string(filepath.Separator) + err := materialize.Materialize(context.Background(), contentOf(t, prepared(t, src, artifact.PackagingTgz)), out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + return + } + require.NoError(t, err) + items, err := os.ReadDir(existing) + require.NoError(t, err) + require.Len(t, items, 1) + assert.Equal(t, "new.txt", items[0].Name()) + assertNoScratch(t, existing) + }) +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index d798a663ed2..0c686dab01a 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -160,6 +160,8 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if dest == "" { return fmt.Errorf("%w: the destination is empty", errUtils.ErrArtifactConfigInvalid) } + // Normalize first so a trailing separator ("out/") cannot make the parent the destination itself. + dest = filepath.Clean(dest) parent := filepath.Dir(dest) if err := os.MkdirAll(parent, parentDirMode); err != nil { return fmt.Errorf("%w: create %q: %w", errUtils.ErrArtifactPackagingInvalid, parent, err) diff --git a/pkg/artifact/produce/repack.go b/pkg/artifact/produce/repack.go index ed199f6265b..be3ae022df3 100644 --- a/pkg/artifact/produce/repack.go +++ b/pkg/artifact/produce/repack.go @@ -83,7 +83,7 @@ func treeOpener(record *artifact.Record, tree string) opener { } file, err := os.Open(filepath.Join(tree, filepath.FromSlash(entryPath))) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: open %q: %w", errUtils.ErrArtifactPackagingInvalid, entryPath, err) } return newVerifyingReader(file, entry.Digest, entry.Size), nil } diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index d9c6c50361c..6fd7a350e26 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -2,7 +2,6 @@ package service_test import ( "context" - "fmt" "os" "path/filepath" "runtime" @@ -364,13 +363,18 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { for _, alias := range []string{"a", "b", "c"} { e.static.Repositories[alias] = gatedList(t, e.repos[alias], gate) } - done := make(chan error, 1) + type listOutcome struct { + count int + err error + } + done := make(chan listOutcome, 1) go func() { result, err := e.svc.List(context.Background(), artifact.ListRequest{}) - if err == nil && len(result.Artifacts) != 3 { - err = fmt.Errorf("got %d artifacts", len(result.Artifacts)) + outcome := listOutcome{err: err} + if err == nil { + outcome.count = len(result.Artifacts) } - done <- err + done <- outcome }() select { case <-gate.entered: @@ -383,7 +387,9 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { case <-time.After(200 * time.Millisecond): } close(gate.release) - require.NoError(t, <-done) + outcome := <-done + require.NoError(t, outcome.err) + assert.Equal(t, 3, outcome.count) assert.Equal(t, int32(1), gate.peak.Load()) } diff --git a/pkg/artifact/service/delegate_test.go b/pkg/artifact/service/delegate_test.go index 5b0f8f7ae17..f0e09fe5195 100644 --- a/pkg/artifact/service/delegate_test.go +++ b/pkg/artifact/service/delegate_test.go @@ -16,6 +16,7 @@ type repoHooks struct { inspect func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) read func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) list func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) + lock func(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) } // delegatingRepository returns a generated MockRepository whose methods forward to base, except @@ -34,9 +35,12 @@ func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHook if hooks.list == nil { hooks.list = base.List } + if hooks.lock == nil { + hooks.lock = base.Lock + } mock := artifact.NewMockRepository(gomock.NewController(t)) mock.EXPECT().Capabilities().DoAndReturn(base.Capabilities).AnyTimes() - mock.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(base.Lock).AnyTimes() + mock.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(hooks.lock).AnyTimes() mock.EXPECT().Resolve(gomock.Any(), gomock.Any()).DoAndReturn(hooks.resolve).AnyTimes() mock.EXPECT().Inspect(gomock.Any(), gomock.Any()).DoAndReturn(hooks.inspect).AnyTimes() mock.EXPECT().OpenDocument(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.OpenDocument).AnyTimes() diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go index 9b90e59f675..0e5f3f2bcf6 100644 --- a/pkg/artifact/service/interrupt_test.go +++ b/pkg/artifact/service/interrupt_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -253,29 +254,27 @@ func TestPublish_TwoWritersWithIdenticalIntentCreateOnce(t *testing.T) { } // unlockedRepository hides the writer lock so the repository's own conditional guard is exercised. -type unlockedRepository struct { - *artifacttest.Repository +// It forwards every other operation to base through the generated repository mock. +func unlockedRepository(t *testing.T, base artifact.Repository) *artifact.MockRepository { + t.Helper() + return delegatingRepository(t, base, repoHooks{lock: func(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + handle := artifact.NewMockLockHandle(gomock.NewController(t)) + handle.EXPECT().Resource().Return(request.Resource).AnyTimes() + handle.EXPECT().Owner().Return("noop").AnyTimes() + handle.EXPECT().Scope().Return(artifact.LockScopeHost).AnyTimes() + handle.EXPECT().Token().Return("noop").AnyTimes() + handle.EXPECT().ExpiresAt().Return(time.Time{}, false).AnyTimes() + handle.EXPECT().Renew(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + handle.EXPECT().Release(gomock.Any()).Return(nil).AnyTimes() + return handle, nil + }}) } -func (u *unlockedRepository) Lock(_ context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { - return noopLock{resource: request.Resource}, nil -} - -type noopLock struct{ resource string } - -func (n noopLock) Resource() string { return n.resource } -func (n noopLock) Owner() string { return "noop" } -func (n noopLock) Scope() artifact.LockScope { return artifact.LockScopeHost } -func (n noopLock) Token() string { return "noop" } -func (n noopLock) ExpiresAt() (time.Time, bool) { return time.Time{}, false } -func (n noopLock) Renew(context.Context, time.Duration) error { return nil } -func (n noopLock) Release(context.Context) error { return nil } - func TestPublish_ConditionalGuardRejectsRacingWritersEvenWithoutAWriterLock(t *testing.T) { const writers = 6 e := newEnv(t, envConfig{}) bare := artifacttest.New(artifacttest.Options{Alias: "bare"}) - e.static.Repositories["bare"] = &unlockedRepository{Repository: bare} + e.static.Repositories["bare"] = unlockedRepository(t, bare) e.static.Info["bare"] = service.RepositoryInfo{Alias: "bare", Capabilities: bare.Capabilities(), Packaging: artifact.PackagingNone} sources := make([]string, writers) diff --git a/pkg/artifact/service/verify.go b/pkg/artifact/service/verify.go index 8330d4df259..8a059b54fd5 100644 --- a/pkg/artifact/service/verify.go +++ b/pkg/artifact/service/verify.go @@ -152,5 +152,8 @@ func gate(ctx context.Context, alias string, repo artifact.Repository, inspectio if err != nil { return nil, err } + if merged.RequireSignature && !merged.HasTrustConstraints() { + return &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationUnchecked}, fmt.Errorf("%w: repository %q has no trusted identities or keys configured, so nothing can be verified; configure artifacts.verification", errUtils.ErrArtifactConfigInvalid, alias) + } return verifySubject(ctx, alias, repo, subject, merged) } diff --git a/pkg/artifact/service/verify_test.go b/pkg/artifact/service/verify_test.go index 76b4a248b56..58f0d7c0be6 100644 --- a/pkg/artifact/service/verify_test.go +++ b/pkg/artifact/service/verify_test.go @@ -169,6 +169,21 @@ func TestConsumption_RequiredVerificationRunsBeforeMaterializationAndForceDoesNo require.NoError(t, err) } +func TestConsumption_RequireSignatureWithoutTrustConstraintsIsRejected(t *testing.T) { + e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) + published := e.publish("dst", e.source("hello")) + _, err := e.svc.Sign(context.Background(), artifact.SignRequest{Repository: "dst", Ref: artifact.Ref{Name: "tools/example", Digest: published.Operation.RecordDigest}}) + require.NoError(t, err) + e.setInfo("dst", func(info *service.RepositoryInfo) { info.Policy = artifact.VerificationPolicy{RequireSignature: true} }) + verifies := e.repos["dst"].Calls("Verify") + + dest := out(t) + _, err = e.svc.Pull(context.Background(), artifact.PullRequest{Repository: "dst", Ref: exampleRef, Destination: dest}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a signature from any signer must not satisfy a policy that trusts no one") + assert.Equal(t, verifies, e.repos["dst"].Calls("Verify"), "the repository is never asked to verify") + assert.True(t, missing(dest), "nothing was materialized") +} + func TestInspect_ReportsVerificationWhenPolicyConstrainsIt(t *testing.T) { e := newEnv(t, envConfig{options: []service.Option{service.WithSigner(releaseSigner)}}) published := e.publish("dst", e.source("hello")) diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 3300df6478e..594f483d5a3 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -373,3 +373,21 @@ func TestSnapshot_CloseWrapsRemovalFailure(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) require.ErrorIs(t, err, os.ErrPermission) } + +func TestCapture_RelativeSourceWithContainedSymlink(t *testing.T) { + base := t.TempDir() + src := filepath.Join(base, "src") + fixtures.WriteFile(t, src, "dir/target.txt", "target", 0o644) + if !fixtures.Symlink(t, src, "link.txt", "dir/target.txt") { + t.Skipf("symlinks are unavailable on this platform") + } + t.Chdir(base) + + snap, err := snapshot.Capture(context.Background(), "src", snapshot.WithStagingRoot(t.TempDir())) + require.NoError(t, err, "a contained link must not be rejected because the source path is relative") + defer snap.Close() + link, ok := snap.Lookup("link.txt") + require.True(t, ok) + assert.Equal(t, artifact.EntrySymlink, link.Type) + assert.Equal(t, "dir/target.txt", link.Target) +} diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go index 97caa95af89..02f7df89958 100644 --- a/pkg/artifact/snapshot/walk.go +++ b/pkg/artifact/snapshot/walk.go @@ -146,7 +146,13 @@ func readDirNames(dir string) ([]string, error) { // path yet resolve outside it. A dangling link is checked through its longest existing ancestor // plus the missing remainder, so it cannot escape once the missing path is created. func (c *capturer) checkLinkContainment() error { - realRoot, err := filepath.EvalSymlinks(c.root) + // linkpath.Resolve reports absolute paths, so the root must be absolute too: EvalSymlinks keeps + // a relative source relative, which would reject every contained link. + absRoot, err := filepath.Abs(c.root) + if err != nil { + return fmt.Errorf("%w: resolve source: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + realRoot, err := filepath.EvalSymlinks(absRoot) if err != nil { return fmt.Errorf("%w: resolve source: %w", errUtils.ErrArtifactPackagingInvalid, err) } diff --git a/pkg/oci/signature/plugin_error_test.go b/pkg/oci/signature/plugin_error_test.go index 017a0181e49..cdb14175910 100644 --- a/pkg/oci/signature/plugin_error_test.go +++ b/pkg/oci/signature/plugin_error_test.go @@ -24,4 +24,12 @@ func TestPluginErrorKeepsBothLegacyClassificationsForOfflineTrust(t *testing.T) err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureConfigInvalid}) require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) require.NotErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + + // Production shape: the trust loader failure is wrapped as invalid, with offline in the cause chain. + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureInvalid, Cause: errUtils.ErrSignatureOffline}) + require.ErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) + require.ErrorIs(t, err, errUtils.ErrPluginSignatureInvalid) + + err = pluginError(&sigcore.Error{Kind: errUtils.ErrSignatureInvalid, Cause: errors.New("bad signature")}) + require.NotErrorIs(t, err, errUtils.ErrPluginOfflineArtifact) } diff --git a/pkg/oci/signature/verify.go b/pkg/oci/signature/verify.go index dfc5eaa2688..0c3f6ada4b7 100644 --- a/pkg/oci/signature/verify.go +++ b/pkg/oci/signature/verify.go @@ -52,7 +52,7 @@ func pluginError(err error) error { switch { case errors.Is(shared.Kind, errUtils.ErrSignatureConfigInvalid): legacy = errUtils.ErrPluginInvalidConfig - case errors.Is(shared.Kind, errUtils.ErrSignatureOffline): + case errors.Is(shared.Kind, errUtils.ErrSignatureOffline) || errors.Is(shared.Cause, errUtils.ErrSignatureOffline): // Offline trust failures were always signature failures too; keep both classifications. legacy = errors.Join(errUtils.ErrPluginOfflineArtifact, errUtils.ErrPluginSignatureInvalid) } diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index daf1398965d..000458d4501 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/static" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/opencontainers/go-digest" @@ -279,3 +280,32 @@ func TestLegacyGraphSpoolsStreamingLayersAndCleansUp(t *testing.T) { assert.NoDirExists(t, spoolDir) graph.close() } + +// ORAS drops foreign and non-distributable layers from every graph copy, which is the legacy +// remote.Write behavior; this guards that the legacy graph keeps relying on it. +func TestLegacyCopyNeverUploadsNonDistributableLayers(t *testing.T) { + distributable := static.NewLayer([]byte("distributable layer"), types.OCILayer) + foreign := static.NewLayer([]byte("foreign layer"), types.DockerForeignLayer) + image, err := mutate.AppendLayers(empty.Image, distributable, foreign) + require.NoError(t, err) + rawManifest, err := image.RawManifest() + require.NoError(t, err) + foreignHash, err := foreign.Digest() + require.NoError(t, err) + distributableHash, err := distributable.Digest() + require.NoError(t, err) + + graph, err := legacyImageGraph(t.Context(), image) + require.NoError(t, err) + defer graph.close() + layout, err := OpenLayout(t.Context(), t.TempDir()) + require.NoError(t, err) + transferred := map[digest.Digest]bool{} + root, err := Copy(t.Context(), Source{Store: graph, Reference: legacyRoot}, Destination{Store: layout, Reference: "latest"}, + WithoutSubjectTraversal(), WithCopyObserver(func(desc ocispec.Descriptor) { transferred[desc.Digest] = true })) + require.NoError(t, err) + + assert.True(t, transferred[digest.Digest(distributableHash.String())], "distributable layers are transferred") + assert.False(t, transferred[digest.Digest(foreignHash.String())], "foreign layers are not uploaded, matching the legacy writer") + assert.Equal(t, digest.FromBytes(rawManifest), root.Digest, "the manifest bytes are unchanged") +} From 59b0753136bb3933dcf9d6df6c5dbf360eada9f3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 07:50:13 -0600 Subject: [PATCH 26/52] fix(oci): guard copy observer set in legacy non-distributable test ORAS copyGraph invokes the PostCopy callback from several goroutines, so the test's unsynchronized map write was a data race (failed CI race shard 1/4). Guard the map with a mutex and document that WithCopyObserver callbacks must be safe for concurrent use. Co-Authored-By: Claude Sonnet 5.5 --- pkg/oci/transport_copy.go | 1 + pkg/oci/transport_hardening_test.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index 9be153be0eb..769d78ef8b1 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -72,6 +72,7 @@ func WithCopyConcurrency(concurrency int) CopyOption { } // WithCopyObserver reports every node that was actually transferred (not skipped as present). +// The observer may be called from several goroutines concurrently, so it must be safe for concurrent use. // //nolint:lintroller // Functional option only. func WithCopyObserver(observe func(ocispec.Descriptor)) CopyOption { diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index 000458d4501..bf65c846d7e 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -300,11 +300,20 @@ func TestLegacyCopyNeverUploadsNonDistributableLayers(t *testing.T) { defer graph.close() layout, err := OpenLayout(t.Context(), t.TempDir()) require.NoError(t, err) + // ORAS invokes the observer from several goroutines, so the recorded set needs a lock. + var transferredMu sync.Mutex transferred := map[digest.Digest]bool{} root, err := Copy(t.Context(), Source{Store: graph, Reference: legacyRoot}, Destination{Store: layout, Reference: "latest"}, - WithoutSubjectTraversal(), WithCopyObserver(func(desc ocispec.Descriptor) { transferred[desc.Digest] = true })) + WithoutSubjectTraversal(), WithCopyObserver(func(desc ocispec.Descriptor) { + transferredMu.Lock() + defer transferredMu.Unlock() + transferred[desc.Digest] = true + })) require.NoError(t, err) + transferredMu.Lock() + defer transferredMu.Unlock() + assert.True(t, transferred[digest.Digest(distributableHash.String())], "distributable layers are transferred") assert.False(t, transferred[digest.Digest(foreignHash.String())], "foreign layers are not uploaded, matching the legacy writer") assert.Equal(t, digest.FromBytes(rawManifest), root.Digest, "the manifest bytes are unchanged") From 08e1ab7f6ca472abaf2a38a7bb452b40fbc65bd5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:49:07 -0600 Subject: [PATCH 27/52] fix(artifact): use t.Skipf in the conformance helper The forbidigo rule bans t.Skip and t.SkipNow, and the non-test conformance helper is not covered by the _test.go exclusion. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/artifacttest/conformance.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index 95a3aabf778..ed2a01eeac3 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -128,7 +128,7 @@ func (c *conformance) publish(t *testing.T, repo artifact.Repository, sp spec, o func (c *conformance) requireWritable(t *testing.T, repo artifact.Repository) { t.Helper() if !repo.Capabilities().Write { - t.Skip("the repository is read-only; capability errors are asserted by the dedicated subtest") + t.Skipf("the repository is read-only; capability errors are asserted by the dedicated subtest") } } @@ -290,7 +290,7 @@ func (c *conformance) conditionalPublish(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) if !repo.Capabilities().ConditionalUpdate { - t.Skip("the repository does not advertise conditional updates") + t.Skipf("the repository does not advertise conditional updates") } ref := artifact.Ref{Name: appName, Version: "1.0.0"} c.publish(t, repo, spec{name: ref.Name, version: ref.Version, content: "base"}, artifact.PublishOptions{}) @@ -452,7 +452,7 @@ func (c *conformance) capabilityErrors(t *testing.T) { repo := c.h.New(t) caps := repo.Capabilities() if caps.Write { - t.Skip("the repository is writable; read-only behavior is not applicable") + t.Skipf("the repository is writable; read-only behavior is not applicable") } src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "a", filePerm) From b3913173dcc28a385a9ec742db63fc350999ec3c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:54:22 -0600 Subject: [PATCH 28/52] fix(artifact): copy caller-owned data at the produce and service boundaries Repack kept the caller's document buffers and Record slices, so editing the request after it returned changed the document blobs that are read lazily at publish time and the file inventory they verify against, even for archive packagings. Prepare kept the request's provenance pointer and the metadata source's kind data, and Describe handed out the configured default labels. Record and Capabilities now have Clone methods and each boundary copies. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/clone.go | 47 ++++++++ pkg/artifact/clone_test.go | 97 ++++++++++++++++ pkg/artifact/produce/files.go | 3 + pkg/artifact/produce/repack.go | 8 +- pkg/artifact/produce/repack_test.go | 105 ++++++++++++++++++ pkg/artifact/service/config_repositories.go | 5 +- .../service/config_repositories_test.go | 35 ++++++ 7 files changed, 296 insertions(+), 4 deletions(-) create mode 100644 pkg/artifact/clone.go create mode 100644 pkg/artifact/clone_test.go diff --git a/pkg/artifact/clone.go b/pkg/artifact/clone.go new file mode 100644 index 00000000000..59bd12b1503 --- /dev/null +++ b/pkg/artifact/clone.go @@ -0,0 +1,47 @@ +package artifact + +import ( + "bytes" + "maps" + "slices" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Clone returns a Record that shares no maps, slices, or pointers with r, so a caller can keep +// either copy and edit the other without changing the first. +func (r *Record) Clone() Record { + defer perf.Track(nil, "artifact.Record.Clone")() + + out := *r + out.Metadata.Labels = maps.Clone(r.Metadata.Labels) + if r.Metadata.Provenance != nil { + provenance := *r.Metadata.Provenance + if provenance.CI != nil { + ci := *provenance.CI + provenance.CI = &ci + } + if provenance.Source != nil { + source := *provenance.Source + provenance.Source = &source + } + out.Metadata.Provenance = &provenance + } + out.Inventory = slices.Clone(r.Inventory) + out.Documents = slices.Clone(r.Documents) + out.KindData = bytes.Clone(r.KindData) + return out +} + +// Clone returns Capabilities that share no slices with c, so the guarantees a repository +// advertises stay static whatever a caller does with the returned value. +func (c *Capabilities) Clone() Capabilities { + defer perf.Track(nil, "artifact.Capabilities.Clone")() + + out := *c + out.Packagings = slices.Clone(c.Packagings) + out.SubjectKinds = slices.Clone(c.SubjectKinds) + out.EvidenceFormats = slices.Clone(c.EvidenceFormats) + out.LockScopes = slices.Clone(c.LockScopes) + return out +} diff --git a/pkg/artifact/clone_test.go b/pkg/artifact/clone_test.go new file mode 100644 index 00000000000..77b2f4c60bc --- /dev/null +++ b/pkg/artifact/clone_test.go @@ -0,0 +1,97 @@ +package artifact_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +func sampleRecord() artifact.Record { + return artifact.Record{ + SchemaVersion: artifact.SchemaVersion, + Name: "team/app", + Version: "1.0.0", + Kind: artifact.KindFiles, + Metadata: artifact.Metadata{ + Description: "sample", + Labels: map[string]string{"a": "1"}, + Provenance: &artifact.Provenance{ + CI: &artifact.CIProvenance{Provider: "github", RunID: "1"}, + Source: &artifact.SourceProvenance{Repository: "repo", Commit: "abc"}, + }, + }, + Inventory: []artifact.Entry{{Path: "a.txt", Type: artifact.EntryFile, Size: 1}, {Path: "b.txt", Type: artifact.EntryFile, Size: 2}}, + Documents: []artifact.Document{{Role: artifact.DocumentReadme, SourcePath: "README.md"}}, + KindData: artifact.KindData(`{"k":1}`), + } +} + +// TestRecord_CloneIsIndependentInBothDirections mutates the clone and then the original, and +// requires that neither edit shows up in the other. +func TestRecord_CloneIsIndependentInBothDirections(t *testing.T) { + tests := []struct { + name string + mutate func(r *artifact.Record) + read func(r *artifact.Record) any + want any + }{ + {"labels", func(r *artifact.Record) { r.Metadata.Labels["a"] = "changed" }, func(r *artifact.Record) any { return r.Metadata.Labels }, map[string]string{"a": "1"}}, + {"inventory", func(r *artifact.Record) { r.Inventory[0].Path = "changed" }, func(r *artifact.Record) any { return r.Inventory[0].Path }, "a.txt"}, + {"inventory last", func(r *artifact.Record) { r.Inventory[1].Size = 99 }, func(r *artifact.Record) any { return r.Inventory[1].Size }, int64(2)}, + {"documents", func(r *artifact.Record) { r.Documents[0].SourcePath = "changed" }, func(r *artifact.Record) any { return r.Documents[0].SourcePath }, "README.md"}, + {"kind data", func(r *artifact.Record) { r.KindData[0] = 'X' }, func(r *artifact.Record) any { return string(r.KindData) }, `{"k":1}`}, + {"CI provenance", func(r *artifact.Record) { r.Metadata.Provenance.CI.RunID = "changed" }, func(r *artifact.Record) any { return r.Metadata.Provenance.CI.RunID }, "1"}, + {"source provenance", func(r *artifact.Record) { r.Metadata.Provenance.Source.Commit = "changed" }, func(r *artifact.Record) any { return r.Metadata.Provenance.Source.Commit }, "abc"}, + {"provenance", func(r *artifact.Record) { r.Metadata.Provenance.CI = nil }, func(r *artifact.Record) any { return r.Metadata.Provenance.CI != nil }, true}, + } + for _, tt := range tests { + t.Run(tt.name+"/clone to original", func(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + tt.mutate(&clone) + assert.Equal(t, tt.want, tt.read(&original)) + }) + t.Run(tt.name+"/original to clone", func(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + tt.mutate(&original) + assert.Equal(t, tt.want, tt.read(&clone)) + }) + } +} + +func TestRecord_CloneKeepsContent(t *testing.T) { + original := sampleRecord() + clone := original.Clone() + assert.Equal(t, original, clone) + + empty := artifact.Record{} + assert.Equal(t, empty, empty.Clone(), "nil maps and slices stay nil") +} + +func TestCapabilities_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.Capabilities { + return artifact.Capabilities{ + Read: true, + Packagings: []artifact.Packaging{artifact.PackagingTar, artifact.PackagingZip}, + SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}, + EvidenceFormats: []artifact.EvidenceFormat{"one", "two"}, + LockScopes: []artifact.LockScope{artifact.LockScopeHost}, + } + } + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + + clone.Packagings[0], clone.SubjectKinds[0], clone.EvidenceFormats[1], clone.LockScopes[0] = artifact.PackagingTgz, "x", "x", artifact.LockScopeRepository + assert.Equal(t, sample(), original, "editing the clone leaves the original alone") + + original = sample() + fresh := sample() + clone = fresh.Clone() + original.Packagings[1], original.SubjectKinds[0], original.EvidenceFormats[0], original.LockScopes[0] = artifact.PackagingNone, "x", "x", artifact.LockScopeRepository + assert.Equal(t, sample(), clone, "editing the original leaves the clone alone") +} diff --git a/pkg/artifact/produce/files.go b/pkg/artifact/produce/files.go index c60960e94ea..00a7d698867 100644 --- a/pkg/artifact/produce/files.go +++ b/pkg/artifact/produce/files.go @@ -156,6 +156,9 @@ func (p *Producer) prepareFrom(ctx context.Context, snap *snapshot.Snapshot, req Documents: documentDescriptors(docs), KindData: id.meta.KindData, } + // The request's provenance and the metadata source's kind data are caller-owned; the sealed + // Record keeps its own copies. + record = record.Clone() if err := record.Seal(); err != nil { return nil, err } diff --git a/pkg/artifact/produce/repack.go b/pkg/artifact/produce/repack.go index be3ae022df3..4c12bde0f5c 100644 --- a/pkg/artifact/produce/repack.go +++ b/pkg/artifact/produce/repack.go @@ -1,6 +1,7 @@ package produce import ( + "bytes" "context" "crypto/sha256" "fmt" @@ -47,7 +48,8 @@ func (p *Producer) Repack(ctx context.Context, request *RepackRequest) (*artifac if request.OperationID == "" || request.PreparedAt.IsZero() { return nil, fmt.Errorf("%w: the operation ID and preparation time are required", errUtils.ErrArtifactConfigInvalid) } - record := request.Record + // The Record is copied so later edits to the request cannot reach the prepared artifact. + record := request.Record.Clone() if err := record.Validate(); err != nil { return nil, err } @@ -90,6 +92,8 @@ func treeOpener(record *artifact.Record, tree string) opener { } // repackDocuments pairs each Record document descriptor with its bytes and verifies the digest. +// The bytes are copied after the check: document blobs are read lazily when the artifact is +// published, so they must not depend on a buffer the caller still owns. func repackDocuments(record *artifact.Record, provided map[artifact.DocumentRole][]byte) ([]stagedDocument, error) { docs := make([]stagedDocument, 0, len(record.Documents)) for _, descriptor := range record.Documents { @@ -100,7 +104,7 @@ func repackDocuments(record *artifact.Record, provided map[artifact.DocumentRole if int64(len(data)) != descriptor.Size || digest.FromBytes(data) != descriptor.Digest { return nil, fmt.Errorf("%w: %s does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, descriptor.Role) } - docs = append(docs, stagedDocument{descriptor: descriptor, data: data}) + docs = append(docs, stagedDocument{descriptor: descriptor, data: bytes.Clone(data)}) } return docs, nil } diff --git a/pkg/artifact/produce/repack_test.go b/pkg/artifact/produce/repack_test.go index 7f48f8e813b..620893562f7 100644 --- a/pkg/artifact/produce/repack_test.go +++ b/pkg/artifact/produce/repack_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -109,3 +110,107 @@ func TestRepack_Failures(t *testing.T) { }) } } + +// TestRepack_PreparedArtifactIsIndependentOfTheRequest proves that what the caller passes in is +// copied at the boundary: editing the request after Repack returns changes nothing the prepared +// artifact serves, and editing the prepared artifact changes nothing in the request. +func TestRepack_PreparedArtifactIsIndependentOfTheRequest(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + original, _ := prepare(t, request(src, artifact.PackagingNone)) + wantDocs := documentsOf(t, original) + require.NotEmpty(t, wantDocs[artifact.DocumentReadme]) + + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + documents := documentsOf(t, original) + req := &produce.RepackRequest{ + Record: original.Record.Clone(), Tree: treeOf(t, original), Documents: documents, + Packaging: packaging, OperationID: "op", PreparedAt: fixedTime, + } + repacked, err := produce.New(produce.WithStagingRoot(t.TempDir())).Repack(context.Background(), req) + require.NoError(t, err) + defer repacked.Close() + wantRecord, err := repacked.Record.Canonical() + require.NoError(t, err) + + // Request to prepared artifact: scribble over everything the caller still owns. + for role := range req.Documents { + for index := range req.Documents[role] { + req.Documents[role][index] = 'X' + } + } + req.Record.Metadata.Labels = map[string]string{"scribble": "x"} + req.Record.Inventory[0].Path = "scribble" + req.Record.Inventory[len(req.Record.Inventory)-1].Digest = "sha256:scribble" + req.Record.Documents[0].SourcePath = "scribble" + + for role, want := range wantDocs { + blob := documentBlob(t, repacked, role) + got := readBlob(t, blob) + assert.Equal(t, want, got, "the %s document served later is the one sealed in its descriptor", role) + assert.Equal(t, blob.Descriptor.Digest, digest.FromBytes(got)) + } + gotRecord, err := repacked.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, wantRecord, gotRecord, "the prepared Record does not follow the request's Record") + if packaging == artifact.PackagingNone { + wantFiles := map[string][]byte{} + for index, blob := range payloadBlobs(original) { + wantFiles[blob.Path] = readBlob(t, &payloadBlobs(original)[index]) + } + gotBlobs := payloadBlobs(repacked) + require.Len(t, gotBlobs, len(wantFiles)) + for index := range gotBlobs { + assert.Equal(t, wantFiles[gotBlobs[index].Path], readBlob(t, &gotBlobs[index]), "payload files still verify against the original inventory: %s", gotBlobs[index].Path) + } + } + + // Prepared artifact to request: editing the result must not reach the request. + req.Record.Inventory[0].Path = "kept" + repacked.Record.Inventory[0].Path = "scribble-too" + assert.Equal(t, "kept", req.Record.Inventory[0].Path) + }) + } +} + +func documentBlob(t *testing.T, prepared *artifact.PreparedArtifact, role artifact.DocumentRole) *artifact.StagedBlob { + t.Helper() + for index := range prepared.Blobs { + if prepared.Blobs[index].Role == artifact.BlobDocument && prepared.Blobs[index].DocumentRole == role { + return &prepared.Blobs[index] + } + } + require.FailNow(t, "the prepared artifact has no such document blob", string(role)) + return nil +} + +// TestPrepare_RecordIsIndependentOfTheRequestAndMetadata proves the pointer and byte-slice fields +// the producer is handed (provenance and kind data) are copied into the sealed Record. +func TestPrepare_RecordIsIndependentOfTheRequestAndMetadata(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + provenance := &artifact.Provenance{ + CI: &artifact.CIProvenance{Provider: "github", RunID: "1"}, + Source: &artifact.SourceProvenance{Commit: "abc"}, + } + kindData := artifact.KindData(`{"k":1}`) + req := request(src, artifact.PackagingTar) + req.Metadata.Provenance = provenance + source := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{KindData: kindData}, nil + } + prepared, _ := prepare(t, req, produce.WithMetadataSource(source)) + want, err := prepared.Record.Canonical() + require.NoError(t, err) + + provenance.CI.RunID = "scribble" + provenance.Source.Commit = "scribble" + kindData[0] = 'X' + + got, err := prepared.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, want, got, "later edits to the request or the metadata do not reach the Record") + assert.Equal(t, "1", prepared.Record.Metadata.Provenance.CI.RunID) + assert.JSONEq(t, `{"k":1}`, string(prepared.Record.KindData)) +} diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go index 1e76b976d17..007f230892d 100644 --- a/pkg/artifact/service/config_repositories.go +++ b/pkg/artifact/service/config_repositories.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "maps" "sort" "sync" @@ -62,13 +63,13 @@ func (c *ConfigRepositories) Describe(alias string) (RepositoryInfo, error) { } info := RepositoryInfo{ Alias: alias, - Capabilities: desc.Capabilities, + Capabilities: desc.Capabilities.Clone(), Packaging: desc.EffectivePackaging(repoCfg.Packaging), RequiredLockScope: plan.Required, Policy: policy, } if repoCfg.Defaults != nil && repoCfg.Defaults.Metadata != nil { - info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: repoCfg.Defaults.Metadata.Labels} + info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: maps.Clone(repoCfg.Defaults.Metadata.Labels)} } if plan.Provider != alias { info.LockProvider = plan.Provider diff --git a/pkg/artifact/service/config_repositories_test.go b/pkg/artifact/service/config_repositories_test.go index 0a2ee720f35..17d27fee91c 100644 --- a/pkg/artifact/service/config_repositories_test.go +++ b/pkg/artifact/service/config_repositories_test.go @@ -120,3 +120,38 @@ func TestConfigRepositories_DrivesTheServiceEndToEnd(t *testing.T) { assert.Equal(t, artifact.OutcomeCreated, result.Outcome) assert.Equal(t, int64(1), constructed.Load()) } + +// TestConfigRepositories_DescribeReturnsIndependentCopies proves a caller editing what Describe +// returned cannot change the registry's capabilities or the configured defaults, which every later +// Describe would otherwise serve. +func TestConfigRepositories_DescribeReturnsIndependentCopies(t *testing.T) { + var constructed atomic.Int64 + labels := map[string]string{"team": "platform"} + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": { + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": "/tmp/shared"}, + Defaults: &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{Labels: labels}}, + }, + }} + repos := service.FromConfig(cfg, configRegistry(t, &constructed)) + pristine, err := repos.Describe("shared") + require.NoError(t, err) + wantPackagings := append([]artifact.Packaging(nil), pristine.Capabilities.Packagings...) + wantLockScopes := append([]artifact.LockScope(nil), pristine.Capabilities.LockScopes...) + require.NotEmpty(t, wantPackagings) + require.NotEmpty(t, wantLockScopes) + + first, err := repos.Describe("shared") + require.NoError(t, err) + first.Capabilities.Packagings[0] = artifact.PackagingZip + first.Capabilities.LockScopes[0] = artifact.LockScopeRepository + first.Defaults.Labels["team"] = "scribble" + + second, err := repos.Describe("shared") + require.NoError(t, err) + assert.Equal(t, wantPackagings, second.Capabilities.Packagings) + assert.Equal(t, wantLockScopes, second.Capabilities.LockScopes) + assert.Equal(t, map[string]string{"team": "platform"}, second.Defaults.Labels) + assert.Equal(t, map[string]string{"team": "platform"}, labels, "the configured defaults are untouched") +} From cc84187adc00ade082c0b99d0338d524a5446570 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:55:02 -0600 Subject: [PATCH 29/52] fix(artifact): isolate the in-memory repository's state and honor delete scope The test double returned capability slices, records, labels, and descriptors that shared storage with its stored state, so a caller edit could change a guarantee or a stored record. It now copies at every boundary. Delete ignored the requested scope: with two references to one record it removed both under a reference scope, reported the same location twice, and never reported the sibling reference. A reference scope now removes only the pinned reference and keeps shared data while another reference uses it, an object or manifest scope reports each removed location once and the siblings as affected, and a scope the repository does not advertise is rejected. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/artifacttest/clone.go | 36 +++ pkg/artifact/artifacttest/read.go | 16 +- pkg/artifact/artifacttest/repository.go | 27 +- pkg/artifact/artifacttest/repository_test.go | 310 +++++++++++++++++++ pkg/artifact/artifacttest/sign.go | 6 +- pkg/artifact/artifacttest/write.go | 69 ++++- 6 files changed, 439 insertions(+), 25 deletions(-) create mode 100644 pkg/artifact/artifacttest/clone.go create mode 100644 pkg/artifact/artifacttest/repository_test.go diff --git a/pkg/artifact/artifacttest/clone.go b/pkg/artifact/artifacttest/clone.go new file mode 100644 index 00000000000..432a5b77cad --- /dev/null +++ b/pkg/artifact/artifacttest/clone.go @@ -0,0 +1,36 @@ +package artifacttest + +import ( + "bytes" + "maps" + "slices" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// The repository never shares maps or slices with its callers: whatever crosses the boundary in +// either direction is copied, so a caller editing its own value can never change a stored record, +// a capability guarantee, or a descriptor after the fact. + +// cloneDescriptor returns a descriptor that does not share maps, slices, or pointers with d. +func cloneDescriptor(in *ocispec.Descriptor) ocispec.Descriptor { + d := *in + d.URLs = slices.Clone(d.URLs) + d.Annotations = maps.Clone(d.Annotations) + d.Data = bytes.Clone(d.Data) + if d.Platform != nil { + platform := *d.Platform + platform.OSFeatures = slices.Clone(platform.OSFeatures) + d.Platform = &platform + } + return d +} + +// cloneEvidence returns evidence whose descriptor does not share storage with e. +func cloneEvidence(in *artifact.SignatureEvidence) artifact.SignatureEvidence { + e := *in + e.Descriptor = cloneDescriptor(&in.Descriptor) + return e +} diff --git a/pkg/artifact/artifacttest/read.go b/pkg/artifact/artifacttest/read.go index 5e641099976..b8192702229 100644 --- a/pkg/artifact/artifacttest/read.go +++ b/pkg/artifact/artifacttest/read.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "maps" + "slices" "sort" "github.com/opencontainers/go-digest" @@ -67,12 +69,12 @@ func (r *Repository) Inspect(_ context.Context, pinned artifact.PinnedReference) if !ok { return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) } - record := s.record + record := s.record.Clone() pinnedOut := r.pinnedFor(pinned.Ref, s) inspection := &artifact.Inspection{Pinned: pinnedOut, Record: &record, Representation: r.representation(s)} held := r.evidence[s.recordDigest] for index := range held { - inspection.Evidence = append(inspection.Evidence, held[index].Evidence) + inspection.Evidence = append(inspection.Evidence, cloneEvidence(&held[index].Evidence)) } return inspection, nil } @@ -118,12 +120,12 @@ func (r *Repository) Read(_ context.Context, pinned artifact.PinnedReference) (* if !ok || !s.payloadAvailable { return nil, fmt.Errorf("%w: the payload of %s is no longer available", errUtils.ErrArtifactNotFound, pinned.RecordDigest) } - record := s.record + record := s.record.Clone() content := &artifact.Content{Record: &record, Representation: r.representation(s)} for index := range s.blobs { held := &s.blobs[index] content.Streams = append(content.Streams, artifact.ContentStream{ - Descriptor: held.descriptor, Role: held.role, Path: held.path, Reader: newVerifyingReader(held.data, held.descriptor.Digest), + Descriptor: cloneDescriptor(&held.descriptor), Role: held.role, Path: held.path, Reader: newVerifyingReader(held.data, held.descriptor.Digest), }) } return content, nil @@ -142,7 +144,7 @@ func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.Pinned r.count("Pin") held := r.evidence[pinned.RecordDigest] for index := range held { - pinned.Evidence = append(pinned.Evidence, held[index].Evidence.Descriptor) + pinned.Evidence = append(pinned.Evidence, cloneDescriptor(&held[index].Evidence.Descriptor)) } return pinned, nil } @@ -183,8 +185,8 @@ func (r *Repository) listEntry(s *stored) artifact.ListEntry { Version: s.record.Version, Kind: s.record.Kind, Description: s.record.Metadata.Description, - Labels: s.record.Metadata.Labels, - Documents: s.record.Documents, + Labels: maps.Clone(s.record.Metadata.Labels), + Documents: slices.Clone(s.record.Documents), } } diff --git a/pkg/artifact/artifacttest/repository.go b/pkg/artifact/artifacttest/repository.go index b67557ba19b..c6270ca22f2 100644 --- a/pkg/artifact/artifacttest/repository.go +++ b/pkg/artifact/artifacttest/repository.go @@ -2,6 +2,7 @@ package artifacttest import ( "fmt" + "strings" "sync" "github.com/opencontainers/go-digest" @@ -112,8 +113,10 @@ func New(opts Options) *Repository { } caps := DefaultCapabilities() if opts.Capabilities != nil { - caps = *opts.Capabilities + caps = opts.Capabilities.Clone() } + // The repository keeps its own copy of the capabilities, never the caller's pointer. + opts.Capabilities = nil return &Repository{ opts: opts, caps: caps, @@ -150,7 +153,7 @@ func DefaultCapabilities() artifact.Capabilities { func (r *Repository) Capabilities() artifact.Capabilities { defer perf.Track(nil, "artifacttest.Repository.Capabilities")() - return r.caps + return r.caps.Clone() } // InjectFailure arms err to be returned from Publish at point, times times (0 means once). @@ -235,6 +238,22 @@ func refKey(name, version string) string { return name + ":" + version } +// refFromKey is the inverse of [refKey]. Names never contain a colon, so the last one separates +// the version. +func refFromKey(key string) artifact.Ref { + cut := strings.LastIndexByte(key, ':') + if cut < 0 { + return artifact.Ref{Name: key} + } + return artifact.Ref{Name: key[:cut], Version: key[cut+1:]} +} + +// locatorForKey is the location of the reference stored under key. +func (r *Repository) locatorForKey(key string) string { + ref := refFromKey(key) + return r.locator(ref.Name, ref.Version) +} + // count records a call; callers hold r.mu. func (r *Repository) count(method string) { r.calls[method]++ @@ -273,10 +292,10 @@ func (r *Repository) representation(s *stored) artifact.Representation { payloadDigests := "" for index := range s.blobs { if s.blobs[index].role == artifact.BlobPayload { - rep.Payload = append(rep.Payload, s.blobs[index].descriptor) + rep.Payload = append(rep.Payload, cloneDescriptor(&s.blobs[index].descriptor)) payloadDigests += s.blobs[index].descriptor.Digest.String() + "\n" } else { - rep.Documents = append(rep.Documents, s.blobs[index].descriptor) + rep.Documents = append(rep.Documents, cloneDescriptor(&s.blobs[index].descriptor)) } } rep.RepresentationDigest = digest.FromString(payloadDigests) diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go new file mode 100644 index 00000000000..bec788426d5 --- /dev/null +++ b/pkg/artifact/artifacttest/repository_test.go @@ -0,0 +1,310 @@ +package artifacttest_test + +import ( + "context" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +const ( + testName = "team/app" + testPerm = 0o644 + aliasName = "team/alias" +) + +// publishOne publishes a single-file artifact and returns its pinned reference. +func publishOne(t *testing.T, repo *artifacttest.Repository) artifact.PinnedReference { + t.Helper() + name, version := testName, "1.0.0" + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "content of "+name+"@"+version, testPerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: name, Version: version, Packaging: artifact.PackagingTar, + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "isolation"}}, + OperationID: "op-" + name + "-" + version, PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: name, Version: version}) + require.NoError(t, err) + return pinned +} + +// TestRepository_CapabilitySlicesAreIsolated proves the repository's guarantees are static: the +// slices a caller passes in or receives back never share storage with the repository. +func TestRepository_CapabilitySlicesAreIsolated(t *testing.T) { + tests := []struct { + name string + mutate func(c *artifact.Capabilities) + read func(c artifact.Capabilities) any + want any + }{ + { + name: "packagings", + mutate: func(c *artifact.Capabilities) { c.Packagings[0] = artifact.PackagingZip }, + read: func(c artifact.Capabilities) any { return c.Packagings }, + want: artifacttest.DefaultCapabilities().Packagings, + }, + { + name: "subject kinds", + mutate: func(c *artifact.Capabilities) { c.SubjectKinds[0] = "other" }, + read: func(c artifact.Capabilities) any { return c.SubjectKinds }, + want: artifacttest.DefaultCapabilities().SubjectKinds, + }, + { + name: "evidence formats", + mutate: func(c *artifact.Capabilities) { c.EvidenceFormats[0] = "other" }, + read: func(c artifact.Capabilities) any { return c.EvidenceFormats }, + want: artifacttest.DefaultCapabilities().EvidenceFormats, + }, + { + name: "lock scopes", + mutate: func(c *artifact.Capabilities) { c.LockScopes[0] = artifact.LockScopeRepository }, + read: func(c artifact.Capabilities) any { return c.LockScopes }, + want: artifacttest.DefaultCapabilities().LockScopes, + }, + } + for _, tt := range tests { + t.Run(tt.name+"/returned value", func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + returned := repo.Capabilities() + tt.mutate(&returned) + assert.Equal(t, tt.want, tt.read(repo.Capabilities()), "mutating the returned slices must not change the repository") + }) + t.Run(tt.name+"/accepted option", func(t *testing.T) { + supplied := artifacttest.DefaultCapabilities() + repo := artifacttest.New(artifacttest.Options{Capabilities: &supplied}) + tt.mutate(&supplied) + assert.Equal(t, tt.want, tt.read(repo.Capabilities()), "mutating the supplied option must not change the repository") + }) + } +} + +func TestRepository_MutatedCapabilitiesDoNotChangeLockGuarantees(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + returned := repo.Capabilities() + returned.LockScopes[0] = artifact.LockScopeRepository + + _, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "r", Owner: "o", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a host-only repository never grants repository scope because a caller edited its copy") +} + +func TestRepository_DeleteScope(t *testing.T) { + aliasRef := artifact.Ref{Name: aliasName, Version: "latest"} + primary := artifact.Ref{Name: testName, Version: "1.0.0"} + + tests := []struct { + name string + caps artifact.DeleteSemantics + scope artifact.DeleteSemantics + pin artifact.Ref + + wantErr error + wantDeleted []string + wantAffected []artifact.Ref + wantRecordAlive bool + wantPrimary bool + wantAlias bool + wantEvidence bool + }{ + { + name: "object scope removes every alias and reports each location once", caps: artifact.DeleteObject, scope: artifact.DeleteObject, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{aliasRef}, + }, + { + name: "object scope through the alias reports the other reference as affected", caps: artifact.DeleteObject, scope: artifact.DeleteObject, pin: aliasRef, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{primary}, + }, + { + name: "manifest scope affects every reference", caps: artifact.DeleteManifest, scope: artifact.DeleteManifest, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0", "mem://memory/team/alias/latest"}, + wantAffected: []artifact.Ref{aliasRef}, + }, + { + name: "reference scope removes only the pinned reference and keeps shared data", caps: artifact.DeleteReference, scope: artifact.DeleteReference, pin: primary, + wantDeleted: []string{"mem://memory/team/app/1.0.0"}, wantRecordAlive: true, wantAlias: true, wantEvidence: true, + }, + { + name: "reference scope through the alias leaves the primary reference", caps: artifact.DeleteReference, scope: artifact.DeleteReference, pin: aliasRef, + wantDeleted: []string{"mem://memory/team/alias/latest"}, wantRecordAlive: true, wantPrimary: true, wantEvidence: true, + }, + {name: "an empty scope is rejected", caps: artifact.DeleteObject, scope: "", pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + {name: "an unknown scope is rejected", caps: artifact.DeleteObject, scope: "everything", pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + {name: "a scope the repository does not advertise is rejected", caps: artifact.DeleteReference, scope: artifact.DeleteObject, pin: primary, wantErr: errUtils.ErrArtifactUnsupportedCapability, wantRecordAlive: true, wantPrimary: true, wantAlias: true, wantEvidence: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = tt.caps + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + require.True(t, repo.MoveRef(aliasRef, pinned.RecordDigest)) + signEvidence(t, repo, &pinned) + + target, err := repo.Resolve(context.Background(), tt.pin) + require.NoError(t, err) + result, err := repo.Delete(context.Background(), target, artifact.DeleteOptions{ExpectedCurrent: &target, Scope: tt.scope}) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + assert.ElementsMatch(t, tt.wantDeleted, result.Deleted, "each removed reference is reported once, by its own location") + assert.Len(t, result.Deleted, len(tt.wantDeleted)) + assert.ElementsMatch(t, tt.wantAffected, result.Affected) + assert.Equal(t, pinned.RecordDigest, result.Digest) + } + + _, primaryAlive := repo.RecordDigestOf(primary) + _, aliasAlive := repo.RecordDigestOf(aliasRef) + assert.Equal(t, tt.wantPrimary, primaryAlive, "primary reference") + assert.Equal(t, tt.wantAlias, aliasAlive, "alias reference") + assert.Equal(t, tt.wantRecordAlive, recordAlive(repo, pinned.RecordDigest), "shared record") + assert.Equal(t, tt.wantEvidence, evidenceAlive(t, repo, &pinned), "shared evidence") + }) + } +} + +func TestRepository_DeleteReferenceScopeRemovesRecordWithItsLastReference(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteReference + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + signEvidence(t, repo, &pinned) + + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteReference}) + require.NoError(t, err) + assert.Equal(t, []string{"mem://memory/team/app/1.0.0"}, result.Deleted) + assert.Empty(t, result.Affected) + assert.False(t, recordAlive(repo, pinned.RecordDigest), "an unreferenced record is no longer retained") + assert.False(t, evidenceAlive(t, repo, &pinned)) +} + +func TestRepository_DeleteReferenceScopeNeedsAMutableReference(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteReference + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + byDigest := pinned + byDigest.Ref = artifact.Ref{Name: testName, Digest: pinned.RecordDigest} + + _, err := repo.Delete(context.Background(), byDigest, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + _, alive := repo.RecordDigestOf(artifact.Ref{Name: testName, Version: "1.0.0"}) + assert.True(t, alive, "a rejected delete changes nothing") +} + +func recordAlive(repo *artifacttest.Repository, recordDigest digest.Digest) bool { + _, err := repo.Inspect(context.Background(), artifact.PinnedReference{RecordDigest: recordDigest, ResolvedDigest: recordDigest}) + return err == nil +} + +func evidenceAlive(t *testing.T, repo *artifacttest.Repository, pinned *artifact.PinnedReference) bool { + t.Helper() + blobs, err := repo.ReadEvidence(context.Background(), pinned) + require.NoError(t, err) + return len(blobs) > 0 +} + +func signEvidence(t *testing.T, repo *artifacttest.Repository, pinned *artifact.PinnedReference) { + t.Helper() + inspection, err := repo.Inspect(context.Background(), *pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) +} + +// TestRepository_StoredStateIsIsolatedFromCallers proves records, labels, and descriptors handed +// to or returned from the repository never alias its stored state, in either direction. +func TestRepository_StoredStateIsIsolatedFromCallers(t *testing.T) { + ctx := context.Background() + ref := artifact.Ref{Name: testName, Version: "1.0.0"} + tests := []struct { + name string + tamper func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) + }{ + {"inspect record labels", func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) { + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + inspection.Record.Metadata.Labels["suite"] = "tampered" + inspection.Record.Inventory[0].Path = "tampered" + inspection.Record.Documents = append(inspection.Record.Documents[:0], artifact.Document{Role: "tampered"}) + }}, + {"read record labels", func(t *testing.T, repo *artifacttest.Repository, pinned artifact.PinnedReference) { + content, err := repo.Read(ctx, pinned) + require.NoError(t, err) + content.Record.Metadata.Labels["suite"] = "tampered" + content.Representation.Payload[0].Digest = digest.FromString("tampered") + }}, + {"list entry labels", func(t *testing.T, repo *artifacttest.Repository, _ artifact.PinnedReference) { + page, err := repo.List(ctx, artifact.ListQuery{}) + require.NoError(t, err) + require.NotEmpty(t, page.Entries) + page.Entries[0].Labels["suite"] = "tampered" + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, repo) + before, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + wantPath := before.Record.Inventory[0].Path + wantDocs := len(before.Record.Documents) + + tt.tamper(t, repo, pinned) + + after, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, map[string]string{"suite": "isolation"}, after.Record.Metadata.Labels) + assert.Equal(t, wantPath, after.Record.Inventory[0].Path) + assert.Len(t, after.Record.Documents, wantDocs) + assert.Equal(t, before.Representation.Payload, after.Representation.Payload) + _, err = repo.Resolve(ctx, ref) + require.NoError(t, err) + }) + } +} + +func TestRepository_PublishCopiesThePreparedRecord(t *testing.T) { + ctx := context.Background() + repo := artifacttest.New(artifacttest.Options{}) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload", testPerm) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(ctx, artifact.PrepareRequest{ + Source: src, Name: testName, Version: "1.0.0", Packaging: artifact.PackagingTar, + Metadata: artifact.Metadata{Labels: map[string]string{"suite": "isolation"}}, + OperationID: "op", PreparedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + _, err = repo.Publish(ctx, prepared, artifact.PublishOptions{}) + require.NoError(t, err) + + prepared.Record.Metadata.Labels["suite"] = "tampered" + prepared.Record.Inventory[0].Path = "tampered" + + pinned, err := repo.Resolve(ctx, artifact.Ref{Name: testName, Version: "1.0.0"}) + require.NoError(t, err) + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, map[string]string{"suite": "isolation"}, inspection.Record.Metadata.Labels, "the stored record is independent of the prepared one") + assert.Equal(t, "data.txt", inspection.Record.Inventory[0].Path) +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index 05620f109d6..60e615728ed 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -207,7 +207,7 @@ func (r *Repository) ReadEvidence(_ context.Context, pinned *artifact.PinnedRefe held := r.evidence[pinned.RecordDigest] blobs := make([]artifact.EvidenceBlob, len(held)) for index := range held { - blobs[index] = artifact.EvidenceBlob{Evidence: held[index].Evidence, Data: bytes.Clone(held[index].Data)} + blobs[index] = artifact.EvidenceBlob{Evidence: cloneEvidence(&held[index].Evidence), Data: bytes.Clone(held[index].Data)} } return blobs, nil } @@ -224,8 +224,8 @@ func (r *Repository) WriteEvidence(_ context.Context, pinned *artifact.PinnedRef } stored := make([]artifact.SignatureEvidence, 0, len(evidence)) for index := range evidence { - r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], artifact.EvidenceBlob{Evidence: evidence[index].Evidence, Data: bytes.Clone(evidence[index].Data)}) - stored = append(stored, evidence[index].Evidence) + r.evidence[pinned.RecordDigest] = append(r.evidence[pinned.RecordDigest], artifact.EvidenceBlob{Evidence: cloneEvidence(&evidence[index].Evidence), Data: bytes.Clone(evidence[index].Data)}) + stored = append(stored, cloneEvidence(&evidence[index].Evidence)) } return stored, nil } diff --git a/pkg/artifact/artifacttest/write.go b/pkg/artifact/artifacttest/write.go index 96094f43dff..972fc3b285d 100644 --- a/pkg/artifact/artifacttest/write.go +++ b/pkg/artifact/artifacttest/write.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "sort" "github.com/opencontainers/go-digest" @@ -30,7 +31,7 @@ func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArt if err := r.trip(FailBeforePayload); err != nil { return nil, err } - record := prepared.Record + record := prepared.Record.Clone() key := refKey(record.Name, record.Version) recordDigest, err := record.RecordDigest() if err != nil { @@ -114,6 +115,9 @@ func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedA blobs := make([]blob, 0, len(prepared.Blobs)) for index := range prepared.Blobs { staged := &prepared.Blobs[index] + if err := ctx.Err(); err != nil { + return nil, err + } reader, err := staged.Open(ctx) if err != nil { return nil, fmt.Errorf("%w: open staged blob %q: %w", errUtils.ErrArtifactPackagingInvalid, staged.Path, err) @@ -132,7 +136,7 @@ func (r *Repository) readBlobs(ctx context.Context, prepared *artifact.PreparedA if int64(len(data)) != staged.Descriptor.Size || digest.FromBytes(data) != staged.Descriptor.Digest { return nil, fmt.Errorf("%w: staged blob %q does not match its descriptor", errUtils.ErrArtifactIntegrityFailed, staged.Path) } - blobs = append(blobs, blob{descriptor: staged.Descriptor, role: staged.Role, documentRole: staged.DocumentRole, path: staged.Path, data: data}) + blobs = append(blobs, blob{descriptor: cloneDescriptor(&staged.Descriptor), role: staged.Role, documentRole: staged.DocumentRole, path: staged.Path, data: data}) } return blobs, nil } @@ -191,8 +195,10 @@ func (r *Repository) checkExpectedCurrent(pinned *artifact.PinnedReference, expe return nil } -// Delete implements [artifact.Repository]. The deletion is scoped to the pinned record: the -// references pointing at it and its exclusively owned payload. +// Delete implements [artifact.Repository]. The requested scope decides what goes: a reference +// scope removes only the pinned reference and keeps the record and its evidence while another +// reference still uses them, while a manifest or object scope removes the record, its evidence, +// and every reference to it. A scope the repository does not advertise is rejected, never widened. // //nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. func (r *Repository) Delete(_ context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { @@ -204,22 +210,63 @@ func (r *Repository) Delete(_ context.Context, pinned artifact.PinnedReference, if r.caps.Delete == artifact.DeleteNone || r.caps.Delete == "" { return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "delete", "delete", "the repository cannot delete") } - s, ok := r.records[pinned.RecordDigest] - if !ok { + if options.Scope != r.caps.Delete { + return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "delete", "delete/"+string(options.Scope), "the repository only deletes with the "+string(r.caps.Delete)+" scope") + } + if _, ok := r.records[pinned.RecordDigest]; !ok { return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) } if err := r.checkExpectedCurrent(&pinned, options.ExpectedCurrent); err != nil { return nil, err } - result := &artifact.DeleteResult{Digest: pinned.RecordDigest} + if options.Scope == artifact.DeleteReference { + return r.deleteReference(&pinned) + } + return r.deleteRecord(&pinned), nil +} + +// deleteReference removes only the pinned reference. The record and its evidence survive while any +// other reference still points at them; callers hold r.mu. +func (r *Repository) deleteReference(pinned *artifact.PinnedReference) (*artifact.DeleteResult, error) { + if pinned.Ref.Version == "" || pinned.Ref.Digest != "" { + return nil, fmt.Errorf("%w: a reference-scoped delete needs a version reference, got %s", errUtils.ErrArtifactRefInvalid, pinned.Ref) + } + key := refKey(pinned.Ref.Name, pinned.Ref.Version) + if current, ok := r.refs[key]; !ok || current != pinned.RecordDigest { + return nil, fmt.Errorf("%w: %s no longer points at %s", errUtils.ErrArtifactConflict, pinned.Ref, pinned.RecordDigest) + } + delete(r.refs, key) + if !r.referenced(pinned.RecordDigest) { + delete(r.records, pinned.RecordDigest) + delete(r.evidence, pinned.RecordDigest) + } + return &artifact.DeleteResult{Digest: pinned.RecordDigest, Deleted: []string{r.locatorForKey(key)}}, nil +} + +// deleteRecord removes the record, its evidence, and every reference to it. Each removed reference +// is reported once by its own location, and every removed reference other than the pinned one is +// reported as affected; callers hold r.mu. +func (r *Repository) deleteRecord(pinned *artifact.PinnedReference) *artifact.DeleteResult { + pinnedKey := "" + if pinned.Ref.Version != "" && pinned.Ref.Digest == "" { + pinnedKey = refKey(pinned.Ref.Name, pinned.Ref.Version) + } + var removed []string for key, d := range r.refs { - if d != pinned.RecordDigest { - continue + if d == pinned.RecordDigest { + removed = append(removed, key) } + } + sort.Strings(removed) + result := &artifact.DeleteResult{Digest: pinned.RecordDigest} + for _, key := range removed { delete(r.refs, key) - result.Deleted = append(result.Deleted, r.locator(s.record.Name, s.record.Version)) + result.Deleted = append(result.Deleted, r.locatorForKey(key)) + if key != pinnedKey { + result.Affected = append(result.Affected, refFromKey(key)) + } } delete(r.records, pinned.RecordDigest) delete(r.evidence, pinned.RecordDigest) - return result, nil + return result } From b1bf16306efdc9597da98cfbcb4e575633d96732 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:55:41 -0600 Subject: [PATCH 30/52] fix(artifact): stop archive extraction when the context is canceled Archive members were copied without a context check, so a cancellation during a large tar, tgz, or zip member kept decompressing and writing it, and a cancellation that arrived as the last member finished still installed the tree. Member bodies are now read through the context, the tar drain is context-aware, and a canceled pull never installs. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/archive.go | 33 +++-- .../materialize/archive_internal_test.go | 121 +++++++++++++++ pkg/artifact/materialize/cancel_test.go | 139 ++++++++++++++++++ pkg/artifact/materialize/extract.go | 4 + pkg/artifact/materialize/replace.go | 15 +- 5 files changed, 296 insertions(+), 16 deletions(-) create mode 100644 pkg/artifact/materialize/archive_internal_test.go create mode 100644 pkg/artifact/materialize/cancel_test.go diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index 7366ced06f7..1604d905f5d 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -82,27 +82,27 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check header, err := tr.Next() if errors.Is(err, io.EOF) { // Drain so the verifying payload reader checks its full digest at end of stream. - _, err = io.Copy(io.Discard, reader) + _, err = copyContext(ctx, io.Discard, reader) return err } if err != nil { return fmt.Errorf("%w: tar: %w", errUtils.ErrArtifactIntegrityFailed, err) } - if err := applyTarHeader(t, tr, header, checker); err != nil { + if err := applyTarHeader(ctx, t, tr, header, checker); err != nil { return err } } } -func applyTarHeader(t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { +func applyTarHeader(ctx context.Context, t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { name := strings.TrimSuffix(path.Clean(header.Name), "/") switch header.Typeflag { case tar.TypeDir: - return applyMember(t, checker, &archiveMember{name: name, kind: memberDir, size: header.Size}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberDir, size: header.Size}) case tar.TypeSymlink: - return applyMember(t, checker, &archiveMember{name: name, kind: memberLink, target: header.Linkname}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberLink, target: header.Linkname}) case tar.TypeReg: - return applyMember(t, checker, &archiveMember{name: name, kind: memberFile, size: header.Size, body: tr}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberFile, size: header.Size, body: tr}) default: return fmt.Errorf("%w: archive member %q has unsupported type %q (hard links, devices, and special files are rejected)", errUtils.ErrArtifactPackagingInvalid, header.Name, header.Typeflag) } @@ -127,7 +127,7 @@ func fillZip(ctx context.Context, t *tree, payload *artifact.ContentStream, scra if err := ctx.Err(); err != nil { return err } - if err := applyZipFile(t, file, checker); err != nil { + if err := applyZipFile(ctx, t, file, checker); err != nil { return err } } @@ -153,25 +153,25 @@ func spoolBounded(ctx context.Context, spool io.Writer, payload *artifact.Conten return size, nil } -func applyZipFile(t *tree, file *zip.File, checker *managedChecker) error { +func applyZipFile(ctx context.Context, t *tree, file *zip.File, checker *managedChecker) error { name := strings.TrimSuffix(path.Clean(file.Name), "/") mode := file.Mode() switch { case mode.IsDir(): - return applyMember(t, checker, &archiveMember{name: name, kind: memberDir}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberDir}) case mode&fs.ModeSymlink != 0: target, err := readZipText(file) if err != nil { return err } - return applyMember(t, checker, &archiveMember{name: name, kind: memberLink, target: target}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberLink, target: target}) case mode.IsRegular(): rc, err := file.Open() if err != nil { return fmt.Errorf("%w: zip member %q: %w", errUtils.ErrArtifactIntegrityFailed, file.Name, err) } defer rc.Close() - return applyMember(t, checker, &archiveMember{name: name, kind: memberFile, size: file.FileInfo().Size(), body: rc}) + return applyMember(ctx, t, checker, &archiveMember{name: name, kind: memberFile, size: file.FileInfo().Size(), body: rc}) default: return fmt.Errorf("%w: archive member %q has an unsupported type", errUtils.ErrArtifactPackagingInvalid, file.Name) } @@ -208,11 +208,18 @@ type archiveMember struct { } // applyMember validates one archive member against the inventory (or the managed directory) and -// writes it through the tree. -func applyMember(t *tree, checker *managedChecker, m *archiveMember) error { +// writes it through the tree. The member body is read through the context, so a cancellation stops +// a large member mid-copy instead of after it. +func applyMember(ctx context.Context, t *tree, checker *managedChecker, m *archiveMember) error { if err := validateMemberName(m.name); err != nil { return err } + if err := ctx.Err(); err != nil { + return err + } + if m.body != nil { + m.body = contextReader{ctx: ctx, reader: m.body} + } if checker.owns(m.name, t) { return checker.check(m) } diff --git a/pkg/artifact/materialize/archive_internal_test.go b/pkg/artifact/materialize/archive_internal_test.go new file mode 100644 index 00000000000..99fa772f836 --- /dev/null +++ b/pkg/artifact/materialize/archive_internal_test.go @@ -0,0 +1,121 @@ +package materialize + +import ( + "archive/zip" + "bytes" + "context" + "io" + "math/rand/v2" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +const ( + internalBigBytes = 2 << 20 + internalMaxAfter = 256 << 10 +) + +// cancelOnBytesReader cancels once it served cancelAfter bytes and counts the bytes served after. +type cancelOnBytesReader struct { + data []byte + cancel context.CancelFunc + cancelAfter int + + served int + afterCanceled int + canceled bool +} + +func (c *cancelOnBytesReader) Read(p []byte) (int, error) { + if len(c.data) == 0 { + return 0, io.EOF + } + n := copy(p, c.data) + c.data = c.data[n:] + c.served += n + if c.canceled { + c.afterCanceled += n + } + if !c.canceled && c.served >= c.cancelAfter { + c.canceled = true + c.cancel() + } + return n, nil +} + +func noise(size int) []byte { + data := make([]byte, size) + random := rand.New(rand.NewPCG(3, 4)) + for index := range data { + data[index] = byte(random.Uint32()) + } + return data +} + +// newTestTree returns a tree over a temporary root whose inventory holds one file entry. +func newTestTree(t *testing.T, name string, body []byte) (*tree, string) { + t.Helper() + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + record := &artifact.Record{Inventory: []artifact.Entry{{Path: name, Type: artifact.EntryFile, Size: int64(len(body)), Mode: 0o644, Digest: digest.FromBytes(body)}}} + return newTree(root, record), dir +} + +func TestApplyMember_StopsInProgressCopyWhenCanceled(t *testing.T) { + body := noise(internalBigBytes) + tests := []struct { + name string + preCanceled bool + }{ + {"pre-canceled", true}, + {"canceled mid-copy", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr, _ := newTestTree(t, "big.bin", body) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tt.preCanceled { + cancel() + } + reader := &cancelOnBytesReader{data: body, cancel: cancel, cancelAfter: 64 << 10} + err := applyMember(ctx, tr, &managedChecker{}, &archiveMember{name: "big.bin", kind: memberFile, size: int64(len(body)), body: reader}) + require.ErrorIs(t, err, context.Canceled) + assert.Less(t, reader.afterCanceled, internalMaxAfter, "the copy stops soon after the cancellation") + assert.Less(t, reader.served, len(body), "the member is not read to the end") + }) + } +} + +func TestApplyZipFile_StopsInflatingWhenCanceled(t *testing.T) { + body := noise(internalBigBytes) + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("big.bin") + require.NoError(t, err) + _, err = w.Write(body) + require.NoError(t, err) + require.NoError(t, zw.Close()) + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, zr.File, 1) + + tr, dir := newTestTree(t, "big.bin", body) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = applyZipFile(ctx, tr, zr.File[0], &managedChecker{}) + require.ErrorIs(t, err, context.Canceled) + info, statErr := os.Stat(filepath.Join(dir, "big.bin")) + if statErr == nil { + assert.Zero(t, info.Size(), "a canceled member writes no bytes") + } +} diff --git a/pkg/artifact/materialize/cancel_test.go b/pkg/artifact/materialize/cancel_test.go new file mode 100644 index 00000000000..b12a550a137 --- /dev/null +++ b/pkg/artifact/materialize/cancel_test.go @@ -0,0 +1,139 @@ +package materialize_test + +import ( + "context" + "io" + "math/rand/v2" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" +) + +// The sizes below make the tests meaningful: the member is far larger than any read buffer, and +// the cancellation arrives long before the end of it. +const ( + bigMemberBytes = 4 << 20 + cancelAfterBytes = 64 << 10 + maxReadsAfterCancel = 256 << 10 +) + +// cancelingReader cancels a context once it served cancelAfter bytes and counts what it serves +// afterwards. When atEOF is set it cancels at the end of the stream instead. +type cancelingReader struct { + inner io.ReadCloser + cancel context.CancelFunc + cancelAfter int + atEOF bool + + served int + servedCancel int + afterCanceled int + canceled bool +} + +func (c *cancelingReader) Read(p []byte) (int, error) { + n, err := c.inner.Read(p) + c.served += n + if c.canceled { + c.afterCanceled += n + } + if !c.canceled && ((!c.atEOF && c.served >= c.cancelAfter) || (c.atEOF && err == io.EOF)) { + c.canceled = true + c.servedCancel = c.served + c.cancel() + } + return n, err +} + +func (c *cancelingReader) Close() error { return c.inner.Close() } + +// noisyFile writes an incompressible file so the packaged payload stays large for every packaging. +func noisyFile(t *testing.T, src string) { + t.Helper() + data := make([]byte, bigMemberBytes) + random := rand.New(rand.NewPCG(1, 2)) + for index := range data { + data[index] = byte(random.Uint32()) + } + fixtures.WriteFile(t, src, "big.bin", string(data), 0o644) +} + +func payloadStream(t *testing.T, content *artifact.Content) *artifact.ContentStream { + t.Helper() + for index := range content.Streams { + if content.Streams[index].Role == artifact.BlobPayload { + return &content.Streams[index] + } + } + require.FailNow(t, "the fixture must have a payload stream") + return nil +} + +func TestMaterialize_CancellationStopsInProgressArchiveMemberCopy(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + noisyFile(t, src) + content := contentOf(t, prepared(t, src, packaging)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := payloadStream(t, content) + reader := &cancelingReader{inner: stream.Reader, cancel: cancel, cancelAfter: cancelAfterBytes} + stream.Reader = reader + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + require.True(t, reader.canceled, "the fixture must reach the cancellation point") + assert.Less(t, reader.afterCanceled, maxReadsAfterCancel, "reading must stop soon after the cancellation, not drain the member") + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr), "a canceled extraction installs nothing") + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_CancellationAtEndOfPayloadNeverInstalls(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + content := contentOf(t, prepared(t, src, packaging)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := payloadStream(t, content) + reader := &cancelingReader{inner: stream.Reader, cancel: cancel, atEOF: true} + stream.Reader = reader + + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.True(t, reader.canceled, "the fixture must reach the end of the payload") + require.ErrorIs(t, err, context.Canceled, "a context canceled while the last member finished must not install") + _, statErr := os.Lstat(out) + assert.True(t, os.IsNotExist(statErr)) + assertNoScratch(t, out) + }) + } +} + +func TestMaterialize_PreCanceledZipWritesNothing(t *testing.T) { + // A zip member is inflated from the local spool; the in-package test covers the cancellation + // arriving mid-member, this one the whole pipeline with a context that is already canceled. + src := t.TempDir() + noisyFile(t, src) + content := contentOf(t, prepared(t, src, artifact.PackagingZip)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := dest(t) + err := materialize.Materialize(ctx, content, out) + require.ErrorIs(t, err, context.Canceled) + _, statErr := os.Lstat(filepath.Join(out, "big.bin")) + assert.True(t, os.IsNotExist(statErr)) +} diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go index 6b99b4a976a..dd92711a102 100644 --- a/pkg/artifact/materialize/extract.go +++ b/pkg/artifact/materialize/extract.go @@ -83,6 +83,10 @@ func Extract(ctx context.Context, content *artifact.Content, stage, scratch stri if err := fill(ctx, t, content, scratch); err != nil { return err } + // The last member may have finished after a cancellation; never report that as success. + if err := ctx.Err(); err != nil { + return err + } if err := checkLinkContainment(stage, record); err != nil { return err } diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 0c686dab01a..909b7d12283 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -192,11 +192,20 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if err := Extract(ctx, content, stage, scratch, opts...); err != nil { return err } + preserveScratch, err = installStaged(ctx, stage, dest, applyOptions(opts).replaceGuard) + return err +} + +// installStaged moves the extracted tree into place. Cancellation is final: a canceled pull never +// installs, even when extraction had just finished. +func installStaged(ctx context.Context, stage, dest string, guard func(dest string) error) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } if err := os.Chmod(stage, parentDirMode); err != nil { - return fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) + return false, fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) } - preserveScratch, err = replaceDestination(stage, dest, applyOptions(opts).replaceGuard) - return err + return replaceDestination(stage, dest, guard) } // replaceDestination atomically exchanges the staged and existing trees. No children are From b1bcccc466f3c3ad7a53188ff1efa32cff555ecf Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 10:56:10 -0600 Subject: [PATCH 31/52] fix(oci): keep the selected root, referrer metadata and credential guard intact - ExtendedCopy with WithoutSubjectTraversal excluded the subject edge for every node, so a referrer no longer reached the selected root and the final tag failed. The exclusion now applies to the selected root alone. - A matched identity whose authenticator resolves to an empty credential is rejected on every authorization instead of sending anonymous requests. - Referrer index descriptors keep Platform and Data and no longer alias the registry response. - The ImageIndex test doubles are generated with mockgen. Co-Authored-By: Claude Sonnet 5.5 --- pkg/oci/failure_test.go | 25 +++++ pkg/oci/identity.go | 35 +++++++ pkg/oci/mock_index_test.go | 147 ++++++++++++++++++++++++++++ pkg/oci/transport_copy.go | 25 ++++- pkg/oci/transport_hardening_test.go | 54 +++++----- pkg/oci/transport_internal_test.go | 14 ++- pkg/oci/transport_legacy.go | 17 +++- pkg/oci/transport_subject_test.go | 51 ++++++++++ 8 files changed, 335 insertions(+), 33 deletions(-) create mode 100644 pkg/oci/mock_index_test.go diff --git a/pkg/oci/failure_test.go b/pkg/oci/failure_test.go index 272b0b03a84..86d68974251 100644 --- a/pkg/oci/failure_test.go +++ b/pkg/oci/failure_test.go @@ -227,3 +227,28 @@ func TestVerifiedBlobChecksExactSizeAndDigest(t *testing.T) { }) } } + +func TestBoundIdentityWithAnEmptyCredentialIsRejectedWhenUsed(t *testing.T) { + t.Parallel() + resource, err := name.NewRepository("registry.example.com/acme/plugin") + require.NoError(t, err) + for _, tc := range []struct { + name string + auth authn.Authenticator + }{ + {"empty config", authn.FromConfig(authn.AuthConfig{})}, + {"blank basic credential", authn.FromConfig(authn.AuthConfig{Auth: "Og=="})}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + controller := gomock.NewController(t) + resolver := NewMockAuthenticationResolver(controller) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(tc.auth, true, nil) + client := NewClient(nil, WithAuthenticationResolver(resolver)) + actual, err := client.authentication(t.Context(), resource) + require.NoError(t, err) + _, err = actual.Authorization() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + }) + } +} diff --git a/pkg/oci/identity.go b/pkg/oci/identity.go index ad4bbd0c676..69a58fe1120 100644 --- a/pkg/oci/identity.go +++ b/pkg/oci/identity.go @@ -3,10 +3,13 @@ package oci import ( "context" "errors" + "fmt" "github.com/google/go-containerregistry/pkg/authn" + orasauth "oras.land/oras-go/v2/registry/remote/auth" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" ) // AuthenticationResolver resolves explicitly bound identities before ambient credentials. @@ -52,5 +55,37 @@ func (c *Client) boundAuthentication(ctx context.Context, resource authn.Resourc if matched && (auth == nil || auth == authn.Anonymous) { return nil, true, errUtils.ErrOCIRegistry } + if matched { + // An authenticator that resolves to an empty credential would otherwise send requests + // unauthenticated, so it is checked whenever it is asked to authorize. + return nonEmptyCredential{Authenticator: auth}, true, nil + } return auth, matched, nil } + +// nonEmptyCredential makes a matched identity fail closed: an authorization that carries no +// credential is an error, never an anonymous request. The check runs on every authorization, so a +// token that refreshes into an empty credential is rejected too. +type nonEmptyCredential struct { + authn.Authenticator +} + +func (n nonEmptyCredential) Authorization() (*authn.AuthConfig, error) { + defer perf.Track(nil, "oci.nonEmptyCredential.Authorization")() + + config, err := n.Authenticator.Authorization() + if err != nil { + return nil, err + } + if config == nil { + return nil, fmt.Errorf("%w: the bound identity resolved to no credential", errUtils.ErrOCIRegistry) + } + credential, err := credentialFrom(authn.FromConfig(*config)) + if err != nil { + return nil, err + } + if credential == orasauth.EmptyCredential { + return nil, fmt.Errorf("%w: the bound identity resolved to an empty credential", errUtils.ErrOCIRegistry) + } + return config, nil +} diff --git a/pkg/oci/mock_index_test.go b/pkg/oci/mock_index_test.go new file mode 100644 index 00000000000..c4b5c017144 --- /dev/null +++ b/pkg/oci/mock_index_test.go @@ -0,0 +1,147 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: ImageIndex) +// +// Generated by this command: +// +// mockgen -destination mock_index_test.go -package oci github.com/google/go-containerregistry/pkg/v1 ImageIndex +// + +// Package oci is a generated GoMock package. +package oci + +import ( + reflect "reflect" + + v1 "github.com/google/go-containerregistry/pkg/v1" + types "github.com/google/go-containerregistry/pkg/v1/types" + gomock "go.uber.org/mock/gomock" +) + +// MockImageIndex is a mock of ImageIndex interface. +type MockImageIndex struct { + ctrl *gomock.Controller + recorder *MockImageIndexMockRecorder + isgomock struct{} +} + +// MockImageIndexMockRecorder is the mock recorder for MockImageIndex. +type MockImageIndexMockRecorder struct { + mock *MockImageIndex +} + +// NewMockImageIndex creates a new mock instance. +func NewMockImageIndex(ctrl *gomock.Controller) *MockImageIndex { + mock := &MockImageIndex{ctrl: ctrl} + mock.recorder = &MockImageIndexMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockImageIndex) EXPECT() *MockImageIndexMockRecorder { + return m.recorder +} + +// Digest mocks base method. +func (m *MockImageIndex) Digest() (v1.Hash, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Digest") + ret0, _ := ret[0].(v1.Hash) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Digest indicates an expected call of Digest. +func (mr *MockImageIndexMockRecorder) Digest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockImageIndex)(nil).Digest)) +} + +// Image mocks base method. +func (m *MockImageIndex) Image(arg0 v1.Hash) (v1.Image, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Image", arg0) + ret0, _ := ret[0].(v1.Image) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Image indicates an expected call of Image. +func (mr *MockImageIndexMockRecorder) Image(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Image", reflect.TypeOf((*MockImageIndex)(nil).Image), arg0) +} + +// ImageIndex mocks base method. +func (m *MockImageIndex) ImageIndex(arg0 v1.Hash) (v1.ImageIndex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageIndex", arg0) + ret0, _ := ret[0].(v1.ImageIndex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageIndex indicates an expected call of ImageIndex. +func (mr *MockImageIndexMockRecorder) ImageIndex(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageIndex", reflect.TypeOf((*MockImageIndex)(nil).ImageIndex), arg0) +} + +// IndexManifest mocks base method. +func (m *MockImageIndex) IndexManifest() (*v1.IndexManifest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexManifest") + ret0, _ := ret[0].(*v1.IndexManifest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IndexManifest indicates an expected call of IndexManifest. +func (mr *MockImageIndexMockRecorder) IndexManifest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexManifest", reflect.TypeOf((*MockImageIndex)(nil).IndexManifest)) +} + +// MediaType mocks base method. +func (m *MockImageIndex) MediaType() (types.MediaType, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MediaType") + ret0, _ := ret[0].(types.MediaType) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MediaType indicates an expected call of MediaType. +func (mr *MockImageIndexMockRecorder) MediaType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockImageIndex)(nil).MediaType)) +} + +// RawManifest mocks base method. +func (m *MockImageIndex) RawManifest() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RawManifest") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RawManifest indicates an expected call of RawManifest. +func (mr *MockImageIndexMockRecorder) RawManifest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RawManifest", reflect.TypeOf((*MockImageIndex)(nil).RawManifest)) +} + +// Size mocks base method. +func (m *MockImageIndex) Size() (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Size") + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Size indicates an expected call of Size. +func (mr *MockImageIndexMockRecorder) Size() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockImageIndex)(nil).Size)) +} diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index 769d78ef8b1..91a99d2844e 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -81,7 +81,8 @@ func WithCopyObserver(observe func(ocispec.Descriptor)) CopyOption { // WithoutSubjectTraversal stops the copy at the selected manifest instead of also copying the // manifest named by its subject field. Legacy writers (for example signature attachments) -// publish an attachment next to a subject that already exists at the destination. +// publish an attachment next to a subject that already exists at the destination. An extended +// copy still reaches the selected manifest through the subject edges of its referrers. // //nolint:lintroller // Functional option only. func WithoutSubjectTraversal() CopyOption { @@ -104,6 +105,26 @@ func (config copyConfig) graphOptions() oras.CopyGraphOptions { if config.noSubjects { options.FindSuccessors = successorsWithoutSubject } + return config.withObserver(options) +} + +// extendedGraphOptions is graphOptions for an extended copy. Extended traversal starts from the +// outermost referrers and reaches the selected root through their subject edges, so the exclusion +// applies to the selected root alone: a referrer still lists the root it attaches to. +func (config copyConfig) extendedGraphOptions(selected *ocispec.Descriptor) oras.CopyGraphOptions { + options := oras.CopyGraphOptions{Concurrency: config.concurrency} + if config.noSubjects { + options.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if desc.Digest == selected.Digest { + return successorsWithoutSubject(ctx, fetcher, desc) + } + return content.Successors(ctx, fetcher, desc) + } + } + return config.withObserver(options) +} + +func (config copyConfig) withObserver(options oras.CopyGraphOptions) oras.CopyGraphOptions { if config.onCopied != nil { options.PostCopy = func(_ context.Context, desc ocispec.Descriptor) error { config.onCopied(desc) @@ -179,7 +200,7 @@ func ExtendedCopy(ctx context.Context, src GraphSource, dst Destination, options var selected ocispec.Descriptor graph := guardedGraphSource{guardedSource: guardedSource{ReadOnlyStore: src.Store, selected: &selected}, graph: src.Store} root, err := oras.ExtendedCopy(ctx, graph, src.Reference, guardedStore{Store: dst.Store}, dst.Reference, - oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.graphOptions()}}) + oras.ExtendedCopyOptions{ExtendedCopyGraphOptions: oras.ExtendedCopyGraphOptions{CopyGraphOptions: config.extendedGraphOptions(&selected)}}) if err == nil { return root, nil } diff --git a/pkg/oci/transport_hardening_test.go b/pkg/oci/transport_hardening_test.go index bf65c846d7e..78d3408def5 100644 --- a/pkg/oci/transport_hardening_test.go +++ b/pkg/oci/transport_hardening_test.go @@ -27,50 +27,46 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "oras.land/oras-go/v2/content/memory" "oras.land/oras-go/v2/registry/remote" errUtils "github.com/cloudposse/atmos/errors" ) -// imageIndex is embedded under a lower-case name so the embedded field does not shadow the -// ImageIndex method of the interface. -type imageIndex = v1.ImageIndex - -// countingIndex counts how often its children are listed. -type countingIndex struct { - imageIndex - listed *int -} - -func (c countingIndex) IndexManifest() (*v1.IndexManifest, error) { - *c.listed++ - return c.imageIndex.IndexManifest() -} - -// sharedParent lists one nested index twice, as a subgraph shared by two parents appears. -type sharedParent struct { - imageIndex - manifest *v1.IndexManifest - raw []byte - child v1.ImageIndex -} - -func (p sharedParent) RawManifest() ([]byte, error) { return p.raw, nil } -func (p sharedParent) IndexManifest() (*v1.IndexManifest, error) { return p.manifest, nil } -func (p sharedParent) ImageIndex(v1.Hash) (v1.ImageIndex, error) { return p.child, nil } +//go:generate go run go.uber.org/mock/mockgen -destination mock_index_test.go -package oci github.com/google/go-containerregistry/pkg/v1 ImageIndex func TestLegacyIndexGraphVisitsASharedIndexOnceAndStopsOnCancel(t *testing.T) { child, err := random.Index(64, 1, 1) require.NoError(t, err) - listed := 0 - shared := countingIndex{imageIndex: child, listed: &listed} childDesc, err := partial.Descriptor(child) require.NoError(t, err) manifest := &v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex, Manifests: []v1.Descriptor{*childDesc, *childDesc}} raw, err := json.Marshal(manifest) require.NoError(t, err) - parent := sharedParent{imageIndex: empty.Index, manifest: manifest, raw: raw, child: shared} + + // The shared child counts how often its own children are listed and otherwise delegates. + listed := 0 + controller := gomock.NewController(t) + shared := NewMockImageIndex(controller) + shared.EXPECT().MediaType().DoAndReturn(child.MediaType).AnyTimes() + shared.EXPECT().Digest().DoAndReturn(child.Digest).AnyTimes() + shared.EXPECT().Size().DoAndReturn(child.Size).AnyTimes() + shared.EXPECT().RawManifest().DoAndReturn(child.RawManifest).AnyTimes() + shared.EXPECT().IndexManifest().DoAndReturn(func() (*v1.IndexManifest, error) { + listed++ + return child.IndexManifest() + }).AnyTimes() + shared.EXPECT().Image(gomock.Any()).DoAndReturn(child.Image).AnyTimes() + shared.EXPECT().ImageIndex(gomock.Any()).DoAndReturn(child.ImageIndex).AnyTimes() + + // The parent lists the shared child twice, as a subgraph shared by two parents appears. Only the + // requested child digest is served, so selecting any other child fails the test. + parent := NewMockImageIndex(controller) + parent.EXPECT().MediaType().Return(types.OCIImageIndex, nil).AnyTimes() + parent.EXPECT().RawManifest().Return(raw, nil).AnyTimes() + parent.EXPECT().IndexManifest().Return(manifest, nil).AnyTimes() + parent.EXPECT().ImageIndex(childDesc.Digest).Return(shared, nil).AnyTimes() graph, err := legacyIndexGraph(t.Context(), parent) require.NoError(t, err) diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index d1b776f9050..f5ea71d6a6d 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/google/go-containerregistry/pkg/authn" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" @@ -104,6 +105,8 @@ func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { {"matched identity without credentials", nil, true, nil}, {"matched identity resolves to anonymous", authn.Anonymous, true, nil}, {"bound authenticator cannot authorize", failingAuthenticator{}, true, nil}, + {"matched identity with an empty credential", authn.FromConfig(authn.AuthConfig{}), true, nil}, + {"matched identity with an empty basic credential", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte(":"))}), true, nil}, } { t.Run(tc.name, func(t *testing.T) { controller := gomock.NewController(t) @@ -178,15 +181,24 @@ func TestNewReferrersIndexRoundTripsDescriptors(t *testing.T) { found := []ocispec.Descriptor{{ MediaType: ocispec.MediaTypeImageManifest, Size: 10, Digest: digest.FromString("a"), ArtifactType: "application/vnd.dev.sigstore.bundle.v0.3+json", Annotations: map[string]string{"k": "v"}, + Data: []byte("inline"), + Platform: &ocispec.Platform{Architecture: "arm64", OS: "linux", OSVersion: "6.1", OSFeatures: []string{"f1", "f2"}, Variant: "v8"}, }} index, err := newReferrersIndex(found) require.NoError(t, err) + // Mutating the registry response afterwards must not reach the index (source to result isolation). + found[0].Data[0] = 'X' + found[0].Platform.OSFeatures[0] = "changed" + found[0].Annotations["k"] = "changed" manifest, err := index.IndexManifest() require.NoError(t, err) require.Len(t, manifest.Manifests, 1) assert.Equal(t, found[0].Digest.String(), manifest.Manifests[0].Digest.String()) assert.Equal(t, found[0].ArtifactType, manifest.Manifests[0].ArtifactType) - assert.Equal(t, found[0].Annotations, manifest.Manifests[0].Annotations) + assert.Equal(t, map[string]string{"k": "v"}, manifest.Manifests[0].Annotations) + assert.Equal(t, []byte("inline"), manifest.Manifests[0].Data, "embedded data survives the conversion") + require.NotNil(t, manifest.Manifests[0].Platform, "the platform survives the conversion") + assert.Equal(t, &v1.Platform{Architecture: "arm64", OS: "linux", OSVersion: "6.1", OSFeatures: []string{"f1", "f2"}, Variant: "v8"}, manifest.Manifests[0].Platform) size, err := index.Size() require.NoError(t, err) raw, err := index.RawManifest() diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go index 84e1cb5ed29..a92b009d7dd 100644 --- a/pkg/oci/transport_legacy.go +++ b/pkg/oci/transport_legacy.go @@ -8,7 +8,9 @@ import ( "errors" "fmt" "io" + "maps" "os" + "slices" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/stream" @@ -291,7 +293,8 @@ func newReferrersIndex(found []ocispec.Descriptor) (*referrersIndex, error) { } manifest.Manifests = append(manifest.Manifests, v1.Descriptor{ MediaType: types.MediaType(found[i].MediaType), Size: found[i].Size, Digest: hash, - Annotations: found[i].Annotations, ArtifactType: found[i].ArtifactType, URLs: found[i].URLs, + Annotations: maps.Clone(found[i].Annotations), ArtifactType: found[i].ArtifactType, + URLs: slices.Clone(found[i].URLs), Data: slices.Clone(found[i].Data), Platform: legacyPlatform(found[i].Platform), }) } raw, err := json.Marshal(manifest) @@ -305,6 +308,18 @@ func newReferrersIndex(found []ocispec.Descriptor) (*referrersIndex, error) { return &referrersIndex{manifest: manifest, raw: raw, hash: hash}, nil } +// legacyPlatform converts an OCI platform to the legacy descriptor's, copying the feature list so +// the index never aliases the registry response. +func legacyPlatform(platform *ocispec.Platform) *v1.Platform { + if platform == nil { + return nil + } + return &v1.Platform{ + Architecture: platform.Architecture, OS: platform.OS, OSVersion: platform.OSVersion, + OSFeatures: slices.Clone(platform.OSFeatures), Variant: platform.Variant, + } +} + //nolint:lintroller // Trivial getters of an in-memory value. func (r *referrersIndex) MediaType() (types.MediaType, error) { return types.OCIImageIndex, nil } diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index af8d3565e0b..b4ced92387b 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -78,3 +78,54 @@ func TestLayoutPush_RecordsOnlyManifestsThatNameASubject(t *testing.T) { }) } } + +// pushSubjectManifest pushes an image manifest that optionally names a subject and returns its descriptor. +func pushSubjectManifest(t *testing.T, layout *oci.Layout, subject *ocispec.Descriptor, note string) ocispec.Descriptor { + t.Helper() + ctx := t.Context() + config := []byte("{}") + configDesc := oci.Descriptor("application/vnd.oci.empty.v1+json", config) + if exists, err := layout.Exists(ctx, configDesc); err == nil && !exists { + require.NoError(t, layout.Push(ctx, configDesc, strings.NewReader(string(config)))) + } + manifest := ocispec.Manifest{Config: configDesc, Layers: []ocispec.Descriptor{}, Subject: subject, Annotations: map[string]string{"note": note}} + manifest.SchemaVersion = 2 + manifest.MediaType = ocispec.MediaTypeImageManifest + raw, err := json.Marshal(manifest) + require.NoError(t, err) + desc := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, layout.Push(ctx, desc, strings.NewReader(string(raw)))) + return desc +} + +func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers(t *testing.T) { + ctx := t.Context() + source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) + require.NoError(t, err) + grandSubject := pushSubjectManifest(t, source, nil, "grand-subject") + root := pushSubjectManifest(t, source, &grandSubject, "root") + signature := pushSubjectManifest(t, source, &root, "signature") + require.NoError(t, source.Tag(ctx, root, "1.0.0")) + + dest, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "dest")) + require.NoError(t, err) + got, err := oci.ExtendedCopy(ctx, oci.GraphSource{Store: source, Reference: "1.0.0"}, oci.Destination{Store: dest, Reference: "1.0.0"}, oci.WithoutSubjectTraversal()) + require.NoError(t, err) + assert.Equal(t, root.Digest, got.Digest) + + for name, want := range map[string]struct { + desc ocispec.Descriptor + wanted bool + }{ + "the selected root": {root, true}, + "the referrer of the root": {signature, true}, + "the subject of the root": {grandSubject, false}, + } { + exists, err := dest.Exists(ctx, want.desc) + require.NoError(t, err, name) + assert.Equal(t, want.wanted, exists, name) + } + resolved, err := dest.Resolve(ctx, "1.0.0") + require.NoError(t, err) + assert.Equal(t, root.Digest, resolved.Digest) +} From 43ced2592c4442759e5b853b8c2dbc80855c026a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 10:57:29 -0600 Subject: [PATCH 32/52] fix(signature): reject Ed25519 trusted keys that cannot verify by digest sigstore-go verifies an Ed25519 message signature only against the full artifact, but the key verifier holds only the subject digest. Such a key validated at configuration time and then failed every bundle. It now fails configuration with ErrSignatureConfigInvalid and the new ErrSignatureKeyUnsupported cause. Co-Authored-By: Claude Sonnet 5.5 --- errors/signature.go | 3 +++ pkg/signature/key.go | 5 +++++ pkg/signature/key_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/errors/signature.go b/errors/signature.go index 1f74c5b9e95..d8868820895 100644 --- a/errors/signature.go +++ b/errors/signature.go @@ -11,6 +11,9 @@ var ( // ErrSignatureConfigInvalid indicates the verification policy itself is unusable (for example an empty identity). ErrSignatureConfigInvalid = errors.New("invalid signature verification configuration") + // ErrSignatureKeyUnsupported indicates a configured trusted key cannot verify bundles by subject digest alone. + ErrSignatureKeyUnsupported = errors.New("signature key type cannot verify by digest") + // ErrSignatureOffline indicates trust material cannot be fetched while verifying offline. ErrSignatureOffline = errors.New("signature trust material is unavailable offline") diff --git a/pkg/signature/key.go b/pkg/signature/key.go index 89c1cf5a589..d636450e132 100644 --- a/pkg/signature/key.go +++ b/pkg/signature/key.go @@ -128,6 +128,11 @@ func (v KeyVerifier) keyRounds() ([]root.TrustedMaterial, error) { if err != nil { return nil, fail(errUtils.ErrSignatureConfigInvalid, err) } + // Message signatures made with Ed25519 can only be verified against the full artifact, and + // this path holds only its digest, so the key would pass here and then reject every bundle. + if _, pure := loaded.(*sigstore.ED25519Verifier); pure { + return nil, fail(errUtils.ErrSignatureConfigInvalid, fmt.Errorf("%w: ed25519 signatures need the full artifact", errUtils.ErrSignatureKeyUnsupported)) + } round := seen[key.Hint] seen[key.Hint]++ if round == len(rounds) { diff --git a/pkg/signature/key_test.go b/pkg/signature/key_test.go index 21eb5b87345..f62b981e41d 100644 --- a/pkg/signature/key_test.go +++ b/pkg/signature/key_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto" "crypto/ecdsa" + "crypto/ed25519" "crypto/elliptic" "crypto/rand" "strings" @@ -148,3 +149,30 @@ func TestKeyVerifierTriesEveryKeyThatSharesAHint(t *testing.T) { _, err = KeyVerifier{Keys: []TrustedKey{wrong, {Hint: signed.Hint, PublicKey: &other.PublicKey}}, LoadRoot: logsOnly(authority)}.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) } + +// Ed25519 message signatures can only be verified against the full artifact, which the digest-based +// verification path never has, so such a key is a configuration error rather than a key that +// validates and then fails every bundle. +func TestKeyVerifierRejectsEd25519KeysAtConfiguration(t *testing.T) { + authority := signaturetest.New(t) + artifact := []byte("key-signed release") + signed := authority.SignWithKey(t, artifact) + edPublic, _, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + + for _, tc := range []struct { + name string + keys []TrustedKey + }{ + {"ed25519 key alone", []TrustedKey{{Hint: signed.Hint, PublicKey: edPublic}}}, + {"ed25519 key beside a supported key", []TrustedKey{{Hint: signed.Hint, PublicKey: signed.PublicKey}, {Hint: "ed", PublicKey: edPublic}}}, + } { + t.Run(tc.name, func(t *testing.T) { + verifier := KeyVerifier{Keys: tc.keys, LoadRoot: logsOnly(authority)} + result, err := verifier.Verify(t.Context(), signed.Bundle, digestOf(artifact), true) + require.ErrorIs(t, err, errUtils.ErrSignatureConfigInvalid) + assert.Contains(t, err.Error(), "ed25519") + assert.Nil(t, result) + }) + } +} From d460fdd568f103454d77d0887b09950ab64acba0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:02:28 -0600 Subject: [PATCH 33/52] fix(materialize): honor cancellation in metadata reads, replace guard and link types - The tar reader reads through the context, so a run of local PAX headers inside Next can be interrupted; cancellation is reported as such, not as an integrity failure. - installStaged checks the context again after the replace guard and before the exchange, so a cancellation during a long guard never swaps the tree. - Symlinks are created after every other member exists, ordered so a link follows the link its target runs through. Windows picks a file or directory link at creation time and would otherwise make a directory link untraversable. - WithOwnerAccess keeps owner read and traversal bits on a temporary tree while the Record keeps its modes. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/archive.go | 18 ++- .../materialize/archive_internal_test.go | 90 ++++++++++++ pkg/artifact/materialize/extract.go | 17 +++ pkg/artifact/materialize/replace.go | 8 +- .../replace_cancel_internal_test.go | 67 +++++++++ pkg/artifact/materialize/tree.go | 128 +++++++++++++++- .../materialize/tree_links_internal_test.go | 137 ++++++++++++++++++ 7 files changed, 454 insertions(+), 11 deletions(-) create mode 100644 pkg/artifact/materialize/replace_cancel_internal_test.go create mode 100644 pkg/artifact/materialize/tree_links_internal_test.go diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index 1604d905f5d..6b438a14c11 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -66,13 +66,16 @@ func fillArchive(ctx context.Context, t *tree, content *artifact.Content, scratc } func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, checker *managedChecker) error { + // Every read goes through the context, metadata included: Next consumes a run of local PAX + // headers before it returns a visible member, so the loop's own check cannot interrupt it. + reader = contextReader{ctx: ctx, reader: reader} if gzipped { gz, err := gzip.NewReader(reader) if err != nil { - return fmt.Errorf("%w: gzip: %w", errUtils.ErrArtifactIntegrityFailed, err) + return archiveReadError(ctx, "gzip", err) } defer gz.Close() - reader = gz + reader = contextReader{ctx: ctx, reader: gz} } tr := tar.NewReader(reader) for { @@ -86,7 +89,7 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check return err } if err != nil { - return fmt.Errorf("%w: tar: %w", errUtils.ErrArtifactIntegrityFailed, err) + return archiveReadError(ctx, "tar", err) } if err := applyTarHeader(ctx, t, tr, header, checker); err != nil { return err @@ -94,6 +97,15 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check } } +// archiveReadError reports a failed archive read. A canceled context is returned as such, because +// the cancellation is what interrupted the read and not damage to the archive. +func archiveReadError(ctx context.Context, format string, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactIntegrityFailed, format, err) +} + func applyTarHeader(ctx context.Context, t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { name := strings.TrimSuffix(path.Clean(header.Name), "/") switch header.Typeflag { diff --git a/pkg/artifact/materialize/archive_internal_test.go b/pkg/artifact/materialize/archive_internal_test.go index 99fa772f836..2732fe1cd01 100644 --- a/pkg/artifact/materialize/archive_internal_test.go +++ b/pkg/artifact/materialize/archive_internal_test.go @@ -1,9 +1,12 @@ package materialize import ( + "archive/tar" "archive/zip" "bytes" "context" + "errors" + "fmt" "io" "math/rand/v2" "os" @@ -14,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" ) @@ -119,3 +123,89 @@ func TestApplyZipFile_StopsInflatingWhenCanceled(t *testing.T) { assert.Zero(t, info.Size(), "a canceled member writes no bytes") } } + +// paxRecord encodes one PAX extended header record, whose leading decimal counts its own digits. +func paxRecord(key, value string) string { + body := " " + key + "=" + value + "\n" + length := len(body) + 1 + for len(fmt.Sprint(length))+len(body) != length { + length = len(fmt.Sprint(length)) + len(body) + } + return fmt.Sprintf("%d%s", length, body) +} + +// rawPaxHeader hand-encodes a local PAX extended header, which archive/tar refuses to write itself. +func rawPaxHeader(records string) []byte { + const block = 512 + header := make([]byte, block) + copy(header[0:], "pax") + copy(header[100:], "0000644\x00") + copy(header[108:], "0000000\x00") + copy(header[116:], "0000000\x00") + copy(header[124:], fmt.Sprintf("%011o\x00", len(records))) + copy(header[136:], "00000000000\x00") + copy(header[148:], " ") + header[156] = tar.TypeXHeader + copy(header[257:], "ustar\x0000") + sum := 0 + for _, b := range header { + sum += int(b) + } + copy(header[148:], fmt.Sprintf("%06o\x00 ", sum)) + padded := make([]byte, (len(records)+block-1)/block*block) + copy(padded, records) + return append(header, padded...) +} + +// successivePaxTar returns a tar stream of count local PAX headers in a row that are followed by one +// regular file. Reading the headers is metadata work that precedes any visible member. +func successivePaxTar(t *testing.T, count int, name string, body []byte) []byte { + t.Helper() + var buf bytes.Buffer + for index := range count { + buf.Write(rawPaxHeader(paxRecord("comment", fmt.Sprintf("header-%d", index)))) + } + writer := tar.NewWriter(&buf) + require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: name, Size: int64(len(body)), Mode: 0o644})) + _, err := writer.Write(body) + require.NoError(t, err) + require.NoError(t, writer.Close()) + return buf.Bytes() +} + +func TestFillTar_CancellationInterruptsMetadataReads(t *testing.T) { + body := []byte("payload") + const ( + headers = 2000 + cancelAfter = 8 << 10 + // One block of read-ahead is the most a reader may serve after the cancellation. + maxAfterCanceled = 4 << 10 + ) + tests := []struct { + name string + preCanceled bool + }{ + {"canceled before the first header", true}, + {"canceled between successive headers", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data := successivePaxTar(t, headers, "file.txt", body) + require.Greater(t, len(data), 100*cancelAfter, "the headers must outlast the cancellation point") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + reader := &cancelOnBytesReader{data: data, cancel: cancel, cancelAfter: cancelAfter} + if tc.preCanceled { + cancel() + } + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + + err := fillTar(ctx, target, reader, false, checker) + + require.ErrorIs(t, err, context.Canceled) + assert.False(t, errors.Is(err, errUtils.ErrArtifactIntegrityFailed), "cancellation is not an integrity failure") + assert.LessOrEqual(t, reader.afterCanceled, maxAfterCanceled, "metadata reads must stop once the context is canceled") + }) + } +} diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go index dd92711a102..56f503111fa 100644 --- a/pkg/artifact/materialize/extract.go +++ b/pkg/artifact/materialize/extract.go @@ -23,6 +23,19 @@ type options struct { // replaceGuard runs immediately before an existing destination is replaced. replaceGuard func(dest string) error + + // ownerAccess keeps owner read and traversal access on the extracted tree. + ownerAccess bool +} + +// WithOwnerAccess keeps owner read access on files and owner read and traversal access on +// directories, whatever modes the Record declares. It is for a temporary tree that is read back +// before it is discarded: a valid Record can declare mode 0000, and an ordinary user could not read +// such a tree. The Record itself is never changed. +func WithOwnerAccess() Option { + defer perf.Track(nil, "materialize.WithOwnerAccess")() + + return func(o *options) { o.ownerAccess = true } } // WithReplaceGuard registers a check that runs right before an existing destination directory is @@ -80,6 +93,7 @@ func Extract(ctx context.Context, content *artifact.Content, stage, scratch stri defer root.Close() t := newTree(root, record) + t.ownerAccess = applied.ownerAccess if err := fill(ctx, t, content, scratch); err != nil { return err } @@ -87,6 +101,9 @@ func Extract(ctx context.Context, content *artifact.Content, stage, scratch stri if err := ctx.Err(); err != nil { return err } + if err := t.createLinks(ctx); err != nil { + return err + } if err := checkLinkContainment(stage, record); err != nil { return err } diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 909b7d12283..801ddbfbd6c 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -205,13 +205,13 @@ func installStaged(ctx context.Context, stage, dest string, guard func(dest stri if err := os.Chmod(stage, parentDirMode); err != nil { return false, fmt.Errorf("%w: set staging mode: %w", errUtils.ErrArtifactPackagingInvalid, err) } - return replaceDestination(stage, dest, guard) + return replaceDestination(ctx, stage, dest, guard) } // replaceDestination atomically exchanges the staged and existing trees. No children are // removed or installed through the destination path, and a failed exchange leaves both intact. // The boolean preserves scratch when a concurrent replacement cannot be safely rolled back. -func replaceDestination(stage, dest string, guard func(dest string) error) (bool, error) { +func replaceDestination(ctx context.Context, stage, dest string, guard func(dest string) error) (bool, error) { info, err := os.Lstat(dest) switch { case os.IsNotExist(err): @@ -236,6 +236,10 @@ func replaceDestination(stage, dest string, guard func(dest string) error) (bool if err := bound.verify(); err != nil { return false, err } + // The guard can run long on a large destination, and cancellation is final: it must not exchange. + if err := ctx.Err(); err != nil { + return false, err + } return exchangeStagedTree(stage, dest, bound.info) } diff --git a/pkg/artifact/materialize/replace_cancel_internal_test.go b/pkg/artifact/materialize/replace_cancel_internal_test.go new file mode 100644 index 00000000000..cf66f7fac16 --- /dev/null +++ b/pkg/artifact/materialize/replace_cancel_internal_test.go @@ -0,0 +1,67 @@ +package materialize + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stagedAndDestination lays out a staged tree and an existing destination that hold different content. +func stagedAndDestination(t *testing.T) (stage, dest string) { + t.Helper() + root := t.TempDir() + stage = filepath.Join(root, "scratch", "tree") + dest = filepath.Join(root, "dest") + require.NoError(t, os.MkdirAll(stage, 0o700)) + require.NoError(t, os.Mkdir(dest, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(stage, "new"), []byte("new"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dest, "original"), []byte("original"), 0o600)) + return stage, dest +} + +func TestInstallStaged_CancellationDuringTheReplaceGuardIsFinal(t *testing.T) { + tests := []struct { + name string + cancelInGuard bool + }{ + {"canceled while the guard runs", true}, + {"not canceled", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if !tc.cancelInGuard && runtime.GOOS == "windows" { + t.Skipf("atomic exchange of an existing destination is unavailable on %s", runtime.GOOS) + } + stage, dest := stagedAndDestination(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + guard := func(string) error { + if tc.cancelInGuard { + cancel() + } + return nil + } + + _, err := installStaged(ctx, stage, dest, guard) + + if !tc.cancelInGuard { + require.NoError(t, err) + data, readErr := os.ReadFile(filepath.Join(dest, "new")) + require.NoError(t, readErr) + assert.Equal(t, "new", string(data)) + return + } + require.ErrorIs(t, err, context.Canceled) + data, readErr := os.ReadFile(filepath.Join(dest, "original")) + require.NoError(t, readErr) + assert.Equal(t, "original", string(data), "the previous content stays in place") + assert.NoFileExists(t, filepath.Join(dest, "new")) + assert.FileExists(t, filepath.Join(stage, "new"), "the staged tree is not installed") + }) + } +} diff --git a/pkg/artifact/materialize/tree.go b/pkg/artifact/materialize/tree.go index b04b3df8cbe..cf84f4567fb 100644 --- a/pkg/artifact/materialize/tree.go +++ b/pkg/artifact/materialize/tree.go @@ -1,6 +1,7 @@ package materialize import ( + "context" "crypto/sha256" "fmt" "io" @@ -8,6 +9,7 @@ import ( "os" "path" "path/filepath" + "slices" "sort" "github.com/opencontainers/go-digest" @@ -19,6 +21,10 @@ import ( const ( stagingDirMode fs.FileMode = 0o700 stagingFileMode fs.FileMode = 0o600 + + // Owner bits that a temporary tree keeps so it can be read back. + ownerFileAccess fs.FileMode = 0o400 + ownerDirAccess fs.FileMode = 0o700 ) // tree writes verified entries below a root and remembers directory modes to apply last. @@ -26,6 +32,14 @@ type tree struct { root *os.Root inventory map[string]*artifact.Entry written map[string]bool + + // links holds the symlinks validated so far. They are created after every other member + // exists (see createLinks), so a platform that types a link by its target sees the target. + links []*artifact.Entry + + // ownerAccess keeps owner read and traversal access on the final modes. The Record keeps the + // original modes; this only protects a temporary tree that is read back before it is discarded. + ownerAccess bool } func newTree(root *os.Root, record *artifact.Record) *tree { @@ -85,16 +99,105 @@ func (t *tree) writeFile(entry *artifact.Entry, src io.Reader) error { return nil } +// symlink records a symlink entry. The link itself is created by createLinks once every file and +// directory exists: Windows chooses between a file link and a directory link when the link is +// created, and a directory link to a target that does not exist yet becomes a file link that can +// never be traversed. func (t *tree) symlink(entry *artifact.Entry) error { - if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { - return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) - } - if err := t.root.Symlink(filepath.FromSlash(entry.Target), filepath.FromSlash(entry.Path)); err != nil { - return fmt.Errorf("%w: this platform cannot create symlink %q -> %q (it is not silently replaced): %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, entry.Target, err) + t.links = append(t.links, entry) + return nil +} + +// createLinks creates the recorded symlinks. A link whose target runs through another link is +// created after that link, so the target resolves to what it will be once the tree is complete. +func (t *tree) createLinks(ctx context.Context) error { + for _, entry := range orderLinks(t.links) { + if err := ctx.Err(); err != nil { + return err + } + if err := t.root.MkdirAll(filepath.FromSlash(path.Dir(entry.Path)), stagingDirMode); err != nil { + return fmt.Errorf("%w: create parent of %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + if err := t.root.Symlink(filepath.FromSlash(entry.Target), filepath.FromSlash(entry.Path)); err != nil { + return fmt.Errorf("%w: this platform cannot create symlink %q -> %q (it is not silently replaced): %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, entry.Target, err) + } } return nil } +// orderLinks sorts links by path and then moves each link after the link its target passes through. +// Links that depend on each other in a cycle keep their path order; they can never resolve anyway. +func orderLinks(links []*artifact.Entry) []*artifact.Entry { + sorted := slices.Clone(links) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + pending, dependents := linkDependencies(sorted) + ordered := make([]*artifact.Entry, 0, len(sorted)) + done := make([]bool, len(sorted)) + var ready []int + for index := range sorted { + if pending[index] == 0 { + ready = append(ready, index) + } + } + for len(ready) > 0 { + index := ready[0] + ready = ready[1:] + done[index] = true + ordered = append(ordered, sorted[index]) + ready = append(ready, releaseDependents(dependents[index], pending)...) + } + for index, link := range sorted { + if !done[index] { + ordered = append(ordered, link) + } + } + return ordered +} + +// linkDependencies counts, for each link, the links that must exist before it, and lists the links +// that wait on each link. +func linkDependencies(sorted []*artifact.Entry) (pending []int, dependents [][]int) { + byPath := make(map[string]int, len(sorted)) + for index, link := range sorted { + byPath[link.Path] = index + } + pending = make([]int, len(sorted)) + dependents = make([][]int, len(sorted)) + for index, link := range sorted { + if dep, ok := linkThrough(link, byPath); ok && dep != index { + pending[index]++ + dependents[dep] = append(dependents[dep], index) + } + } + return pending, dependents +} + +// releaseDependents counts one satisfied dependency for each waiting link and returns the links +// that are now ready. +func releaseDependents(waiting []int, pending []int) []int { + var ready []int + for _, index := range waiting { + pending[index]-- + if pending[index] == 0 { + ready = append(ready, index) + } + } + return ready +} + +// linkThrough finds the nearest link that the target of link passes through, given the cleaned +// target path relative to the archive root. +func linkThrough(link *artifact.Entry, byPath map[string]int) (int, bool) { + target := path.Clean(path.Dir(link.Path) + "/" + link.Target) + for target != "." && target != "/" && target != "" { + if index, ok := byPath[target]; ok { + return index, true + } + target = path.Dir(target) + } + return 0, false +} + // complete checks that every inventory entry was produced and applies final modes: files first, // then directories deepest-first so a restrictive directory mode never blocks earlier chmods. func (t *tree) complete() error { @@ -113,9 +216,22 @@ func (t *tree) complete() error { if entry.Type == artifact.EntrySymlink { continue } - if err := t.root.Chmod(filepath.FromSlash(entryPath), fs.FileMode(entry.Mode)); err != nil { + if err := t.root.Chmod(filepath.FromSlash(entryPath), t.finalMode(entry)); err != nil { return fmt.Errorf("%w: set mode %#o on %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Mode, entryPath, err) } } return nil } + +// finalMode is the mode applied to an entry: its recorded mode, plus owner access when the tree is a +// temporary one that is read back (read for files, read and traversal for directories). +func (t *tree) finalMode(entry *artifact.Entry) fs.FileMode { + mode := fs.FileMode(entry.Mode) + if !t.ownerAccess { + return mode + } + if entry.Type == artifact.EntryDirectory { + return mode | ownerDirAccess + } + return mode | ownerFileAccess +} diff --git a/pkg/artifact/materialize/tree_links_internal_test.go b/pkg/artifact/materialize/tree_links_internal_test.go new file mode 100644 index 00000000000..8ea5bee55da --- /dev/null +++ b/pkg/artifact/materialize/tree_links_internal_test.go @@ -0,0 +1,137 @@ +package materialize + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +func linkEntry(name, target string) *artifact.Entry { + return &artifact.Entry{Path: name, Type: artifact.EntrySymlink, Target: target} +} + +func linkPaths(links []*artifact.Entry) []string { + paths := make([]string, 0, len(links)) + for _, link := range links { + paths = append(paths, link.Path) + } + return paths +} + +func TestOrderLinks_CreatesALinkAfterTheLinkItsTargetPassesThrough(t *testing.T) { + tests := []struct { + name string + links []*artifact.Entry + want []string + }{ + {"independent links keep path order", []*artifact.Entry{linkEntry("b", "x"), linkEntry("a", "y")}, []string{"a", "b"}}, + {"a link to a directory that sorts after it", []*artifact.Entry{linkEntry("alias", "z-dir")}, []string{"alias"}}, + {"a chain is created from its end", []*artifact.Entry{linkEntry("a", "b"), linkEntry("b", "c"), linkEntry("c", "dir")}, []string{"c", "b", "a"}}, + {"a target below a link", []*artifact.Entry{linkEntry("x", "y/sub"), linkEntry("y", "dir")}, []string{"y", "x"}}, + {"a relative target in a subdirectory", []*artifact.Entry{linkEntry("d/a", "../e"), linkEntry("e", "dir")}, []string{"e", "d/a"}}, + {"a cycle keeps path order and terminates", []*artifact.Entry{linkEntry("q", "p"), linkEntry("p", "q")}, []string{"p", "q"}}, + {"a link to itself terminates", []*artifact.Entry{linkEntry("s", "s")}, []string{"s"}}, + {"no links", nil, []string{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + original := linkPaths(tc.links) + got := linkPaths(orderLinks(tc.links)) + assert.Equal(t, tc.want, got) + assert.Equal(t, original, linkPaths(tc.links), "the caller's slice is not reordered") + }) + } +} + +func TestTree_CreatesSymlinksOnlyAfterEveryOtherMemberExists(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + alias := linkEntry("alias", "z-dir") + zdir := &artifact.Entry{Path: "z-dir", Type: artifact.EntryDirectory, Mode: 0o755} + target := newTree(root, &artifact.Record{Inventory: []artifact.Entry{*alias, *zdir}}) + + require.NoError(t, target.symlink(alias)) + _, err = os.Lstat(filepath.Join(dir, "alias")) + require.ErrorIs(t, err, os.ErrNotExist, "recording a link must not create it yet") + + require.NoError(t, target.mkdir(zdir)) + if err := target.createLinks(t.Context()); err != nil { + t.Skipf("this platform cannot create symlinks: %v", err) + } + info, err := os.Lstat(filepath.Join(dir, "alias")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&os.ModeSymlink) + entries, err := os.ReadDir(filepath.Join(dir, "alias")) + require.NoError(t, err, "the link must be traversable as a directory") + assert.Empty(t, entries) +} + +func TestTree_OwnerAccessKeepsATemporaryTreeReadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + body := []byte("sealed") + tests := []struct { + name string + ownerAccess bool + wantFile os.FileMode + wantDir os.FileMode + }{ + {"recorded modes are applied as declared", false, 0o000, 0o000}, + {"owner access is kept on top of the recorded modes", true, 0o400, 0o700}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { _ = root.Close() }) + record := &artifact.Record{Inventory: []artifact.Entry{ + {Path: "d", Type: artifact.EntryDirectory, Mode: 0}, + {Path: "d/f", Type: artifact.EntryFile, Size: int64(len(body)), Mode: 0, Digest: digest.FromBytes(body)}, + }} + target := newTree(root, record) + target.ownerAccess = tc.ownerAccess + for index := range record.Inventory { + entry := &record.Inventory[index] + _, err := target.entryFor(entry.Path) + require.NoError(t, err) + if entry.Type == artifact.EntryDirectory { + require.NoError(t, target.mkdir(entry)) + continue + } + require.NoError(t, target.writeFile(entry, bytes.NewReader(body))) + } + + require.NoError(t, target.complete()) + assert.Zero(t, record.Inventory[0].Mode, "the Record keeps the modes it declared") + assert.Zero(t, record.Inventory[1].Mode, "the Record keeps the modes it declared") + t.Cleanup(func() { + _ = os.Chmod(filepath.Join(dir, "d"), 0o700) + _ = os.Chmod(filepath.Join(dir, "d", "f"), 0o600) + }) + + dirInfo, err := os.Stat(filepath.Join(dir, "d")) + require.NoError(t, err) + assert.Equal(t, tc.wantDir, dirInfo.Mode().Perm()) + if tc.ownerAccess { + fileInfo, err := os.Stat(filepath.Join(dir, "d", "f")) + require.NoError(t, err) + assert.Equal(t, tc.wantFile, fileInfo.Mode().Perm()) + data, err := os.ReadFile(filepath.Join(dir, "d", "f")) + require.NoError(t, err) + assert.Equal(t, body, data) + } + }) + } +} From d9e3aa10719bee63bfd3198aff31d2c3b78eef9b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:06:12 -0600 Subject: [PATCH 34/52] fix(snapshot): recheck metadata after the final hash, honor cancellation and judge links as captured - The final recheck reads the file metadata again after hashing, so a mode change that leaves bytes and mtime alone is rejected as a source change. - Capture checks the context before staging and before returning, so an already canceled capture of an empty directory no longer succeeds. - Link containment resolves the captured links (linkpath.Escapes) instead of the live source, which another process may change after the recheck. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/linkpath/linkpath.go | 37 ++++++ pkg/artifact/linkpath/linkpath_test.go | 25 ++++ pkg/artifact/snapshot/copy.go | 19 +++ .../snapshot/recheck_internal_test.go | 120 ++++++++++++++++++ pkg/artifact/snapshot/snapshot.go | 26 +++- pkg/artifact/snapshot/walk.go | 29 ++--- 6 files changed, 232 insertions(+), 24 deletions(-) create mode 100644 pkg/artifact/snapshot/recheck_internal_test.go diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index 549917a518f..15e116c9aad 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -40,6 +40,43 @@ func Resolve(target string) (string, bool) { return filepath.Join(append([]string{w.resolved}, w.missing...)...), true } +// Escapes reports whether the symlink at from leaves the tree that links describes. The tree is +// recorded, not read from disk: links maps the slash-separated path of every symlink, relative to +// the tree root, to its target. Resolution follows the recorded links exactly like Resolve follows +// physical ones (a dangling tail is resolved lexically), so the answer cannot change when the +// files the links were read from change afterwards. A link chain that loops does not escape. +func Escapes(links map[string]string, from string) bool { + defer perf.Track(nil, "linkpath.Escapes")() + + pending := splitComponents(from) + var resolved []string + for hops := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case ".": + case parentComponent: + if len(resolved) == 0 { + return true + } + resolved = resolved[:len(resolved)-1] + default: + current := strings.Join(append(resolved, component), "/") + target, isLink := links[current] + if !isLink { + resolved = append(resolved, component) + continue + } + hops++ + if hops > maxLinkHops { + return false + } + pending = append(splitComponents(target), pending...) + } + } + return false +} + // walker resolves a path one component at a time. type walker struct { // base is the filesystem root, with its volume on Windows. diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go index 687820dc279..a7b3c42e231 100644 --- a/pkg/artifact/linkpath/linkpath_test.go +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -91,3 +91,28 @@ func TestWithin(t *testing.T) { }) } } + +func TestEscapes(t *testing.T) { + tests := []struct { + name string + links map[string]string + from string + want bool + }{ + {"link to a sibling stays inside", map[string]string{"a": "b"}, "a", false}, + {"link above the root escapes", map[string]string{"a": ".."}, "a", true}, + {"link to a missing path below a link to the root escapes through it", map[string]string{"d": ".", "L": "d/../outside/missing"}, "L", true}, + {"the same link inside the root when the directory link goes deeper", map[string]string{"d": "sub", "L": "d/../outside/missing"}, "L", false}, + {"chained links that climb out escape", map[string]string{"a/b": "..", "c": "a/b/../.."}, "c", true}, + {"chained links that stay inside", map[string]string{"a/b": "..", "c": "a/b/x"}, "c", false}, + {"a link inside a linked directory is read through the link", map[string]string{"d": "sub", "sub/L": "../.."}, "d/L", true}, + {"a dangling link beyond the root escapes", map[string]string{"a": "x/../../y"}, "a", true}, + {"a loop does not escape and terminates", map[string]string{"p": "q", "q": "p"}, "p", false}, + {"a plain name is not a link", map[string]string{}, "plain", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, linkpath.Escapes(tc.links, tc.from)) + }) + } +} diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index 95637ba02b6..9c4b52d419b 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -163,6 +163,25 @@ func (c *capturer) recheckFile(record *sourceRecord, now os.FileInfo) error { if !sameFileState(record.info, opened) { return changedError(record.rel, errSizeOrTimeChanged) } + if err := c.rehash(file, record); err != nil { + return err + } + if hook := c.opts.onRehashed; hook != nil { + hook(record.rel) + } + // A mode change leaves the bytes and the digest alone, so the metadata is read once more. + final, err := file.Stat() + if err != nil { + return changedError(record.rel, err) + } + if !sameFileState(record.info, final) { + return changedError(record.rel, errSizeOrTimeChanged) + } + return nil +} + +// rehash reads the source file again and compares its size and digest with the staged copy. +func (c *capturer) rehash(file *os.File, record *sourceRecord) error { // Read at most one byte past the recorded size so a file that grew after the stat is rejected // without being read in full. hasher := sha256.New() diff --git a/pkg/artifact/snapshot/recheck_internal_test.go b/pkg/artifact/snapshot/recheck_internal_test.go new file mode 100644 index 00000000000..a455784492d --- /dev/null +++ b/pkg/artifact/snapshot/recheck_internal_test.go @@ -0,0 +1,120 @@ +package snapshot + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// withRehashHook registers a callback invoked after a source file was hashed again and before the +// recheck inspects its metadata one last time. +func withRehashHook(hook func(path string)) Option { + return func(o *options) { o.onRehashed = hook } +} + +func TestCapture_RejectsAModeChangeDuringTheFinalHash(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("file modes cannot be changed independently of content on %s", runtime.GOOS) + } + src := t.TempDir() + file := filepath.Join(src, "tool.sh") + require.NoError(t, os.WriteFile(file, []byte("#!/bin/sh\n"), 0o644)) + require.NoError(t, os.Chmod(file, 0o644)) + + // Byte count, digest, and modification time all stay the same: only the mode changes. + hook := func(rel string) { + assert.Equal(t, "tool.sh", rel) + require.NoError(t, os.Chmod(file, 0o755)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(hook)) + + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} + +func TestCapture_StaysQuietWhenNothingChangesDuringTheFinalHash(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("a"), 0o644)) + var seen []string + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(func(rel string) { seen = append(seen, rel) })) + require.NoError(t, err) + defer snap.Close() + assert.Equal(t, []string{"a.txt"}, seen) +} + +func TestCapture_RejectsCancellationBeforeAndAfterAnEmptyDirectory(t *testing.T) { + tests := []struct { + name string + do func(t *testing.T, stagingRoot string) error + }{ + {"already canceled", func(t *testing.T, stagingRoot string) error { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := Capture(ctx, t.TempDir(), WithStagingRoot(stagingRoot)) + return err + }}, + {"canceled after the last file was rechecked", func(t *testing.T, stagingRoot string) error { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("a"), 0o644)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + _, err := Capture(ctx, src, WithStagingRoot(stagingRoot), withRehashHook(func(string) { cancel() })) + return err + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stagingRoot := t.TempDir() + require.ErrorIs(t, tc.do(t, stagingRoot), context.Canceled) + leftovers, err := os.ReadDir(stagingRoot) + require.NoError(t, err) + assert.Empty(t, leftovers, "a canceled capture leaves no staging directory behind") + }) + } +} + +// The containment check reads the captured links, not the source: the source may change after the +// final recheck, and the snapshot still records the links as they were captured. +func TestCheckLinkContainment_UsesTheCapturedLinkTargets(t *testing.T) { + root := t.TempDir() + // On disk the directory link now points deeper, which would make the chain below look contained. + require.NoError(t, os.Mkdir(filepath.Join(root, "sub"), 0o755)) + if err := os.Symlink("sub", filepath.Join(root, "d")); err != nil { + t.Skipf("this platform cannot create symlinks: %v", err) + } + require.NoError(t, os.Symlink("d/../outside/missing", filepath.Join(root, "L"))) + + tests := []struct { + name string + sources []sourceRecord + wantErr error + }{ + {"the captured chain escapes", []sourceRecord{ + {abs: filepath.Join(root, "d"), rel: "d", target: "."}, + {abs: filepath.Join(root, "L"), rel: "L", target: "d/../outside/missing"}, + }, errUtils.ErrArtifactUnsafePath}, + {"the captured chain stays inside", []sourceRecord{ + {abs: filepath.Join(root, "d"), rel: "d", target: "sub"}, + {abs: filepath.Join(root, "L"), rel: "L", target: "d/../outside/missing"}, + }, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &capturer{root: root, sources: tc.sources} + err := c.checkLinkContainment() + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + assert.Contains(t, err.Error(), `"L"`) + return + } + require.NoError(t, err) + }) + } +} diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index dd6d78eeb75..bde0c296e14 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -47,6 +47,7 @@ type options struct { stagingRoot string limits Limits onCaptured func(path string) + onRehashed func(path string) } // WithStagingRoot sets the directory that holds the private staging directory. The default is the @@ -159,6 +160,10 @@ func Capture(ctx context.Context, source string, opts ...Option) (*Snapshot, err if applied.limits.MaxEntries <= 0 { applied.limits.MaxEntries = DefaultMaxEntries } + // Cancellation is final even when no entry would ever check it, such as an empty directory. + if err := ctx.Err(); err != nil { + return nil, err + } root, err := os.MkdirTemp(applied.stagingRoot, stagingPattern) if err != nil { return nil, fmt.Errorf("%w: create staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) @@ -193,19 +198,28 @@ func captureInto(ctx context.Context, source, root string, opts *options) (*Snap if err != nil { return nil, err } - if err := c.recheck(); err != nil { + if err := c.verify(info.IsDir()); err != nil { return nil, err } - if info.IsDir() { + sort.Slice(c.entries, func(i, j int) bool { return c.entries[i].Path < c.entries[j].Path }) + return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), staged: c.staged}, nil +} + +// verify checks the finished capture: the source is unchanged, no link leaves it, the inventory is +// valid, and the capture was not canceled while it ran. +func (c *capturer) verify(isDir bool) error { + if err := c.recheck(); err != nil { + return err + } + if isDir { if err := c.checkLinkContainment(); err != nil { - return nil, err + return err } } if err := c.finish(); err != nil { - return nil, err + return err } - sort.Slice(c.entries, func(i, j int) bool { return c.entries[i].Path < c.entries[j].Path }) - return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), staged: c.staged}, nil + return c.ctx.Err() } // resolveSource resolves a user-chosen top-level source (which may itself be a symlink) and checks diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go index 02f7df89958..a30c2b5b6b9 100644 --- a/pkg/artifact/snapshot/walk.go +++ b/pkg/artifact/snapshot/walk.go @@ -141,28 +141,21 @@ func readDirNames(dir string) ([]string, error) { return names, nil } -// checkLinkContainment resolves every captured symlink physically and rejects any that leaves the -// source. A lexical target check is not enough, because a chain of links can clean to a contained -// path yet resolve outside it. A dangling link is checked through its longest existing ancestor -// plus the missing remainder, so it cannot escape once the missing path is created. +// checkLinkContainment resolves every captured symlink through the captured links and rejects any +// that leaves the source. A lexical check of each target is not enough, because a chain of links +// can clean to a contained path yet resolve outside it. A dangling link is resolved through its +// existing part plus the missing remainder, so it cannot escape once the missing path is created. +// The check reads the links as captured and never the source, which may have changed since. func (c *capturer) checkLinkContainment() error { - // linkpath.Resolve reports absolute paths, so the root must be absolute too: EvalSymlinks keeps - // a relative source relative, which would reject every contained link. - absRoot, err := filepath.Abs(c.root) - if err != nil { - return fmt.Errorf("%w: resolve source: %w", errUtils.ErrArtifactPackagingInvalid, err) - } - realRoot, err := filepath.EvalSymlinks(absRoot) - if err != nil { - return fmt.Errorf("%w: resolve source: %w", errUtils.ErrArtifactPackagingInvalid, err) + links := make(map[string]string) + for index := range c.sources { + if record := &c.sources[index]; record.target != "" { + links[record.rel] = record.target + } } for index := range c.sources { record := &c.sources[index] - if record.target == "" { - continue - } - resolved, reachable := linkpath.Resolve(record.abs) - if reachable && !linkpath.Within(realRoot, resolved) { + if record.target != "" && linkpath.Escapes(links, record.rel) { return fmt.Errorf("%w: symlink %q resolves outside the source", errUtils.ErrArtifactUnsafePath, record.rel) } } From c4a39fcd3ad1642d8df1327db32a40c24b0a821a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:08:20 -0600 Subject: [PATCH 35/52] fix(produce): classify stream, archive-open and metadata-source failures Read and close failures of a staged file, a missing staged archive, and a MetadataSource error now carry ErrArtifactPackagingInvalid while keeping their cause (and the source path for metadata), so callers can match them through the artifact sentinels. io.EOF and the digest-mismatch classification are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/produce/assemble.go | 8 ++- pkg/artifact/produce/context_internal_test.go | 42 +++++++++++++++ pkg/artifact/produce/files.go | 6 ++- pkg/artifact/produce/produce_test.go | 53 +++++++++++++++++++ pkg/artifact/produce/repack.go | 11 +++- 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/pkg/artifact/produce/assemble.go b/pkg/artifact/produce/assemble.go index d95eaef1452..bdd514abb38 100644 --- a/pkg/artifact/produce/assemble.go +++ b/pkg/artifact/produce/assemble.go @@ -121,7 +121,13 @@ func (a *assembly) archiveBlob(ctx context.Context, packaging artifact.Packaging return artifact.StagedBlob{ Descriptor: ocispec.Descriptor{MediaType: mediaType, Digest: digest.NewDigest(digest.SHA256, hasher), Size: info.Size()}, Role: artifact.BlobPayload, - Open: func(context.Context) (io.ReadCloser, error) { return os.Open(path) }, + Open: func(context.Context) (io.ReadCloser, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%w: open staged archive: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return file, nil + }, }, nil } diff --git a/pkg/artifact/produce/context_internal_test.go b/pkg/artifact/produce/context_internal_test.go index 8b457ad7782..9f904a46b3b 100644 --- a/pkg/artifact/produce/context_internal_test.go +++ b/pkg/artifact/produce/context_internal_test.go @@ -1,14 +1,19 @@ package produce import ( + "bytes" "context" "errors" "io" + "os" "strings" "testing" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/archive" ) @@ -25,3 +30,40 @@ func TestArchiveMemberStopsReadingAfterCancellation(t *testing.T) { _, err = reader.Read(buf) require.True(t, errors.Is(err, context.Canceled)) } + +func TestVerifyingReader_ClassifiesStreamFailures(t *testing.T) { + body := []byte("payload") + hash := digest.FromBytes(body) + closedFile := func(t *testing.T) *os.File { + t.Helper() + file, err := os.CreateTemp(t.TempDir(), "closed-*") + require.NoError(t, err) + require.NoError(t, file.Close()) + return file + } + + t.Run("a read failure keeps its cause and gains the packaging sentinel", func(t *testing.T) { + reader := newVerifyingReader(closedFile(t), hash, int64(len(body))) + _, err := reader.Read(make([]byte, 8)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrClosed) + }) + t.Run("a close failure keeps its cause and gains the packaging sentinel", func(t *testing.T) { + err := newVerifyingReader(closedFile(t), hash, int64(len(body))).Close() + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrClosed) + }) + t.Run("end of stream stays unchanged when the content matches", func(t *testing.T) { + reader := newVerifyingReader(io.NopCloser(bytes.NewReader(body)), hash, int64(len(body))) + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, body, got) + require.NoError(t, reader.Close()) + }) + t.Run("a digest mismatch at the end keeps the integrity classification", func(t *testing.T) { + reader := newVerifyingReader(io.NopCloser(bytes.NewReader([]byte("tampered"))), hash, int64(len(body))) + _, err := io.ReadAll(reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.NotErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) +} diff --git a/pkg/artifact/produce/files.go b/pkg/artifact/produce/files.go index 00a7d698867..625670f472f 100644 --- a/pkg/artifact/produce/files.go +++ b/pkg/artifact/produce/files.go @@ -121,7 +121,11 @@ func (p *Producer) producerMetadata(ctx context.Context, source string) (Produce if p.metadata == nil { return ProducerMetadata{}, nil } - return p.metadata(ctx, source) + meta, err := p.metadata(ctx, source) + if err != nil { + return ProducerMetadata{}, fmt.Errorf("%w: read producer metadata for %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) + } + return meta, nil } // identity is the resolved name, version, and producer metadata of one preparation. diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 3d87445cf94..1c313bdf842 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -641,3 +641,56 @@ func TestRegisterAddsFilesProducer(t *testing.T) { require.ErrorIs(t, produce.Register(registry), errUtils.ErrArtifactConflict) assert.Equal(t, []string{artifact.KindFiles}, registry.Kinds()) } + +func TestFilesProducer_MetadataSourceFailureIsClassifiedAndKeepsItsCause(t *testing.T) { + cause := errors.New("provider unavailable") + tests := []struct { + name string + cause error + }{ + {"a provider failure", cause}, + {"a canceled lookup", context.Canceled}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + source := func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{}, tc.cause + } + _, err := produce.New(produce.WithStagingRoot(t.TempDir()), produce.WithMetadataSource(source)).Prepare(t.Context(), *request(src, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, tc.cause) + assert.Contains(t, err.Error(), src, "the source path is part of the error") + }) + } +} + +func TestFilesProducer_ArchivePayloadOpenFailureIsClassified(t *testing.T) { + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + prepared, staging := prepare(t, request(src, artifact.PackagingTgz)) + + var removed int + require.NoError(t, filepath.WalkDir(staging, func(path string, entry os.DirEntry, err error) error { + if err == nil && !entry.IsDir() && filepath.Base(path) == "payload.tgz" { + removed++ + return os.Remove(path) + } + return err + })) + require.Equal(t, 1, removed, "exactly one staged archive must exist") + + var checked int + for index := range prepared.Blobs { + blob := &prepared.Blobs[index] + if blob.Role != artifact.BlobPayload { + continue + } + checked++ + _, err := blob.Open(t.Context()) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, os.ErrNotExist) + } + require.Equal(t, 1, checked) +} diff --git a/pkg/artifact/produce/repack.go b/pkg/artifact/produce/repack.go index 4c12bde0f5c..e9413cc780a 100644 --- a/pkg/artifact/produce/repack.go +++ b/pkg/artifact/produce/repack.go @@ -132,12 +132,19 @@ func (v *verifyingReader) Read(p []byte) (int, error) { if v.read != v.size || digest.NewDigest(digest.SHA256, v.hasher) != v.expected { return n, fmt.Errorf("%w: file content does not match its inventory digest", errUtils.ErrArtifactIntegrityFailed) } + return n, err } - return n, err + if err != nil { + return n, fmt.Errorf("%w: read staged file: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return n, nil } func (v *verifyingReader) Close() error { defer perf.Track(nil, "produce.verifyingReader.Close")() - return v.rc.Close() + if err := v.rc.Close(); err != nil { + return fmt.Errorf("%w: close staged file: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + return nil } From 221b3cffe65b5fcbc6c51c0233400da56cba9971 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:19:58 -0600 Subject: [PATCH 36/52] fix(artifact): add Metadata and VerificationPolicy clones and test both isolation directions Record.Clone now delegates to Metadata.Clone, and VerificationPolicy.Clone copies the trust-set slices so a description handed to a caller never aliases its configuration. The Capabilities clone test now mutates the source it clones. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/clone.go | 49 ++++++++++++++++++++-------- pkg/artifact/clone_test.go | 65 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/pkg/artifact/clone.go b/pkg/artifact/clone.go index 59bd12b1503..93c24f228f1 100644 --- a/pkg/artifact/clone.go +++ b/pkg/artifact/clone.go @@ -14,19 +14,7 @@ func (r *Record) Clone() Record { defer perf.Track(nil, "artifact.Record.Clone")() out := *r - out.Metadata.Labels = maps.Clone(r.Metadata.Labels) - if r.Metadata.Provenance != nil { - provenance := *r.Metadata.Provenance - if provenance.CI != nil { - ci := *provenance.CI - provenance.CI = &ci - } - if provenance.Source != nil { - source := *provenance.Source - provenance.Source = &source - } - out.Metadata.Provenance = &provenance - } + out.Metadata = r.Metadata.Clone() out.Inventory = slices.Clone(r.Inventory) out.Documents = slices.Clone(r.Documents) out.KindData = bytes.Clone(r.KindData) @@ -45,3 +33,38 @@ func (c *Capabilities) Clone() Capabilities { out.LockScopes = slices.Clone(c.LockScopes) return out } + +// Clone returns Metadata that shares no map or pointer with m. +func (m *Metadata) Clone() Metadata { + defer perf.Track(nil, "artifact.Metadata.Clone")() + + out := *m + out.Labels = maps.Clone(m.Labels) + if m.Provenance != nil { + provenance := *m.Provenance + if provenance.CI != nil { + ci := *provenance.CI + provenance.CI = &ci + } + if provenance.Source != nil { + source := *provenance.Source + provenance.Source = &source + } + out.Provenance = &provenance + } + return out +} + +// Clone returns a VerificationPolicy that shares no slice with p, so editing the trust sets of a +// description handed to a caller never reaches the configured policy. +func (p *VerificationPolicy) Clone() VerificationPolicy { + defer perf.Track(nil, "artifact.VerificationPolicy.Clone")() + + out := *p + out.TrustSets = slices.Clone(p.TrustSets) + for index := range out.TrustSets { + out.TrustSets[index].Identities = slices.Clone(out.TrustSets[index].Identities) + out.TrustSets[index].Keys = slices.Clone(out.TrustSets[index].Keys) + } + return out +} diff --git a/pkg/artifact/clone_test.go b/pkg/artifact/clone_test.go index 77b2f4c60bc..884b016ff6e 100644 --- a/pkg/artifact/clone_test.go +++ b/pkg/artifact/clone_test.go @@ -90,8 +90,69 @@ func TestCapabilities_CloneIsIndependentInBothDirections(t *testing.T) { assert.Equal(t, sample(), original, "editing the clone leaves the original alone") original = sample() - fresh := sample() - clone = fresh.Clone() + clone = original.Clone() original.Packagings[1], original.SubjectKinds[0], original.EvidenceFormats[0], original.LockScopes[0] = artifact.PackagingNone, "x", "x", artifact.LockScopeRepository assert.Equal(t, sample(), clone, "editing the original leaves the clone alone") } + +func TestMetadata_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.Metadata { return sampleRecord().Metadata } + mutations := map[string]func(m *artifact.Metadata){ + "labels": func(m *artifact.Metadata) { m.Labels["a"] = "changed" }, + "CI provenance": func(m *artifact.Metadata) { m.Provenance.CI.RunID = "changed" }, + "source provenance": func(m *artifact.Metadata) { m.Provenance.Source.Commit = "changed" }, + "provenance": func(m *artifact.Metadata) { m.Provenance.CI = nil }, + } + for name, mutate := range mutations { + t.Run(name+"/clone to original", func(t *testing.T) { + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + mutate(&clone) + assert.Equal(t, sample(), original) + }) + t.Run(name+"/original to clone", func(t *testing.T) { + original := sample() + clone := original.Clone() + mutate(&original) + assert.Equal(t, sample(), clone) + }) + } + assert.Equal(t, artifact.Metadata{}, (&artifact.Metadata{}).Clone(), "nil maps and pointers stay nil") +} + +func TestVerificationPolicy_CloneIsIndependentInBothDirections(t *testing.T) { + sample := func() artifact.VerificationPolicy { + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: "first", Issuer: "issuer-1"}, {Identity: "second", Issuer: "issuer-2"}}, Keys: []string{"a.pub"}}, + {Keys: []string{"b.pub", "c.pub"}}, + }, + Offline: true, + } + } + mutations := map[string]func(p *artifact.VerificationPolicy){ + "trust set order": func(p *artifact.VerificationPolicy) { p.TrustSets[0], p.TrustSets[1] = p.TrustSets[1], p.TrustSets[0] }, + "first identity": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Identities[0].Identity = "changed" }, + "last identity issuer": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Identities[1].Issuer = "changed" }, + "first key": func(p *artifact.VerificationPolicy) { p.TrustSets[0].Keys[0] = "changed" }, + "last key": func(p *artifact.VerificationPolicy) { p.TrustSets[1].Keys[1] = "changed" }, + } + for name, mutate := range mutations { + t.Run(name+"/clone to original", func(t *testing.T) { + original := sample() + clone := original.Clone() + require.Equal(t, original, clone) + mutate(&clone) + assert.Equal(t, sample(), original) + }) + t.Run(name+"/original to clone", func(t *testing.T) { + original := sample() + clone := original.Clone() + mutate(&original) + assert.Equal(t, sample(), clone) + }) + } + assert.Equal(t, artifact.VerificationPolicy{}, (&artifact.VerificationPolicy{}).Clone(), "an empty policy stays empty") +} From 36a10c972cde7d3c114a8f6e481bb5f2ee6b363d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:20:58 -0600 Subject: [PATCH 37/52] fix(service): keep cancellation, description copies and staging readable - A default listing where one repository completed and another stopped on cancellation now reports the cancellation instead of an incomplete success. - StaticRepositories.Describe returns deep copies of the table entry and of the repository capabilities, so edits cannot reach later descriptions. - The copy staging tree keeps owner access while it is repacked, so a Record with mode 0000 can still be copied; the Record keeps its modes. - Service tests record journal entries through the generated MockJournal. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/service/consume_test.go | 43 +++++++++++++ pkg/artifact/service/copy.go | 4 +- pkg/artifact/service/copy_test.go | 61 +++++++++++++++++++ pkg/artifact/service/helpers_test.go | 28 ++++++--- pkg/artifact/service/query.go | 15 ++++- pkg/artifact/service/repositories.go | 12 ++++ pkg/artifact/service/static.go | 8 ++- pkg/artifact/service/static_test.go | 91 ++++++++++++++++++++++++++++ 8 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 pkg/artifact/service/static_test.go diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index 6fd7a350e26..fa77e9db59c 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "sync" "sync/atomic" "testing" "time" @@ -331,6 +332,48 @@ func TestList_OnlyFailingRepositoriesFailTheListing(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "when no repository can answer, the listing fails") } +func TestEnumeration_CancellationAfterOneDefaultRepositoryCompletesIsNotSuccess(t *testing.T) { + tests := []struct { + name string + run func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) + }{ + {"list", func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) { + return svc.List(ctx, artifact.ListRequest{}) + }}, + {"search", func(ctx context.Context, svc *service.Service) (*artifact.ListResult, error) { + return svc.Search(ctx, artifact.SearchRequest{Query: "app"}) + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "beta": {}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1"}, nil) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Alpha completes first; beta is canceled only after alpha returned its page. + alphaDone := make(chan struct{}) + var once sync.Once + e.static.Repositories["alpha"] = delegatingRepository(t, e.repos["alpha"], repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + page, err := e.repos["alpha"].List(ctx, query) + once.Do(func() { close(alphaDone) }) + return page, err + }}) + e.static.Repositories["beta"] = delegatingRepository(t, e.repos["beta"], repoHooks{list: func(ctx context.Context, _ artifact.ListQuery) (*artifact.ListPage, error) { + <-alphaDone + cancel() + return nil, ctx.Err() + }}) + + result, err := tc.run(ctx, e.svc) + + require.ErrorIs(t, err, context.Canceled, "an incomplete listing must not be reported as a success") + require.NotNil(t, result) + assert.Len(t, result.Artifacts, 1, "what alpha returned stays visible beside the error") + }) + } +} + type listGate struct { inflight atomic.Int32 peak atomic.Int32 diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index e36f64c69c5..ac25d29cb44 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -214,7 +214,9 @@ func (s *Service) stageCopy(ctx context.Context, op *operation, plan *copyPlan) } } files := filepath.Join(tree, "files") - if err := materialize.Materialize(ctx, content, files); err != nil { + // The tree is read back by the repack and then discarded, so it keeps owner read and traversal + // access even when the Record declares a restrictive mode; the Record itself is repacked unchanged. + if err := materialize.Materialize(ctx, content, files, materialize.WithOwnerAccess()); err != nil { cleanup() return nil, nil, err } diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index 77e29f1a584..55931cdfbad 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "slices" "sync/atomic" "testing" @@ -11,11 +12,13 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/artifact/artifacttest" "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" "github.com/cloudposse/atmos/pkg/artifact/service" ) @@ -389,3 +392,61 @@ func TestCopy_EvidenceIsNotWrittenToARepointedDestination(t *testing.T) { }) } } + +// TestCopy_StagingKeepsOwnerAccessWhileRepacking proves a Record that declares mode 0000 can still +// be repacked: the temporary tree stays readable until the repack finished, and the Record the +// repack receives keeps the declared modes. +func TestCopy_StagingKeepsOwnerAccessWhileRepacking(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skipf("permission bits are not enforced for this process on %s (uid %d)", runtime.GOOS, os.Geteuid()) + } + var repacked bool + repacker := service.NewMockRepacker(gomock.NewController(t)) + repacker.EXPECT().Repack(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request *produce.RepackRequest) (*artifact.PreparedArtifact, error) { + repacked = true + data, err := os.ReadFile(filepath.Join(request.Tree, "data.txt")) + require.NoError(t, err, "a file declared with mode 0000 stays readable for the repack") + assert.Equal(t, "hello", string(data)) + entries, err := os.ReadDir(filepath.Join(request.Tree, "bin")) + require.NoError(t, err, "a directory declared with mode 0000 stays traversable for the repack") + assert.NotEmpty(t, entries) + for _, entry := range request.Record.Inventory { + if entry.Type == artifact.EntryFile || entry.Type == artifact.EntryDirectory { + assert.Zero(t, entry.Mode, "the Record keeps the declared mode for %s", entry.Path) + } + } + return nil, errTransport + }) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingTgz, service.WithRepacker(repacker)) + e.publish("src", e.source("hello")) + base := e.repos["src"] + e.static.Repositories["src"] = delegatingRepository(t, base, repoHooks{ + inspect: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + inspection, err := base.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + record := inspection.Record.Clone() + for index := range record.Inventory { + record.Inventory[index].Mode = 0 + } + require.NoError(t, record.Seal(), "the content digest follows the declared modes") + inspection.Record = &record + return inspection, nil + }, + read: func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + content, err := base.Read(ctx, pinned) + if err != nil { + return nil, err + } + content.Record = nil + return content, nil + }, + }) + + _, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) + + require.ErrorIs(t, err, errTransport) + require.True(t, repacked, "the copy must reach the repack") + e.assertStagingClean() +} diff --git a/pkg/artifact/service/helpers_test.go b/pkg/artifact/service/helpers_test.go index ae533c2b328..6274a2a51ff 100644 --- a/pkg/artifact/service/helpers_test.go +++ b/pkg/artifact/service/helpers_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/artifact/artifacttest" @@ -45,19 +46,20 @@ func (c *fakeClock) Set(t time.Time) { c.now = t } -// recordingJournal records lifecycle events. -type recordingJournal struct { +// journalLog collects the lifecycle events that the generated MockJournal receives. It is a +// synchronized collector only; it does not implement service.Journal. +type journalLog struct { mu sync.Mutex entries []service.JournalEntry } -func (j *recordingJournal) Record(_ context.Context, entry *service.JournalEntry) { +func (j *journalLog) add(entry *service.JournalEntry) { j.mu.Lock() defer j.mu.Unlock() j.entries = append(j.entries, *entry) } -func (j *recordingJournal) phases() []service.JournalPhase { +func (j *journalLog) phases() []service.JournalPhase { j.mu.Lock() defer j.mu.Unlock() phases := make([]service.JournalPhase, 0, len(j.entries)) @@ -67,12 +69,21 @@ func (j *recordingJournal) phases() []service.JournalPhase { return phases } -func (j *recordingJournal) snapshot() []service.JournalEntry { +func (j *journalLog) snapshot() []service.JournalEntry { j.mu.Lock() defer j.mu.Unlock() return append([]service.JournalEntry(nil), j.entries...) } +// newRecordingJournal returns a generated MockJournal that records every entry into the log. +func newRecordingJournal(t *testing.T) (*service.MockJournal, *journalLog) { + t.Helper() + log := &journalLog{} + journal := service.NewMockJournal(gomock.NewController(t)) + journal.EXPECT().Record(gomock.Any(), gomock.Any()).Do(func(_ context.Context, entry *service.JournalEntry) { log.add(entry) }).AnyTimes() + return journal, log +} + // temporaryError is a retryable failure. type temporaryError struct{ message string } @@ -85,7 +96,7 @@ type env struct { repos map[string]*artifacttest.Repository static *service.StaticRepositories clock *fakeClock - journal *recordingJournal + journal *journalLog staging string sleeps []time.Duration sleepMu sync.Mutex @@ -105,11 +116,12 @@ func newEnv(t *testing.T, config envConfig) *env { if config.repos == nil { config.repos = map[string]artifacttest.Options{"dst": {Alias: "dst"}} } + journal, log := newRecordingJournal(t) e := &env{ t: t, repos: map[string]*artifacttest.Repository{}, clock: &fakeClock{now: startTime}, - journal: &recordingJournal{}, + journal: log, staging: t.TempDir(), } e.static = &service.StaticRepositories{Repositories: map[string]artifact.Repository{}, Info: config.infos} @@ -126,7 +138,7 @@ func newEnv(t *testing.T, config envConfig) *env { options := []service.Option{ service.WithClock(e.clock.Now), service.WithIDGenerator(func() string { return fmt.Sprintf("op-%d", counter.Add(1)) }), - service.WithJournal(e.journal), + service.WithJournal(journal), service.WithSleep(func(_ context.Context, d time.Duration) error { e.sleepMu.Lock() e.sleeps = append(e.sleeps, d) diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go index aaa640bb824..170166a3bb6 100644 --- a/pkg/artifact/service/query.go +++ b/pkg/artifact/service/query.go @@ -213,7 +213,20 @@ func (s *Service) runEnumeration(ctx context.Context, request *enumeration, resu succeeded++ } sortEntries(result.Artifacts) - if explicit && len(errs) > 0 || succeeded == 0 && len(errs) > 0 { + return enumerationError(ctx, explicit, succeeded, errs) +} + +// enumerationError decides whether the collected per-repository failures fail the listing. +// Cancellation is never partial success: a repository that stopped because the caller canceled left +// the listing incomplete, whatever the others returned. +func enumerationError(ctx context.Context, explicit bool, succeeded int, errs []error) error { + if len(errs) == 0 { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return errors.Join(append([]error{ctxErr}, errs...)...) + } + if explicit || succeeded == 0 { return errors.Join(errs...) } return nil diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go index 6d6f8ca2f21..80341700226 100644 --- a/pkg/artifact/service/repositories.go +++ b/pkg/artifact/service/repositories.go @@ -37,6 +37,18 @@ type RepositoryInfo struct { Policy artifact.VerificationPolicy } +// Clone returns a RepositoryInfo that shares no map, slice, or pointer with i, so a caller editing a +// description it received cannot change the configuration that produced it. +func (i *RepositoryInfo) Clone() RepositoryInfo { + defer perf.Track(nil, "service.RepositoryInfo.Clone")() + + out := *i + out.Capabilities = i.Capabilities.Clone() + out.Defaults = i.Defaults.Clone() + out.Policy = i.Policy.Clone() + return out +} + // Repositories resolves repository aliases. Describe must be free of network access and // credential resolution so dry runs never authenticate; Open may resolve identities. type Repositories interface { diff --git a/pkg/artifact/service/static.go b/pkg/artifact/service/static.go index b0124df0a63..fcb64619ee1 100644 --- a/pkg/artifact/service/static.go +++ b/pkg/artifact/service/static.go @@ -20,10 +20,14 @@ func (s *StaticRepositories) Describe(alias string) (RepositoryInfo, error) { if !ok { return RepositoryInfo{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) } - info := s.Info[alias] + // The table entry and the repository's capabilities are copied, so editing the returned + // description never reaches the table or the repository that every later call reads. + configured := s.Info[alias] + info := configured.Clone() info.Alias = alias if info.Capabilities.Packagings == nil && !info.Capabilities.Read && !info.Capabilities.Write { - info.Capabilities = repo.Capabilities() + reported := repo.Capabilities() + info.Capabilities = reported.Clone() } if info.Packaging == "" && len(info.Capabilities.Packagings) > 0 { info.Packaging = info.Capabilities.Packagings[0] diff --git a/pkg/artifact/service/static_test.go b/pkg/artifact/service/static_test.go new file mode 100644 index 00000000000..9b5837c9e32 --- /dev/null +++ b/pkg/artifact/service/static_test.go @@ -0,0 +1,91 @@ +package service_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func staticPolicy() artifact.VerificationPolicy { + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: "release@example.com", Issuer: "https://issuer.example.com"}}, Keys: []string{"release.pub"}}, + }, + } +} + +func configuredInfo() service.RepositoryInfo { + return service.RepositoryInfo{ + Capabilities: artifact.Capabilities{Read: true, Packagings: []artifact.Packaging{artifact.PackagingTar, artifact.PackagingZip}, LockScopes: []artifact.LockScope{artifact.LockScopeHost}}, + Defaults: artifact.Metadata{ + Description: "defaults", + Labels: map[string]string{"team": "platform"}, + Provenance: &artifact.Provenance{CI: &artifact.CIProvenance{Provider: "github"}}, + }, + Policy: staticPolicy(), + } +} + +func reportedCapabilities() artifact.Capabilities { + return artifact.Capabilities{Read: true, Write: true, Packagings: []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingNone}, SubjectKinds: []artifact.SubjectKind{artifact.SubjectRecord}} +} + +// TestStaticRepositories_DescribeReturnsIndependentCopies proves a caller editing what Describe +// returned cannot change the table or the repository, which every later Describe would serve. +func TestStaticRepositories_DescribeReturnsIndependentCopies(t *testing.T) { + tests := []struct { + name string + // info builds the table entry; an empty one falls back to what the repository reports. + info func() service.RepositoryInfo + mutate func(info *service.RepositoryInfo) + }{ + {"defaults labels and provenance", configuredInfo, func(info *service.RepositoryInfo) { + info.Defaults.Labels["team"] = "scribble" + info.Defaults.Provenance.CI.Provider = "scribble" + }}, + {"configured capabilities", configuredInfo, func(info *service.RepositoryInfo) { + info.Capabilities.Packagings[0] = artifact.PackagingNone + info.Capabilities.LockScopes[0] = artifact.LockScopeRepository + }}, + {"policy trust sets", configuredInfo, func(info *service.RepositoryInfo) { + info.Policy.TrustSets[0].Identities[0].Identity = "scribble" + info.Policy.TrustSets[0].Keys[0] = "scribble" + }}, + {"capabilities reported by the repository", func() service.RepositoryInfo { return service.RepositoryInfo{} }, func(info *service.RepositoryInfo) { + info.Capabilities.Packagings[0] = artifact.PackagingZip + info.Capabilities.SubjectKinds[0] = "scribble" + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // The repository hands out the same backing slices on every call, as a cached description would. + build := func() (*service.StaticRepositories, map[string]service.RepositoryInfo, *artifact.Capabilities) { + caps := reportedCapabilities() + repo := artifact.NewMockRepository(gomock.NewController(t)) + repo.EXPECT().Capabilities().DoAndReturn(func() artifact.Capabilities { return caps }).AnyTimes() + table := map[string]service.RepositoryInfo{"repo": tc.info()} + return &service.StaticRepositories{Repositories: map[string]artifact.Repository{"repo": repo}, Info: table}, table, &caps + } + untouched, _, _ := build() + want, err := untouched.Describe("repo") + require.NoError(t, err) + + static, table, repoCaps := build() + first, err := static.Describe("repo") + require.NoError(t, err) + tc.mutate(&first) + + second, err := static.Describe("repo") + require.NoError(t, err) + assert.Equal(t, want, second, "a later description is not affected by edits to an earlier one") + assert.Equal(t, tc.info(), table["repo"], "the configured table entry is untouched") + assert.Equal(t, reportedCapabilities(), *repoCaps, "the repository's own capabilities are untouched") + }) + } +} From 25c5ca2f98908f0c834f20640e592b0ac8c94c71 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:22:52 -0600 Subject: [PATCH 38/52] test(artifacttest): recheck the subject after the signer returns Sign releases the lock while the signer runs, so a concurrent Delete could remove the record and the append then recreated evidence for it. The subject is checked again under the lock before the evidence is stored. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/artifacttest/repository_test.go | 33 ++++++++++++++++++++ pkg/artifact/artifacttest/sign.go | 4 +++ 2 files changed, 37 insertions(+) diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go index bec788426d5..f6fcff8e380 100644 --- a/pkg/artifact/artifacttest/repository_test.go +++ b/pkg/artifact/artifacttest/repository_test.go @@ -8,6 +8,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -308,3 +309,35 @@ func TestRepository_PublishCopiesThePreparedRecord(t *testing.T) { assert.Equal(t, map[string]string{"suite": "isolation"}, inspection.Record.Metadata.Labels, "the stored record is independent of the prepared one") assert.Equal(t, "data.txt", inspection.Record.Inventory[0].Path) } + +// TestRepository_SignDoesNotStoreEvidenceForARecordDeletedWhileSigning proves the subject is checked +// again once the signer returned: Sign releases the lock while the signer runs, so a concurrent +// Delete can remove the record in between. +func TestRepository_SignDoesNotStoreEvidenceForARecordDeletedWhileSigning(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Delete = artifact.DeleteObject + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + pinned := publishOne(t, repo) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + + fake := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return(fake.Formats()).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), subject).DoAndReturn(func(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + // The record disappears while the signer is still working. + _, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteObject}) + require.NoError(t, err) + return fake.Sign(ctx, subject) + }) + + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Empty(t, evidence) + assert.False(t, recordAlive(repo, pinned.RecordDigest)) + assert.False(t, evidenceAlive(t, repo, &pinned), "no evidence may be left for a deleted record") +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index 60e615728ed..ad94d28e902 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -108,6 +108,10 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject } r.mu.Lock() defer r.mu.Unlock() + // The lock was released while the signer ran, so a Delete may have removed the subject since. + if err := r.checkSubject(subject); err != nil { + return nil, err + } r.evidence[subject.Digest] = append(r.evidence[subject.Digest], artifact.EvidenceBlob{Evidence: evidence, Data: bytes.Clone(payload.Data)}) return []artifact.SignatureEvidence{evidence}, nil } From 2b11023e925c483a0689a149322df2bc726b279c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:26:54 -0600 Subject: [PATCH 39/52] test(service): replace the hand-written repointing repository with a generated mock The repointing double now delegates through the generated MockRepository, with a publish hook beside the existing resolve hook, instead of embedding a concrete repository and overriding methods by hand. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/service/copy_test.go | 49 +++++++++++++++------------ pkg/artifact/service/delegate_test.go | 6 +++- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index 55931cdfbad..8d4ed883d28 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -341,27 +341,32 @@ func TestCopy_UnchangedDestinationStillReceivesLaterSignatures(t *testing.T) { assert.Len(t, held, 1) } -// repointingRepository answers Resolve with a different digest once a publish completed, as if -// another writer forced the reference elsewhere after the destination lock was released. -type repointingRepository struct { - *artifacttest.Repository - published atomic.Bool -} - -func (r *repointingRepository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { - result, err := r.Repository.Publish(ctx, prepared, options) - if err == nil { - r.published.Store(true) - } - return result, err -} - -func (r *repointingRepository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { - pinned, err := r.Repository.Resolve(ctx, ref) - if err == nil && r.published.Load() { - pinned.ResolvedDigest = digest.FromString("another writer") - } - return pinned, err +// repointedRepository returns a generated repository that answers Resolve with a different digest +// once a publish completed, as if another writer forced the reference elsewhere after the +// destination lock was released. It keeps the evidence writer of the base repository. +func repointedRepository(t *testing.T, base *artifacttest.Repository) artifact.Repository { + t.Helper() + var published atomic.Bool + mock := delegatingRepository(t, base, repoHooks{ + publish: func(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + result, err := base.Publish(ctx, prepared, options) + if err == nil { + published.Store(true) + } + return result, err + }, + resolve: func(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + pinned, err := base.Resolve(ctx, ref) + if err == nil && published.Load() { + pinned.ResolvedDigest = digest.FromString("another writer") + } + return pinned, err + }, + }) + return struct { + artifact.Repository + artifact.EvidenceWriter + }{mock, base} } func TestCopy_EvidenceIsNotWrittenToARepointedDestination(t *testing.T) { @@ -382,7 +387,7 @@ func TestCopy_EvidenceIsNotWrittenToARepointedDestination(t *testing.T) { require.NoError(t, err) dst := e.repos["dst"] if tt.repoint { - e.static.Repositories["dst"] = &repointingRepository{Repository: dst} + e.static.Repositories["dst"] = repointedRepository(t, dst) } result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) diff --git a/pkg/artifact/service/delegate_test.go b/pkg/artifact/service/delegate_test.go index f0e09fe5195..4e32f3f7e98 100644 --- a/pkg/artifact/service/delegate_test.go +++ b/pkg/artifact/service/delegate_test.go @@ -17,6 +17,7 @@ type repoHooks struct { read func(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) list func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) lock func(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) + publish func(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) } // delegatingRepository returns a generated MockRepository whose methods forward to base, except @@ -38,6 +39,9 @@ func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHook if hooks.lock == nil { hooks.lock = base.Lock } + if hooks.publish == nil { + hooks.publish = base.Publish + } mock := artifact.NewMockRepository(gomock.NewController(t)) mock.EXPECT().Capabilities().DoAndReturn(base.Capabilities).AnyTimes() mock.EXPECT().Lock(gomock.Any(), gomock.Any()).DoAndReturn(hooks.lock).AnyTimes() @@ -45,7 +49,7 @@ func delegatingRepository(t *testing.T, base artifact.Repository, hooks repoHook mock.EXPECT().Inspect(gomock.Any(), gomock.Any()).DoAndReturn(hooks.inspect).AnyTimes() mock.EXPECT().OpenDocument(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.OpenDocument).AnyTimes() mock.EXPECT().Read(gomock.Any(), gomock.Any()).DoAndReturn(hooks.read).AnyTimes() - mock.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Publish).AnyTimes() + mock.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(hooks.publish).AnyTimes() mock.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(hooks.list).AnyTimes() mock.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(base.Delete).AnyTimes() mock.EXPECT().Pin(gomock.Any(), gomock.Any()).DoAndReturn(base.Pin).AnyTimes() From 4fc07d3df544e137f18a89f4baec22749be79448 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:35:13 -0600 Subject: [PATCH 40/52] refactor(materialize): name the archive read stage parameter for what it is Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/archive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index 6b438a14c11..bd661884d33 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -99,11 +99,11 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check // archiveReadError reports a failed archive read. A canceled context is returned as such, because // the cancellation is what interrupted the read and not damage to the archive. -func archiveReadError(ctx context.Context, format string, err error) error { +func archiveReadError(ctx context.Context, stage string, err error) error { if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } - return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactIntegrityFailed, format, err) + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactIntegrityFailed, stage, err) } func applyTarHeader(ctx context.Context, t *tree, tr *tar.Reader, header *tar.Header, checker *managedChecker) error { From 83c1efac42717939b54014f730fd9f2a604f2460 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 12:33:46 -0600 Subject: [PATCH 41/52] test(produce): match the %q-quoted source path in the metadata failure test On Windows %q doubles backslashes, so the raw temp-dir path never matched the error text. Compare against the quoted form instead. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/produce/produce_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 1c313bdf842..709701757fd 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "errors" + "fmt" "io" "os" "path/filepath" @@ -661,7 +662,8 @@ func TestFilesProducer_MetadataSourceFailureIsClassifiedAndKeepsItsCause(t *test _, err := produce.New(produce.WithStagingRoot(t.TempDir()), produce.WithMetadataSource(source)).Prepare(t.Context(), *request(src, artifact.PackagingNone)) require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) require.ErrorIs(t, err, tc.cause) - assert.Contains(t, err.Error(), src, "the source path is part of the error") + // The path is quoted with %q, which doubles backslashes on Windows. + assert.Contains(t, err.Error(), fmt.Sprintf("%q", src), "the source path is part of the error") }) } } From 63349ff20e5bc2ca8eb71d5b504fc71861e1bc6b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 18:24:17 -0600 Subject: [PATCH 42/52] fix(artifact): address CodeRabbit review findings on the transport layer OCI layout and transport: - Keep a displaced attachment manifest as an unnamed root when its tag moves, so it stays discoverable as a referrer after the layout is reopened. - Resolve a named reference only when index.json lists it, so a tag whose write failed is never accepted as proof of persistence by recovery. - Wrap bound-identity authorization failures with ErrOCIRegistry and reject credentials the registry client cannot send (username or password alone). Symlink handling: - linkpath.Resolve no longer cleans parent components before following links, and reports ErrSymlinkLoop separately from other failures; link containment rejects unresolvable targets and still tolerates loops. - Record every link met while expanding a target when ordering link creation. Service and snapshot: - Journal PhaseReleased only when the lock release succeeded. - Classify capture cancellation and deadline errors, keeping the context error. Tests: clone the expected payload before tampering; separate the symlink probe from the operation under test; add regression tests for each behavior change. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 4 + pkg/artifact/artifacttest/repository_test.go | 4 +- pkg/artifact/linkpath/linkpath.go | 95 ++++++++++++------- pkg/artifact/linkpath/linkpath_test.go | 55 +++++++---- pkg/artifact/materialize/links.go | 12 ++- pkg/artifact/materialize/tree.go | 61 ++++++++++-- .../materialize/tree_links_internal_test.go | 55 ++++++++++- pkg/artifact/service/commit.go | 7 +- pkg/artifact/service/mutate_test.go | 5 +- pkg/artifact/snapshot/snapshot.go | 18 +++- pkg/artifact/snapshot/snapshot_test.go | 29 ++++++ pkg/oci/identity.go | 12 ++- pkg/oci/transport_internal_test.go | 27 ++++++ pkg/oci/transport_layout.go | 72 +++++++++++++- pkg/oci/transport_subject_test.go | 54 +++++++++++ 15 files changed, 435 insertions(+), 75 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 36a6281ebea..74addb484fd 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2129,6 +2129,10 @@ var ( // reconciled: the completed reference could not be proven to hold, or to lack, the expected content. ErrArtifactPublicationAmbiguous = errors.New("artifact publication outcome is ambiguous") + // ErrSymlinkLoop is returned when resolving a path follows more symlinks than the bound allows, which + // means the links form a loop. + ErrSymlinkLoop = errors.New("symlink loop") + // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The // underlying I/O error is wrapped alongside it for diagnostics. ErrArtifactStreamFailed = errors.New("artifact stream failed") diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go index f6fcff8e380..82e0a2f49bf 100644 --- a/pkg/artifact/artifacttest/repository_test.go +++ b/pkg/artifact/artifacttest/repository_test.go @@ -2,6 +2,7 @@ package artifacttest_test import ( "context" + "slices" "testing" "time" @@ -269,6 +270,7 @@ func TestRepository_StoredStateIsIsolatedFromCallers(t *testing.T) { require.NoError(t, err) wantPath := before.Record.Inventory[0].Path wantDocs := len(before.Record.Documents) + wantPayload := slices.Clone(before.Representation.Payload) tt.tamper(t, repo, pinned) @@ -277,7 +279,7 @@ func TestRepository_StoredStateIsIsolatedFromCallers(t *testing.T) { assert.Equal(t, map[string]string{"suite": "isolation"}, after.Record.Metadata.Labels) assert.Equal(t, wantPath, after.Record.Inventory[0].Path) assert.Len(t, after.Record.Documents, wantDocs) - assert.Equal(t, before.Representation.Payload, after.Representation.Payload) + assert.Equal(t, wantPayload, after.Representation.Payload) _, err = repo.Resolve(ctx, ref) require.NoError(t, err) }) diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index 15e116c9aad..7a715adc054 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -4,40 +4,61 @@ package linkpath import ( "errors" + "fmt" "io/fs" "os" "path/filepath" "strings" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" ) // maxLinkHops bounds the symlinks followed while resolving one path, so a loop terminates. -const maxLinkHops = 255 +const ( + maxLinkHops = 255 + parentComponent = ".." +) // Resolve returns the physical absolute path that target refers to. It follows every symlink it // meets, like filepath.EvalSymlinks. Unlike EvalSymlinks, it also resolves a path whose tail does // not exist: the longest existing prefix is resolved physically, and the missing remainder is // joined lexically. A dangling link therefore still reports where it would land once the missing -// path is created. Resolve reports false when the path loops, passes through a non-directory, or -// cannot be inspected. -func Resolve(target string) (string, bool) { +// path is created. Parent components are applied only after the symlinks before them are resolved, +// so "link/../x" is not collapsed to a sibling of "link". +// +// Resolve fails with [errUtils.ErrSymlinkLoop] when the links form a loop, and with another error +// when the path passes through a non-directory or cannot be inspected. Callers decide separately +// what each failure means. +func Resolve(target string) (string, error) { defer perf.Track(nil, "linkpath.Resolve")() - absolute, err := filepath.Abs(target) + absolute, err := absolutePath(target) if err != nil { - return "", false + return "", fmt.Errorf("%w: make %q absolute: %w", errUtils.ErrArtifactUnsafePath, target, err) } volume := filepath.VolumeName(absolute) w := &walker{base: volume + string(filepath.Separator), pending: splitComponents(absolute[len(volume):])} w.resolved = w.base for len(w.pending) > 0 { - done, ok := w.step() - if done { - return filepath.Join(append([]string{w.resolved}, w.missing...)...), ok + if err := w.step(); err != nil { + return "", err } } - return filepath.Join(append([]string{w.resolved}, w.missing...)...), true + return filepath.Join(append([]string{w.resolved}, w.missing...)...), nil +} + +// absolutePath makes target absolute without cleaning it: filepath.Abs would collapse parent +// components lexically, before any symlink in front of them has been resolved. +func absolutePath(target string) (string, error) { + if filepath.IsAbs(target) { + return target, nil + } + working, err := os.Getwd() + if err != nil { + return "", err + } + return working + string(filepath.Separator) + target, nil } // Escapes reports whether the symlink at from leaves the tree that links describes. The tree is @@ -95,60 +116,68 @@ type walker struct { hops int } -// step consumes one component. It reports done when resolution ended, and ok when it succeeded. -func (w *walker) step() (done, ok bool) { +// step consumes one component. It returns nil to continue and an error when resolution failed. +func (w *walker) step() error { component := w.pending[0] w.pending = w.pending[1:] if len(w.missing) > 0 { - switch component { - case ".": - case "..": - w.missing = w.missing[:len(w.missing)-1] - default: - w.missing = append(w.missing, component) - } - return false, true + w.stepMissing(component) + return nil } - if component == "." || component == ".." { + if component == "." || component == parentComponent { w.resolved = stepDot(w.resolved, component) - return false, true + return nil } next := filepath.Join(w.resolved, component) info, err := os.Lstat(next) if errors.Is(err, fs.ErrNotExist) { w.missing = append(w.missing, component) - return false, true + return nil } if err != nil { - return true, false + return fmt.Errorf("%w: inspect %q: %w", errUtils.ErrArtifactUnsafePath, next, err) } if info.Mode()&fs.ModeSymlink == 0 { if !info.IsDir() && len(w.pending) > 0 { - return true, false + return fmt.Errorf("%w: %q is not a directory", errUtils.ErrArtifactUnsafePath, next) } w.resolved = next - return false, true + return nil } return w.follow(next) } +// stepMissing resolves the missing tail lexically until it returns to an existing directory. +func (w *walker) stepMissing(component string) { + switch component { + case ".": + case parentComponent: + w.missing = w.missing[:len(w.missing)-1] + default: + w.missing = append(w.missing, component) + } +} + // follow splices the target of the symlink at link in front of the pending components. -func (w *walker) follow(link string) (done, ok bool) { +func (w *walker) follow(link string) error { w.hops++ + if w.hops > maxLinkHops { + return fmt.Errorf("%w: more than %d links followed at %q", errUtils.ErrSymlinkLoop, maxLinkHops, link) + } components, absolute, err := linkComponents(link) - if err != nil || w.hops > maxLinkHops { - return true, false + if err != nil { + return fmt.Errorf("%w: read link %q: %w", errUtils.ErrArtifactUnsafePath, link, err) } if absolute { w.resolved = w.base } w.pending = append(components, w.pending...) - return false, true + return nil } -// stepDot applies a "." or ".." component to a fully resolved directory. +// stepDot applies a current or parent directory component to a fully resolved directory. func stepDot(resolved, component string) string { - if component == ".." { + if component == parentComponent { return filepath.Dir(resolved) } return resolved @@ -170,7 +199,7 @@ func Within(root, target string) bool { defer perf.Track(nil, "linkpath.Within")() relative, err := filepath.Rel(root, target) - return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) + return err == nil && relative != parentComponent && !strings.HasPrefix(relative, parentComponent+string(filepath.Separator)) } // splitComponents splits a path into its non-empty components. diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go index a7b3c42e231..1756dbdf543 100644 --- a/pkg/artifact/linkpath/linkpath_test.go +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/fixtures" "github.com/cloudposse/atmos/pkg/artifact/linkpath" ) @@ -34,31 +35,49 @@ func TestResolve(t *testing.T) { require.True(t, fixtures.Symlink(t, root, "tree/loop-b", "loop-a")) tests := []struct { - name string - path string - want string - reachable bool + name string + path string + want string + wantErr error }{ - {"plain file", filepath.Join(root, "tree", "real", "file.txt"), filepath.Join(root, "tree", "real", "file.txt"), true}, - {"link to directory", filepath.Join(root, "tree", "to-real"), filepath.Join(root, "tree", "real"), true}, - {"dangling tail stays lexical under the link target", filepath.Join(root, "tree", "dangling"), filepath.Join(root, "tree", "real", "missing", "deeper"), true}, - {"dangling tail walking out through a link", filepath.Join(root, "tree", "escapes"), filepath.Join(filepath.Dir(root), "outside", "missing"), true}, - {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), true}, - {"missing tail returns to physical resolution", filepath.Join(root, "tree", "missing-return"), filepath.Join(filepath.Dir(root), "outside"), true}, - {"loop", filepath.Join(root, "tree", "loop-a"), "", false}, - {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", false}, + {"plain file", filepath.Join(root, "tree", "real", "file.txt"), filepath.Join(root, "tree", "real", "file.txt"), nil}, + {"link to directory", filepath.Join(root, "tree", "to-real"), filepath.Join(root, "tree", "real"), nil}, + {"dangling tail stays lexical under the link target", filepath.Join(root, "tree", "dangling"), filepath.Join(root, "tree", "real", "missing", "deeper"), nil}, + {"dangling tail walking out through a link", filepath.Join(root, "tree", "escapes"), filepath.Join(filepath.Dir(root), "outside", "missing"), nil}, + {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), nil}, + {"missing tail returns to physical resolution", filepath.Join(root, "tree", "missing-return"), filepath.Join(filepath.Dir(root), "outside"), nil}, + {"loop", filepath.Join(root, "tree", "loop-a"), "", errUtils.ErrSymlinkLoop}, + {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", errUtils.ErrArtifactUnsafePath}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, reachable := linkpath.Resolve(tt.path) - assert.Equal(t, tt.reachable, reachable) - if tt.reachable { - assert.Equal(t, tt.want, got) + got, err := linkpath.Resolve(tt.path) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return } + require.NoError(t, err) + assert.Equal(t, tt.want, got) }) } } +// A parent component after a symlink applies to the link's physical target, not to the link's +// directory: "jump/../secret" is a sibling of the directory jump points into. +func TestResolve_ParentComponentFollowsTheLinkPhysically(t *testing.T) { + root := realTemp(t) + fixtures.WriteFile(t, root, "elsewhere/deep/keep.txt", "x", 0o644) + if !fixtures.Symlink(t, root, "tree/jump", "../elsewhere/deep") { + t.Skip("this platform cannot create symlinks") + } + // Written as a joined string so the path keeps its ".." component. + input := filepath.Join(root, "tree", "jump") + string(filepath.Separator) + ".." + string(filepath.Separator) + "secret" + + got, err := linkpath.Resolve(input) + require.NoError(t, err) + assert.Equal(t, filepath.Join(root, "elsewhere", "secret"), got) +} + func TestResolve_AbsoluteLinkTarget(t *testing.T) { root := realTemp(t) target := filepath.Join(root, "target") @@ -67,8 +86,8 @@ func TestResolve_AbsoluteLinkTarget(t *testing.T) { if err := os.Symlink(target, link); err != nil { t.Skipf("symlinks are unavailable here: %v", err) } - got, reachable := linkpath.Resolve(filepath.Join(link, "missing")) - assert.True(t, reachable) + got, err := linkpath.Resolve(filepath.Join(link, "missing")) + require.NoError(t, err) assert.Equal(t, filepath.Join(target, "missing"), got) } diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go index bbe99cd5f39..66386dd7a07 100644 --- a/pkg/artifact/materialize/links.go +++ b/pkg/artifact/materialize/links.go @@ -1,6 +1,7 @@ package materialize import ( + "errors" "fmt" "path/filepath" @@ -23,8 +24,15 @@ func checkLinkContainment(stage string, record *artifact.Record) error { if entry.Type != artifact.EntrySymlink { continue } - resolved, reachable := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) - if reachable && !linkpath.Within(realRoot, resolved) { + resolved, err := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) + switch { + case errors.Is(err, errUtils.ErrSymlinkLoop): + // A loop never reaches a path outside the root, so it is not an escape. + continue + case err != nil: + // Any other failure leaves the target unknown, and an unknown target cannot be shown to stay inside. + return fmt.Errorf("%w: symlink %q cannot be resolved: %w", errUtils.ErrArtifactUnsafePath, entry.Path, err) + case !linkpath.Within(realRoot, resolved): return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) } } diff --git a/pkg/artifact/materialize/tree.go b/pkg/artifact/materialize/tree.go index cf84f4567fb..bccdb97f00f 100644 --- a/pkg/artifact/materialize/tree.go +++ b/pkg/artifact/materialize/tree.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "sort" + "strings" "github.com/opencontainers/go-digest" @@ -164,7 +165,7 @@ func linkDependencies(sorted []*artifact.Entry) (pending []int, dependents [][]i pending = make([]int, len(sorted)) dependents = make([][]int, len(sorted)) for index, link := range sorted { - if dep, ok := linkThrough(link, byPath); ok && dep != index { + for _, dep := range linksThrough(link, index, sorted, byPath) { pending[index]++ dependents[dep] = append(dependents[dep], index) } @@ -185,17 +186,57 @@ func releaseDependents(waiting []int, pending []int) []int { return ready } -// linkThrough finds the nearest link that the target of link passes through, given the cleaned -// target path relative to the archive root. -func linkThrough(link *artifact.Entry, byPath map[string]int) (int, bool) { - target := path.Clean(path.Dir(link.Path) + "/" + link.Target) - for target != "." && target != "/" && target != "" { - if index, ok := byPath[target]; ok { - return index, true +// maxLinkExpansion bounds the links followed while expanding one target, so a cycle terminates. +const maxLinkExpansion = 255 + +// linksThrough lists every other link that the target of link passes through. The target is expanded +// the way the filesystem resolves it: a link met on the way is replaced by its own target, and a +// parent component applies to the directory reached after that replacement, never to the name before +// it. Cleaning the path lexically first would drop a link that a parent component follows. +func linksThrough(link *artifact.Entry, self int, links []*artifact.Entry, byPath map[string]int) []int { + pending := splitSlash(path.Dir(link.Path) + "/" + link.Target) + var deps []int + var directory []string + seen := make(map[int]bool) + for hops := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case "", ".": + continue + case "..": + if len(directory) > 0 { + directory = directory[:len(directory)-1] + } + continue + } + current := strings.Join(append(slices.Clone(directory), component), "/") + index, isLink := byPath[current] + if !isLink { + directory = append(directory, component) + continue } - target = path.Dir(target) + if index != self && !seen[index] { + seen[index] = true + deps = append(deps, index) + } + hops++ + if hops > maxLinkExpansion { + break + } + target := links[index].Target + if path.IsAbs(target) { + directory = nil + } + pending = append(splitSlash(target), pending...) } - return 0, false + return deps +} + +// splitSlash splits a slash-separated path into its components, keeping empty and dot components so +// the caller decides what they mean. +func splitSlash(value string) []string { + return strings.Split(value, "/") } // complete checks that every inventory entry was produced and applies final modes: files first, diff --git a/pkg/artifact/materialize/tree_links_internal_test.go b/pkg/artifact/materialize/tree_links_internal_test.go index 8ea5bee55da..0d64aa87238 100644 --- a/pkg/artifact/materialize/tree_links_internal_test.go +++ b/pkg/artifact/materialize/tree_links_internal_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" ) @@ -37,6 +38,10 @@ func TestOrderLinks_CreatesALinkAfterTheLinkItsTargetPassesThrough(t *testing.T) {"a chain is created from its end", []*artifact.Entry{linkEntry("a", "b"), linkEntry("b", "c"), linkEntry("c", "dir")}, []string{"c", "b", "a"}}, {"a target below a link", []*artifact.Entry{linkEntry("x", "y/sub"), linkEntry("y", "dir")}, []string{"y", "x"}}, {"a relative target in a subdirectory", []*artifact.Entry{linkEntry("d/a", "../e"), linkEntry("e", "dir")}, []string{"e", "d/a"}}, + // "m" reaches deep/c only after "z" is expanded, because the parent component applies to z's target. + {"a parent component after a link", []*artifact.Entry{linkEntry("m", "z/../c"), linkEntry("z", "deep/b"), linkEntry("deep/c", "dir")}, []string{"deep/c", "z", "m"}}, + // "t" passes through "r" (a link to "s"), and "s" is a link that no prefix of t's target names. + {"every link met while expanding the target", []*artifact.Entry{linkEntry("t", "r/f"), linkEntry("r", "s/sub"), linkEntry("s", "dir")}, []string{"s", "r", "t"}}, {"a cycle keeps path order and terminates", []*artifact.Entry{linkEntry("q", "p"), linkEntry("p", "q")}, []string{"p", "q"}}, {"a link to itself terminates", []*artifact.Entry{linkEntry("s", "s")}, []string{"s"}}, {"no links", nil, []string{}}, @@ -65,9 +70,8 @@ func TestTree_CreatesSymlinksOnlyAfterEveryOtherMemberExists(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist, "recording a link must not create it yet") require.NoError(t, target.mkdir(zdir)) - if err := target.createLinks(t.Context()); err != nil { - t.Skipf("this platform cannot create symlinks: %v", err) - } + requireSymlinkSupport(t) + require.NoError(t, target.createLinks(t.Context())) info, err := os.Lstat(filepath.Join(dir, "alias")) require.NoError(t, err) assert.NotZero(t, info.Mode()&os.ModeSymlink) @@ -135,3 +139,48 @@ func TestTree_OwnerAccessKeepsATemporaryTreeReadable(t *testing.T) { }) } } + +// requireSymlinkSupport skips the test only when the platform cannot create symlinks at all. It runs +// its own probe so a failure of the operation under test fails the test instead of skipping it. +func requireSymlinkSupport(t *testing.T) { + t.Helper() + probe := t.TempDir() + if err := os.WriteFile(filepath.Join(probe, "target.txt"), []byte("x"), 0o600); err != nil { + t.Fatalf("write the symlink probe target: %v", err) + } + if err := os.Symlink("target.txt", filepath.Join(probe, "link")); err != nil { + t.Skipf("this platform cannot create symlinks: %v", err) + } +} + +// A link whose target cannot be resolved for a reason other than a loop is rejected, because an unknown +// target cannot be shown to stay inside the package; a loop never leaves the root, so it is accepted. +func TestCheckLinkContainment_ResolutionFailures(t *testing.T) { + requireSymlinkSupport(t) + tests := []struct { + name string + links map[string]string + wantErr bool + }{ + {name: "through a regular file", links: map[string]string{"bad": "file.txt/child"}, wantErr: true}, + {name: "loop", links: map[string]string{"a": "b", "b": "a"}}, + {name: "inside the root", links: map[string]string{"ok": "file.txt"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stage := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(stage, "file.txt"), []byte("x"), 0o600)) + record := &artifact.Record{} + for name, target := range tt.links { + require.NoError(t, os.Symlink(target, filepath.Join(stage, name))) + record.Inventory = append(record.Inventory, artifact.Entry{Path: name, Type: artifact.EntrySymlink}) + } + err := checkLinkContainment(stage, record) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + return + } + require.NoError(t, err) + }) + } +} diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go index 29f99e54b35..5c2c660b4ae 100644 --- a/pkg/artifact/service/commit.go +++ b/pkg/artifact/service/commit.go @@ -97,8 +97,11 @@ func (s *Service) commit(ctx context.Context, pub *publication) (err error) { } s.event(ctx, pub, PhaseLocked, 0, nil) defer func() { - _ = locks.release(ctx, pub.op) - s.event(ctx, pub, PhaseReleased, 0, nil) + // A failed release already raised its recovery warning with the owner and resource; the + // journal must not claim the lock was released, and the committed outcome is unchanged. + if releaseErr := locks.release(ctx, pub.op); releaseErr == nil { + s.event(ctx, pub, PhaseReleased, 0, nil) + } if err != nil { s.event(ctx, pub, PhaseFailed, 0, err) } diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go index 359b77a2082..07436eac227 100644 --- a/pkg/artifact/service/mutate_test.go +++ b/pkg/artifact/service/mutate_test.go @@ -213,11 +213,14 @@ func TestLocks_ReleaseFailureIsReportedForOperatorRecovery(t *testing.T) { handle.EXPECT().Release(gomock.Any()).Return(errTransport) repos := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{"dst": {Capabilities: caps, Packaging: artifact.PackagingNone}}} e := newEnv(t, envConfig{}) - svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging)) + journal, log := newRecordingJournal(t) + svc, err := service.New(repos, nil, service.WithStagingRoot(e.staging), service.WithJournal(journal)) require.NoError(t, err) result, err := svc.Publish(context.Background(), e.publishRequest("dst", e.source("hello"))) require.NoError(t, err, "the publication itself succeeded") + assert.Contains(t, log.phases(), service.PhaseCommitted) + assert.NotContains(t, log.phases(), service.PhaseReleased, "a failed release must not be journaled as released") var warning artifact.Diagnostic for _, diagnostic := range result.Diagnostics { if diagnostic.Code == "lock-release-failed" { diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index bde0c296e14..c77666eacc2 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -2,6 +2,7 @@ package snapshot import ( "context" + "errors" "fmt" "io" "os" @@ -162,7 +163,7 @@ func Capture(ctx context.Context, source string, opts ...Option) (*Snapshot, err } // Cancellation is final even when no entry would ever check it, such as an empty directory. if err := ctx.Err(); err != nil { - return nil, err + return nil, classifyInterruption(err) } root, err := os.MkdirTemp(applied.stagingRoot, stagingPattern) if err != nil { @@ -171,11 +172,24 @@ func Capture(ctx context.Context, source string, opts ...Option) (*Snapshot, err snap, err := captureInto(ctx, source, root, &applied) if err != nil { _ = os.RemoveAll(root) - return nil, err + return nil, classifyInterruption(err) } return snap, nil } +// classifyInterruption gives a cancellation or deadline error the packaging sentinel so callers see +// a classified artifact error, and keeps the context error in the chain. Every other error, and one +// that already carries a sentinel, is returned unchanged. +func classifyInterruption(err error) error { + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + return err + } + if errors.Is(err, errUtils.ErrArtifactPackagingInvalid) { + return err + } + return fmt.Errorf("%w: capture interrupted: %w", errUtils.ErrArtifactPackagingInvalid, err) +} + func captureInto(ctx context.Context, source, root string, opts *options) (*Snapshot, error) { dataDir := filepath.Join(root, dataDirName) if err := os.Mkdir(dataDir, stagingDirPerm); err != nil { diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 594f483d5a3..6ebf58bef8c 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -315,6 +315,35 @@ func TestCapture_CancellationRemovesStaging(t *testing.T) { assertEmptyDir(t, staging) } +// Cancellation and deadline errors carry the packaging sentinel and keep the context error in the chain, +// whether the context ended before capture began or while it ran. +func TestCapture_CancellationIsClassifiedAndKeepsItsCause(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + fixtures.WriteFile(t, src, "b.txt", "b", 0o644) + + t.Run("canceled before capture", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.Canceled) + }) + t.Run("canceled during capture", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithCaptureHook(func(string) { cancel() })) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.Canceled) + }) + t.Run("deadline exceeded", func(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + _, err := snapshot.Capture(ctx, src, snapshot.WithStagingRoot(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.ErrorIs(t, err, context.DeadlineExceeded) + }) +} + func TestSnapshot_CloseRemovesStagingAndIsIdempotent(t *testing.T) { src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "a", 0o644) diff --git a/pkg/oci/identity.go b/pkg/oci/identity.go index 69a58fe1120..8c5da84d692 100644 --- a/pkg/oci/identity.go +++ b/pkg/oci/identity.go @@ -75,7 +75,7 @@ func (n nonEmptyCredential) Authorization() (*authn.AuthConfig, error) { config, err := n.Authenticator.Authorization() if err != nil { - return nil, err + return nil, errors.Join(errUtils.ErrOCIRegistry, err) } if config == nil { return nil, fmt.Errorf("%w: the bound identity resolved to no credential", errUtils.ErrOCIRegistry) @@ -87,5 +87,15 @@ func (n nonEmptyCredential) Authorization() (*authn.AuthConfig, error) { if credential == orasauth.EmptyCredential { return nil, fmt.Errorf("%w: the bound identity resolved to an empty credential", errUtils.ErrOCIRegistry) } + if !canSend(credential) { + return nil, fmt.Errorf("%w: the bound identity resolved to an incomplete credential: a token, or both a username and a password, is required", errUtils.ErrOCIRegistry) + } return config, nil } + +// canSend reports whether the registry client can turn the credential into an Authorization header: a +// token, or a username together with a password. A username or password on its own is rejected +// instead of letting the request go out unauthenticated. +func canSend(credential orasauth.Credential) bool { + return credential.AccessToken != "" || credential.RefreshToken != "" || (credential.Username != "" && credential.Password != "") +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index f5ea71d6a6d..26cd3abe779 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -107,6 +107,9 @@ func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { {"bound authenticator cannot authorize", failingAuthenticator{}, true, nil}, {"matched identity with an empty credential", authn.FromConfig(authn.AuthConfig{}), true, nil}, {"matched identity with an empty basic credential", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte(":"))}), true, nil}, + {"matched identity with a username only", authn.FromConfig(authn.AuthConfig{Username: "identity"}), true, nil}, + {"matched identity with a password only", authn.FromConfig(authn.AuthConfig{Password: "bound"}), true, nil}, + {"matched identity with a username-only basic credential", authn.FromConfig(authn.AuthConfig{Auth: base64.StdEncoding.EncodeToString([]byte("identity:"))}), true, nil}, } { t.Run(tc.name, func(t *testing.T) { controller := gomock.NewController(t) @@ -126,6 +129,30 @@ func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { assert.Zero(t, requests.Load(), "no request may reach the registry when the identity cannot be resolved") } +// A bound identity must be able to produce an Authorization header: a token, or a username with a +// password. These shapes are accepted. +func TestNonEmptyCredentialAcceptsSendableCredentials(t *testing.T) { + for name, config := range map[string]authn.AuthConfig{ + "username and password": {Username: "identity", Password: "bound"}, + "access token": {RegistryToken: "token"}, + "refresh token": {IdentityToken: "refresh"}, + "encoded basic": {Auth: base64.StdEncoding.EncodeToString([]byte("identity:bound"))}, + } { + t.Run(name, func(t *testing.T) { + got, err := nonEmptyCredential{Authenticator: authn.FromConfig(config)}.Authorization() + require.NoError(t, err) + assert.NotNil(t, got) + }) + } +} + +// An authorization failure from the bound identity carries the registry sentinel and its cause. +func TestNonEmptyCredentialClassifiesAuthorizationFailures(t *testing.T) { + _, err := nonEmptyCredential{Authenticator: failingAuthenticator{}}.Authorization() + require.ErrorIs(t, err, errUtils.ErrOCIRegistry) + assert.ErrorIs(t, err, os.ErrPermission) +} + func TestBoundIdentityTakesPrecedenceOverAmbientCredentials(t *testing.T) { var seen atomic.Value server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 7b580689c21..20424707a66 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -12,6 +12,7 @@ import ( "slices" "sync" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/content/oci" "oras.land/oras-go/v2/errdef" @@ -189,7 +190,32 @@ func (l *Layout) Resolve(ctx context.Context, reference string) (ocispec.Descrip defer perf.Track(nil, "oci.Layout.Resolve")() resolved, err := l.store.Resolve(ctx, reference) - return resolved, layoutError(err) + if err != nil { + return resolved, layoutError(err) + } + if !l.persisted(reference, resolved.Digest) { + // The resolver is updated before index.json is written, so a failed write leaves a tag that + // resolves now and disappears when the layout is reopened. Never report such a tag as published. + return ocispec.Descriptor{}, layoutError(errdef.ErrNotFound) + } + return resolved, nil +} + +// persisted reports whether a reference is backed by index.json. A digest reference always is, because +// it names content, not a tag; a named reference is only when the persisted index lists it for the +// digest it resolved to. +func (l *Layout) persisted(reference string, resolved digest.Digest) bool { + if _, err := digest.Parse(reference); err == nil { + return true + } + l.mu.Lock() + defer l.mu.Unlock() + for i := range l.index { + if l.index[i].Annotations[ocispec.AnnotationRefName] == reference && l.index[i].Digest == resolved { + return true + } + } + return false } // Predecessors lists manifests in the layout that reference the node. @@ -222,7 +248,49 @@ func (l *Layout) Tag(ctx context.Context, desc ocispec.Descriptor, reference str named.Annotations = make(map[string]string, len(desc.Annotations)+1) maps.Copy(named.Annotations, desc.Annotations) named.Annotations[ocispec.AnnotationRefName] = reference - return l.commitIndex(withReference(slices.Clone(l.index), named)) + displaced, moved := displacedByReference(l.index, reference, desc.Digest) + entries := withReference(slices.Clone(l.index), named) + if moved { + var err error + if entries, err = l.keepDisplacedAttachment(ctx, entries, displaced); err != nil { + return err + } + } + return l.commitIndex(entries) +} + +// displacedByReference returns the root that reference names when it points at a different digest than +// the one about to be tagged: moving the tag replaces that root's entry in index.json. +func displacedByReference(entries []ocispec.Descriptor, reference string, tagged digest.Digest) (ocispec.Descriptor, bool) { + for i := range entries { + if entries[i].Annotations[ocispec.AnnotationRefName] == reference && entries[i].Digest != tagged { + return entries[i], true + } + } + return ocispec.Descriptor{}, false +} + +// keepDisplacedAttachment lists a root that lost its tag as an unnamed root when it is an attachment, +// a manifest with a subject. Without its own entry the manifest would no longer be reachable from +// index.json, and reopening the layout would lose it as a referrer of its subject. A manifest without +// a subject, or one that is still listed, needs nothing. +func (l *Layout) keepDisplacedAttachment(ctx context.Context, entries []ocispec.Descriptor, displaced ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if displaced.MediaType != ocispec.MediaTypeImageManifest && displaced.MediaType != ocispec.MediaTypeImageIndex { + return entries, nil + } + for i := range entries { + if entries[i].Digest == displaced.Digest { + return entries, nil + } + } + subject, err := readSubject(ctx, l.store, &displaced) + if err != nil { + return nil, layoutError(err) + } + if subject == nil { + return entries, nil + } + return append(entries, ocispec.Descriptor{MediaType: displaced.MediaType, Digest: displaced.Digest, Size: displaced.Size}), nil } func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []ocispec.Descriptor { diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index b4ced92387b..f0d99c88a89 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -10,6 +10,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/errdef" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/oci" @@ -98,6 +99,59 @@ func pushSubjectManifest(t *testing.T, layout *oci.Layout, subject *ocispec.Desc return desc } +// Moving a tag off an attachment manifest must not drop the attachment from the layout: it stays an +// unnamed root, so reopening the layout still finds it as a referrer of its subject. +func TestTagMovePreservesTheDisplacedAttachmentRoot(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + subject := pushSubjectManifest(t, layout, nil, "subject") + attachment := pushSubjectManifest(t, layout, &subject, "attachment") + replacement := pushSubjectManifest(t, layout, nil, "replacement") + require.NoError(t, layout.Tag(ctx, attachment, "moving")) + + require.NoError(t, layout.Tag(ctx, replacement, "moving")) + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + resolved, err := reopened.Resolve(ctx, "moving") + require.NoError(t, err) + assert.Equal(t, replacement.Digest, resolved.Digest, "the tag moved to the new manifest") + referrers, err := reopened.Predecessors(ctx, subject) + require.NoError(t, err) + digests := make([]string, 0, len(referrers)) + for _, referrer := range referrers { + digests = append(digests, referrer.Digest.String()) + } + assert.Contains(t, digests, attachment.Digest.String(), "the displaced attachment is still discoverable after reopening") + assert.Contains(t, layoutRoots(t, directory), attachment.Digest.String()) +} + +// A tag that could not be written to index.json would vanish when the layout is reopened, so it must not +// resolve: otherwise a recovery check would report success for a tag that was never persisted. +func TestResolveDoesNotTrustATagThatWasNotPersisted(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + root := pushSubjectManifest(t, layout, nil, "root") + + // Replace index.json with a non-empty directory so the atomic write cannot land on any platform. + indexPath := filepath.Join(directory, "index.json") + require.NoError(t, os.Remove(indexPath)) + require.NoError(t, os.MkdirAll(filepath.Join(indexPath, "blocker"), 0o755)) + + require.Error(t, layout.Tag(ctx, root, "1.0.0"), "the index write fails") + _, err = layout.Resolve(ctx, "1.0.0") + require.Error(t, err, "an unpersisted tag does not resolve") + require.ErrorIs(t, err, errdef.ErrNotFound) + + resolved, err := layout.Resolve(ctx, root.Digest.String()) + require.NoError(t, err, "a digest reference still resolves") + assert.Equal(t, root.Digest, resolved.Digest) +} + func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers(t *testing.T) { ctx := t.Context() source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) From 12efe5fd4ded80394be3ba67850a7015273667c7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 19:26:20 -0600 Subject: [PATCH 43/52] fix(artifact): address the second round of CodeRabbit findings - Normalize the destination before inspecting it, so a trailing separator cannot make a symlink destination pass the symlink check. - Convert long absolute paths to extended-length form before MoveFile on Windows. - Construct a repository outside the shared cache mutex with per-alias state, so opening a cached repository is not blocked by another repository's construction and a waiting caller stops on its own context. - Retain the source root's identity during capture and reject a replacement, which a symlink swapped in before the walk would otherwise hide from every per-entry check. Each change has a regression test; the snapshot test was confirmed to fail without the check. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/exchange_windows.go | 15 ++- pkg/artifact/materialize/longpath.go | 21 ++++ .../materialize/longpath_internal_test.go | 31 +++++ pkg/artifact/materialize/materialize_test.go | 25 ++++ pkg/artifact/materialize/replace.go | 5 + pkg/artifact/service/config_repositories.go | 79 +++++++++--- .../service/config_repositories_test.go | 118 ++++++++++++++++++ pkg/artifact/snapshot/copy.go | 17 +++ .../snapshot/recheck_internal_test.go | 33 +++++ pkg/artifact/snapshot/snapshot.go | 6 +- pkg/artifact/snapshot/walk.go | 15 ++- 11 files changed, 340 insertions(+), 25 deletions(-) create mode 100644 pkg/artifact/materialize/longpath.go create mode 100644 pkg/artifact/materialize/longpath_internal_test.go diff --git a/pkg/artifact/materialize/exchange_windows.go b/pkg/artifact/materialize/exchange_windows.go index 8bdde7e1fbb..9bfcac6df81 100644 --- a/pkg/artifact/materialize/exchange_windows.go +++ b/pkg/artifact/materialize/exchange_windows.go @@ -5,6 +5,7 @@ package materialize import ( "fmt" "os" + "path/filepath" "golang.org/x/sys/windows" @@ -16,13 +17,23 @@ func exchangeDirectories(_ *os.File, _ string, _ *os.File, _ string) error { } func renameExclusive(from, to string) error { - source, err := windows.UTF16PtrFromString(from) + source, err := moveFilePath(from) if err != nil { return err } - target, err := windows.UTF16PtrFromString(to) + target, err := moveFilePath(to) if err != nil { return err } return windows.MoveFile(source, target) } + +// moveFilePath returns the UTF-16 form of path as MoveFile accepts it: an absolute path beyond MAX_PATH +// gets the extended-length prefix, and a short path keeps its value. +func moveFilePath(path string) (*uint16, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return nil, err + } + return windows.UTF16PtrFromString(extendedLengthPath(absolute)) +} diff --git a/pkg/artifact/materialize/longpath.go b/pkg/artifact/materialize/longpath.go new file mode 100644 index 00000000000..876eb9beaab --- /dev/null +++ b/pkg/artifact/materialize/longpath.go @@ -0,0 +1,21 @@ +package materialize + +import "strings" + +// maxPathChars is the Windows MAX_PATH limit, which counts the terminating NUL. +const maxPathChars = 260 + +// extendedLengthPath converts a long absolute Windows path to the extended-length form that APIs such +// as MoveFile accept beyond MAX_PATH. A path shorter than the limit, and one that already has an +// extended-length prefix, is returned unchanged. A UNC path (\\server\share) becomes \\?\UNC\server\share. +// The path must already be absolute and use backslashes: the prefix switches off the normalization +// that would otherwise convert slashes and resolve dot segments. +func extendedLengthPath(absolute string) string { + if len(absolute) < maxPathChars || strings.HasPrefix(absolute, `\\?\`) { + return absolute + } + if strings.HasPrefix(absolute, `\\`) { + return `\\?\UNC\` + absolute[2:] + } + return `\\?\` + absolute +} diff --git a/pkg/artifact/materialize/longpath_internal_test.go b/pkg/artifact/materialize/longpath_internal_test.go new file mode 100644 index 00000000000..f293ef3dd7b --- /dev/null +++ b/pkg/artifact/materialize/longpath_internal_test.go @@ -0,0 +1,31 @@ +package materialize + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtendedLengthPath(t *testing.T) { + long := `C:\` + strings.Repeat(`a\`, 130) + "dir" + longUNC := `\\server\share\` + strings.Repeat(`a\`, 130) + "dir" + tests := []struct { + name string + in string + want string + }{ + {"short drive path is unchanged", `C:\tmp\out`, `C:\tmp\out`}, + {"short UNC path is unchanged", `\\server\share\out`, `\\server\share\out`}, + {"long drive path gets the prefix", long, `\\?\` + long}, + {"long UNC path gets the UNC prefix", longUNC, `\\?\UNC\` + longUNC[2:]}, + {"an extended-length path is unchanged", `\\?\` + long, `\\?\` + long}, + {"the boundary length is converted", `C:\` + strings.Repeat("b", maxPathChars-3), `\\?\C:\` + strings.Repeat("b", maxPathChars-3)}, + {"one below the boundary is unchanged", `C:\` + strings.Repeat("b", maxPathChars-4), `C:\` + strings.Repeat("b", maxPathChars-4)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, extendedLengthPath(tt.in)) + }) + } +} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index d7bbc85077a..3a31b0bde71 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -169,6 +169,31 @@ func TestMaterialize_ReplacesRestrictiveDestination(t *testing.T) { assertNoScratch(t, out) } +// A trailing separator makes the filesystem resolve a symlink as a directory, so the destination is +// normalized before it is inspected: a symlink destination is rejected whether or not it ends in one. +func TestInspectDestination_RejectsASymlinkWithATrailingSeparator(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + root := t.TempDir() + // The target holds exactly the inventory, so following the link would report a match. + fixtures.WriteFile(t, root, "target/a.txt", "a", 0o644) + if !fixtures.Symlink(t, root, "linked", "target") { + t.Skip("this platform cannot create symlinks") + } + link := filepath.Join(root, "linked") + + for name, dest := range map[string]string{ + "plain": link, + "trailing separator": link + string(filepath.Separator), + } { + t.Run(name, func(t *testing.T) { + _, err := materialize.InspectDestination(dest, &p.Record) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + }) + } +} + func TestInspectDestination_States(t *testing.T) { src := t.TempDir() fixtures.WriteFile(t, src, "a.txt", "a", 0o644) diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 801ddbfbd6c..19e0ee85367 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -52,6 +52,11 @@ const ( func InspectDestination(dest string, record *artifact.Record) (DestinationState, error) { defer perf.Track(nil, "materialize.InspectDestination")() + // A trailing separator makes the filesystem resolve a symlink as a directory, which would let a + // matching link target pass the symlink check below. Inspect the cleaned path instead. + if dest != "" { + dest = filepath.Clean(dest) + } info, err := os.Lstat(dest) switch { case os.IsNotExist(err): diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go index 007f230892d..70c6986f558 100644 --- a/pkg/artifact/service/config_repositories.go +++ b/pkg/artifact/service/config_repositories.go @@ -2,6 +2,7 @@ package service import ( "context" + "errors" "fmt" "maps" "sort" @@ -23,7 +24,17 @@ type ConfigRepositories struct { registry *artifact.RepositoryRegistry options []artifact.RepositoryOption mu sync.Mutex - opened map[string]artifact.Repository + opened map[string]*openedRepository +} + +// An openedRepository is the construction state of one alias. Its done channel closes when +// construction ends, and the repository and error are valid after that. The shared mutex guards only +// the map of these entries, never a construction: building a repository may resolve credentials and +// wait on a user. +type openedRepository struct { + done chan struct{} + repo artifact.Repository + err error } var _ Repositories = (*ConfigRepositories)(nil) @@ -36,7 +47,7 @@ func FromConfig(cfg *schema.ArtifactConfig, registry *artifact.RepositoryRegistr if registry == nil { registry = artifact.DefaultRepositoryRegistry() } - return &ConfigRepositories{cfg: cfg, registry: registry, options: opts, opened: map[string]artifact.Repository{}} + return &ConfigRepositories{cfg: cfg, registry: registry, options: opts, opened: map[string]*openedRepository{}} } // Describe implements [Repositories]. For a repository that delegates locking, the lock scopes of @@ -106,25 +117,61 @@ func containsScope(scopes []artifact.LockScope, scope artifact.LockScope) bool { return false } -// Open implements [Repositories]. Each repository is constructed once. +// Open implements [Repositories]. Each repository is constructed once, and constructing one does not +// block opening another: callers waiting for the same alias share its construction and stop waiting +// when their own context ends. A failed construction is not cached, so a later Open retries it. func (c *ConfigRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { defer perf.Track(nil, "service.ConfigRepositories.Open")() - c.mu.Lock() - defer c.mu.Unlock() - if repo, ok := c.opened[alias]; ok { - return repo, nil - } - repoCfg, ok := c.cfg.Repositories[alias] - if !ok { - return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + for { + c.mu.Lock() + if entry, ok := c.opened[alias]; ok { + c.mu.Unlock() + repo, retry, err := c.await(ctx, entry) + if retry { + continue + } + return repo, err + } + repoCfg, ok := c.cfg.Repositories[alias] + if !ok { + c.mu.Unlock() + return nil, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + entry := &openedRepository{done: make(chan struct{})} + c.opened[alias] = entry + c.mu.Unlock() + return c.construct(ctx, alias, &repoCfg, entry) } - repo, err := c.registry.New(ctx, alias, &repoCfg, c.options...) - if err != nil { - return nil, err +} + +// construct builds the repository outside the shared mutex and publishes the outcome to every caller +// waiting on entry. A failure removes the entry so a later Open can try again. +func (c *ConfigRepositories) construct(ctx context.Context, alias string, repoCfg *schema.ArtifactRepositoryConfig, entry *openedRepository) (artifact.Repository, error) { + entry.repo, entry.err = c.registry.New(ctx, alias, repoCfg, c.options...) + if entry.err != nil { + c.mu.Lock() + if c.opened[alias] == entry { + delete(c.opened, alias) + } + c.mu.Unlock() } - c.opened[alias] = repo - return repo, nil + close(entry.done) + return entry.repo, entry.err +} + +// await waits for another caller's construction of entry, or for ctx to end. It reports retry when that +// construction ended because the other caller's own context did, which says nothing about this caller. +func (c *ConfigRepositories) await(ctx context.Context, entry *openedRepository) (artifact.Repository, bool, error) { + select { + case <-entry.done: + case <-ctx.Done(): + return nil, false, ctx.Err() + } + if entry.err != nil && ctx.Err() == nil && (errors.Is(entry.err, context.Canceled) || errors.Is(entry.err, context.DeadlineExceeded)) { + return nil, true, nil + } + return entry.repo, false, entry.err } // Aliases implements [Repositories]. diff --git a/pkg/artifact/service/config_repositories_test.go b/pkg/artifact/service/config_repositories_test.go index 17d27fee91c..d4445b39f37 100644 --- a/pkg/artifact/service/config_repositories_test.go +++ b/pkg/artifact/service/config_repositories_test.go @@ -2,8 +2,10 @@ package service_test import ( "context" + "sync" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -155,3 +157,119 @@ func TestConfigRepositories_DescribeReturnsIndependentCopies(t *testing.T) { assert.Equal(t, map[string]string{"team": "platform"}, second.Defaults.Labels) assert.Equal(t, map[string]string{"team": "platform"}, labels, "the configured defaults are untouched") } + +// blockingRegistry builds repositories like configRegistry, but the construction of "slow" waits for +// release, so a test can hold one alias mid-construction. +func blockingRegistry(t *testing.T, started chan<- struct{}, release <-chan struct{}, constructed *atomic.Int64) *artifact.RepositoryRegistry { + t.Helper() + registry := artifact.NewRepositoryRegistry() + desc, ok := artifact.BuiltinKind(artifact.RepositoryKindLocalDir) + require.True(t, ok) + require.NoError(t, registry.Register(desc, func(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + constructed.Add(1) + if alias == "slow" { + started <- struct{}{} + <-release + } + return artifacttest.New(artifacttest.Options{Alias: alias}), nil + })) + return registry +} + +func blockingConfig() *schema.ArtifactConfig { + return &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "fast": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/fast"}}, + "slow": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": "/tmp/slow"}}, + }} +} + +// While one repository is being constructed (it may be resolving credentials), opening an already +// cached repository must not wait for it. +func TestConfigRepositories_OpenOfACachedRepositoryIsNotBlockedByAnotherConstruction(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 1), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + cached, err := repos.Open(context.Background(), "fast") + require.NoError(t, err) + + slowDone := make(chan error, 1) + go func() { + _, openErr := repos.Open(context.Background(), "slow") + slowDone <- openErr + }() + <-started + + again := make(chan artifact.Repository, 1) + go func() { + repo, _ := repos.Open(context.Background(), "fast") + again <- repo + }() + select { + case repo := <-again: + assert.Same(t, cached, repo) + case <-time.After(2 * time.Second): + close(release) + t.Fatal("opening a cached repository waited for another repository's construction") + } + close(release) + require.NoError(t, <-slowDone) +} + +// A caller waiting for a repository that another caller is constructing stops waiting when its own +// context ends, and the construction still completes for everyone else. +func TestConfigRepositories_AWaitingOpenHonorsItsContext(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 1), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + + firstDone := make(chan artifact.Repository, 1) + go func() { + repo, _ := repos.Open(context.Background(), "slow") + firstDone <- repo + }() + <-started + + ctx, cancel := context.WithCancel(context.Background()) + waiter := make(chan error, 1) + go func() { + _, err := repos.Open(ctx, "slow") + waiter <- err + }() + cancel() + select { + case err := <-waiter: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + close(release) + t.Fatal("a canceled waiter kept waiting for the construction") + } + close(release) + require.NotNil(t, <-firstDone) + assert.Equal(t, int64(1), constructed.Load(), "the waiter did not start a second construction") +} + +// Callers that race for the same alias share one construction and one repository. +func TestConfigRepositories_ConcurrentOpensOfOneAliasConstructOnce(t *testing.T) { + var constructed atomic.Int64 + started, release := make(chan struct{}, 8), make(chan struct{}) + repos := service.FromConfig(blockingConfig(), blockingRegistry(t, started, release, &constructed)) + + const callers = 8 + results := make([]artifact.Repository, callers) + var wg sync.WaitGroup + for i := range results { + wg.Add(1) + go func() { + defer wg.Done() + results[i], _ = repos.Open(context.Background(), "slow") + }() + } + <-started + close(release) + wg.Wait() + for _, repo := range results { + require.NotNil(t, repo) + assert.Same(t, results[0], repo) + } + assert.Equal(t, int64(1), constructed.Load()) +} diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index 9c4b52d419b..112a1cb5e14 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -103,6 +103,9 @@ func sameFileState(a, b os.FileInfo) bool { // recheck verifies every captured source entry still matches what was staged: type, size, // modification time, directory listing, link target, and file digest. func (c *capturer) recheck() error { + if err := c.recheckRoot(); err != nil { + return err + } for index := range c.sources { if err := c.ctx.Err(); err != nil { return err @@ -123,6 +126,20 @@ func (c *capturer) recheck() error { return nil } +// recheckRoot verifies that the source root is still the directory or file that was resolved. If another +// process replaced it, for example with a symlink, the children would have been read from the replacement +// and every per-entry check would agree with them, so the root's own identity is compared. +func (c *capturer) recheckRoot() error { + now, err := os.Lstat(c.root) + if err != nil { + return changedError(c.root, err) + } + if !os.SameFile(c.rootInfo, now) { + return fmt.Errorf("%w: the source %q was replaced while it was being captured", errUtils.ErrArtifactSourceChanged, c.root) + } + return nil +} + func (c *capturer) recheckSource(record *sourceRecord) error { now, err := os.Lstat(record.abs) if err != nil { diff --git a/pkg/artifact/snapshot/recheck_internal_test.go b/pkg/artifact/snapshot/recheck_internal_test.go index a455784492d..7e19b2ee126 100644 --- a/pkg/artifact/snapshot/recheck_internal_test.go +++ b/pkg/artifact/snapshot/recheck_internal_test.go @@ -39,6 +39,39 @@ func TestCapture_RejectsAModeChangeDuringTheFinalHash(t *testing.T) { assert.Nil(t, snap) } +// withResolvedHook registers a callback invoked once the source has been resolved and before it is read. +func withResolvedHook(hook func(path string)) Option { + return func(o *options) { o.onResolved = hook } +} + +// A source directory replaced by a symlink to another directory after it was resolved would be read +// through the link: every child record describes the link's target, so the per-file checks agree with +// each other. The root's own identity is retained and verified, so the capture is rejected. +func TestCapture_RejectsReplacementOfTheSourceRootBeforeItIsRead(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("a directory cannot be swapped for a symlink portably on %s", runtime.GOOS) + } + parent := t.TempDir() + src := filepath.Join(parent, "src") + require.NoError(t, os.Mkdir(src, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("selected"), 0o644)) + outside := filepath.Join(parent, "outside") + require.NoError(t, os.Mkdir(outside, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("not selected"), 0o644)) + + swapped := false + hook := func(string) { + swapped = true + require.NoError(t, os.Rename(src, filepath.Join(parent, "moved"))) + require.NoError(t, os.Symlink(outside, src)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withResolvedHook(hook)) + + require.True(t, swapped, "the hook ran") + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} + func TestCapture_StaysQuietWhenNothingChangesDuringTheFinalHash(t *testing.T) { src := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(src, "a.txt"), []byte("a"), 0o644)) diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index c77666eacc2..413646e2b05 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -49,6 +49,7 @@ type options struct { limits Limits onCaptured func(path string) onRehashed func(path string) + onResolved func(path string) } // WithStagingRoot sets the directory that holds the private staging directory. The default is the @@ -199,11 +200,14 @@ func captureInto(ctx context.Context, source, root string, opts *options) (*Snap if err != nil { return nil, err } + if hook := opts.onResolved; hook != nil { + hook(resolved) + } staging, err := os.Lstat(root) if err != nil { return nil, fmt.Errorf("%w: inspect staging directory: %w", errUtils.ErrArtifactPackagingInvalid, err) } - c := &capturer{staging: staging, ctx: ctx, opts: opts, root: resolved, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} + c := &capturer{staging: staging, ctx: ctx, opts: opts, root: resolved, rootInfo: info, dataDir: dataDir, staged: make(map[string]string), dirs: make(map[string][]string)} if info.IsDir() { err = c.walkDir(resolved, "") } else { diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go index a30c2b5b6b9..ab17aa1e5be 100644 --- a/pkg/artifact/snapshot/walk.go +++ b/pkg/artifact/snapshot/walk.go @@ -30,12 +30,15 @@ type capturer struct { opts *options root string dataDir string - entries []artifact.Entry - staged map[string]string - dirs map[string][]string - sources []sourceRecord - total int64 - counter int + + // rootInfo identifies the source root as it was resolved, so a replacement is detected. + rootInfo os.FileInfo + entries []artifact.Entry + staged map[string]string + dirs map[string][]string + sources []sourceRecord + total int64 + counter int // staging identifies the private staging directory, which is skipped when it lies inside the source. staging os.FileInfo From 711c998313fcd008d169fb53b62799931ce22371 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 21:08:05 -0600 Subject: [PATCH 44/52] fix(artifact): address the third round of CodeRabbit findings Reject reads without the read capability, classify signer failures and a corrupt gzip trailer, resolve a relative destination to an absolute path, accept links that run through a regular file when they stay inside the tree, fill a missing username or password from an encoded Auth, anchor the layout root, read the resolver and index under one lock, fall back to the referrers tag schema on 400 or 406, serve native referrers from the test registry, and prepare index children before reading the index manifest. Schema 1 images are refused explicitly. Each behavior change has a regression test. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 2 + pkg/artifact/artifacttest/read.go | 21 +++ pkg/artifact/artifacttest/repository_test.go | 48 ++++++ pkg/artifact/artifacttest/sign.go | 2 +- pkg/artifact/linkpath/linkpath.go | 7 +- pkg/artifact/linkpath/linkpath_test.go | 2 +- pkg/artifact/materialize/archive.go | 6 +- .../materialize/archive_internal_test.go | 23 +++ pkg/artifact/materialize/links.go | 14 ++ pkg/artifact/materialize/materialize_test.go | 24 +++ pkg/artifact/materialize/replace.go | 8 +- .../materialize/tree_links_internal_test.go | 7 +- pkg/oci/client.go | 3 +- pkg/oci/ocitest/registry.go | 22 ++- pkg/oci/referrers_fallback_test.go | 139 ++++++++++++++++++ pkg/oci/transport_internal_test.go | 9 +- pkg/oci/transport_layout.go | 27 ++-- pkg/oci/transport_legacy.go | 101 ++++++++----- pkg/oci/transport_legacy_children_test.go | 80 ++++++++++ pkg/oci/transport_registry.go | 65 ++++++-- pkg/oci/transport_roundtrip_test.go | 2 +- pkg/oci/transport_subject_test.go | 64 ++++++++ 22 files changed, 600 insertions(+), 76 deletions(-) create mode 100644 pkg/oci/referrers_fallback_test.go create mode 100644 pkg/oci/transport_legacy_children_test.go diff --git a/errors/errors.go b/errors/errors.go index 74addb484fd..f7cd45d8f15 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1608,6 +1608,8 @@ var ( ErrArtifactStoreInvalidArgs = errors.New("invalid artifact store arguments") ErrArtifactMetadataFailed = errors.New("failed to load artifact metadata") ErrArtifactIntegrityFailed = errors.New("artifact integrity check failed") + ErrArtifactSigningFailed = errors.New("failed to sign artifact") + ErrPathThroughNonDirectory = errors.New("path passes through a non-directory") // CI cache errors. ErrCacheUnavailable = errors.New("CI cache is not available in this environment") diff --git a/pkg/artifact/artifacttest/read.go b/pkg/artifact/artifacttest/read.go index b8192702229..24d5aa9409c 100644 --- a/pkg/artifact/artifacttest/read.go +++ b/pkg/artifact/artifacttest/read.go @@ -21,6 +21,9 @@ import ( func (r *Repository) Resolve(_ context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { defer perf.Track(nil, "artifacttest.Repository.Resolve")() + if err := r.requireRead("resolve"); err != nil { + return artifact.PinnedReference{}, err + } pinned, err := r.resolve(ref) r.mu.Lock() hook := r.opts.OnResolved @@ -65,6 +68,9 @@ func (r *Repository) Inspect(_ context.Context, pinned artifact.PinnedReference) r.mu.Lock() defer r.mu.Unlock() r.count("Inspect") + if err := r.requireRead("inspect"); err != nil { + return nil, err + } s, ok := r.records[pinned.RecordDigest] if !ok { return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) @@ -88,6 +94,9 @@ func (r *Repository) OpenDocument(_ context.Context, pinned artifact.PinnedRefer r.mu.Lock() defer r.mu.Unlock() r.count("OpenDocument") + if err := r.requireRead("open-document"); err != nil { + return nil, err + } s, ok := r.records[pinned.RecordDigest] if !ok { return nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, pinned.RecordDigest) @@ -116,6 +125,9 @@ func (r *Repository) Read(_ context.Context, pinned artifact.PinnedReference) (* r.mu.Lock() defer r.mu.Unlock() r.count("Read") + if err := r.requireRead("read"); err != nil { + return nil, err + } s, ok := r.records[pinned.RecordDigest] if !ok || !s.payloadAvailable { return nil, fmt.Errorf("%w: the payload of %s is no longer available", errUtils.ErrArtifactNotFound, pinned.RecordDigest) @@ -254,3 +266,12 @@ func (v *verifyingReader) Close() error { return nil } + +// requireRead rejects a read operation when the repository does not advertise the read capability, even +// if writes are enabled: a write-only repository must not serve reads. +func (r *Repository) requireRead(operation string) error { + if !r.caps.Read { + return artifact.NewUnsupportedCapability(r.opts.Alias, operation, "read", "the repository is not readable") + } + return nil +} diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go index 82e0a2f49bf..0d9803e0ce2 100644 --- a/pkg/artifact/artifacttest/repository_test.go +++ b/pkg/artifact/artifacttest/repository_test.go @@ -2,6 +2,7 @@ package artifacttest_test import ( "context" + "errors" "slices" "testing" "time" @@ -343,3 +344,50 @@ func TestRepository_SignDoesNotStoreEvidenceForARecordDeletedWhileSigning(t *tes assert.False(t, recordAlive(repo, pinned.RecordDigest)) assert.False(t, evidenceAlive(t, repo, &pinned), "no evidence may be left for a deleted record") } + +// A repository that does not advertise the read capability rejects every read entrypoint, even when +// writes are enabled. +func TestRepository_ReadsAreRejectedWithoutTheReadCapability(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + repo := artifacttest.New(artifacttest.Options{Capabilities: &caps}) + ctx := context.Background() + + checks := map[string]func() error{ + "Resolve": func() error { _, err := repo.Resolve(ctx, artifact.Ref{Name: testName, Version: "1.0.0"}); return err }, + "Inspect": func() error { _, err := repo.Inspect(ctx, artifact.PinnedReference{}); return err }, + "OpenDocument": func() error { + _, err := repo.OpenDocument(ctx, artifact.PinnedReference{}, artifact.DocumentRole("readme")) + return err + }, + "Read": func() error { _, err := repo.Read(ctx, artifact.PinnedReference{}); return err }, + } + for name, call := range checks { + t.Run(name, func(t *testing.T) { + var capability *artifact.CapabilityError + require.ErrorAs(t, call(), &capability) + assert.Equal(t, "read", capability.Capability) + }) + } +} + +// A failure from the signer carries the signing sentinel and keeps its cause. +func TestRepository_SignClassifiesSignerFailures(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, repo) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} + cause := errors.New("the signing service refused the request") + fake := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return(fake.Formats()).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), subject).Return(nil, cause) + + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + + require.ErrorIs(t, err, errUtils.ErrArtifactSigningFailed) + assert.ErrorIs(t, err, cause) +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index ad94d28e902..56efd1c513f 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -93,7 +93,7 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject } payload, err := signer.Sign(ctx, subject) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactSigningFailed, err) } if !r.caps.HasEvidenceFormat(payload.Format) { return nil, artifact.NewUnsupportedCapability(r.opts.Alias, "sign", "evidence/"+string(payload.Format), "the repository cannot store this evidence format") diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index 7a715adc054..e20aac6b8f3 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -27,8 +27,9 @@ const ( // path is created. Parent components are applied only after the symlinks before them are resolved, // so "link/../x" is not collapsed to a sibling of "link". // -// Resolve fails with [errUtils.ErrSymlinkLoop] when the links form a loop, and with another error -// when the path passes through a non-directory or cannot be inspected. Callers decide separately +// Resolve fails with [errUtils.ErrSymlinkLoop] when the links form a loop, with +// [errUtils.ErrPathThroughNonDirectory] when the path passes through a non-directory, and with another +// error when it cannot be inspected. Callers decide separately // what each failure means. func Resolve(target string) (string, error) { defer perf.Track(nil, "linkpath.Resolve")() @@ -139,7 +140,7 @@ func (w *walker) step() error { } if info.Mode()&fs.ModeSymlink == 0 { if !info.IsDir() && len(w.pending) > 0 { - return fmt.Errorf("%w: %q is not a directory", errUtils.ErrArtifactUnsafePath, next) + return fmt.Errorf("%w: %w: %q is not a directory", errUtils.ErrArtifactUnsafePath, errUtils.ErrPathThroughNonDirectory, next) } w.resolved = next return nil diff --git a/pkg/artifact/linkpath/linkpath_test.go b/pkg/artifact/linkpath/linkpath_test.go index 1756dbdf543..cc39e4ff87f 100644 --- a/pkg/artifact/linkpath/linkpath_test.go +++ b/pkg/artifact/linkpath/linkpath_test.go @@ -47,7 +47,7 @@ func TestResolve(t *testing.T) { {"missing path is lexical", filepath.Join(root, "tree", "absent", "..", "other"), filepath.Join(root, "tree", "other"), nil}, {"missing tail returns to physical resolution", filepath.Join(root, "tree", "missing-return"), filepath.Join(filepath.Dir(root), "outside"), nil}, {"loop", filepath.Join(root, "tree", "loop-a"), "", errUtils.ErrSymlinkLoop}, - {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", errUtils.ErrArtifactUnsafePath}, + {"through a regular file", filepath.Join(root, "tree", "real", "file.txt", "child"), "", errUtils.ErrPathThroughNonDirectory}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index bd661884d33..ba90473722e 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -85,8 +85,10 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check header, err := tr.Next() if errors.Is(err, io.EOF) { // Drain so the verifying payload reader checks its full digest at end of stream. - _, err = copyContext(ctx, io.Discard, reader) - return err + if _, err = copyContext(ctx, io.Discard, reader); err != nil { + return archiveReadError(ctx, "tar trailer", err) + } + return nil } if err != nil { return archiveReadError(ctx, "tar", err) diff --git a/pkg/artifact/materialize/archive_internal_test.go b/pkg/artifact/materialize/archive_internal_test.go index 2732fe1cd01..39d1bf84caa 100644 --- a/pkg/artifact/materialize/archive_internal_test.go +++ b/pkg/artifact/materialize/archive_internal_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "archive/zip" "bytes" + "compress/gzip" "context" "errors" "fmt" @@ -209,3 +210,25 @@ func TestFillTar_CancellationInterruptsMetadataReads(t *testing.T) { }) } } + +// A gzip trailer that fails its checksum is only noticed while the tail of the stream is drained after +// the last tar member; it must still be reported as an integrity failure, not as a bare gzip error. +func TestFillTar_ClassifiesACorruptGzipTrailer(t *testing.T) { + body := []byte("payload") + var archive bytes.Buffer + gz := gzip.NewWriter(&archive) + tw := tar.NewWriter(gz) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file.txt", Typeflag: tar.TypeReg, Size: int64(len(body)), Mode: 0o644})) + _, err := tw.Write(body) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + data := archive.Bytes() + data[len(data)-8] ^= 0xff // The CRC32 in the gzip trailer. + + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + err = fillTar(t.Context(), target, bytes.NewReader(data), true, checker) + + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go index 66386dd7a07..808ae4471fa 100644 --- a/pkg/artifact/materialize/links.go +++ b/pkg/artifact/materialize/links.go @@ -19,6 +19,12 @@ func checkLinkContainment(stage string, record *artifact.Record) error { if err != nil { return fmt.Errorf("%w: resolve staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) } + recorded := make(map[string]string) + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type == artifact.EntrySymlink { + recorded[entry.Path] = entry.Target + } + } for index := range record.Inventory { entry := &record.Inventory[index] if entry.Type != artifact.EntrySymlink { @@ -29,6 +35,14 @@ func checkLinkContainment(stage string, record *artifact.Record) error { case errors.Is(err, errUtils.ErrSymlinkLoop): // A loop never reaches a path outside the root, so it is not an escape. continue + case errors.Is(err, errUtils.ErrPathThroughNonDirectory): + // The target runs through a regular file, so it can never resolve to anything. Capture accepts + // such a link when its recorded path stays inside the tree, and so does materialization: the + // same lexical check, over the recorded links, decides. + if linkpath.Escapes(recorded, entry.Path) { + return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) + } + continue case err != nil: // Any other failure leaves the target unknown, and an unknown target cannot be shown to stay inside. return fmt.Errorf("%w: symlink %q cannot be resolved: %w", errUtils.ErrArtifactUnsafePath, entry.Path, err) diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index 3a31b0bde71..280cf8f6890 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -150,6 +150,30 @@ func TestMaterialize_ReplacesWithoutMergingAndStaysInsideDestination(t *testing. _ = linkOK } +// A destination of "." names the working directory, whose parent is not ".": the destination is made +// absolute before its parent is derived, so an empty working directory is replaced without force. +func TestMaterialize_ReplacesAnEmptyWorkingDirectoryNamedByDot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the working directory cannot be replaced while it is in use on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + work := filepath.Join(t.TempDir(), "work") + require.NoError(t, os.Mkdir(work, 0o755)) + t.Chdir(work) + + state, err := materialize.InspectDestination(".", &p.Record) + require.NoError(t, err) + require.Equal(t, materialize.DestinationEmpty, state) + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), ".")) + + data, err := os.ReadFile(filepath.Join(work, "new.txt")) + require.NoError(t, err) + assert.Equal(t, "new", string(data)) + assertNoScratch(t, work) +} + func TestMaterialize_ReplacesRestrictiveDestination(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("atomic directory replacement is unavailable on Windows") diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index 19e0ee85367..c28b6fe2314 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -165,8 +165,12 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op if dest == "" { return fmt.Errorf("%w: the destination is empty", errUtils.ErrArtifactConfigInvalid) } - // Normalize first so a trailing separator ("out/") cannot make the parent the destination itself. - dest = filepath.Clean(dest) + // Normalize first so a trailing separator ("out/") cannot make the parent the destination itself, + // and make the path absolute so a relative destination such as "." has a real parent directory. + dest, err := filepath.Abs(dest) + if err != nil { + return fmt.Errorf("%w: resolve destination: %w", errUtils.ErrArtifactPackagingInvalid, err) + } parent := filepath.Dir(dest) if err := os.MkdirAll(parent, parentDirMode); err != nil { return fmt.Errorf("%w: create %q: %w", errUtils.ErrArtifactPackagingInvalid, parent, err) diff --git a/pkg/artifact/materialize/tree_links_internal_test.go b/pkg/artifact/materialize/tree_links_internal_test.go index 0d64aa87238..00e27153c97 100644 --- a/pkg/artifact/materialize/tree_links_internal_test.go +++ b/pkg/artifact/materialize/tree_links_internal_test.go @@ -162,7 +162,10 @@ func TestCheckLinkContainment_ResolutionFailures(t *testing.T) { links map[string]string wantErr bool }{ - {name: "through a regular file", links: map[string]string{"bad": "file.txt/child"}, wantErr: true}, + // Capture accepts a link whose target runs through a regular file while its recorded path stays inside + // the tree, so materialization accepts it too; one that leaves the tree is rejected. + {name: "through a regular file, inside the tree", links: map[string]string{"dangling": "file.txt/child"}}, + {name: "through a regular file, leaving the tree", links: map[string]string{"bad": "file.txt/../../outside"}, wantErr: true}, {name: "loop", links: map[string]string{"a": "b", "b": "a"}}, {name: "inside the root", links: map[string]string{"ok": "file.txt"}}, } @@ -173,7 +176,7 @@ func TestCheckLinkContainment_ResolutionFailures(t *testing.T) { record := &artifact.Record{} for name, target := range tt.links { require.NoError(t, os.Symlink(target, filepath.Join(stage, name))) - record.Inventory = append(record.Inventory, artifact.Entry{Path: name, Type: artifact.EntrySymlink}) + record.Inventory = append(record.Inventory, artifact.Entry{Path: name, Type: artifact.EntrySymlink, Target: target}) } err := checkLinkContainment(stage, record) if tt.wantErr { diff --git a/pkg/oci/client.go b/pkg/oci/client.go index 063fe854882..019d597a684 100644 --- a/pkg/oci/client.go +++ b/pkg/oci/client.go @@ -136,7 +136,8 @@ func (c *Client) Tags(ctx context.Context, repo name.Repository) ([]string, erro // WriteImage uploads an image and its blobs, preserving the image's existing raw bytes. // Rejected writes never retry anonymously, and an unknown outcome is reconciled by -// resolving the destination rather than writing again. +// resolving the destination rather than writing again. Only OCI and Docker schema 2 images +// are supported: a schema 1 image is refused with an integrity error before any layer is read. func (c *Client) WriteImage(ctx context.Context, ref name.Reference, img v1.Image) error { defer perf.Track(c.config, "oci.Client.WriteImage")() diff --git a/pkg/oci/ocitest/registry.go b/pkg/oci/ocitest/registry.go index ef882d5d521..48854f61491 100644 --- a/pkg/oci/ocitest/registry.go +++ b/pkg/oci/ocitest/registry.go @@ -164,8 +164,8 @@ type Server struct { server *httptest.Server } -// NewServer starts an empty in-process OCI registry (OCI 1.1 referrers included). When -// credential is non-nil, every request must carry exactly that Basic identity; any other +// NewServer starts an empty in-process OCI registry with the native OCI 1.1 referrers API enabled. +// When credential is non-nil, every request must carry exactly that Basic identity; any other // request receives a 401 challenge, so a client that downgraded to anonymous or reused // another registry's credentials fails visibly. func NewServer(t *testing.T, credential *Credential) *Server { @@ -173,7 +173,23 @@ func NewServer(t *testing.T, credential *Credential) *Server { t.Helper() - handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + return newServer(t, credential, true) +} + +// NewServerWithoutReferrers is NewServer for a registry without the OCI 1.1 referrers API, so attachment +// discovery has to use the referrers tag-schema fallback. Use it for fallback-only tests. +func NewServerWithoutReferrers(t *testing.T, credential *Credential) *Server { + defer perf.Track(nil, "ocitest.NewServerWithoutReferrers")() + + t.Helper() + + return newServer(t, credential, false) +} + +func newServer(t *testing.T, credential *Credential, referrers bool) *Server { + t.Helper() + + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0)), registry.WithReferrersSupport(referrers)) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if credential != nil { user, password, ok := r.BasicAuth() diff --git a/pkg/oci/referrers_fallback_test.go b/pkg/oci/referrers_fallback_test.go new file mode 100644 index 00000000000..f0bfb2129f2 --- /dev/null +++ b/pkg/oci/referrers_fallback_test.go @@ -0,0 +1,139 @@ +package oci_test + +import ( + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/registry" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/keyring" + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" +) + +// registryClient returns a client that can present the identity for host. +func registryClient(t *testing.T, host string, credential *ocitest.Credential) *oci.Client { + t.Helper() + store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) + require.NoError(t, err) + chain := oci.CredentialKeychain{Store: store} + if credential != nil { + require.NoError(t, chain.Login(host, &authn.AuthConfig{Username: credential.Username, Password: credential.Password})) + } + return oci.NewClient(nil, oci.WithKeychain(chain)) +} + +// pushAttachment pushes a subject manifest and one attachment that names it, and returns both descriptors. +func pushAttachment(t *testing.T, store oci.RepositoryStore) (subject, attachment ocispec.Descriptor) { + t.Helper() + ctx := t.Context() + config := []byte("{}") + configDesc := oci.Descriptor("application/vnd.oci.empty.v1+json", config) + require.NoError(t, store.Push(ctx, configDesc, strings.NewReader(string(config)))) + push := func(subject *ocispec.Descriptor, note string) ocispec.Descriptor { + manifest := ocispec.Manifest{Config: configDesc, Layers: []ocispec.Descriptor{}, Subject: subject, Annotations: map[string]string{"note": note}} + manifest.SchemaVersion = 2 + manifest.MediaType = ocispec.MediaTypeImageManifest + raw, err := json.Marshal(manifest) + require.NoError(t, err) + desc := oci.Descriptor(ocispec.MediaTypeImageManifest, raw) + require.NoError(t, store.Push(ctx, desc, strings.NewReader(string(raw)))) + return desc + } + subject = push(nil, "subject") + attachment = push(&subject, "attachment") + return subject, attachment +} + +func referrerDigests(referrers []ocispec.Descriptor) []string { + digests := make([]string, 0, len(referrers)) + for _, referrer := range referrers { + digests = append(digests, referrer.Digest.String()) + } + return digests +} + +// The fixture serves the native OCI 1.1 referrers API by default and the tag-schema fallback on request; +// attachment discovery works against both. +func TestReferrersDiscoverAttachmentsOnNativeAndFallbackRegistries(t *testing.T) { + credential := &ocitest.Credential{Username: "publisher", Password: "s3cret"} + for name, start := range map[string]func(*testing.T, *ocitest.Credential) *ocitest.Server{ + "native": ocitest.NewServer, + "fallback": ocitest.NewServerWithoutReferrers, + } { + t.Run(name, func(t *testing.T) { + server := start(t, credential) + client := registryClient(t, server.Host, credential) + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + + referrers, err := store.Referrers(t.Context(), subject, "") + + require.NoError(t, err) + assert.Equal(t, []string{attachment.Digest.String()}, referrerDigests(referrers)) + }) + } +} + +// Some registries answer the referrers API with 400 or 406 instead of 404. That means the API is not +// available, so discovery falls back to the tag schema; other statuses stay errors. +func TestReferrersFallBackToTheTagSchemaOnBadRequestAndNotAcceptable(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusNotAcceptable} { + t.Run(http.StatusText(status), func(t *testing.T) { + var refuse atomic.Bool + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if refuse.Load() && strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(status) + return + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + refuse.Store(true) + // A new store has not learned whether the registry supports the referrers API, as in a later process. + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + + referrers, err := fresh.Referrers(t.Context(), subject, "") + + require.NoError(t, err) + assert.Equal(t, []string{attachment.Digest.String()}, referrerDigests(referrers)) + }) + } +} + +func TestReferrersKeepOtherStatusesAsErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + + _, err = store.Referrers(t.Context(), oci.Descriptor(ocispec.MediaTypeImageManifest, []byte("x")), "") + + require.Error(t, err) +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index 26cd3abe779..0fd3c1f9cbe 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -133,10 +133,11 @@ func TestRepositoryFailsClosedBeforeAnyRequest(t *testing.T) { // password. These shapes are accepted. func TestNonEmptyCredentialAcceptsSendableCredentials(t *testing.T) { for name, config := range map[string]authn.AuthConfig{ - "username and password": {Username: "identity", Password: "bound"}, - "access token": {RegistryToken: "token"}, - "refresh token": {IdentityToken: "refresh"}, - "encoded basic": {Auth: base64.StdEncoding.EncodeToString([]byte("identity:bound"))}, + "username and password": {Username: "identity", Password: "bound"}, + "access token": {RegistryToken: "token"}, + "refresh token": {IdentityToken: "refresh"}, + "encoded basic": {Auth: base64.StdEncoding.EncodeToString([]byte("identity:bound"))}, + "username with encoded auth": {Username: "publisher", Auth: base64.StdEncoding.EncodeToString([]byte("publisher:secret"))}, } { t.Run(name, func(t *testing.T) { got, err := nonEmptyCredential{Authenticator: authn.FromConfig(config)}.Authorization() diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 20424707a66..91515e1104f 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -51,14 +51,21 @@ var ( func OpenLayout(ctx context.Context, directory string) (*Layout, error) { defer perf.Track(nil, "oci.OpenLayout")() - store, err := oci.NewWithContext(ctx, directory) + // Resolve the root once and give the same absolute path to the store and the layout: the store + // anchors itself, so a relative root here would let a later working-directory change send index + // commits to a different directory than the blobs. + root, err := filepath.Abs(directory) + if err != nil { + return nil, errors.Join(errUtils.ErrOCILayout, err) + } + store, err := oci.NewWithContext(ctx, root) if err != nil { return nil, errors.Join(errUtils.ErrOCILayout, err) } // The ORAS store would otherwise list every pushed manifest in index.json. store.AutoSaveIndex = false store.AutoGC = false - layout := &Layout{root: filepath.Clean(directory), store: store} + layout := &Layout{root: root, store: store} if err := layout.loadIndex(); err != nil { return nil, err } @@ -189,11 +196,15 @@ func (l *Layout) recordRootLocked(desc ocispec.Descriptor) error { func (l *Layout) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { defer perf.Track(nil, "oci.Layout.Resolve")() + // The resolver and the committed index are observed under one lock, so a concurrent Tag cannot move the + // reference between the two reads and make an existing tag look unpersisted. + l.mu.Lock() + defer l.mu.Unlock() resolved, err := l.store.Resolve(ctx, reference) if err != nil { return resolved, layoutError(err) } - if !l.persisted(reference, resolved.Digest) { + if !l.persistedLocked(reference, resolved.Digest) { // The resolver is updated before index.json is written, so a failed write leaves a tag that // resolves now and disappears when the layout is reopened. Never report such a tag as published. return ocispec.Descriptor{}, layoutError(errdef.ErrNotFound) @@ -201,15 +212,13 @@ func (l *Layout) Resolve(ctx context.Context, reference string) (ocispec.Descrip return resolved, nil } -// persisted reports whether a reference is backed by index.json. A digest reference always is, because -// it names content, not a tag; a named reference is only when the persisted index lists it for the -// digest it resolved to. -func (l *Layout) persisted(reference string, resolved digest.Digest) bool { +// persistedLocked reports whether a reference is backed by index.json. A digest reference always is, +// because it names content, not a tag; a named reference is only when the persisted index lists it for +// the digest it resolved to. The caller holds l.mu. +func (l *Layout) persistedLocked(reference string, resolved digest.Digest) bool { if _, err := digest.Parse(reference); err == nil { return true } - l.mu.Lock() - defer l.mu.Unlock() for i := range l.index { if l.index[i].Annotations[ocispec.AnnotationRefName] == reference && l.index[i].Digest == resolved { return true diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go index a92b009d7dd..6e4bd07535f 100644 --- a/pkg/oci/transport_legacy.go +++ b/pkg/oci/transport_legacy.go @@ -13,6 +13,7 @@ import ( "slices" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/opencontainers/go-digest" @@ -148,6 +149,15 @@ func (g *legacyGraph) addBytes(mediaType string, raw []byte) ocispec.Descriptor } func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Descriptor, error) { + mediaType, err := image.MediaType() + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + // A schema 1 manifest has no config or layer descriptors in the shape this graph serves, so it is + // refused up front instead of failing later in Manifest() or after its layers were already read. + if mediaType == types.DockerManifestSchema1 || mediaType == types.DockerManifestSchema1Signed { + return ocispec.Descriptor{}, fmt.Errorf("%w: schema 1 images are not supported; convert the image to OCI or Docker schema 2", errUtils.ErrOCIIntegrity) + } if err := g.spoolStreamingLayers(ctx, image); err != nil { return ocispec.Descriptor{}, err } @@ -155,10 +165,6 @@ func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Des if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) } - mediaType, err := image.MediaType() - if err != nil { - return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) - } manifest, err := image.Manifest() if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) @@ -189,10 +195,32 @@ func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Des // addIndex records an index and, once per distinct index digest, every child below it. Cancellation // is checked before each accessor call and child, so a canceled write stops preparing the graph. +// +// A child can be an image with a streaming layer, whose digest, and so the index manifest that lists +// it, exists only after the stream was read once. The children are therefore prepared first, through +// [partial.Manifests], which evaluates them lazily, and the index manifest is read afterwards. An index +// whose manifest is already computable is recognized beforehand, so a shared subgraph is traversed once. func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispec.Descriptor, error) { if err := ctx.Err(); err != nil { return ocispec.Descriptor{}, err } + if raw, err := index.RawManifest(); err == nil { + if done, found := g.indexes[digest.FromBytes(raw)]; found { + return done, nil + } + } + children, err := partial.Manifests(index) + if err != nil { + return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + } + for _, child := range children { + if err := ctx.Err(); err != nil { + return ocispec.Descriptor{}, err + } + if err := g.addChild(ctx, child); err != nil { + return ocispec.Descriptor{}, err + } + } raw, err := index.RawManifest() if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) @@ -204,49 +232,52 @@ func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispe if done, found := g.indexes[digest.FromBytes(raw)]; found { return done, nil } - if err := ctx.Err(); err != nil { - return ocispec.Descriptor{}, err - } - manifest, err := index.IndexManifest() - if err != nil { - return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) - } - for i := range manifest.Manifests { - if err := ctx.Err(); err != nil { - return ocispec.Descriptor{}, err - } - if err := g.addChild(ctx, index, &manifest.Manifests[i]); err != nil { - return ocispec.Descriptor{}, err - } - } // The index is marked complete only after every child was added successfully. desc := g.addBytes(string(mediaType), raw) g.indexes[desc.Digest] = desc return desc, nil } -// addChild records one child manifest or nested index of a legacy index. -func (g *legacyGraph) addChild(ctx context.Context, index v1.ImageIndex, child *v1.Descriptor) error { - switch { - case child.MediaType.IsIndex(): - nested, err := index.ImageIndex(child.Digest) - if err != nil { - return errors.Join(errUtils.ErrOCIIntegrity, err) - } - _, err = g.addIndex(ctx, nested) +// addChild records one child of a legacy index: an image, a nested index, or a layer that is a blob in +// its own right. +func (g *legacyGraph) addChild(ctx context.Context, child partial.Describable) error { + switch typed := child.(type) { + case v1.ImageIndex: + _, err := g.addIndex(ctx, typed) return err - case child.MediaType.IsImage(): - nested, err := index.Image(child.Digest) - if err != nil { - return errors.Join(errUtils.ErrOCIIntegrity, err) - } - _, err = g.addImage(ctx, nested) + case v1.Image: + _, err := g.addImage(ctx, typed) return err + case v1.Layer: + return g.addLayer(ctx, typed) default: - return fmt.Errorf("%w: unsupported child media type %q", errUtils.ErrOCIIntegrity, child.MediaType) + mediaType, _ := child.MediaType() + return fmt.Errorf("%w: unsupported child media type %q", errUtils.ErrOCIIntegrity, mediaType) } } +// addLayer serves a layer that an index lists as a child. A streaming layer is spooled first, because +// its digest exists only after it was read. +func (g *legacyGraph) addLayer(ctx context.Context, layer v1.Layer) error { + hash, err := layer.Digest() + if errors.Is(err, stream.ErrNotComputed) { + if err := g.spoolLayer(ctx, layer); err != nil { + return err + } + hash, err = layer.Digest() + } + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + key := digest.Digest(hash.String()) + if path, found := g.spooled[key]; found { + g.blobs[key] = func() (io.ReadCloser, error) { return os.Open(path) } + return nil + } + g.blobs[key] = layer.Compressed + return nil +} + // Fetch streams a known blob. func (g *legacyGraph) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { if err := ctx.Err(); err != nil { diff --git a/pkg/oci/transport_legacy_children_test.go b/pkg/oci/transport_legacy_children_test.go new file mode 100644 index 00000000000..453653918e7 --- /dev/null +++ b/pkg/oci/transport_legacy_children_test.go @@ -0,0 +1,80 @@ +package oci + +import ( + "bytes" + "io" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// An index child that is an image with a streaming layer has no digest until the stream is read once, so +// every child is prepared before the index manifest, whose descriptors depend on it, is read. +func TestLegacyIndexGraphPreparesStreamingChildrenBeforeTheIndexManifest(t *testing.T) { + payload := bytes.Repeat([]byte("streamed layer bytes "), 64) + image, err := mutate.AppendLayers(empty.Image, stream.NewLayer(io.NopCloser(bytes.NewReader(payload)))) + require.NoError(t, err) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: image}) + + graph, err := legacyIndexGraph(t.Context(), index) + require.NoError(t, err) + t.Cleanup(graph.close) + + raw, err := index.RawManifest() + require.NoError(t, err, "the index manifest exists once its children were prepared") + assert.Equal(t, digest.FromBytes(raw), graph.root.Digest, "the root describes the manifest the index now serves") + layers, err := image.Layers() + require.NoError(t, err) + hash, err := layers[0].Digest() + require.NoError(t, err) + exists, err := graph.Exists(t.Context(), ocispec.Descriptor{Digest: digest.Digest(hash.String())}) + require.NoError(t, err) + assert.True(t, exists, "the streamed layer is served from its spool") +} + +// A child that is a layer rather than an image or an index is a blob the registry must receive too. +func TestLegacyIndexGraphServesLayerChildren(t *testing.T) { + blob := []byte("an artifact blob that is neither an image nor an index") + layer := static.NewLayer(blob, types.MediaType("application/vnd.example.blob.v1")) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: layer}) + + graph, err := legacyIndexGraph(t.Context(), index) + require.NoError(t, err) + t.Cleanup(graph.close) + + hash, err := layer.Digest() + require.NoError(t, err) + reader, err := graph.Fetch(t.Context(), ocispec.Descriptor{Digest: digest.Digest(hash.String())}) + require.NoError(t, err) + defer reader.Close() + got, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, blob, got) +} + +// Schema 1 images have no manifest the legacy graph can carry as OCI content, so they are refused with an +// explicit error instead of failing later in Manifest(). +func TestLegacyImageGraphRefusesSchemaOneImages(t *testing.T) { + image, err := random.Image(64, 1) + require.NoError(t, err) + for _, mediaType := range []types.MediaType{types.DockerManifestSchema1, types.DockerManifestSchema1Signed} { + t.Run(string(mediaType), func(t *testing.T) { + graph, err := legacyImageGraph(t.Context(), mutate.MediaType(image, mediaType)) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Contains(t, err.Error(), "schema 1") + assert.Nil(t, graph) + }) + } +} diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go index 6e6771a9192..4c6e0f2816d 100644 --- a/pkg/oci/transport_registry.go +++ b/pkg/oci/transport_registry.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/base64" + "errors" "io" "net/http" "strings" @@ -15,6 +16,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/errcode" "oras.land/oras-go/v2/registry/remote/retry" errUtils "github.com/cloudposse/atmos/errors" @@ -123,20 +125,36 @@ func credentialFrom(authenticator authn.Authenticator) (auth.Credential, error) RefreshToken: config.IdentityToken, AccessToken: config.RegistryToken, } - if config.Auth != "" && credential.Username == "" && credential.Password == "" { - decoded, err := base64.StdEncoding.DecodeString(config.Auth) - if err != nil { - return auth.EmptyCredential, wrapRegistry(err) + // An encoded Auth fills whichever of the username and password is missing, the way the registry + // transport does; a username or password given explicitly keeps its value. + if config.Auth != "" && (credential.Username == "" || credential.Password == "") { + if err := fillFromEncodedAuth(&credential, config.Auth); err != nil { + return auth.EmptyCredential, err } - user, password, found := strings.Cut(string(decoded), ":") - if !found { - return auth.EmptyCredential, errUtils.ErrOCIRegistry - } - credential.Username, credential.Password = user, password } return credential, nil } +// fillFromEncodedAuth decodes a base64 "user:password" value and fills the username and password the +// credential does not already carry. +func fillFromEncodedAuth(credential *auth.Credential, encoded string) error { + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return wrapRegistry(err) + } + user, password, found := strings.Cut(string(decoded), ":") + if !found { + return errUtils.ErrOCIRegistry + } + if credential.Username == "" { + credential.Username = user + } + if credential.Password == "" { + credential.Password = password + } + return nil +} + // registryTransport retries only idempotent reads. Mutations are sent exactly once; // an unknown outcome is reconciled by resolving the destination instead. type registryTransport struct { @@ -234,15 +252,38 @@ func (s *registryStore) Delete(ctx context.Context, target ocispec.Descriptor) e } func (s *registryStore) Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { + found, err := s.listReferrers(ctx, subject, artifactType) + if err != nil && referrersAPIUnavailable(err) { + // The registry answered the referrers API with 400 or 406 instead of 404: the API is not available + // there. Mark it unsupported, which switches this repository to the tag schema, and list again. If the + // capability was already settled the first answer stands. + if capabilityErr := s.repo.SetReferrersCapability(false); capabilityErr == nil { + found, err = s.listReferrers(ctx, subject, artifactType) + } + } + if err != nil { + return nil, wrapRegistry(err) + } + return found, nil +} + +func (s *registryStore) listReferrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { var found []ocispec.Descriptor err := s.repo.Referrers(ctx, subject, artifactType, func(page []ocispec.Descriptor) error { found = append(found, page...) return nil }) - if err != nil { - return nil, wrapRegistry(err) + return found, err +} + +// referrersAPIUnavailable reports whether err is a registry response that says the referrers API is not +// available: 400 Bad Request or 406 Not Acceptable. Every other error, including other statuses, is a failure. +func referrersAPIUnavailable(err error) bool { + var response *errcode.ErrorResponse + if !errors.As(err, &response) { + return false } - return found, nil + return response.StatusCode == http.StatusBadRequest || response.StatusCode == http.StatusNotAcceptable } var ( diff --git a/pkg/oci/transport_roundtrip_test.go b/pkg/oci/transport_roundtrip_test.go index f1e32120543..ce182409ce9 100644 --- a/pkg/oci/transport_roundtrip_test.go +++ b/pkg/oci/transport_roundtrip_test.go @@ -41,7 +41,7 @@ func pluginFixtureDirectory() string { // that can only present that identity. func authenticatedRegistry(t *testing.T, password string) (*ocitest.Server, *oci.Client) { t.Helper() - server := ocitest.NewServer(t, &ocitest.Credential{Username: "publisher", Password: password}) + server := ocitest.NewServerWithoutReferrers(t, &ocitest.Credential{Username: "publisher", Password: password}) store, err := keyring.New(keyring.Config{Type: keyring.TypeMemory}) require.NoError(t, err) chain := oci.CredentialKeychain{Store: store} diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index f0d99c88a89..9f5becb534e 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -152,6 +153,69 @@ func TestResolveDoesNotTrustATagThatWasNotPersisted(t *testing.T) { assert.Equal(t, root.Digest, resolved.Digest) } +// A relative layout root is anchored when the layout is opened: changing the working directory afterwards +// must not send index commits to a different directory than the one holding the blobs. +func TestOpenLayoutAnchorsARelativeRoot(t *testing.T) { + ctx := t.Context() + base := t.TempDir() + other := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(other, "layout"), 0o755)) + t.Chdir(base) + layout, err := oci.OpenLayout(ctx, "layout") + require.NoError(t, err) + root := pushSubjectManifest(t, layout, nil, "root") + + t.Chdir(other) + require.NoError(t, layout.Tag(ctx, root, "1.0.0")) + + assert.Contains(t, layoutRoots(t, filepath.Join(base, "layout")), root.Digest.String(), "the tag landed in the layout that was opened") + _, err = os.Stat(filepath.Join(other, "layout", "index.json")) + assert.ErrorIs(t, err, os.ErrNotExist, "an unrelated layout in the new working directory is untouched") +} + +// A reference that exists throughout a concurrent Tag keeps resolving: the resolver and the committed index +// are read under one lock, so the tag moving between two digests never looks unpersisted. +func TestResolveIsAtomicWithConcurrentTags(t *testing.T) { + ctx := t.Context() + layout, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "layout")) + require.NoError(t, err) + first := pushSubjectManifest(t, layout, nil, "first") + second := pushSubjectManifest(t, layout, nil, "second") + require.NoError(t, layout.Tag(ctx, first, "moving")) + + const rounds = 300 + var wg sync.WaitGroup + failures := make(chan error, rounds) + wg.Add(2) + go func() { + defer wg.Done() + for i := range rounds { + target := first + if i%2 == 0 { + target = second + } + if err := layout.Tag(ctx, target, "moving"); err != nil { + failures <- err + return + } + } + }() + go func() { + defer wg.Done() + for range rounds { + if _, err := layout.Resolve(ctx, "moving"); err != nil { + failures <- err + return + } + } + }() + wg.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } +} + func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers(t *testing.T) { ctx := t.Context() source, err := oci.OpenLayout(ctx, filepath.Join(t.TempDir(), "source")) From cc7a396c27615030ac09a4d604d59d9140edd854 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 22:25:48 -0600 Subject: [PATCH 45/52] refactor(artifact): keep link containment and legacy graph building under the complexity limit Extract single-purpose helpers so checkLinkContainment, legacyGraph.addImage and legacyGraph.addIndex stay at or below the revive cyclomatic limit. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/materialize/links.go | 58 +++++++++++++++---------- pkg/oci/transport_legacy.go | 71 ++++++++++++++++++++++--------- 2 files changed, 87 insertions(+), 42 deletions(-) diff --git a/pkg/artifact/materialize/links.go b/pkg/artifact/materialize/links.go index 808ae4471fa..57357e7b2bb 100644 --- a/pkg/artifact/materialize/links.go +++ b/pkg/artifact/materialize/links.go @@ -19,36 +19,50 @@ func checkLinkContainment(stage string, record *artifact.Record) error { if err != nil { return fmt.Errorf("%w: resolve staging root: %w", errUtils.ErrArtifactPackagingInvalid, err) } + recorded := recordedLinks(record) + for index := range record.Inventory { + entry := &record.Inventory[index] + if entry.Type != artifact.EntrySymlink { + continue + } + if err := checkLinkEntry(stage, realRoot, recorded, entry); err != nil { + return err + } + } + return nil +} + +// recordedLinks maps each recorded symlink path to its recorded target. +func recordedLinks(record *artifact.Record) map[string]string { recorded := make(map[string]string) for index := range record.Inventory { if entry := &record.Inventory[index]; entry.Type == artifact.EntrySymlink { recorded[entry.Path] = entry.Target } } - for index := range record.Inventory { - entry := &record.Inventory[index] - if entry.Type != artifact.EntrySymlink { - continue - } - resolved, err := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) - switch { - case errors.Is(err, errUtils.ErrSymlinkLoop): - // A loop never reaches a path outside the root, so it is not an escape. - continue - case errors.Is(err, errUtils.ErrPathThroughNonDirectory): - // The target runs through a regular file, so it can never resolve to anything. Capture accepts - // such a link when its recorded path stays inside the tree, and so does materialization: the - // same lexical check, over the recorded links, decides. - if linkpath.Escapes(recorded, entry.Path) { - return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) - } - continue - case err != nil: - // Any other failure leaves the target unknown, and an unknown target cannot be shown to stay inside. - return fmt.Errorf("%w: symlink %q cannot be resolved: %w", errUtils.ErrArtifactUnsafePath, entry.Path, err) - case !linkpath.Within(realRoot, resolved): + return recorded +} + +// checkLinkEntry decides whether one staged symlink stays inside the package root. +func checkLinkEntry(stage, realRoot string, recorded map[string]string, entry *artifact.Entry) error { + resolved, err := linkpath.Resolve(filepath.Join(stage, filepath.FromSlash(entry.Path))) + switch { + case errors.Is(err, errUtils.ErrSymlinkLoop): + // A loop never reaches a path outside the root, so it is not an escape. + return nil + case errors.Is(err, errUtils.ErrPathThroughNonDirectory): + // The target runs through a regular file, so it can never resolve to anything. Capture accepts + // such a link when its recorded path stays inside the tree, and so does materialization: the + // same lexical check, over the recorded links, decides. + if linkpath.Escapes(recorded, entry.Path) { return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) } + return nil + case err != nil: + // Any other failure leaves the target unknown, and an unknown target cannot be shown to stay inside. + return fmt.Errorf("%w: symlink %q cannot be resolved: %w", errUtils.ErrArtifactUnsafePath, entry.Path, err) + case !linkpath.Within(realRoot, resolved): + return fmt.Errorf("%w: symlink %q resolves outside the package", errUtils.ErrArtifactUnsafePath, entry.Path) } return nil } diff --git a/pkg/oci/transport_legacy.go b/pkg/oci/transport_legacy.go index 6e4bd07535f..bc3b3267129 100644 --- a/pkg/oci/transport_legacy.go +++ b/pkg/oci/transport_legacy.go @@ -153,10 +153,8 @@ func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Des if err != nil { return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) } - // A schema 1 manifest has no config or layer descriptors in the shape this graph serves, so it is - // refused up front instead of failing later in Manifest() or after its layers were already read. - if mediaType == types.DockerManifestSchema1 || mediaType == types.DockerManifestSchema1Signed { - return ocispec.Descriptor{}, fmt.Errorf("%w: schema 1 images are not supported; convert the image to OCI or Docker schema 2", errUtils.ErrOCIIntegrity) + if err := refuseSchema1(mediaType); err != nil { + return ocispec.Descriptor{}, err } if err := g.spoolStreamingLayers(ctx, image); err != nil { return ocispec.Descriptor{}, err @@ -176,6 +174,23 @@ func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Des g.blobs[digest.Digest(manifest.Config.Digest.String())] = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(rawConfig)), nil } + g.registerLayers(image, manifest) + return g.addBytes(string(mediaType), raw), nil +} + +// refuseSchema1 rejects a schema 1 manifest. It has no config or layer descriptors in the shape this +// graph serves, so it is refused up front instead of failing later in Manifest() or after its layers +// were already read. +func refuseSchema1(mediaType types.MediaType) error { + if mediaType == types.DockerManifestSchema1 || mediaType == types.DockerManifestSchema1Signed { + return fmt.Errorf("%w: schema 1 images are not supported; convert the image to OCI or Docker schema 2", errUtils.ErrOCIIntegrity) + } + return nil +} + +// registerLayers serves each layer of an image, from its spool file when it was spooled and from the +// image otherwise. +func (g *legacyGraph) registerLayers(image v1.Image, manifest *v1.Manifest) { for i := range manifest.Layers { hash := manifest.Layers[i].Digest if path, found := g.spooled[digest.Digest(hash.String())]; found { @@ -190,7 +205,6 @@ func (g *legacyGraph) addImage(ctx context.Context, image v1.Image) (ocispec.Des return layer.Compressed() } } - return g.addBytes(string(mediaType), raw), nil } // addIndex records an index and, once per distinct index digest, every child below it. Cancellation @@ -204,22 +218,11 @@ func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispe if err := ctx.Err(); err != nil { return ocispec.Descriptor{}, err } - if raw, err := index.RawManifest(); err == nil { - if done, found := g.indexes[digest.FromBytes(raw)]; found { - return done, nil - } - } - children, err := partial.Manifests(index) - if err != nil { - return ocispec.Descriptor{}, errors.Join(errUtils.ErrOCIIntegrity, err) + if done, found := g.completedIndex(index); found { + return done, nil } - for _, child := range children { - if err := ctx.Err(); err != nil { - return ocispec.Descriptor{}, err - } - if err := g.addChild(ctx, child); err != nil { - return ocispec.Descriptor{}, err - } + if err := g.addChildren(ctx, index); err != nil { + return ocispec.Descriptor{}, err } raw, err := index.RawManifest() if err != nil { @@ -238,6 +241,34 @@ func (g *legacyGraph) addIndex(ctx context.Context, index v1.ImageIndex) (ocispe return desc, nil } +// completedIndex returns the descriptor of an index that was already added, when its manifest is +// computable beforehand. +func (g *legacyGraph) completedIndex(index v1.ImageIndex) (ocispec.Descriptor, bool) { + raw, err := index.RawManifest() + if err != nil { + return ocispec.Descriptor{}, false + } + done, found := g.indexes[digest.FromBytes(raw)] + return done, found +} + +// addChildren prepares every child of an index, checking cancellation before each one. +func (g *legacyGraph) addChildren(ctx context.Context, index v1.ImageIndex) error { + children, err := partial.Manifests(index) + if err != nil { + return errors.Join(errUtils.ErrOCIIntegrity, err) + } + for _, child := range children { + if err := ctx.Err(); err != nil { + return err + } + if err := g.addChild(ctx, child); err != nil { + return err + } + } + return nil +} + // addChild records one child of a legacy index: an image, a nested index, or a layer that is a blob in // its own right. func (g *legacyGraph) addChild(ctx context.Context, child partial.Describable) error { From 2831f0a1859dab232123baef644f67c9f836fb17 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 06:46:29 -0600 Subject: [PATCH 46/52] fix(artifact): address the fourth round of CodeRabbit findings - Keep the source Close error in WriteDeterministic and join it with any copy error. - Bound the second lock acquisition in the leaked-ownership conformance check. - Bound cumulative PAX header bytes beneath the tar reader, so successive headers cannot grow without limit before Next returns a visible member. - Report an empty destination that matches an empty inventory as a match, and allow an unrecorded directory that is an ancestor of an inventory entry. - Leave a single entry when a tag moves onto an index that already has an unnamed root. - Discover predecessors through the referrers adapter so its tag-schema fallback applies, and settle the referrers capability before deleting an attachment, without retrying the DELETE. - Verify the whole fetched body against its descriptor size and digest in readSubject, even when the subject is found early. Each change has a regression test that failed before the fix. Co-Authored-By: Claude Sonnet 5.5 --- pkg/archive/deterministic.go | 9 +- pkg/archive/deterministic_test.go | 23 +++++ pkg/artifact/artifacttest/conformance.go | 7 +- pkg/artifact/artifacttest/conformance_test.go | 35 ++++++++ pkg/artifact/materialize/archive.go | 45 +++++++++- .../materialize/archive_internal_test.go | 30 +++++++ pkg/artifact/materialize/materialize_test.go | 45 ++++++++++ pkg/artifact/materialize/replace.go | 29 ++++++- pkg/oci/referrers_fallback_test.go | 86 +++++++++++++++++++ pkg/oci/transport_internal_test.go | 28 ++++++ pkg/oci/transport_layout.go | 5 +- pkg/oci/transport_registry.go | 26 +++++- pkg/oci/transport_roundtrip_test.go | 30 +++++++ pkg/oci/transport_subject.go | 68 +++++++++++++-- 14 files changed, 450 insertions(+), 16 deletions(-) diff --git a/pkg/archive/deterministic.go b/pkg/archive/deterministic.go index 0029b00caf2..a869fc58f10 100644 --- a/pkg/archive/deterministic.go +++ b/pkg/archive/deterministic.go @@ -215,7 +215,7 @@ func writeZipMember(zw *zip.Writer, member *Member) error { // copyMemberContent copies exactly member.Size bytes and fails if the source is shorter or longer, // so an archive never describes bytes other than the ones it carries. -func copyMemberContent(dst io.Writer, member *Member) error { +func copyMemberContent(dst io.Writer, member *Member) (err error) { if member.Open == nil { return memberError(member.Path, errUtils.ErrArchiveWriteFailed) } @@ -223,7 +223,12 @@ func copyMemberContent(dst io.Writer, member *Member) error { if err != nil { return writeFailedError(member.Path, err) } - defer src.Close() + // A failed close can mean the source did not verify, so it is kept alongside any copy error. + defer func() { + if closeErr := src.Close(); closeErr != nil { + err = errors.Join(err, writeFailedError(member.Path, closeErr)) + } + }() written, err := io.Copy(dst, io.LimitReader(src, member.Size)) if err != nil { diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go index 8dce801194d..618517f99a6 100644 --- a/pkg/archive/deterministic_test.go +++ b/pkg/archive/deterministic_test.go @@ -204,3 +204,26 @@ func TestWriteDeterministic_PropagatesWriterFailures(t *testing.T) { }) } } + +var errTestSourceClose = errors.New("source close failed") + +type closeFailReader struct { + io.Reader +} + +func (closeFailReader) Close() error { return errTestSourceClose } + +func TestWriteDeterministic_SurfacesSourceCloseFailure(t *testing.T) { + for _, format := range []string{FormatTar, FormatZip} { + t.Run(format, func(t *testing.T) { + member := Member{ + Path: "a", Kind: MemberFile, Size: 2, + Open: func() (io.ReadCloser, error) { return closeFailReader{strings.NewReader("ab")}, nil }, + } + err := WriteDeterministic(io.Discard, format, []Member{member}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArchiveWriteFailed) + assert.ErrorIs(t, err, errTestSourceClose) + }) + } +} diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index ed2a01eeac3..0a82751968b 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -409,6 +409,9 @@ func unsupportedSigner(t *testing.T) artifact.Signer { return signer } +// leakCheckTimeout bounds the second acquisition so a leaked claim fails the test instead of hanging it. +const leakCheckTimeout = 5 * time.Second + func (c *conformance) lock(t *testing.T) { repo := c.h.New(t) ctx := context.Background() @@ -443,7 +446,9 @@ func (c *conformance) lock(t *testing.T) { // Such a claim is left for deliberate operator recovery, so leak-freedom is not promised. return } - again, err := repo.Lock(ctx, second) + retry, retryCancel := context.WithTimeout(ctx, leakCheckTimeout) + defer retryCancel() + again, err := repo.Lock(retry, second) require.NoError(t, err, "a canceled acquisition leaked no ownership") require.NoError(t, again.Release(ctx)) } diff --git a/pkg/artifact/artifacttest/conformance_test.go b/pkg/artifact/artifacttest/conformance_test.go index ba61ae92da3..758079b93e1 100644 --- a/pkg/artifact/artifacttest/conformance_test.go +++ b/pkg/artifact/artifacttest/conformance_test.go @@ -2,6 +2,7 @@ package artifacttest_test import ( "context" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -69,3 +70,37 @@ func TestRepository_LockRejectsLeasesAndWrongTokens(t *testing.T) { require.ErrorIs(t, handle.Release(ctx), errUtils.ErrArtifactConflict, "a stale handle cannot release a lock it no longer owns") assert.Empty(t, repo.HeldLocks()) } + +// deadlineRecorder notes every competing-owner Lock call made without a deadline. +type deadlineRecorder struct { + artifact.Repository + unbounded atomic.Int32 +} + +func (r *deadlineRecorder) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + if request.Owner == "owner-b" { + if _, ok := ctx.Deadline(); !ok { + r.unbounded.Add(1) + } + } + return r.Repository.Lock(ctx, request) +} + +func TestConformance_LockAcquisitionsAreBounded(t *testing.T) { + var recorders []*deadlineRecorder + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + New: func(*testing.T) artifact.Repository { + recorder := &deadlineRecorder{Repository: artifacttest.New(artifacttest.Options{})} + recorders = append(recorders, recorder) + return recorder + }, + }) + require.NotEmpty(t, recorders) + for _, recorder := range recorders { + assert.Zero(t, recorder.unbounded.Load(), "a competing acquisition must carry a deadline so a leaked claim cannot hang the suite") + } +} diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index ba90473722e..324b4d1d982 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -19,6 +19,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" ) // Names of the distribution members generated inside a packaged representation. @@ -37,6 +38,11 @@ const ( // The chunk size of context-aware copies. copyChunkSize = 32 * 1024 + + // The most decompressed bytes one call to the tar reader's Next may consume before it returns a + // visible member. Successive local PAX headers are consumed inside a single call, so this bounds + // the metadata a hostile archive can make the extractor read without producing a member. + maxTarMetadataBytes = 1 << 20 ) // fillArchive extracts the single payload stream of a tar, tgz, or zip representation. @@ -77,12 +83,15 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check defer gz.Close() reader = contextReader{ctx: ctx, reader: gz} } - tr := tar.NewReader(reader) + budget := &metadataBudget{reader: reader} + tr := tar.NewReader(budget) for { if err := ctx.Err(); err != nil { return err } + budget.arm(maxTarMetadataBytes) header, err := tr.Next() + budget.disarm() if errors.Is(err, io.EOF) { // Drain so the verifying payload reader checks its full digest at end of stream. if _, err = copyContext(ctx, io.Discard, reader); err != nil { @@ -99,6 +108,40 @@ func fillTar(ctx context.Context, t *tree, reader io.Reader, gzipped bool, check } } +// metadataBudget bounds the bytes read while it is armed. The tar reader is armed only inside Next, so +// member bodies, which are bounded by their own declared sizes, do not count against it. +type metadataBudget struct { + reader io.Reader + remaining int64 + armed bool +} + +func (b *metadataBudget) arm(limit int64) { + b.remaining = limit + b.armed = true +} + +func (b *metadataBudget) disarm() { + b.armed = false +} + +func (b *metadataBudget) Read(p []byte) (int, error) { + defer perf.Track(nil, "materialize.metadataBudget.Read")() + + if !b.armed { + return b.reader.Read(p) + } + if b.remaining <= 0 { + return 0, fmt.Errorf("%w: tar metadata exceeds %d bytes before a member", errUtils.ErrArtifactTooLarge, int64(maxTarMetadataBytes)) + } + if int64(len(p)) > b.remaining { + p = p[:b.remaining] + } + n, err := b.reader.Read(p) + b.remaining -= int64(n) + return n, err +} + // archiveReadError reports a failed archive read. A canceled context is returned as such, because // the cancellation is what interrupted the read and not damage to the archive. func archiveReadError(ctx context.Context, stage string, err error) error { diff --git a/pkg/artifact/materialize/archive_internal_test.go b/pkg/artifact/materialize/archive_internal_test.go index 39d1bf84caa..d510b98800d 100644 --- a/pkg/artifact/materialize/archive_internal_test.go +++ b/pkg/artifact/materialize/archive_internal_test.go @@ -232,3 +232,33 @@ func TestFillTar_ClassifiesACorruptGzipTrailer(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) } + +// The context never cancels here: the metadata budget alone must stop a run of PAX headers that +// outgrows it, before Next returns a visible member. +func TestFillTar_BoundsSuccessiveMetadataHeaders(t *testing.T) { + body := []byte("payload") + tests := []struct { + name string + headers int + wantErr bool + }{ + {"headers within the budget", 100, false}, + {"headers beyond the budget", 2000, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data := successivePaxTar(t, tc.headers, "file.txt", body) + target, _ := newTestTree(t, "file.txt", body) + checker := &managedChecker{documents: map[string]artifact.Document{}, seen: map[string]bool{}} + + err := fillTar(t.Context(), target, bytes.NewReader(data), false, checker) + + if !tc.wantErr { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + assert.NotErrorIs(t, err, context.Canceled) + }) + } +} diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index 280cf8f6890..8ef75ee6f2c 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -744,3 +744,48 @@ func TestMaterialize_TrailingSeparatorDestination(t *testing.T) { assertNoScratch(t, existing) }) } + +func TestInspectDestination_EmptyRecordMatchesAnEmptyDestination(t *testing.T) { + p := prepared(t, t.TempDir(), artifact.PackagingNone) + require.Empty(t, p.Record.Inventory, "the sealed Record is empty") + out := dest(t) + + require.NoError(t, materialize.Materialize(context.Background(), contentOf(t, p), out)) + state, err := materialize.InspectDestination(out, &p.Record) + + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "a repeat pull of an empty Record is recognized as a match") + + fixtures.WriteFile(t, out, "stray.txt", "x", 0o644) + state, err = materialize.InspectDestination(out, &p.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "content under an empty Record is a difference") +} + +func TestInspectDestination_AllowsAnUnrecordedAncestorDirectory(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "sub/a.txt", "a", 0o644) + p := prepared(t, src, artifact.PackagingNone) + record := p.Record + record.Inventory = nil + for _, entry := range p.Record.Inventory { + if entry.Type != artifact.EntryDirectory { + record.Inventory = append(record.Inventory, entry) + } + } + require.Len(t, record.Inventory, 1, "the inventory names the file but not its directory") + + root := t.TempDir() + matching := filepath.Join(root, "matching") + fixtures.WriteFile(t, matching, "sub/a.txt", "a", 0o644) + state, err := materialize.InspectDestination(matching, &record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "a directory that only holds recorded entries is not a difference") + + unrelated := filepath.Join(root, "unrelated") + fixtures.WriteFile(t, unrelated, "sub/a.txt", "a", 0o644) + require.NoError(t, os.Mkdir(filepath.Join(unrelated, "other"), 0o755)) + state, err = materialize.InspectDestination(unrelated, &record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationDiffers, state, "an unrelated extra directory is still a difference") +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index c28b6fe2314..c4b21f2931e 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "runtime" @@ -73,6 +74,10 @@ func InspectDestination(dest string, record *artifact.Record) (DestinationState, return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) } if len(children) == 0 { + // An empty Record is satisfied by an empty directory, so a repeat pull of it is a match. + if len(record.Inventory) == 0 { + return DestinationMatches, nil + } return DestinationEmpty, nil } if treeMatches(dest, record) { @@ -89,6 +94,7 @@ func treeMatches(dir string, record *artifact.Record) bool { for _, entry := range record.Inventory { want[entry.Path] = entry } + ancestors := requiredAncestors(want) matched := 0 walkErr := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { if err != nil { @@ -101,8 +107,16 @@ func treeMatches(dir string, record *artifact.Record) bool { if err != nil { return err } - entry, ok := want[filepath.ToSlash(relative)] - if !ok || !entryMatches(p, d, &entry) { + key := filepath.ToSlash(relative) + entry, ok := want[key] + if !ok { + // A directory the inventory does not record is still required when an entry lives under it. + if d.IsDir() && ancestors[key] { + return nil + } + return errMismatch + } + if !entryMatches(p, d, &entry) { return errMismatch } matched++ @@ -111,6 +125,17 @@ func treeMatches(dir string, record *artifact.Record) bool { return walkErr == nil && matched == len(want) } +// requiredAncestors returns the directories that hold at least one inventory entry. +func requiredAncestors(want map[string]artifact.Entry) map[string]bool { + ancestors := make(map[string]bool) + for name := range want { + for dir := path.Dir(name); dir != "." && dir != "/" && !ancestors[dir]; dir = path.Dir(dir) { + ancestors[dir] = true + } + } + return ancestors +} + func entryMatches(p string, d fs.DirEntry, entry *artifact.Entry) bool { info, err := d.Info() if err != nil { diff --git a/pkg/oci/referrers_fallback_test.go b/pkg/oci/referrers_fallback_test.go index f0bfb2129f2..c0b3b9c035f 100644 --- a/pkg/oci/referrers_fallback_test.go +++ b/pkg/oci/referrers_fallback_test.go @@ -6,11 +6,14 @@ import ( "log" "net/http" "net/http/httptest" + "path/filepath" "strings" + "sync" "sync/atomic" "testing" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" @@ -137,3 +140,86 @@ func TestReferrersKeepOtherStatusesAsErrors(t *testing.T) { require.Error(t, err) } + +// Predecessor discovery during an extended copy uses the same fallback as Referrers, so a registry that +// answers the referrers API with 400 still yields its attachments without a prior Referrers call. +func TestExtendedCopyDiscoversAttachmentsThroughTheReferrersFallback(t *testing.T) { + var refuse atomic.Bool + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if refuse.Load() && strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(http.StatusBadRequest) + return + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + subject, attachment := pushAttachment(t, store) + require.NoError(t, store.Tag(t.Context(), subject, "1.0.0")) + refuse.Store(true) + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + layout, err := oci.OpenLayout(t.Context(), filepath.Join(t.TempDir(), "layout")) + require.NoError(t, err) + + _, err = oci.ExtendedCopy(t.Context(), oci.GraphSource{Store: fresh, Reference: "1.0.0"}, oci.Destination{Store: layout, Reference: "1.0.0"}) + + require.NoError(t, err) + copied, err := layout.Exists(t.Context(), attachment) + require.NoError(t, err) + assert.True(t, copied, "the attachment was discovered through the tag-schema fallback") +} + +// Deleting an attachment needs the registry's referrers capability to keep the tag-schema index current. +// A 400 on the capability probe settles it through the fallback; the DELETE itself is sent exactly once. +func TestDeleteAttachmentSettlesTheReferrersCapabilityThroughTheFallback(t *testing.T) { + // The registry first answers the referrers API with 404, so pushes maintain the tag-schema index, and + // then with 400, the way a registry that rejects the endpoint does. + var referrersStatus atomic.Int32 + referrersStatus.Store(http.StatusNotFound) + var mu sync.Mutex + var deletes []string + backing := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/referrers/") { + w.WriteHeader(int(referrersStatus.Load())) + return + } + if r.Method == http.MethodDelete { + mu.Lock() + deletes = append(deletes, r.URL.Path) + mu.Unlock() + } + backing.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "http://") + client := registryClient(t, host, nil) + store, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + _, attachment := pushAttachment(t, store) + referrersStatus.Store(http.StatusBadRequest) + ref, err := name.NewDigest(host + "/acme/plugin@" + attachment.Digest.String()) + require.NoError(t, err) + + require.NoError(t, client.Delete(t.Context(), ref)) + + mu.Lock() + defer mu.Unlock() + attachmentDeletes := 0 + for _, deleted := range deletes { + if strings.HasSuffix(deleted, "/manifests/"+attachment.Digest.String()) { + attachmentDeletes++ + } + } + assert.Equal(t, 1, attachmentDeletes, "the DELETE of the attachment is sent exactly once") + fresh, err := client.Repository(t.Context(), oci.Repository{Registry: host, Path: "acme/plugin"}) + require.NoError(t, err) + exists, err := fresh.Exists(t.Context(), attachment) + require.NoError(t, err) + assert.False(t, exists, "the attachment is deleted") +} diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index 0fd3c1f9cbe..88fb644e887 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -319,3 +319,31 @@ func TestReadSubjectSkipsLongStringsAndFindsSubject(t *testing.T) { }) } } + +func TestReadSubjectVerifiesTheWholeBodyBeforeAcceptingEitherResult(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + declared := `{"subject":` + subject + `,"annotations":{"k":"v"}}` + tampered := `{"subject":` + subject + `,"annotations":{"k":"w"}}` + tests := []struct { + name string + desc ocispec.Descriptor + body string + }{ + {"subject found early but the tail is tampered", Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)), tampered}, + {"subject found early but the body is longer", Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)), declared + " "}, + {"no subject and the digest differs", Descriptor(ocispec.MediaTypeImageManifest, []byte(`{"a":1}`)), `{"a":2}`}, + {"no subject and the body is shorter than declared", ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString(`{"a":1} `), Size: 10}, `{"a":1}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + store.EXPECT().Fetch(gomock.Any(), tc.desc).Return(io.NopCloser(strings.NewReader(tc.body)), nil) + + got, err := readSubject(t.Context(), store, &tc.desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) + }) + } +} diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 91515e1104f..a009fd1fd73 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -307,7 +307,10 @@ func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []oci for i := range entries { if entries[i].Annotations[ocispec.AnnotationRefName] == reference { entries[i] = named - return entries + // An unnamed root for the same index would leave two entries for it, so the tag takes its place. + return slices.DeleteFunc(entries, func(entry ocispec.Descriptor) bool { + return entry.Digest == named.Digest && entry.Annotations[ocispec.AnnotationRefName] == "" + }) } } for i := range entries { diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go index 4c6e0f2816d..af59a58b95b 100644 --- a/pkg/oci/transport_registry.go +++ b/pkg/oci/transport_registry.go @@ -223,8 +223,8 @@ func (s *registryStore) Tag(ctx context.Context, desc ocispec.Descriptor, refere } func (s *registryStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - found, err := s.repo.Predecessors(ctx, node) - return found, wrapRegistry(err) + // Referrers applies the 400/406 tag-schema fallback, so discovery works without a prior Referrers call. + return s.Referrers(ctx, node, "") } func (s *registryStore) Tags(ctx context.Context) ([]string, error) { @@ -248,7 +248,17 @@ func (s *registryStore) Delete(ctx context.Context, target ocispec.Descriptor) e if target.MediaType == "" { return wrapRegistry(s.repo.Manifests().Delete(ctx, target)) } - return wrapRegistry(s.repo.Delete(ctx, target)) + err := s.repo.Delete(ctx, target) + if err != nil && referrersProbeUnavailable(err) { + // Deleting an attachment first probes the referrers API to decide whether the tag-schema index needs + // updating. A 400 or 406 on that probe fails the call before any DELETE is sent, so settling the + // capability as unsupported and calling again never repeats a DELETE. If the capability was already + // settled the first answer stands. + if capabilityErr := s.repo.SetReferrersCapability(false); capabilityErr == nil { + err = s.repo.Delete(ctx, target) + } + } + return wrapRegistry(err) } func (s *registryStore) Referrers(ctx context.Context, subject ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) { @@ -286,6 +296,16 @@ func referrersAPIUnavailable(err error) bool { return response.StatusCode == http.StatusBadRequest || response.StatusCode == http.StatusNotAcceptable } +// referrersProbeUnavailable reports whether err is a referrers API answer of 400 or 406 to a read of the +// referrers endpoint. A response to any other request, a DELETE included, never qualifies. +func referrersProbeUnavailable(err error) bool { + var response *errcode.ErrorResponse + if !errors.As(err, &response) || !referrersAPIUnavailable(err) { + return false + } + return response.Method == http.MethodGet && response.URL != nil && strings.Contains(response.URL.Path, "/referrers/") +} + var ( _ RepositoryStore = (*registryStore)(nil) _ Transport = (*Client)(nil) diff --git a/pkg/oci/transport_roundtrip_test.go b/pkg/oci/transport_roundtrip_test.go index ce182409ce9..408365ba6f5 100644 --- a/pkg/oci/transport_roundtrip_test.go +++ b/pkg/oci/transport_roundtrip_test.go @@ -268,6 +268,36 @@ func TestLayoutHoldsSeveralReferencesWithoutWeakeningLoadIndex(t *testing.T) { assert.Equal(t, []string{"plugin-1.2.3", "two-platform"}, reopened.References()) } +func TestLayoutMovingATagOntoAnUnnamedRootLeavesASingleEntry(t *testing.T) { + ctx := t.Context() + twoPlatformDirectory, twoPlatformDigest := twoPlatformLayout(t) + releaseStore, err := oci.OpenLayoutReadOnly(ctx, pluginFixtureDirectory()) + require.NoError(t, err) + otherStore, err := oci.OpenLayoutReadOnly(ctx, twoPlatformDirectory) + require.NoError(t, err) + + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + _, err = oci.Copy(ctx, oci.Source{Store: releaseStore, Reference: pluginReleaseIndexDigest}, oci.Destination{Store: layout, Reference: "release"}) + require.NoError(t, err) + // The second index is listed as an unnamed root first, then the tag moves onto it. + _, err = oci.Copy(ctx, oci.Source{Store: otherStore, Reference: twoPlatformDigest}, oci.Destination{Store: layout, Reference: twoPlatformDigest}) + require.NoError(t, err) + moved, err := otherStore.Resolve(ctx, twoPlatformDigest) + require.NoError(t, err) + require.NoError(t, layout.Tag(ctx, moved, "release")) + + index, err := oci.LoadIndex(directory, twoPlatformDigest) + require.NoError(t, err, "the moved tag leaves exactly one root entry") + require.NotNil(t, index) + + server, client := authenticatedRegistry(t, "secret") + ref, err := name.NewTag(server.Host + "/acme/plugin:moved") + require.NoError(t, err) + require.NoError(t, oci.ImportIndex(ctx, client, directory, ref)) +} + // TestLegacyWriteIndexPublishesTheExistingReleaseBytes proves the bridge serializes a legacy // index's raw bytes: publishing the committed plugin release through the old Registry API must yield // byte-identical blobs and the pinned digest, never a rebuilt release. diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go index b7926854390..e1ea382ab20 100644 --- a/pkg/oci/transport_subject.go +++ b/pkg/oci/transport_subject.go @@ -10,9 +10,11 @@ import ( "io" "strings" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" ) // maxManifestKeyBytes bounds how much of an object key the subject scan retains. A longer key can @@ -20,24 +22,78 @@ import ( const maxManifestKeyBytes = 64 // readSubject streams a stored manifest or index and returns its subject, or nil when it has none. -// The blob was digest-verified when it was pushed, so the scan is bounded by the descriptor size -// and keeps constant memory: every other top-level field, including arbitrarily long strings, is -// skipped byte by byte without being buffered. A large manifest without a subject is therefore -// never rejected. +// The scan is bounded by the descriptor size and the whole body is verified against its size and +// digest before any result is returned. It keeps constant memory: every other top-level field, +// including arbitrarily long strings, is skipped byte by byte without being buffered. A large +// manifest without a subject is therefore never rejected. func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { if desc.Size < 0 { return nil, errUtils.ErrOCIIntegrity } + if err := desc.Digest.Validate(); err != nil { + return nil, wrapIntegrity(err) + } reader, err := store.Fetch(ctx, *desc) if err != nil { return nil, err } defer reader.Close() - scanner := bufio.NewReader(io.LimitReader(reader, desc.Size)) + check := &bodyCheck{verifier: desc.Digest.Verifier()} + scanner := bufio.NewReader(io.TeeReader(io.LimitReader(reader, desc.Size), check)) if err := expectByte(scanner, '{'); err != nil { return nil, integrityError(err) } - return scanForSubject(scanner) + subject, err := scanForSubject(scanner) + if err != nil { + return nil, err + } + // The scan may stop at the subject or at the closing brace; either way the rest of the body is read + // so the size and digest cover every byte before the result is trusted. + if _, err := io.Copy(io.Discard, scanner); err != nil { + return nil, integrityError(err) + } + if err := check.finish(reader, desc.Size); err != nil { + return nil, err + } + return subject, nil +} + +// bodyCheck accumulates the size and digest of the bytes a scan consumed. +type bodyCheck struct { + verifier digest.Verifier + size int64 +} + +func (b *bodyCheck) Write(p []byte) (int, error) { + defer perf.Track(nil, "oci.bodyCheck.Write")() + + b.size += int64(len(p)) + return b.verifier.Write(p) +} + +// finish reports an integrity failure unless exactly want bytes were read, nothing follows them in +// source, and they match the descriptor digest. +func (b *bodyCheck) finish(source io.Reader, want int64) error { + if b.size != want { + return fmt.Errorf("%w: the manifest has %d bytes but %d were declared", errUtils.ErrOCIIntegrity, b.size, want) + } + var extra [1]byte + for { + n, err := source.Read(extra[:]) + if n > 0 { + return fmt.Errorf("%w: the manifest is longer than the declared %d bytes", errUtils.ErrOCIIntegrity, want) + } + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return integrityError(err) + } + } + if !b.verifier.Verified() { + return fmt.Errorf("%w: the manifest does not match its digest", errUtils.ErrOCIIntegrity) + } + return nil } // scanForSubject reads the members of a manifest object, after its opening brace, until it finds From 20abf4a9dfd1aefb6beeb9a1af4fff4fedb1cbf1 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 07:51:20 -0600 Subject: [PATCH 47/52] fix(oci): stop verifying a subject manifest body when the context is canceled readSubject drained and hashed the rest of a manifest body even after the context was canceled, because stores such as ORAS local storage ignore the fetch context. Every read, including the final extra-byte probe, now goes through a context-aware reader, a canceled read returns the context error instead of an integrity error, and success is never returned after cancellation. Uncanceled reads still verify the full body. Co-Authored-By: Claude Sonnet 5.5 --- pkg/oci/transport_internal_test.go | 63 ++++++++++++++++++++++++++++++ pkg/oci/transport_subject.go | 42 ++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index 88fb644e887..974abcd3518 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -1,6 +1,7 @@ package oci import ( + "context" "encoding/base64" "errors" "io" @@ -347,3 +348,65 @@ func TestReadSubjectVerifiesTheWholeBodyBeforeAcceptingEitherResult(t *testing.T }) } } + +// cancelAfterFirstReadReader serves a body and cancels the context once the first chunk was read. It +// ignores the context otherwise, like ORAS local storage, and counts the bytes it served. +type cancelAfterFirstReadReader struct { + reader io.Reader + cancel context.CancelFunc + served int64 +} + +func (c *cancelAfterFirstReadReader) Read(p []byte) (int, error) { + n, err := c.reader.Read(p) + c.served += int64(n) + c.cancel() + return n, err +} + +func (c *cancelAfterFirstReadReader) Close() error { return nil } + +func TestReadSubjectStopsReadingWhenTheContextIsCanceled(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + padding := strings.Repeat("p", 4<<20) + tests := []struct { + name string + manifest string + }{ + {"subject found early with a large tail", `{"subject":` + subject + `,"annotations":{"k":"` + padding + `"}}`}, + {"no subject and a large body", `{"annotations":{"k":"` + padding + `"}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(tc.manifest)) + body := &cancelAfterFirstReadReader{reader: strings.NewReader(tc.manifest), cancel: cancel} + store.EXPECT().Fetch(gomock.Any(), desc).Return(body, nil) + + got, err := readSubject(ctx, store, &desc) + + require.ErrorIs(t, err, context.Canceled) + assert.NotErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) + assert.Less(t, body.served, int64(len(tc.manifest)), "the canceled read must not consume the whole body") + }) + } +} + +func TestReadSubjectStillRejectsAWrongDigestWhenTheContextStaysActive(t *testing.T) { + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + declared := `{"subject":` + subject + `,"annotations":{"k":"v"}}` + tampered := `{"subject":` + subject + `,"annotations":{"k":"w"}}` + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + desc := Descriptor(ocispec.MediaTypeImageManifest, []byte(declared)) + store.EXPECT().Fetch(gomock.Any(), desc).Return(io.NopCloser(strings.NewReader(tampered)), nil) + + got, err := readSubject(t.Context(), store, &desc) + + require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) + assert.Nil(t, got) +} diff --git a/pkg/oci/transport_subject.go b/pkg/oci/transport_subject.go index e1ea382ab20..df83494d5e1 100644 --- a/pkg/oci/transport_subject.go +++ b/pkg/oci/transport_subject.go @@ -38,6 +38,21 @@ func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) ( return nil, err } defer reader.Close() + // Some stores, such as ORAS local storage, ignore the fetch context, so every read is guarded here. + subject, err := verifySubject(ctx, contextReader{ctx: ctx, reader: reader}, desc) + if err != nil { + return nil, canceledOr(ctx, err) + } + // A cancellation that raced the final reads must not be reported as a verified result. + if err := ctx.Err(); err != nil { + return nil, err + } + return subject, nil +} + +// verifySubject scans the body for the subject and then reads the remainder so the size and digest +// cover every byte. +func verifySubject(ctx context.Context, reader io.Reader, desc *ocispec.Descriptor) (*ocispec.Descriptor, error) { check := &bodyCheck{verifier: desc.Digest.Verifier()} scanner := bufio.NewReader(io.TeeReader(io.LimitReader(reader, desc.Size), check)) if err := expectByte(scanner, '{'); err != nil { @@ -52,12 +67,39 @@ func readSubject(ctx context.Context, store Fetcher, desc *ocispec.Descriptor) ( if _, err := io.Copy(io.Discard, scanner); err != nil { return nil, integrityError(err) } + if err := ctx.Err(); err != nil { + return nil, err + } if err := check.finish(reader, desc.Size); err != nil { return nil, err } return subject, nil } +// canceledOr returns the context error when the context ended, since a canceled read is not an +// integrity failure; otherwise it returns err unchanged. +func canceledOr(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("reading the manifest subject: %w", ctxErr) + } + return err +} + +// contextReader fails a read as soon as the context is done. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (c contextReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "oci.contextReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.reader.Read(p) +} + // bodyCheck accumulates the size and digest of the bytes a scan consumed. type bodyCheck struct { verifier digest.Verifier From 7d50671e347d60552c72b2462fa20b5892ae2495 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 18:20:42 -0600 Subject: [PATCH 48/52] fix(artifacts): preserve historical merge resolutions after stack repair --- pkg/artifact/materialize/materialize_test.go | 36 ++++++++++++++----- pkg/artifact/service/query.go | 22 ++++++------ pkg/artifact/service/sign.go | 19 ++++++---- pkg/auth/deferred/manager_test.go | 3 +- pkg/store/authbridge/resolver_scoped_test.go | 3 +- .../hashicorp_vault_lifetime_test.go | 6 ++-- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index 8ef75ee6f2c..cf05c8fa154 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -366,6 +366,17 @@ func sealedRecord(t *testing.T, inventory []artifact.Entry) artifact.Record { return record } +// unsealedRecord builds a Record around an inventory without sealing it. The record layer rejects +// inventories whose symlink targets escape the package, so such a record can never be sealed; +// this helper lets a test prove Materialize still refuses one that arrives unsealed. +func unsealedRecord(inventory []artifact.Entry) artifact.Record { + return artifact.Record{ + SchemaVersion: artifact.SchemaVersion, + Name: "pkg/crafted", Version: "1", Kind: artifact.KindFiles, + CreatedAt: artifact.NewTimestamp(fixedTime), Inventory: inventory, + } +} + func fileEntry(path, content string, mode uint32) artifact.Entry { return artifact.Entry{Path: path, Type: artifact.EntryFile, Size: int64(len(content)), Mode: mode, Digest: digest.FromString(content)} } @@ -475,11 +486,13 @@ func requireSymlinks(t *testing.T) { func TestMaterialize_ChainedSymlinkEscapeIsRejected(t *testing.T) { requireSymlinks(t) - record := sealedRecord(t, []artifact.Entry{ + record := unsealedRecord([]artifact.Entry{ {Path: "a", Type: artifact.EntryDirectory, Mode: 0o755}, {Path: "a/b", Type: artifact.EntrySymlink, Target: ".."}, {Path: "c", Type: artifact.EntrySymlink, Target: "a/b/../.."}, }) + // The record layer rejects the escape before any content is written, so it cannot be sealed. + require.ErrorIs(t, record.Seal(), errUtils.ErrArtifactUnsafePath) content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} out := dest(t) err := materialize.Materialize(context.Background(), content, out) @@ -515,14 +528,18 @@ func TestMaterialize_DanglingChainedSymlinkEscapeIsRejected(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - record := sealedRecord(t, tt.inventory) - content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} - err := materialize.Materialize(context.Background(), content, dest(t)) if tt.wantErr { - require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + // The record layer rejects the escape, so the record cannot be sealed, and + // Materialize must still refuse it when it arrives unsealed. + record := unsealedRecord(tt.inventory) + require.ErrorIs(t, record.Seal(), errUtils.ErrArtifactUnsafePath) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + require.ErrorIs(t, materialize.Materialize(context.Background(), content, dest(t)), errUtils.ErrArtifactUnsafePath) return } - require.NoError(t, err, "a dangling link that stays inside the tree is allowed") + record := sealedRecord(t, tt.inventory) + content := &artifact.Content{Record: &record, Representation: artifact.Representation{Packaging: artifact.PackagingNone}} + require.NoError(t, materialize.Materialize(context.Background(), content, dest(t)), "a dangling link that stays inside the tree is allowed") }) } } @@ -610,12 +627,13 @@ func TestMaterialize_ReplaceGuard(t *testing.T) { out := dest(t) fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(tt.guard)) - if tt.wantErr != nil { + switch { + case tt.wantErr != nil: require.ErrorIs(t, err, tt.wantErr) - } else if runtime.GOOS == "windows" { + case runtime.GOOS == "windows": require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) tt.wantStale = true - } else { + default: require.NoError(t, err) } _, staleErr := os.Stat(filepath.Join(out, "stale.txt")) diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go index 170166a3bb6..172f0eac578 100644 --- a/pkg/artifact/service/query.go +++ b/pkg/artifact/service/query.go @@ -86,16 +86,7 @@ func (s *Service) showDocument(ctx context.Context, op *operation, request *arti return fmt.Errorf("open %s of %s in repository %q: %w", request.Role, request.Ref, request.Repository, err) } defer stream.Reader.Close() - matched := false - if inspection.Record != nil { - for _, document := range inspection.Record.Documents { - if document.Role == request.Role && document == stream.Document { - matched = true - break - } - } - } - if !matched { + if !documentMatchesRecord(inspection.Record, request.Role, stream.Document) { return fmt.Errorf("%w: document does not match the pinned record", errUtils.ErrArtifactIntegrityFailed) } content, err := readDocument(stream) @@ -108,6 +99,17 @@ func (s *Service) showDocument(ctx context.Context, op *operation, request *arti return nil } +func documentMatchesRecord(record *artifact.Record, role artifact.DocumentRole, document artifact.Document) bool { + if record != nil { + for _, expected := range record.Documents { + if expected.Role == role && expected == document { + return true + } + } + } + return false +} + func validateShow(request *artifact.ShowDocumentRequest) error { if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { return err diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go index f929b4e1f1f..b126bf60034 100644 --- a/pkg/artifact/service/sign.go +++ b/pkg/artifact/service/sign.go @@ -47,12 +47,8 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig } defer func() { _ = locks.release(ctx, op) }() - current, err := repo.Resolve(ctx, request.Ref) - if err != nil { - return fmt.Errorf("resolve %s after locking: %w", request.Ref, err) - } - if current.ResolvedDigest != pinned.ResolvedDigest || current.RecordDigest != pinned.RecordDigest { - return fmt.Errorf("%w: subject %s changed while acquiring its lock", errUtils.ErrArtifactConflict, request.Ref) + if err := verifyLockedSubject(ctx, repo, request.Ref, &pinned); err != nil { + return err } evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, resource)}) if err != nil { @@ -65,6 +61,17 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig return nil } +func verifyLockedSubject(ctx context.Context, repo artifact.Repository, ref artifact.Ref, pinned *artifact.PinnedReference) error { + current, err := repo.Resolve(ctx, ref) + if err != nil { + return fmt.Errorf("resolve %s after locking: %w", ref, err) + } + if current.ResolvedDigest != pinned.ResolvedDigest || current.RecordDigest != pinned.RecordDigest { + return fmt.Errorf("%w: subject %s changed while acquiring its lock", errUtils.ErrArtifactConflict, ref) + } + return nil +} + // validateSign requires a repository, a valid digest reference, and a configured signer. func (s *Service) validateSign(request *artifact.SignRequest) error { if err := validateRepositoryRef(request.Repository, request.Ref); err != nil { diff --git a/pkg/auth/deferred/manager_test.go b/pkg/auth/deferred/manager_test.go index c3a7cd9bd88..73f62318ec9 100644 --- a/pkg/auth/deferred/manager_test.go +++ b/pkg/auth/deferred/manager_test.go @@ -173,7 +173,8 @@ func TestManagerDelegatesScopedIdentityOwnership(t *testing.T) { err := operation(ctx, &schema.AuthContext{Environment: map[string]string{"VAULT_TOKEN": "synthetic-reader"}}) events = append(events, "released") return err - }) + }, + ) err := manager.WithIsolatedIdentity(t.Context(), "reader", func(ctx context.Context, auth *schema.AuthContext) error { events = append(events, "operation") require.Same(t, t.Context(), ctx) diff --git a/pkg/store/authbridge/resolver_scoped_test.go b/pkg/store/authbridge/resolver_scoped_test.go index a64c59cc705..4cd57c2b844 100644 --- a/pkg/store/authbridge/resolver_scoped_test.go +++ b/pkg/store/authbridge/resolver_scoped_test.go @@ -40,7 +40,8 @@ func TestScopedResolverKeepsIdentityOwnership(t *testing.T) { err := operation(ctx, tt.auth) events = append(events, "released") return errors.Join(err, tt.cleanupError) - }) + }, + ) resolver := NewResolver(manager, &schema.ConfigAndStacksInfo{}) err := resolver.WithEnvironmentAuthContext(t.Context(), "reader", func(ctx context.Context, auth *store.EnvironmentAuthConfig) error { events = append(events, "operation") diff --git a/pkg/store/providers/hashicorp_vault_lifetime_test.go b/pkg/store/providers/hashicorp_vault_lifetime_test.go index f481ea77a36..b6c503b48ab 100644 --- a/pkg/store/providers/hashicorp_vault_lifetime_test.go +++ b/pkg/store/providers/hashicorp_vault_lifetime_test.go @@ -25,7 +25,8 @@ func scopedVault(t *testing.T, cleanup error) (*VaultStore, *fakeVaultKV, *int) err := operation(ctx, &store.EnvironmentAuthConfig{Environment: map[string]string{"VAULT_TOKEN": "synthetic-identity"}}) *releases++ return errors.Join(err, cleanup) - }).AnyTimes() + }, + ).AnyTimes() s.SetAuthContext(resolver, "openbao") return s, fake, releases } @@ -130,7 +131,8 @@ func TestVaultOperationLifetimeRejectsMissingCredentials(t *testing.T) { return operation(ctx, nil) } return operation(ctx, &store.EnvironmentAuthConfig{Environment: environment}) - }) + }, + ) s.SetAuthContext(resolver, "openbao") value, err := s.GetKey("key") require.ErrorIs(t, err, store.ErrAuthContextNotAvailable) From db7a71278be086ad70126946c85ec8f9c705a9ca Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 18:28:26 -0600 Subject: [PATCH 49/52] test(errors): make CI color rendering terminal capability explicit --- errors/formatter_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/errors/formatter_test.go b/errors/formatter_test.go index 50171afc82c..6bfbc1aceba 100644 --- a/errors/formatter_test.go +++ b/errors/formatter_test.go @@ -248,6 +248,8 @@ func TestShouldUseColorInCI(t *testing.T) { } func TestFormatInCIUsesColorUnlessDisabled(t *testing.T) { + // TERM=dumb deliberately disables automatic styling, even in CI. + t.Setenv("TERM", "xterm-256color") t.Setenv("CI", "true") t.Setenv("NO_COLOR", "") From f57b5405e169710de9f68269ac9dff265e21bf1d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 20:27:34 -0600 Subject: [PATCH 50/52] fix(artifact): enforce bound reads and consistent admission checks Respect read capabilities in conformance, reject corrupt evidence and repeated managed directories, bind replacement guards to open roots, recheck pathname identity after hashing, and normalize immutability policy metadata. --- pkg/artifact/artifacttest/conformance.go | 17 +++++ pkg/artifact/artifacttest/conformance_test.go | 10 +++ pkg/artifact/artifacttest/repository_test.go | 36 ++++++++++ pkg/artifact/artifacttest/sign.go | 3 + pkg/artifact/materialize/archive.go | 5 +- pkg/artifact/materialize/extract.go | 7 +- pkg/artifact/materialize/materialize_test.go | 66 +++++++++++++++++-- pkg/artifact/materialize/replace.go | 58 +++++++++------- .../replace_cancel_internal_test.go | 2 +- pkg/artifact/service/commit.go | 4 ++ pkg/artifact/service/copy_test.go | 31 +++++++++ pkg/artifact/service/publish_test.go | 2 + pkg/artifact/service/pull.go | 9 +-- pkg/artifact/service/repositories.go | 7 +- pkg/artifact/snapshot/copy.go | 12 ++++ .../snapshot/recheck_internal_test.go | 18 +++++ 16 files changed, 243 insertions(+), 44 deletions(-) diff --git a/pkg/artifact/artifacttest/conformance.go b/pkg/artifact/artifacttest/conformance.go index 0a82751968b..d0be52cad36 100644 --- a/pkg/artifact/artifacttest/conformance.go +++ b/pkg/artifact/artifacttest/conformance.go @@ -132,6 +132,15 @@ func (c *conformance) requireWritable(t *testing.T, repo artifact.Repository) { } } +func (c *conformance) requireReadable(t *testing.T, repo artifact.Repository) { + t.Helper() + if !repo.Capabilities().Read { + _, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: firstVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + t.Skipf("successful reads require the read capability; unsupported Resolve was checked") + } +} + func (c *conformance) capabilities(t *testing.T) { repo := c.h.New(t) first, second := repo.Capabilities(), repo.Capabilities() @@ -142,6 +151,7 @@ func (c *conformance) capabilities(t *testing.T) { func (c *conformance) multipleNamesAndVersions(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) items := []struct{ name, version string }{{appName, "1.0.0"}, {appName, "2.0.0"}, {"other/tool", "1.0.0"}} var all []published for _, item := range items { @@ -164,6 +174,7 @@ func (c *conformance) multipleNamesAndVersions(t *testing.T) { func (c *conformance) roundTrip(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "round trip"}, artifact.PublishOptions{}) pinned, err := repo.Resolve(context.Background(), item.ref) require.NoError(t, err) @@ -182,6 +193,7 @@ func (c *conformance) roundTrip(t *testing.T) { func (c *conformance) metadataAndDocuments(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "docs"}, artifact.PublishOptions{}) pinned, err := repo.Resolve(context.Background(), item.ref) require.NoError(t, err) @@ -201,6 +213,7 @@ func (c *conformance) metadataAndDocuments(t *testing.T) { func (c *conformance) digestReferences(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "digest"}, artifact.PublishOptions{}) byVersion, err := repo.Resolve(context.Background(), item.ref) require.NoError(t, err) @@ -260,6 +273,7 @@ func (c *conformance) enumeration(t *testing.T) { func (c *conformance) replacement(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) first := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "first"}, artifact.PublishOptions{}) oldPinned, err := repo.Resolve(context.Background(), first.ref) require.NoError(t, err) @@ -314,6 +328,7 @@ func (c *conformance) conditionalPublish(t *testing.T) { func (c *conformance) deletion(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) keep := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "keep"}, artifact.PublishOptions{}) drop := c.publish(t, repo, spec{name: appName, version: secondVersion, content: "drop"}, artifact.PublishOptions{}) pinned, err := repo.Resolve(context.Background(), drop.ref) @@ -337,6 +352,7 @@ func (c *conformance) deletion(t *testing.T) { func (c *conformance) pin(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "pin"}, artifact.PublishOptions{}) resolved, err := repo.Resolve(context.Background(), item.ref) require.NoError(t, err) @@ -352,6 +368,7 @@ func (c *conformance) pin(t *testing.T) { func (c *conformance) signVerify(t *testing.T) { repo := c.h.New(t) c.requireWritable(t, repo) + c.requireReadable(t, repo) item := c.publish(t, repo, spec{name: appName, version: firstVersion, content: "sign"}, artifact.PublishOptions{}) pinned, err := repo.Resolve(context.Background(), item.ref) require.NoError(t, err) diff --git a/pkg/artifact/artifacttest/conformance_test.go b/pkg/artifact/artifacttest/conformance_test.go index 758079b93e1..bb7c42bd083 100644 --- a/pkg/artifact/artifacttest/conformance_test.go +++ b/pkg/artifact/artifacttest/conformance_test.go @@ -104,3 +104,13 @@ func TestConformance_LockAcquisitionsAreBounded(t *testing.T) { assert.Zero(t, recorder.unbounded.Load(), "a competing acquisition must carry a deadline so a leaked claim cannot hang the suite") } } + +func TestInMemoryRepositoryConformance_WriteWithoutRead(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + caps.Read = false + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(*testing.T) artifact.Repository { + return artifacttest.New(artifacttest.Options{Capabilities: &caps}) + }, + }) +} diff --git a/pkg/artifact/artifacttest/repository_test.go b/pkg/artifact/artifacttest/repository_test.go index 0d9803e0ce2..4ef08502940 100644 --- a/pkg/artifact/artifacttest/repository_test.go +++ b/pkg/artifact/artifacttest/repository_test.go @@ -391,3 +391,39 @@ func TestRepository_SignClassifiesSignerFailures(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactSigningFailed) assert.ErrorIs(t, err, cause) } + +func TestRepository_VerifyRejectsEvidenceIntegrityMismatch(t *testing.T) { + source := artifacttest.New(artifacttest.Options{}) + pinned := publishOne(t, source) + signEvidence(t, source, &pinned) + valid, err := source.ReadEvidence(t.Context(), &pinned) + require.NoError(t, err) + require.Len(t, valid, 1) + signer := &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} + for _, mismatch := range []string{"size", "digest"} { + t.Run(mismatch, func(t *testing.T) { + repo := artifacttest.New(artifacttest.Options{}) + target := publishOne(t, repo) + corrupt := valid[0] + if mismatch == "size" { + corrupt.Evidence.Descriptor.Size++ + } else { + corrupt.Evidence.Descriptor.Digest = digest.FromString("different bytes") + } + _, err := repo.WriteEvidence(t.Context(), &target, []artifact.EvidenceBlob{corrupt}, nil) + require.NoError(t, err) + result, err := repo.Verify(t.Context(), valid[0].Evidence.Subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Empty(t, result.Claims) + assert.Empty(t, result.EvidenceDigests) + require.NotEmpty(t, result.Diagnostics) + assert.Equal(t, "evidence-integrity", result.Diagnostics[0].Code) + _, err = repo.WriteEvidence(t.Context(), &target, valid, nil) + require.NoError(t, err) + result, err = repo.Verify(t.Context(), valid[0].Evidence.Subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Len(t, result.Claims, 1) + assert.Equal(t, []digest.Digest{valid[0].Evidence.Descriptor.Digest}, result.EvidenceDigests) + }) + } +} diff --git a/pkg/artifact/artifacttest/sign.go b/pkg/artifact/artifacttest/sign.go index 56efd1c513f..fa946717fb8 100644 --- a/pkg/artifact/artifacttest/sign.go +++ b/pkg/artifact/artifacttest/sign.go @@ -167,6 +167,9 @@ func (r *Repository) authenticate(subject artifact.SignatureSubject, held *artif if !r.caps.HasEvidenceFormat(held.Evidence.Format) { return skip("unsupported-evidence", "evidence in an unsupported format was ignored") } + if held.Evidence.Descriptor.Size != int64(len(held.Data)) || held.Evidence.Descriptor.Digest != digest.FromBytes(held.Data) { + return skip("evidence-integrity", "evidence whose bytes do not match its descriptor was ignored") + } var parsed fakeEvidence if err := json.Unmarshal(held.Data, &parsed); err != nil { return skip("malformed-evidence", "malformed evidence was ignored") diff --git a/pkg/artifact/materialize/archive.go b/pkg/artifact/materialize/archive.go index 324b4d1d982..ca6dbcbd5a2 100644 --- a/pkg/artifact/materialize/archive.go +++ b/pkg/artifact/materialize/archive.go @@ -352,9 +352,10 @@ func (m *managedChecker) owns(name string, t *tree) bool { func (m *managedChecker) check(member *archiveMember) error { name := member.name if name == managedParent || name == managedDir { - if member.kind != memberDir { - return fmt.Errorf("%w: %q must be a directory", errUtils.ErrArtifactPackagingInvalid, name) + if member.kind != memberDir || m.seen[name] { + return fmt.Errorf("%w: %q must be a unique directory", errUtils.ErrArtifactPackagingInvalid, name) } + m.seen[name] = true return nil } want, ok := m.expected(name) diff --git a/pkg/artifact/materialize/extract.go b/pkg/artifact/materialize/extract.go index 56f503111fa..ee72be405d2 100644 --- a/pkg/artifact/materialize/extract.go +++ b/pkg/artifact/materialize/extract.go @@ -22,7 +22,7 @@ type options struct { maxBytes int64 // replaceGuard runs immediately before an existing destination is replaced. - replaceGuard func(dest string) error + replaceGuard func(dest *os.Root) error // ownerAccess keeps owner read and traversal access on the extracted tree. ownerAccess bool @@ -41,8 +41,9 @@ func WithOwnerAccess() Option { // WithReplaceGuard registers a check that runs right before an existing destination directory is // replaced, after extraction finished and the destination was bound. A caller uses it to re-check // the destination state it decided on earlier, because the destination may have changed during the -// download. A non-nil error aborts the replacement and leaves the destination untouched. -func WithReplaceGuard(guard func(dest string) error) Option { +// download. The guard must inspect through the supplied root, never reopen its Name. +// The caller must not close the root. A non-nil error aborts the replacement and leaves the destination untouched. +func WithReplaceGuard(guard func(dest *os.Root) error) Option { defer perf.Track(nil, "materialize.WithReplaceGuard")() return func(o *options) { o.replaceGuard = guard } diff --git a/pkg/artifact/materialize/materialize_test.go b/pkg/artifact/materialize/materialize_test.go index cf05c8fa154..6364383b0d5 100644 --- a/pkg/artifact/materialize/materialize_test.go +++ b/pkg/artifact/materialize/materialize_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "errors" + "fmt" "io" "os" "path/filepath" @@ -615,12 +616,12 @@ func TestMaterialize_ReplaceGuard(t *testing.T) { errRefused := errors.New("guard refused") tests := []struct { name string - guard func(dest string) error + guard func(dest *os.Root) error wantErr error wantStale bool }{ - {"guard accepts", func(string) error { return nil }, nil, false}, - {"guard refuses", func(string) error { return errRefused }, errRefused, true}, + {"guard accepts", func(*os.Root) error { return nil }, nil, false}, + {"guard refuses", func(*os.Root) error { return errRefused }, errRefused, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -645,7 +646,7 @@ func TestMaterialize_ReplaceGuard(t *testing.T) { t.Run("guard is not consulted for an absent destination", func(t *testing.T) { called := false out := dest(t) - err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(func(string) error { + err := materialize.Materialize(context.Background(), contentOf(t, p), out, materialize.WithReplaceGuard(func(*os.Root) error { called = true return errRefused })) @@ -669,10 +670,10 @@ func TestMaterialize_DestinationSwappedForASymlinkNeverRedirectsRemoval(t *testi fixtures.WriteFile(t, out, "stale.txt", "stale", 0o644) fixtures.WriteFile(t, outside, "keep.txt", "keep", 0o644) - swap := materialize.WithReplaceGuard(func(dest string) error { + swap := materialize.WithReplaceGuard(func(dest *os.Root) error { // Simulate an attacker replacing the checked destination right after it was inspected. - require.NoError(t, os.Rename(dest, filepath.Join(root, "moved-away"))) - require.NoError(t, os.Symlink(outside, dest)) + require.NoError(t, os.Rename(dest.Name(), filepath.Join(root, "moved-away"))) + require.NoError(t, os.Symlink(outside, dest.Name())) return nil }) err := materialize.Materialize(context.Background(), contentOf(t, p), out, swap) @@ -807,3 +808,54 @@ func TestInspectDestination_AllowsAnUnrecordedAncestorDirectory(t *testing.T) { require.NoError(t, err) assert.Equal(t, materialize.DestinationDiffers, state, "an unrelated extra directory is still a difference") } + +func TestMaterialize_RepeatedManagedDirectories(t *testing.T) { + for _, name := range []string{".atmos/", ".atmos/artifact/"} { + for _, populated := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/populated=%t", name, populated), func(t *testing.T) { + var inventory []artifact.Entry + var members []tarMember + if populated { + inventory = []artifact.Entry{fileEntry("payload.txt", "payload", 0o600)} + members = append(members, tarMember{header: tar.Header{Name: "payload.txt", Typeflag: tar.TypeReg, Mode: 0o600}, body: "payload"}) + } + directory := tarMember{header: tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0o755}} + members = append(members, directory, directory) + record := sealedRecord(t, inventory) + content := archiveContent(&record, artifact.PackagingTgz, tgzOf(t, members)) + t.Cleanup(func() { _ = content.Close() }) + require.ErrorIs(t, materialize.Materialize(t.Context(), content, dest(t)), errUtils.ErrArtifactPackagingInvalid) + }) + } + } +} + +func TestMaterialize_ReplaceGuardInspectsBoundDirectoryDuringPathSwap(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("a directory with an open handle cannot be renamed on Windows") + } + src := t.TempDir() + fixtures.WriteFile(t, src, "new.txt", "new", 0o644) + p := prepared(t, src, artifact.PackagingTgz) + out := dest(t) + fixtures.WriteFile(t, out, "keep.txt", "unapproved", 0o644) + guard := materialize.WithReplaceGuard(func(root *os.Root) error { + moved := out + "-moved" + require.NoError(t, os.Rename(out, moved)) + require.NoError(t, os.Mkdir(out, 0o755)) + state, err := materialize.InspectBoundDestination(root, &p.Record) + require.NoError(t, err) + require.NoError(t, os.Remove(out)) + require.NoError(t, os.Rename(moved, out)) + assert.Equal(t, materialize.DestinationDiffers, state) + if state == materialize.DestinationDiffers { + return errUtils.ErrArtifactConflict + } + return nil + }) + require.ErrorIs(t, materialize.Materialize(t.Context(), contentOf(t, p), out, guard), errUtils.ErrArtifactConflict) + data, err := os.ReadFile(filepath.Join(out, "keep.txt")) + require.NoError(t, err) + assert.Equal(t, "unapproved", string(data)) + assertNoScratch(t, out) +} diff --git a/pkg/artifact/materialize/replace.go b/pkg/artifact/materialize/replace.go index c4b21f2931e..b2d3bf0eefb 100644 --- a/pkg/artifact/materialize/replace.go +++ b/pkg/artifact/materialize/replace.go @@ -69,18 +69,30 @@ func InspectDestination(dest string, record *artifact.Record) (DestinationState, case !info.IsDir(): return DestinationDiffers, fmt.Errorf("%w: destination %q exists and is not a directory", errUtils.ErrArtifactConflict, dest) } - children, err := os.ReadDir(dest) + bound, err := bindDestination(dest, info) if err != nil { - return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, dest, err) + return DestinationDiffers, err + } + defer bound.root.Close() + return InspectBoundDestination(bound.root, record) +} + +// InspectBoundDestination classifies the directory held by root without resolving its pathname. +// The caller retains ownership of root and must keep it open throughout inspection. +func InspectBoundDestination(root *os.Root, record *artifact.Record) (DestinationState, error) { + defer perf.Track(nil, "materialize.InspectBoundDestination")() + + children, err := fs.ReadDir(root.FS(), rootDirName) + if err != nil { + return DestinationDiffers, fmt.Errorf("%w: destination %q: %w", errUtils.ErrArtifactPackagingInvalid, root.Name(), err) } if len(children) == 0 { - // An empty Record is satisfied by an empty directory, so a repeat pull of it is a match. if len(record.Inventory) == 0 { return DestinationMatches, nil } return DestinationEmpty, nil } - if treeMatches(dest, record) { + if treeMatches(root, record) { return DestinationMatches, nil } return DestinationDiffers, nil @@ -89,25 +101,21 @@ func InspectDestination(dest string, record *artifact.Record) (DestinationState, // treeMatches reports whether dir holds exactly the Record inventory: the same paths, types, link // targets, file sizes and digests, and (off Windows) permission bits. Any unreadable entry counts // as a difference. -func treeMatches(dir string, record *artifact.Record) bool { +func treeMatches(root *os.Root, record *artifact.Record) bool { want := make(map[string]artifact.Entry, len(record.Inventory)) for _, entry := range record.Inventory { want[entry.Path] = entry } ancestors := requiredAncestors(want) matched := 0 - walkErr := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + walkErr := fs.WalkDir(root.FS(), rootDirName, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } - if p == dir { + if p == rootDirName { return nil } - relative, err := filepath.Rel(dir, p) - if err != nil { - return err - } - key := filepath.ToSlash(relative) + key := p entry, ok := want[key] if !ok { // A directory the inventory does not record is still required when an entry lives under it. @@ -116,7 +124,7 @@ func treeMatches(dir string, record *artifact.Record) bool { } return errMismatch } - if !entryMatches(p, d, &entry) { + if !entryMatches(root, p, d, &entry) { return errMismatch } matched++ @@ -136,7 +144,7 @@ func requiredAncestors(want map[string]artifact.Entry) map[string]bool { return ancestors } -func entryMatches(p string, d fs.DirEntry, entry *artifact.Entry) bool { +func entryMatches(root *os.Root, p string, d fs.DirEntry, entry *artifact.Entry) bool { info, err := d.Info() if err != nil { return false @@ -145,19 +153,19 @@ func entryMatches(p string, d fs.DirEntry, entry *artifact.Entry) bool { case artifact.EntryDirectory: return info.IsDir() && modeMatches(info, entry) case artifact.EntrySymlink: - return linkMatches(p, info, entry) + return linkMatches(root, p, info, entry) default: - return fileMatches(p, info, entry) + return fileMatches(root, p, info, entry) } } -func linkMatches(p string, info os.FileInfo, entry *artifact.Entry) bool { - target, err := os.Readlink(p) +func linkMatches(root *os.Root, p string, info os.FileInfo, entry *artifact.Entry) bool { + target, err := root.Readlink(p) return err == nil && info.Mode()&os.ModeSymlink != 0 && filepath.ToSlash(target) == entry.Target } -func fileMatches(p string, info os.FileInfo, entry *artifact.Entry) bool { - return info.Mode().IsRegular() && info.Size() == entry.Size && modeMatches(info, entry) && fileDigest(p) == entry.Digest +func fileMatches(root *os.Root, p string, info os.FileInfo, entry *artifact.Entry) bool { + return info.Mode().IsRegular() && info.Size() == entry.Size && modeMatches(info, entry) && fileDigest(root, p) == entry.Digest } func modeMatches(info os.FileInfo, entry *artifact.Entry) bool { @@ -167,8 +175,8 @@ func modeMatches(info os.FileInfo, entry *artifact.Entry) bool { return uint32(info.Mode().Perm()) == entry.Mode } -func fileDigest(p string) digest.Digest { - file, err := os.Open(p) +func fileDigest(root *os.Root, p string) digest.Digest { + file, err := root.Open(p) if err != nil { return "" } @@ -232,7 +240,7 @@ func Materialize(ctx context.Context, content *artifact.Content, dest string, op // installStaged moves the extracted tree into place. Cancellation is final: a canceled pull never // installs, even when extraction had just finished. -func installStaged(ctx context.Context, stage, dest string, guard func(dest string) error) (bool, error) { +func installStaged(ctx context.Context, stage, dest string, guard func(dest *os.Root) error) (bool, error) { if err := ctx.Err(); err != nil { return false, err } @@ -245,7 +253,7 @@ func installStaged(ctx context.Context, stage, dest string, guard func(dest stri // replaceDestination atomically exchanges the staged and existing trees. No children are // removed or installed through the destination path, and a failed exchange leaves both intact. // The boolean preserves scratch when a concurrent replacement cannot be safely rolled back. -func replaceDestination(ctx context.Context, stage, dest string, guard func(dest string) error) (bool, error) { +func replaceDestination(ctx context.Context, stage, dest string, guard func(dest *os.Root) error) (bool, error) { info, err := os.Lstat(dest) switch { case os.IsNotExist(err): @@ -263,7 +271,7 @@ func replaceDestination(ctx context.Context, stage, dest string, guard func(dest } defer bound.root.Close() if guard != nil { - if err := guard(dest); err != nil { + if err := guard(bound.root); err != nil { return false, err } } diff --git a/pkg/artifact/materialize/replace_cancel_internal_test.go b/pkg/artifact/materialize/replace_cancel_internal_test.go index cf66f7fac16..d0241135df5 100644 --- a/pkg/artifact/materialize/replace_cancel_internal_test.go +++ b/pkg/artifact/materialize/replace_cancel_internal_test.go @@ -40,7 +40,7 @@ func TestInstallStaged_CancellationDuringTheReplaceGuardIsFinal(t *testing.T) { stage, dest := stagedAndDestination(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() - guard := func(string) error { + guard := func(*os.Root) error { if tc.cancelInGuard { cancel() } diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go index 5c2c660b4ae..6483fde9fe9 100644 --- a/pkg/artifact/service/commit.go +++ b/pkg/artifact/service/commit.go @@ -48,6 +48,10 @@ func newIncoming(prepared *artifact.PreparedArtifact) (*incoming, error) { in.size += prepared.Blobs[index].Descriptor.Size } } + in.inspection = &artifact.Inspection{ + Record: in.record, + Representation: artifact.Representation{Packaging: prepared.Packaging, Payload: in.payload}, + } return in, nil } diff --git a/pkg/artifact/service/copy_test.go b/pkg/artifact/service/copy_test.go index 8d4ed883d28..098bbb4206b 100644 --- a/pkg/artifact/service/copy_test.go +++ b/pkg/artifact/service/copy_test.go @@ -2,6 +2,7 @@ package service_test import ( "context" + "fmt" "os" "path/filepath" "runtime" @@ -455,3 +456,33 @@ func TestCopy_StagingKeepsOwnerAccessWhileRepacking(t *testing.T) { require.True(t, repacked, "the copy must reach the repack") e.assertStagingClean() } + +func TestCopy_InspectionPolicySeesIncomingReleaseOnBothPaths(t *testing.T) { + for _, native := range []bool{false, true} { + t.Run(fmt.Sprintf("native=%t", native), func(t *testing.T) { + var sawIncoming bool + policy := service.ImmutabilityFunc(func(c service.Candidate) (bool, string) { + require.NotNil(t, c.Inspection) + if c.Existing { + return false, "" + } + sawIncoming = true + assert.Equal(t, native, c.Prepared == nil) + return c.Inspection.Record.Metadata.Labels["format"] == "plugin", "incoming plugin" + }) + e := copyEnv(t, artifact.PackagingNone, artifact.PackagingNone, service.WithImmutabilityPolicy(policy)) + e.publish("dst", e.source("generic")) + request := e.publishRequest("src", e.source("plugin")) + request.Metadata.Labels = map[string]string{"format": "plugin"} + _, err := e.svc.Publish(t.Context(), request) + require.NoError(t, err) + if native { + e.static.Repositories["dst"] = newNativeRepository(t, e.repos["dst"]) + } + _, err = e.svc.Copy(t.Context(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst", Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "incoming plugin") + assert.True(t, sawIncoming) + }) + } +} diff --git a/pkg/artifact/service/publish_test.go b/pkg/artifact/service/publish_test.go index 8ec3f684177..5a38342c700 100644 --- a/pkg/artifact/service/publish_test.go +++ b/pkg/artifact/service/publish_test.go @@ -214,6 +214,8 @@ func TestPublish_ImmutabilityPolicyMockSeesBothSides(t *testing.T) { assert.NotNil(t, c.Inspection) } else { sawIncoming = true + require.NotNil(t, c.Inspection) + assert.Equal(t, "changed", c.Inspection.Record.Metadata.Description) assert.NotNil(t, c.Prepared) } return false, "" diff --git a/pkg/artifact/service/pull.go b/pkg/artifact/service/pull.go index 4d9ade84709..fe125659cc1 100644 --- a/pkg/artifact/service/pull.go +++ b/pkg/artifact/service/pull.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "os" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -67,7 +68,7 @@ func (s *Service) download(ctx context.Context, op *operation, request *artifact if err := bindContent(content, source.inspection); err != nil { return err } - guard := materialize.WithReplaceGuard(func(dest string) error { + guard := materialize.WithReplaceGuard(func(dest *os.Root) error { return recheckDestination(dest, request.Force, source.inspection.Record) }) if err := materialize.Materialize(ctx, content, request.Destination, guard); err != nil { @@ -109,13 +110,13 @@ func checkDestination(op *operation, request *artifact.PullRequest, record *arti // recheckDestination repeats the conflict decision right before the destination is replaced, // because files may have appeared in it while the artifact was downloading. An empty destination // and a destination that already matches are always replaceable; different content needs force. -func recheckDestination(dest string, force bool, record *artifact.Record) error { - state, err := materialize.InspectDestination(dest, record) +func recheckDestination(dest *os.Root, force bool, record *artifact.Record) error { + state, err := materialize.InspectBoundDestination(dest, record) if err != nil { return err } if state == materialize.DestinationDiffers && !force { - return fmt.Errorf("%w: destination %q changed during the download and now holds different content; use force to replace it", errUtils.ErrArtifactConflict, dest) + return fmt.Errorf("%w: destination %q changed during the download and now holds different content; use force to replace it", errUtils.ErrArtifactConflict, dest.Name()) } return nil } diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go index 80341700226..f4a31209561 100644 --- a/pkg/artifact/service/repositories.go +++ b/pkg/artifact/service/repositories.go @@ -130,10 +130,13 @@ type Candidate struct { // Repository is the repository alias involved. Repository string - // Inspection is the metadata view, set for stored artifacts and for copy sources. + // Inspection is the metadata view for every candidate, including prepared publications + // and native copies. Policies must classify through this view; Record may be nil for + // foreign graphs. Pinned is unset until a prepared publication is stored. Inspection *artifact.Inspection - // Prepared is the artifact about to be published, when one was prepared. + // Prepared optionally provides staged content. It is nil for native copies and stored + // artifacts, so policies must not rely on it to classify an incoming release. Prepared *artifact.PreparedArtifact } diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index 112a1cb5e14..b35e1fe1bd7 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -194,6 +194,18 @@ func (c *capturer) recheckFile(record *sourceRecord, now os.FileInfo) error { if !sameFileState(record.info, final) { return changedError(record.rel, errSizeOrTimeChanged) } + return recheckPath(record) +} + +// recheckPath rejects a replacement of the pathname while its original descriptor was hashed. +func recheckPath(record *sourceRecord) error { + current, err := os.Lstat(record.abs) + if err != nil { + return changedError(record.rel, err) + } + if !os.SameFile(record.info, current) || !sameFileState(record.info, current) { + return changedError(record.rel, errSizeOrTimeChanged) + } return nil } diff --git a/pkg/artifact/snapshot/recheck_internal_test.go b/pkg/artifact/snapshot/recheck_internal_test.go index 7e19b2ee126..93394636c85 100644 --- a/pkg/artifact/snapshot/recheck_internal_test.go +++ b/pkg/artifact/snapshot/recheck_internal_test.go @@ -151,3 +151,21 @@ func TestCheckLinkContainment_UsesTheCapturedLinkTargets(t *testing.T) { }) } } + +func TestCapture_RejectsPathReplacementDuringTheFinalHash(t *testing.T) { + src := t.TempDir() + file := filepath.Join(src, "data.txt") + require.NoError(t, os.WriteFile(file, []byte("original"), 0o644)) + info, err := os.Stat(file) + require.NoError(t, err) + replacement := filepath.Join(t.TempDir(), "replacement") + require.NoError(t, os.WriteFile(replacement, []byte("original"), info.Mode())) + require.NoError(t, os.Chtimes(replacement, info.ModTime(), info.ModTime())) + hook := func(rel string) { + require.Equal(t, "data.txt", rel) + require.NoError(t, os.Rename(replacement, file)) + } + snap, err := Capture(t.Context(), src, WithStagingRoot(t.TempDir()), withRehashHook(hook)) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) + assert.Nil(t, snap) +} From d9bad8d6c33c9a4fca9979dd0cedc47004453b5c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 20:30:37 -0600 Subject: [PATCH 51/52] fix(oci): preserve layout classification and token precedence Classify all read-only layout source errors, honor registry tokens before decoding Basic auth, and track guarded write recovery work. --- pkg/oci/transport_copy.go | 4 +++ pkg/oci/transport_internal_test.go | 52 ++++++++++++++++++++++++++++++ pkg/oci/transport_layout.go | 29 +++++++++++++++-- pkg/oci/transport_registry.go | 4 +++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/pkg/oci/transport_copy.go b/pkg/oci/transport_copy.go index 91a99d2844e..0f8474eb698 100644 --- a/pkg/oci/transport_copy.go +++ b/pkg/oci/transport_copy.go @@ -344,6 +344,8 @@ func (g guardedStore) Exists(ctx context.Context, target ocispec.Descriptor) (bo } func (g guardedStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { + defer perf.Track(nil, "oci.guardedStore.Push")() + if err := ctx.Err(); err != nil { return err } @@ -364,6 +366,8 @@ func (g guardedStore) Push(ctx context.Context, expected ocispec.Descriptor, con // PushReference forwards to the wrapped store when it can push a manifest under a reference, // which lets registry destinations write the root manifest directly under its tag. func (g guardedStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + defer perf.Track(nil, "oci.guardedStore.PushReference")() + if err := ctx.Err(); err != nil { return err } diff --git a/pkg/oci/transport_internal_test.go b/pkg/oci/transport_internal_test.go index 974abcd3518..8f4631c40b5 100644 --- a/pkg/oci/transport_internal_test.go +++ b/pkg/oci/transport_internal_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/errcode" @@ -410,3 +411,54 @@ func TestReadSubjectStillRejectsAWrongDigestWhenTheContextStaysActive(t *testing require.ErrorIs(t, err, errUtils.ErrOCIIntegrity) assert.Nil(t, got) } + +func TestReadOnlyLayoutClassifiesEveryReadError(t *testing.T) { + store := NewMockRepositoryStore(gomock.NewController(t)) + store.EXPECT().Fetch(gomock.Any(), gomock.Any()).Return(nil, os.ErrPermission) + store.EXPECT().Exists(gomock.Any(), gomock.Any()).Return(false, os.ErrPermission) + store.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(ocispec.Descriptor{}, os.ErrPermission) + store.EXPECT().Predecessors(gomock.Any(), gomock.Any()).Return(nil, os.ErrPermission) + layout := &readOnlyLayout{store: store} + _, fetchErr := layout.Fetch(t.Context(), ocispec.Descriptor{}) + _, existsErr := layout.Exists(t.Context(), ocispec.Descriptor{}) + _, resolveErr := layout.Resolve(t.Context(), "missing") + _, predecessorsErr := layout.Predecessors(t.Context(), ocispec.Descriptor{}) + for _, err := range []error{fetchErr, existsErr, resolveErr, predecessorsErr} { + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.ErrorIs(t, err, os.ErrPermission) + } +} + +func TestReadOnlyLayoutMissingReferenceIsNotRegistryFailure(t *testing.T) { + directory := t.TempDir() + _, err := OpenLayout(t.Context(), directory) + require.NoError(t, err) + source, err := OpenLayoutReadOnly(t.Context(), directory) + require.NoError(t, err) + destination, err := OpenLayout(t.Context(), t.TempDir()) + require.NoError(t, err) + _, err = Copy(t.Context(), Source{Store: source, Reference: "missing"}, Destination{Store: destination, Reference: "copy"}) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + assert.ErrorIs(t, err, errdef.ErrNotFound) + assert.NotErrorIs(t, err, errUtils.ErrOCIRegistry) +} + +func TestRepositoryRegistryTokenIgnoresMalformedBasicAuth(t *testing.T) { + repository := Repository{Registry: "registry.example.com", Path: "acme/plugin"} + resource, err := repository.legacy() + require.NoError(t, err) + resolver := NewMockAuthenticationResolver(gomock.NewController(t)) + resolver.EXPECT().Resolve(gomock.Any(), resource).Return(authn.FromConfig(authn.AuthConfig{RegistryToken: "valid-token", Auth: "!"}), true, nil) + client := NewClient(nil, WithAuthenticationResolver(resolver)) + opened, err := client.Repository(t.Context(), repository) + require.NoError(t, err) + store, ok := opened.(*registryStore) + require.True(t, ok) + authClient, ok := store.repo.Client.(*auth.Client) + require.True(t, ok) + credential, err := authClient.Credential(t.Context(), repository.Registry) + require.NoError(t, err) + assert.Equal(t, "valid-token", credential.AccessToken) + assert.Empty(t, credential.Username) + assert.Empty(t, credential.Password) +} diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index a009fd1fd73..6d80337e6e0 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -44,7 +44,7 @@ type Layout struct { var ( _ GraphStore = (*Layout)(nil) - _ ReadOnlyGraphStore = (*oci.ReadOnlyStore)(nil) + _ ReadOnlyGraphStore = (*readOnlyLayout)(nil) ) // OpenLayout opens or creates a layout directory. @@ -81,7 +81,32 @@ func OpenLayoutReadOnly(ctx context.Context, directory string) (ReadOnlyGraphSto if err != nil { return nil, errors.Join(errUtils.ErrOCILayout, err) } - return store, nil + return &readOnlyLayout{store: store}, nil +} + +// readOnlyLayout preserves the layout error classification on every source operation. +type readOnlyLayout struct { + store ReadOnlyGraphStore +} + +func (l *readOnlyLayout) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := l.store.Fetch(ctx, target) + return reader, layoutError(err) +} + +func (l *readOnlyLayout) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + exists, err := l.store.Exists(ctx, target) + return exists, layoutError(err) +} + +func (l *readOnlyLayout) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + desc, err := l.store.Resolve(ctx, reference) + return desc, layoutError(err) +} + +func (l *readOnlyLayout) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + predecessors, err := l.store.Predecessors(ctx, node) + return predecessors, layoutError(err) } func (l *Layout) indexPath() string { return filepath.Join(l.root, ocispec.ImageIndexFile) } diff --git a/pkg/oci/transport_registry.go b/pkg/oci/transport_registry.go index af59a58b95b..9cbf2987741 100644 --- a/pkg/oci/transport_registry.go +++ b/pkg/oci/transport_registry.go @@ -125,6 +125,10 @@ func credentialFrom(authenticator authn.Authenticator) (auth.Credential, error) RefreshToken: config.IdentityToken, AccessToken: config.RegistryToken, } + // A registry bearer token takes precedence over Basic credentials, even malformed Auth. + if credential.AccessToken != "" { + return credential, nil + } // An encoded Auth fills whichever of the username and password is missing, the way the registry // transport does; a username or password given explicitly keeps its value. if config.Auth != "" && (credential.Username == "" || credential.Password == "") { From a237e652599f2cb8b6f35ed5fb00acbd5726b6d3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 20:31:32 -0600 Subject: [PATCH 52/52] test(snapshot): respect Windows open-file replacement restriction --- pkg/artifact/snapshot/recheck_internal_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/artifact/snapshot/recheck_internal_test.go b/pkg/artifact/snapshot/recheck_internal_test.go index 93394636c85..30cc6ae8566 100644 --- a/pkg/artifact/snapshot/recheck_internal_test.go +++ b/pkg/artifact/snapshot/recheck_internal_test.go @@ -153,6 +153,9 @@ func TestCheckLinkContainment_UsesTheCapturedLinkTargets(t *testing.T) { } func TestCapture_RejectsPathReplacementDuringTheFinalHash(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows denies replacement while the source descriptor is open") + } src := t.TempDir() file := filepath.Join(src, "data.txt") require.NoError(t, os.WriteFile(file, []byte("original"), 0o644))