From 942fc5224d51e4c0adcea23fd5618ccdf26eda0f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 10:47:41 -0500 Subject: [PATCH 01/53] feat(artifacts): add aws/s3 artifact repository (A05b) Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/options.go | 30 ++- pkg/artifact/options_test.go | 20 ++ pkg/artifact/s3/build.go | 111 ++++++++ pkg/artifact/s3/conformance_test.go | 38 +++ pkg/artifact/s3/delete.go | 114 ++++++++ pkg/artifact/s3/delete_test.go | 121 +++++++++ pkg/artifact/s3/doc.go | 66 +++++ pkg/artifact/s3/edge_test.go | 293 +++++++++++++++++++++ pkg/artifact/s3/emulator_test.go | 52 ++++ pkg/artifact/s3/errors.go | 79 ++++++ pkg/artifact/s3/evidence.go | 304 ++++++++++++++++++++++ pkg/artifact/s3/faults.go | 62 +++++ pkg/artifact/s3/helpers_test.go | 390 ++++++++++++++++++++++++++++ pkg/artifact/s3/identity.go | 37 +++ pkg/artifact/s3/identity_test.go | 179 +++++++++++++ pkg/artifact/s3/integrity_test.go | 278 ++++++++++++++++++++ pkg/artifact/s3/internal_test.go | 139 ++++++++++ pkg/artifact/s3/job.go | 154 +++++++++++ pkg/artifact/s3/keys.go | 86 ++++++ pkg/artifact/s3/list_test.go | 150 +++++++++++ pkg/artifact/s3/location_test.go | 194 ++++++++++++++ pkg/artifact/s3/lock.go | 283 ++++++++++++++++++++ pkg/artifact/s3/lock_test.go | 321 +++++++++++++++++++++++ pkg/artifact/s3/objects.go | 166 ++++++++++++ pkg/artifact/s3/options.go | 124 +++++++++ pkg/artifact/s3/plan.go | 146 +++++++++++ pkg/artifact/s3/publish.go | 248 ++++++++++++++++++ pkg/artifact/s3/publish_test.go | 214 +++++++++++++++ pkg/artifact/s3/read.go | 291 +++++++++++++++++++++ pkg/artifact/s3/recovery_test.go | 248 ++++++++++++++++++ pkg/artifact/s3/register.go | 58 +++++ pkg/artifact/s3/register_test.go | 119 +++++++++ pkg/artifact/s3/repository.go | 292 +++++++++++++++++++++ pkg/artifact/s3/reserve.go | 191 ++++++++++++++ pkg/artifact/s3/resolve.go | 200 ++++++++++++++ pkg/artifact/s3/roundtrip_test.go | 130 ++++++++++ pkg/artifact/s3/service_test.go | 158 +++++++++++ pkg/artifact/s3/signature_test.go | 224 ++++++++++++++++ pkg/artifact/s3/state.go | 217 ++++++++++++++++ pkg/artifact/s3/upload.go | 61 +++++ 40 files changed, 6587 insertions(+), 1 deletion(-) create mode 100644 pkg/artifact/s3/build.go create mode 100644 pkg/artifact/s3/conformance_test.go create mode 100644 pkg/artifact/s3/delete.go create mode 100644 pkg/artifact/s3/delete_test.go create mode 100644 pkg/artifact/s3/doc.go create mode 100644 pkg/artifact/s3/edge_test.go create mode 100644 pkg/artifact/s3/emulator_test.go create mode 100644 pkg/artifact/s3/errors.go create mode 100644 pkg/artifact/s3/evidence.go create mode 100644 pkg/artifact/s3/faults.go create mode 100644 pkg/artifact/s3/helpers_test.go create mode 100644 pkg/artifact/s3/identity.go create mode 100644 pkg/artifact/s3/identity_test.go create mode 100644 pkg/artifact/s3/integrity_test.go create mode 100644 pkg/artifact/s3/internal_test.go create mode 100644 pkg/artifact/s3/job.go create mode 100644 pkg/artifact/s3/keys.go create mode 100644 pkg/artifact/s3/list_test.go create mode 100644 pkg/artifact/s3/location_test.go create mode 100644 pkg/artifact/s3/lock.go create mode 100644 pkg/artifact/s3/lock_test.go create mode 100644 pkg/artifact/s3/objects.go create mode 100644 pkg/artifact/s3/options.go create mode 100644 pkg/artifact/s3/plan.go create mode 100644 pkg/artifact/s3/publish.go create mode 100644 pkg/artifact/s3/publish_test.go create mode 100644 pkg/artifact/s3/read.go create mode 100644 pkg/artifact/s3/recovery_test.go create mode 100644 pkg/artifact/s3/register.go create mode 100644 pkg/artifact/s3/register_test.go create mode 100644 pkg/artifact/s3/repository.go create mode 100644 pkg/artifact/s3/reserve.go create mode 100644 pkg/artifact/s3/resolve.go create mode 100644 pkg/artifact/s3/roundtrip_test.go create mode 100644 pkg/artifact/s3/service_test.go create mode 100644 pkg/artifact/s3/signature_test.go create mode 100644 pkg/artifact/s3/state.go create mode 100644 pkg/artifact/s3/upload.go diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index e6f44c02a8f..4ad3b35e5e7 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -145,6 +145,14 @@ type S3Options struct { // KeyPattern overrides the object key below the prefix. KeyPattern string `mapstructure:"key_pattern"` + + // Endpoint overrides the S3 endpoint URL, for example an S3-compatible emulator such as Floci. + // It never carries credentials; those come from the repository identity. + Endpoint string `mapstructure:"endpoint"` + + // PathStyle forces path-style (true) or virtual-hosted-style (false) addressing. Nil keeps the + // default: path-style whenever an endpoint override applies, virtual-hosted otherwise. + PathStyle *bool `mapstructure:"path_style"` } // Validate implements Options. @@ -160,10 +168,30 @@ func (o *S3Options) Validate() error { if o.Region != "" && !regionPattern.MatchString(o.Region) { errs = append(errs, fmt.Errorf("%w: region %q is not a valid region name", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Region))) } - errs = append(errs, validatePrefix(o.Prefix), validateS3Tags(o.Tags), validateKeyPattern(o.KeyPattern, "")) + errs = append(errs, validatePrefix(o.Prefix), validateS3Tags(o.Tags), validateKeyPattern(o.KeyPattern, ""), validateEndpoint(o.Endpoint)) return errors.Join(errs...) } +// validateEndpoint checks an S3 endpoint override: an absolute http or https URL with a host and +// no embedded credentials. It performs no network access. +func validateEndpoint(endpoint string) error { + if endpoint == "" { + return nil + } + parsed, err := url.Parse(endpoint) + switch { + case err != nil: + return fmt.Errorf("%w: endpoint %q is not a valid URL: %w", errUtils.ErrArtifactConfigInvalid, endpoint, err) + case parsed.Scheme != "http" && parsed.Scheme != "https": + return fmt.Errorf("%w: endpoint %q must use the http or https scheme", errUtils.ErrArtifactConfigInvalid, endpoint) + case parsed.Host == "": + return fmt.Errorf("%w: endpoint %q has no host", errUtils.ErrArtifactConfigInvalid, endpoint) + case parsed.User != nil: + return fmt.Errorf("%w: endpoint must not embed credentials; reference an auth identity with `identity` instead", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + // ValidateForPackaging implements PackagingValidator. func (o *S3Options) ValidateForPackaging(packaging Packaging) error { defer perf.Track(nil, "artifact.S3Options.ValidateForPackaging")() diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index 7225967d632..7b715b41e71 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -77,6 +77,11 @@ func TestOptionsValidate(t *testing.T) { {name: "s3 empty tag key", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"": "v"}}, wantErr: "must be 1-128 characters"}, {name: "s3 too many tags", options: &S3Options{Bucket: "acme-artifacts", Tags: elevenTags()}, wantErr: "exceed the S3 limit"}, {name: "s3 bad key pattern", options: &S3Options{Bucket: "acme-artifacts", KeyPattern: "../x"}, wantErr: "segment"}, + {name: "s3 emulator endpoint", options: &S3Options{Bucket: "acme-artifacts", Endpoint: "http://localhost:4566"}}, + {name: "s3 endpoint without scheme", options: &S3Options{Bucket: "acme-artifacts", Endpoint: "localhost:4566"}, wantErr: "http or https"}, + {name: "s3 endpoint with file scheme", options: &S3Options{Bucket: "acme-artifacts", Endpoint: "file:///tmp/x"}, wantErr: "http or https"}, + {name: "s3 endpoint without host", options: &S3Options{Bucket: "acme-artifacts", Endpoint: "https://"}, wantErr: "no host"}, + {name: "s3 endpoint with credentials", options: &S3Options{Bucket: "acme-artifacts", Endpoint: "https://user:pass@example.com"}, wantErr: "credentials"}, {name: "registry ok", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/atmos"}}, {name: "registry with port", options: &OCIRegistryOptions{Registry: "localhost:5000"}}, {name: "registry max port", options: &OCIRegistryOptions{Registry: "localhost:65535"}}, @@ -211,3 +216,18 @@ func TestValidationCreatesNothing(t *testing.T) { require.NoError(t, err) assert.Empty(t, entries, "validation must not create destination directories") } + +func TestDecodeOptions_S3EndpointAndPathStyle(t *testing.T) { + var options S3Options + require.NoError(t, DecodeOptions(map[string]any{"bucket": "acme-artifacts", "endpoint": "http://localhost:4566", "path_style": true}, &options)) + assert.Equal(t, "http://localhost:4566", options.Endpoint) + require.NotNil(t, options.PathStyle) + assert.True(t, *options.PathStyle) + + var defaults S3Options + require.NoError(t, DecodeOptions(map[string]any{"bucket": "acme-artifacts"}, &defaults)) + assert.Nil(t, defaults.PathStyle, "an omitted path_style keeps the endpoint-dependent default") + + err := DecodeOptions(map[string]any{"bucket": "acme-artifacts", "path_style": "yes please"}, &S3Options{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} diff --git a/pkg/artifact/s3/build.go b/pkg/artifact/s3/build.go new file mode 100644 index 00000000000..ab24a76dce0 --- /dev/null +++ b/pkg/artifact/s3/build.go @@ -0,0 +1,111 @@ +package s3 + +import ( + "context" + "slices" + + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// build uploads, writes, and completes. It is the part of the sequence after the pending state is +// durable. +func (r *Repository) build(ctx context.Context, job *publishJob, plan *publishPlan, pending *refState) (*artifact.PublishResult, error) { + items, err := job.plan() + if err != nil { + return nil, err + } + if err := r.uploadAll(ctx, job, items); err != nil { + return nil, wrapWrite("upload payload", err) + } + if err := r.writeRecord(ctx, job.recordDigest, job.recordBytes); err != nil { + return nil, wrapWrite("write record", err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + rep := r.representation(job) + if err := r.writeRepresentation(ctx, rep); err != nil { + return nil, wrapWrite("write representation", err) + } + if err := r.settings.fault(faultAfterMetadata); err != nil { + return nil, err + } + completed := pending.ref.Completed(job.recordDigest) + if _, err := r.writeReference(ctx, &completed, blob.IfMatch(pending.etag)); err != nil { + return nil, wrapWrite("complete reference", err) + } + if err := r.settings.fault(faultAfterCommit); err != nil { + return nil, err + } + r.retire(ctx, job, plan) + return r.result(job.record, job.recordDigest, rep, plan.admission.Outcome(), plan.scope), nil +} + +// representation builds the representation record for the job. +func (r *Repository) representation(job *publishJob) *layout.Representation { + return &layout.Representation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: artifact.RepositoryKindS3, Name: job.record.Name, Version: job.record.Version, + RecordDigest: job.recordDigest, OperationID: job.opID, Packaging: job.prepared.Packaging, Location: job.location, + Payload: job.payload, Documents: job.documents, RepresentationDigest: job.representationDigest(), + } +} + +// result builds the publish result for a stored artifact. +func (r *Repository) result(record *artifact.Record, recordDigest digest.Digest, rep *layout.Representation, outcome artifact.Outcome, scope artifact.LockScope) *artifact.PublishResult { + locator := r.uri(rep.Location) + return &artifact.PublishResult{ + Outcome: outcome, + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, recordDigest), + Representation: rep.Artifact(locator), + Location: locator, + LockScope: scope, + } +} + +// retire removes what the completed publication replaced and the journal it owned. Everything here +// is best-effort: the reference is already complete, and a leftover is garbage, not an +// inconsistency. Only exact keys the retired state owned are removed, and never a key the new +// payload uses. +func (r *Repository) retire(ctx context.Context, job *publishJob, plan *publishPlan) { + cleanup, cancel := detached(ctx) + defer cancel() + keep, err := setOfRecord(job.record).keys(job.prepared.Packaging, job.location) + if err != nil { + return + } + if plan.abandoned != nil { + r.removePayload(cleanup, plan.abandoned, keep) + } + if plan.prior != nil { + r.retirePrior(cleanup, plan.prior, job, keep) + } + _ = r.deleteObject(cleanup, layout.OperationPath(job.opID)) +} + +// retirePrior removes the payload, Record, and evidence of the replaced publication. +func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job *publishJob, keep []string) { + if prior.RecordDigest == "" || prior.RecordDigest == job.recordDigest { + return + } + if prior.Location != "" { + if record, _, err := r.loadRecord(ctx, prior.RecordDigest); err == nil { + r.removePayload(ctx, &ownedPayload{packaging: prior.Packaging, location: prior.Location, set: setOfRecord(record)}, keep) + } + } + _ = r.deleteEvidence(ctx, prior.RecordDigest) + _ = r.deleteObject(ctx, layout.RecordPath(prior.RecordDigest)) +} + +// removePayload deletes the objects of a payload that the new publication does not reuse. +func (r *Repository) removePayload(ctx context.Context, owned *ownedPayload, keep []string) { + keys, err := owned.set.keys(owned.packaging, owned.location) + if err != nil { + return + } + keys = slices.DeleteFunc(keys, func(key string) bool { return slices.Contains(keep, key) }) + _ = r.deleteAll(ctx, keys) +} diff --git a/pkg/artifact/s3/conformance_test.go b/pkg/artifact/s3/conformance_test.go new file mode 100644 index 00000000000..6f4fd703d3e --- /dev/null +++ b/pkg/artifact/s3/conformance_test.go @@ -0,0 +1,38 @@ +package s3 + +import ( + "testing" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +func TestRepositoryConformance(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRepo(t, spec.withOpts(WithVerifier(&artifacttest.FakeVerifier{}))) + }, + Packaging: spec.packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} + +// Without a verifier the repository advertises no evidence formats and Sign returns a capability +// error instead of a successful no-op. +func TestRepositoryConformance_WithoutVerifier(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec), func(t *testing.T) { + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { return newRepo(t, spec) }, + Packaging: spec.packaging, + }) + }) + } +} diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go new file mode 100644 index 00000000000..7737b8cb7b4 --- /dev/null +++ b/pkg/artifact/s3/delete.go @@ -0,0 +1,114 @@ +package s3 + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Repository]. Deletion is scoped to the selected artifact: its +// reference, its Record, its representation, its detached evidence, and the payload objects it +// exclusively owns. Objects the repository does not manage (a user's extra object below a payload +// prefix) stay in place, and nothing outside the repository prefix is touched. +// +// The reference is removed first, with a conditional delete on the observed ETag, so the artifact +// disappears atomically; an interruption afterwards leaves garbage that a later publication of the +// same reference clears, never a reference that points at missing bytes. A pending publication is +// never deleted: its owner must repair it with the same intent. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (result *artifact.DeleteResult, err error) { + defer perf.Track(nil, "s3.Repository.Delete")() + + if options.Scope != "" && options.Scope != artifact.DeleteObject { + return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/"+string(options.Scope), + "S3 repositories delete the reference and the payload it owns together (object scope)") + } + record, _, err := r.loadRecord(ctx, pinned.RecordDigest) + if err != nil { + return nil, err + } + if options.Lock == nil { + lock, lockErr := r.acquire(ctx, record.Name+":"+record.Version, r.settings.owner, "") + if lockErr != nil { + return nil, lockErr + } + defer func() { + if releaseErr := lock.Release(ctx); err == nil { + err = releaseErr + } + }() + } + return r.deleteLocked(ctx, &pinned, &options) +} + +func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedReference, options *artifact.DeleteOptions) (*artifact.DeleteResult, error) { + v, err := r.openView(ctx, pinned) + if err != nil { + return nil, err + } + stored := v.ref.ref + if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != stored.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is %s but the delete expected %s", errUtils.ErrArtifactConflict, stored.Name, stored.Version, stored.RecordDigest, options.ExpectedCurrent.ResolvedDigest) + } + keys, err := setOfRecord(v.record).keys(v.rep.Packaging, v.rep.Location) + if err != nil { + return nil, deleteFailed(err) + } + if err := r.removeReference(ctx, v.ref); err != nil { + return nil, err + } + if err := r.deleteAll(ctx, keys); err != nil { + return nil, deleteFailed(err) + } + if err := r.deleteEvidence(ctx, stored.RecordDigest); err != nil { + return nil, deleteFailed(err) + } + for _, owned := range []string{layout.RecordPath(stored.RecordDigest), layout.RepresentationPath(stored.Key())} { + if err := r.deleteObject(ctx, owned); err != nil { + return nil, deleteFailed(err) + } + } + r.tidyOperation(ctx, stored.OperationID) + return &artifact.DeleteResult{Deleted: r.deletedLocators(v, keys), Digest: stored.RecordDigest}, nil +} + +// removeReference deletes the reference only while its ETag is unchanged. A lost response is settled +// by reading the reference: an absent one proves the delete happened. +func (r *Repository) removeReference(ctx context.Context, state *refState) error { + err := r.deleteConditional(ctx, layout.ReferencePath(state.ref.Key()), state.etag) + if err == nil { + return nil + } + if blob.IsAmbiguous(err) { + settle, cancel := detached(ctx) + defer cancel() + if current, readErr := r.loadReference(settle, state.ref.Key()); readErr == nil && current == nil { + return nil + } + } + return fmt.Errorf("%w: remove reference: %w", errUtils.ErrArtifactDeleteFailed, err) +} + +// deletedLocators reports what a deletion removed: the archive objects, or the payload prefix for +// per-file packaging. +func (r *Repository) deletedLocators(v *view, keys []string) []string { + if !isArchive(v.rep.Packaging) { + return []string{r.uri(v.rep.Location) + "/"} + } + locators := make([]string, 0, len(keys)) + for _, key := range keys { + locators = append(locators, r.uri(key)) + } + return locators +} + +// deleteFailed marks a failure to remove an artifact's objects. +func deleteFailed(err error) error { + return fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) +} diff --git a/pkg/artifact/s3/delete_test.go b/pkg/artifact/s3/delete_test.go new file mode 100644 index 00000000000..db5940160ae --- /dev/null +++ b/pkg/artifact/s3/delete_test.go @@ -0,0 +1,121 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Deletion is scoped to the selected artifact: its reference, Record, representation, evidence, and +// owned payload objects. Sibling versions and unmanaged objects stay. +func TestDelete_IsScopedToTheSelectedArtifact(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t).withOpts(WithVerifier(&artifacttest.FakeVerifier{})) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{version: "1.0.0", files: map[string]string{"data.txt": "keep"}}, artifact.PublishOptions{}) + drop := publish(t, repo, &prepareSpec{version: "2.0.0", files: map[string]string{"data.txt": "drop"}}, artifact.PublishOptions{}) + signer := newSigner() + _, err := repo.Sign(context.Background(), recordSubject(t, repo, &drop.Pinned), signer, artifact.SignOptions{}) + require.NoError(t, err) + spec.fake.Seed("unrelated/user-object.txt", []byte("mine"), nil) + if spec.packaging == artifact.PackagingNone { + spec.fake.Seed("team/app/2.0.0/extra-user-file.txt", []byte("also mine"), nil) + } + spec.fake.ResetRequests() + + result, err := repo.Delete(context.Background(), drop.Pinned, artifact.DeleteOptions{ExpectedCurrent: &drop.Pinned, Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.NotEmpty(t, result.Deleted) + assert.Equal(t, drop.Pinned.RecordDigest, result.Digest) + + _, err = repo.Resolve(context.Background(), ref(testName, "2.0.0")) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assertIntact(t, repo, ref(testName, "1.0.0")) + assert.Equal(t, []string{testName + ":1.0.0"}, listNames(t, repo)) + _, ok := spec.fake.Object("unrelated/user-object.txt") + assert.True(t, ok, "objects outside the artifact are untouched") + for _, key := range bucketKeys(spec.fake) { + assert.NotContains(t, key, drop.Pinned.RecordDigest.Encoded(), "the Record and its evidence are gone") + } + if spec.packaging == artifact.PackagingNone { + _, ok = spec.fake.Object("team/app/2.0.0/extra-user-file.txt") + assert.True(t, ok, "an unmanaged object inside the payload prefix stays") + } + + deletes := spec.fake.RequestsFor(s3test.OpDeleteObject) + require.NotEmpty(t, deletes) + assert.Equal(t, layout.ReferencePath(layout.ReferenceKey(testName, "2.0.0")), deletes[0].Key, "the reference is removed first") + assert.NotEmpty(t, deletes[0].Header.Get("If-Match"), "and conditionally on the observed ETag") + }) + } +} + +func TestDelete_RejectsStaleTargetsScopesAndPendingPublications(t *testing.T) { + spec := noneSpec().fresh(t) + var fault armedFault + repo := newRepo(t, spec.withOpts(fault.option())) + done := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + before := bucketKeys(spec.fake) + + stale := done.Pinned + stale.ResolvedDigest = digest.FromString("stale") + _, err := repo.Delete(context.Background(), done.Pinned, artifact.DeleteOptions{ExpectedCurrent: &stale}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + + _, err = repo.Delete(context.Background(), done.Pinned, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Equal(t, before, bucketKeys(spec.fake), "rejected deletes changed nothing") + + fault.hook = crashAt(faultAfterMetadata) + _, err = repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "changed"}, opID: "op-pending"}), + artifact.PublishOptions{Force: true, ExpectedPrior: &done.Pinned}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Delete(context.Background(), done.Pinned, artifact.DeleteOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "a pending publication is never deleted; its owner repairs it") +} + +// The reference removal is conditional: a reference that moved on is a failed delete, not a removal +// of someone else's state. +func TestDelete_ReferenceRemovalIsConditional(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + state, err := repo.loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + require.NotNil(t, state) + + err = repo.removeReference(context.Background(), &refState{ref: state.ref, etag: `"not-the-current-etag"`}) + require.ErrorIs(t, err, errUtils.ErrArtifactDeleteFailed) + require.True(t, blob.IsPreconditionFailed(err)) + assertIntact(t, repo, defaultRef()) + + require.NoError(t, repo.removeReference(context.Background(), state)) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +// After a delete the reference can be published again, even if cleanup left leftovers behind. +func TestDelete_ThenRepublish(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + _, err := repo.Delete(context.Background(), first.Pinned, artifact.DeleteOptions{}) + require.NoError(t, err) + assert.Empty(t, payloadKeys(spec.fake, "")) + assert.Empty(t, managedKeys(spec.fake, "", layout.ManagedRoot+"/refs")) + + again := publish(t, repo, &prepareSpec{opID: "op-again"}, artifact.PublishOptions{}) + assert.Equal(t, artifact.OutcomeCreated, again.Outcome) + assertIntact(t, repo, defaultRef()) +} diff --git a/pkg/artifact/s3/doc.go b/pkg/artifact/s3/doc.go new file mode 100644 index 00000000000..522aaa264df --- /dev/null +++ b/pkg/artifact/s3/doc.go @@ -0,0 +1,66 @@ +// Package s3 implements the aws/s3 artifact repository behind [artifact.Repository]. It stores the +// same record layout as the local repositories (package layout) and follows the same publication +// lifecycle, over a transport-only [blob.Store]. Production repositories build the AWS SDK store +// lazily from configuration; tests inject any store, typically the in-process fake S3 server. +// +// # Object layout +// +// Below the configured bucket and prefix the managed records live in the reserved +// ".atmos-artifacts/v1/" namespace: reference, Record, representation, operation journal, detached +// evidence, and lock-claim objects. The payload lives outside it at the location rendered once, at +// publication, from the key_pattern. +// +// - packaging none stores one object per file below /, with the distribution documents +// and manifest.json adjacent at /.atmos/artifact/. Directories, symlinks, and modes +// are not S3 concepts: the Record inventory carries them and materialization restores them. +// - packaging tar, tgz, or zip stores one archive object at with adjacent +// .metadata.json and .docs/README.md|CHANGELOG.md objects. +// +// The configured S3 object tags are written on every object a publication creates (payload, +// documents, records, journals, claims, evidence). Portable labels stay in the Record and are +// never copied into tags. +// +// # Publication and conditional writes +// +// A publication reserves its location and records a pending operation journal, marks the +// reference pending, uploads the payload and documents, writes the immutable Record and the +// representation, and only then flips the reference to completed. Reference ownership is a +// conditional write: If-None-Match for a first publication and If-Match with the observed ETag for +// every later state change, so a concurrent writer loses with a conflict instead of overwriting. +// ETags are concurrency tokens only; every consumed byte is verified against the digest in the +// Record. A response that was lost after the request was sent is reconciled by reading the +// reference back, never assumed to have failed. An endpoint that cannot enforce a required +// condition fails the operation with an unsupported-capability error and never overwrites. +// Streaming uploads use bounded multipart transfers and abort the upload on cancellation. +// +// An interrupted or failed force replacement leaves the reference pending, and so unavailable to +// readers, until a retry with the same intent repairs it; there is no rollback. A pending +// operation of a different intent is never stolen. +// +// # Locks and recovery +// +// Lock provides repository-scope mutation coordination through owner claim objects under the +// managed namespace. A claim is created with If-None-Match, carries an ownership token, and is +// released with a token check and a conditional delete. V1 claims never expire and are never +// stolen: a paused or crashed writer cannot become an unfenced stale owner. S3 Object Lock +// retention is unrelated and is not used. +// +// A writer that crashes while holding a claim leaves it behind. Every later attempt then fails +// with a recovery diagnostic that names the claim object, its resource, the owner, and the +// operation. The operator confirms the owning writer has stopped and deletes that one claim object +// (for example with the AWS console or "aws s3 rm"); nothing else needs to change. The brief +// repository-wide claim taken while a location is reserved is recovered the same way. +// +// # Signatures +// +// Evidence is stored as digest-named objects below the signature directory of the canonical Record +// digest. Adding evidence never changes the Record, the reference, or the payload. Cryptographic +// verification is delegated to an injected [artifact.Verifier]; without one the repository +// advertises no evidence formats. +// +// # Identity +// +// A configured identity is resolved lazily on first use, through the injected resolver, and an +// identity that cannot be resolved fails closed: there is no fallback to ambient credentials. +// Construction, capability description, and dry runs perform no network access. +package s3 diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go new file mode 100644 index 00000000000..d7346a90b65 --- /dev/null +++ b/pkg/artifact/s3/edge_test.go @@ -0,0 +1,293 @@ +package s3 + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// A same-intent retry under a different packaging cannot reuse the journaled location (its +// extension would disagree), so it publishes at a freshly rendered location and retires the +// abandoned attempt's partial payload instead of orphaning it. +func TestResumedPublicationRetiresAnAbandonedLocation(t *testing.T) { + tests := []struct { + name string + crashed *rspec + retry *rspec + abandoned []string + wantKeys []string + }{ + { + name: "tgz to zip", crashed: archiveSpec(artifact.PackagingTgz), retry: archiveSpec(artifact.PackagingZip), + abandoned: []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json"}, + wantKeys: []string{"team/app/1.0.0.zip", "team/app/1.0.0.zip.docs", "team/app/1.0.0.zip.metadata.json"}, + }, + { + name: "archive to files", crashed: archiveSpec(artifact.PackagingTgz), retry: noneSpec(), + abandoned: []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := s3test.NewServer(t) + crashed := newRepo(t, tt.crashed.with(func(s *rspec) { s.fake = fake }).withOpts(withFaults(crashAt(faultAfterMetadata)))) + _, err := crashed.Publish(context.Background(), prepare(t, crashed, &prepareSpec{opID: "op-first"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + for _, key := range tt.abandoned { + _, ok := fake.Object(key) + require.True(t, ok, "the crashed attempt left %s behind", key) + } + + retry := newRepo(t, tt.retry.with(func(s *rspec) { s.fake = fake })) + _, err = retry.Publish(context.Background(), prepare(t, retry, &prepareSpec{opID: "op-retry"}), artifact.PublishOptions{}) + require.NoError(t, err, "the same intent repairs the pending reference whatever the packaging") + for _, key := range tt.abandoned { + _, ok := fake.Object(key) + assert.False(t, ok, "the abandoned object %s was retired, not orphaned", key) + } + assertIntact(t, retry, defaultRef()) + assert.Empty(t, managedKeys(fake, "", layout.ManagedRoot+"/operations")) + }) + } +} + +// A Record object that already exists with different bytes is corruption, never replaced. +func TestPublish_ExistingRecordWithDifferentBytesIsRejected(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + recordDigest, err := prepared.Record.RecordDigest() + require.NoError(t, err) + spec.fake.Seed(layout.RecordPath(recordDigest), []byte("squatter"), nil) + + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + object, ok := spec.fake.Object(layout.RecordPath(recordDigest)) + require.True(t, ok) + assert.Equal(t, []byte("squatter"), object.Data, "the existing object is left as found") + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the reference never completed") +} + +// Managed documents that disagree with where they live are corruption. +func TestTamperedManagedDocuments(t *testing.T) { + setup := func(t *testing.T) (*Repository, *s3test.Server, artifact.PinnedReference) { + repo, spec, pinned := setupTampering(t) + _ = spec + return repo, spec.fake, pinned + } + otherRecord := func(t *testing.T, repo *Repository) digest.Digest { + return publish(t, repo, &prepareSpec{name: "other/app"}, artifact.PublishOptions{}).Pinned.RecordDigest + } + t.Run("a reference naming another artifact's record", func(t *testing.T) { + repo, fake, _ := setup(t) + other := otherRecord(t, repo) + state, err := repo.loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + forged := *state.ref + forged.RecordDigest = other + raw, err := forged.Marshal() + require.NoError(t, err) + fake.Seed(layout.ReferencePath(forged.Key()), raw, nil) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("a reference stored under another reference's key", func(t *testing.T) { + repo, fake, _ := setup(t) + otherRecord(t, repo) + state, err := repo.loadReference(context.Background(), layout.ReferenceKey("other/app", testVersion)) + require.NoError(t, err) + raw, err := state.ref.Marshal() + require.NoError(t, err) + fake.Seed(layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), raw, nil) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("a representation stored under another reference's key", func(t *testing.T) { + repo, fake, pinned := setup(t) + otherRecord(t, repo) + rep, err := repo.loadRepresentation(context.Background(), layout.ReferenceKey("other/app", testVersion)) + require.NoError(t, err) + raw, err := rep.Marshal() + require.NoError(t, err) + fake.Seed(layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)), raw, nil) + _, err = repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("an archive representation with two payloads", func(t *testing.T) { + repo, fake, pinned := setup(t) + rep, err := repo.loadRepresentation(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + rep.Payload = append(rep.Payload, rep.Payload[0]) + raw, err := rep.Marshal() + require.NoError(t, err) + fake.Seed(layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)), raw, nil) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("an operation journal describing another operation", func(t *testing.T) { + repo, spec, _ := setupTampering(t) + fake := spec.fake + other := publishPending(t, spec, "op-real") + raw, _, err := repo.readObject(context.Background(), layout.OperationPath(other), layout.MaxDocumentBytes) + require.NoError(t, err) + require.NotNil(t, raw) + fake.Seed(layout.OperationPath("op-claimed"), raw, nil) + _, err = repo.loadOperation(context.Background(), "op-claimed") + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("an oversize reference", func(t *testing.T) { + repo, fake, _ := setup(t) + fake.Seed(layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), bytes.Repeat([]byte("x"), int(layout.MaxDocumentBytes)+1), nil) + _, err := repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + }) +} + +func setupTampering(t *testing.T) (*Repository, *rspec, artifact.PinnedReference) { + t.Helper() + spec := archiveSpec(artifact.PackagingTgz).fresh(t) + repo := newRepo(t, spec) + return repo, spec, publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned +} + +// publishPending leaves a crashed publication behind and returns its operation ID. +func publishPending(t *testing.T, spec *rspec, opID string) string { + t.Helper() + crashed := newRepo(t, spec.withOpts(withFaults(crashAt(faultAfterMetadata)))) + _, err := crashed.Publish(context.Background(), prepare(t, crashed, &prepareSpec{name: "pending/app", opID: opID}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + return opID +} + +// A digest reference to a Record whose reference was deleted is not found. +func TestResolveByDigest_ReferenceDeletedUnderneath(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + require.NoError(t, repo.deleteObject(context.Background(), layout.ReferencePath(layout.ReferenceKey(testName, testVersion)))) + _, err := repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: pinned.RecordDigest}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + require.ErrorContains(t, err, "reference was deleted") +} + +// The generation recheck reports a reference that vanished while its bytes were read. +func TestRead_GuardReportsAVanishedReference(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NoError(t, repo.deleteObject(context.Background(), layout.ReferencePath(layout.ReferenceKey(testName, testVersion)))) + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactNotFound) +} + +// A failure while deleting payload objects leaves the reference gone and garbage behind, never a +// reference that points at missing bytes, and a republish clears the garbage. +func TestDelete_PartialFailureLeavesGarbageNotADanglingReference(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{files: map[string]string{"a.txt": "a", "b.txt": "b"}}, artifact.PublishOptions{}).Pinned + spec.fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, KeyPrefix: "team/app/1.0.0/b.txt", Status: 403, Code: "AccessDenied", Times: -1}) + + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDeleteFailed) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the reference was removed first") + + spec.fake.ClearFaults() + again := publish(t, repo, &prepareSpec{files: map[string]string{"a.txt": "a2", "b.txt": "b2"}, opID: "op-again"}, artifact.PublishOptions{}) + assert.Equal(t, artifact.OutcomeCreated, again.Outcome, "leftover payload objects are still owned by the reference's representation") + assertIntact(t, repo, defaultRef()) +} + +// A lost response to the reference delete is settled by reading the reference. +func TestDelete_LostResponseIsReconciled(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + spec.fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, KeyPrefix: layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), Action: s3test.DropAfterCommit}) + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{}) + require.NoError(t, err) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +// A create that never reached the endpoint is retried transparently. +func TestLock_CreateThatNeverArrivedIsRetried(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.DropBeforeCommit}) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err) + require.NoError(t, handle.Release(context.Background())) +} + +// More evidence than the read bound is reported, not silently dropped. +func TestVerify_BoundsTheNumberOfEvidenceObjectsRead(t *testing.T) { + spec := signedSpec(t, noneSpec()) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + for index := 0; index < maxEvidenceFiles+3; index++ { + stored, err := layout.NewEvidence(subject, &artifact.EvidencePayload{Format: "fake", MediaType: "application/json", Data: []byte(strings.Repeat("x", index+1))}) + require.NoError(t, err) + raw, err := stored.Marshal() + require.NoError(t, err) + require.NoError(t, repo.putBytes(context.Background(), layout.SignaturePath(subject.Kind, subject.Digest, stored.Digest()), raw)) + } + loaded, diagnostics, err := repo.loadEvidence(context.Background(), subject.Digest) + require.NoError(t, err) + assert.Len(t, loaded, maxEvidenceFiles) + require.Len(t, diagnostics, 1) + assert.Equal(t, "evidence-truncated", diagnostics[0].Code) +} + +func TestSign_RejectsOversizeEvidence(t *testing.T) { + repo := newRepo(t, signedSpec(t, noneSpec())) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + huge := &oversizeSigner{} + _, err := repo.Sign(context.Background(), recordSubject(t, repo, &pinned), huge, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +// oversizeSigner returns evidence past the bounded-read limit. +type oversizeSigner struct{} + +func (*oversizeSigner) Formats() []artifact.EvidenceFormat { + return []artifact.EvidenceFormat{artifacttest.FakeEvidenceFormat} +} + +func (*oversizeSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + return &artifact.EvidencePayload{Format: artifacttest.FakeEvidenceFormat, MediaType: "application/json", Data: make([]byte, artifact.MaxEvidenceSize+1)}, nil +} + +// The configured owner is recorded in claims and journals, and a part size reaches the SDK store. +func TestOptions_OwnerAndPartSize(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + cfg := s3Config(map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1"}) + repo, err := New("owned", cfg, WithOwner("ci-runner-7"), WithPartSize(smallPart), WithConcurrency(2), WithLockWait(time.Second)) + require.NoError(t, err) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource}) + require.NoError(t, err) + assert.Equal(t, "ci-runner-7", handle.Owner(), "an unset request owner falls back to the configured one") + require.NoError(t, handle.Release(context.Background())) + + publish(t, repo, &prepareSpec{files: bigFiles()}, artifact.PublishOptions{}) + assert.GreaterOrEqual(t, fake.CompletedUploads(), 1, "the configured part size forced multipart uploads") + assert.NotEmpty(t, defaultOwner()) +} diff --git a/pkg/artifact/s3/emulator_test.go b/pkg/artifact/s3/emulator_test.go new file mode 100644 index 00000000000..92096c200e8 --- /dev/null +++ b/pkg/artifact/s3/emulator_test.go @@ -0,0 +1,52 @@ +package s3 + +import ( + "crypto/rand" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/schema" +) + +// TestEmulatorConformance runs the shared repository contract suite against a real S3-compatible +// endpoint (Floci, MinIO, or AWS). It is skipped unless ATMOS_BLOB_S3_ENDPOINT and +// ATMOS_BLOB_S3_BUCKET are set (ATMOS_BLOB_S3_REGION is optional); credentials come from the usual +// AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY variables and the bucket must already exist. The +// repositories talk through the production SDK client, so the run also proves that the endpoint +// enforces conditional writes (the conditional-write probe fails the run otherwise). Every subtest +// uses a unique key prefix, so runs and packagings never collide. +func TestEmulatorConformance(t *testing.T) { + emulator := s3test.EmulatorFromEnv(t) + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz} { + t.Run(string(packaging), func(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + var raw [6]byte + _, err := rand.Read(raw[:]) + require.NoError(t, err) + cfg := &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindS3, Packaging: string(packaging), + Options: map[string]any{ + "bucket": emulator.Bucket, "endpoint": emulator.Endpoint, "region": emulator.Region, + "prefix": "atmos-artifacts-conformance/" + hex.EncodeToString(raw[:]), + "tags": map[string]any{"purpose": "conformance"}, + }, + } + repo, err := New("emulator", cfg, WithVerifier(&artifacttest.FakeVerifier{}), WithPartSize(5<<20)) + require.NoError(t, err) + return repo + }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} diff --git a/pkg/artifact/s3/errors.go b/pkg/artifact/s3/errors.go new file mode 100644 index 00000000000..dfe3e02c485 --- /dev/null +++ b/pkg/artifact/s3/errors.go @@ -0,0 +1,79 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// Transport operations, used to choose the generic sentinel of an unclassified failure. +const ( + opRead = "read" + opWrite = "write" + opDelete = "delete" +) + +// mapErr translates a blob failure into the artifact vocabulary while keeping the original error in +// the chain, so callers can still match both with errors.Is. Context errors pass through untouched. +func (r *Repository) mapErr(op, what string, err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return err + case blob.IsConditionalUnsupported(err): + unsupported := artifact.NewUnsupportedCapability(r.alias, op, "conditional-writes", + fmt.Sprintf("the S3 endpoint cannot enforce the conditional write needed for %s; use an endpoint that honors If-None-Match and If-Match, or AWS S3", what)) + unsupported.Cause = err + return unsupported + case blob.IsPreconditionFailed(err): + return fmt.Errorf("%w: %s changed concurrently: %w", errUtils.ErrArtifactConflict, what, err) + case blob.IsAmbiguous(err): + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactPublicationAmbiguous, what, err) + case errors.Is(err, errUtils.ErrBlobIdentityUnresolved): + return fmt.Errorf("repository %q cannot authenticate to S3 for %s: %w", r.alias, what, err) + default: + return fmt.Errorf("%w: %s: %w", genericSentinel(op), what, err) + } +} + +func genericSentinel(op string) error { + switch op { + case opRead: + return errUtils.ErrArtifactDownloadFailed + case opDelete: + return errUtils.ErrArtifactDeleteFailed + default: + return errUtils.ErrArtifactUploadFailed + } +} + +// isArtifactError reports whether err already carries an artifact sentinel the caller matches on, +// so a generic wrapper does not hide it. +func isArtifactError(err error) bool { + for _, sentinel := range []error{ + errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactIntegrityFailed, + errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactUnsafePath, errUtils.ErrArtifactTooLarge, + errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactRecordInvalid, errUtils.ErrArtifactPublicationAmbiguous, + errUtils.ErrArtifactNotFound, errUtils.ErrBlobIdentityUnresolved, + context.Canceled, context.DeadlineExceeded, + } { + if errors.Is(err, sentinel) { + return true + } + } + return isCrash(err) +} + +// wrapWrite classifies a publication failure. Context errors and already-typed artifact errors pass +// through so callers keep matching on them. +func wrapWrite(step string, err error) error { + if isArtifactError(err) { + return err + } + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactUploadFailed, step, err) +} diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go new file mode 100644 index 00000000000..a10e13811af --- /dev/null +++ b/pkg/artifact/s3/evidence.go @@ -0,0 +1,304 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxEvidenceFiles bounds how many evidence objects one verification or export reads. More than +// this is reported, not silently dropped. +const maxEvidenceFiles = 64 + +// loadedEvidence is one stored evidence object that parsed and matched its name. +type loadedEvidence struct { + evidence *layout.Evidence + locator string +} + +// loadEvidence reads the evidence stored for a Record subject. Malformed or oversize objects are +// reported as diagnostics and skipped; they can never override valid matching evidence. A failure +// to list or read the store is an error, because "no evidence" must never mean "could not look". +func (r *Repository) loadEvidence(ctx context.Context, subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic, error) { + dir := layout.SignatureDir(artifact.SubjectRecord, subject) + "/" + objects, err := r.listPrefix(ctx, dir) + if err != nil { + return nil, nil, err + } + var ( + loaded []loadedEvidence + diags []artifact.Diagnostic + ) + for index := range objects { + stem, ok := strings.CutSuffix(strings.TrimPrefix(objects[index].Key, dir), ".json") + if !ok || strings.Contains(stem, "/") { + continue + } + if len(loaded) >= maxEvidenceFiles { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: fmt.Sprintf("more than %d evidence files; the rest were not read", maxEvidenceFiles)}) + break + } + item, problem := r.readEvidenceObject(ctx, objects[index].Key, stem) + switch { + case errors.Is(problem, context.Canceled), errors.Is(problem, context.DeadlineExceeded): + return nil, nil, problem + case problem != nil: + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", r.uri(objects[index].Key), problem)}) + default: + loaded = append(loaded, loadedEvidence{evidence: item, locator: r.uri(objects[index].Key)}) + } + } + return loaded, diags, nil +} + +func (r *Repository) readEvidenceObject(ctx context.Context, key, hexDigest string) (*layout.Evidence, error) { + want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) + if err := want.Validate(); err != nil { + return nil, err + } + raw, _, err := r.readObject(ctx, key, layout.MaxEvidenceFileBytes) + if err != nil { + return nil, err + } + if raw == nil { + return nil, fmt.Errorf("%w: evidence object disappeared", errUtils.ErrArtifactNotFound) + } + return layout.DecodeEvidence(raw, want) +} + +// evidenceRefs converts loaded evidence into descriptors. +func (r *Repository) evidenceRefs(loaded []loadedEvidence) []artifact.SignatureEvidence { + out := make([]artifact.SignatureEvidence, 0, len(loaded)) + for index := range loaded { + out = append(out, loaded[index].evidence.Reference(loaded[index].locator)) + } + return out +} + +// requireCurrentRecord checks that digest names the current completed Record of its reference and +// returns the canonical bytes. Evidence only ever binds to a live subject. +func (r *Repository) requireCurrentRecord(ctx context.Context, recordDigest digest.Digest) ([]byte, error) { + record, raw, err := r.loadRecord(ctx, recordDigest) + if err != nil { + return nil, err + } + if _, err := r.resolveRef(ctx, artifact.Ref{Name: record.Name, Digest: recordDigest}); err != nil { + return nil, err + } + return raw, nil +} + +// checkSubject validates a signature subject for this repository. +func (r *Repository) checkSubject(operation string, subject artifact.SignatureSubject) error { + if err := subject.Validate(); err != nil { + return err + } + if caps := r.Capabilities(); !caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.alias, operation, "subject/"+string(subject.Kind), + "S3 repositories bind detached evidence to the canonical Record only") + } + return nil +} + +// Sign implements [artifact.Repository]: it asks the signer for evidence over the Record subject +// and stores it as an object named by the evidence digest below the subject's directory. Evidence +// lives beside the Record, so signing never changes the Record, the reference, or the payload. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "s3.Repository.Sign")() + + if err := r.checkSubject("sign", subject); err != nil { + return nil, err + } + if signer == nil { + return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) + } + raw, err := r.requireCurrentRecord(ctx, subject.Digest) + if err != nil { + return nil, err + } + if subject.Size != 0 && subject.Size != int64(len(raw)) { + return nil, fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, len(raw)) + } + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, err + } + if payload == nil { + return nil, fmt.Errorf("%w: the signer returned no evidence", errUtils.ErrArtifactVerificationFailed) + } + stored, err := r.storeEvidence(ctx, subject, payload) + if err != nil { + return nil, err + } + return []artifact.SignatureEvidence{stored}, nil +} + +// storeEvidence validates and writes one evidence payload. +func (r *Repository) storeEvidence(ctx context.Context, subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (artifact.SignatureEvidence, error) { + if caps := r.Capabilities(); !caps.HasEvidenceFormat(payload.Format) { + return artifact.SignatureEvidence{}, artifact.NewUnsupportedCapability(r.alias, "sign", "evidence/"+string(payload.Format), + fmt.Sprintf("the repository stores and verifies %v evidence; configure a verifier for this format", caps.EvidenceFormats)) + } + evidence, err := layout.NewEvidence(subject, payload) + if err != nil { + return artifact.SignatureEvidence{}, err + } + raw, err := evidence.Marshal() + if err != nil { + return artifact.SignatureEvidence{}, err + } + key := layout.SignaturePath(subject.Kind, subject.Digest, evidence.Digest()) + if err := r.putImmutable(ctx, key, raw, contentTypeJSON, layout.MaxEvidenceFileBytes); err != nil { + return artifact.SignatureEvidence{}, wrapWrite("store evidence", err) + } + return evidence.Reference(r.uri(key)), nil +} + +// deleteEvidence removes every evidence object stored for a Record subject. +func (r *Repository) deleteEvidence(ctx context.Context, subject digest.Digest) error { + objects, err := r.listPrefix(ctx, layout.SignatureDir(artifact.SubjectRecord, subject)+"/") + if err != nil { + return err + } + keys := make([]string, 0, len(objects)) + for index := range objects { + keys = append(keys, objects[index].Key) + } + return r.deleteAll(ctx, keys) +} + +// Verify implements [artifact.Repository]. It authenticates the stored evidence with the injected +// verifier; it never trusts a cached verdict. No evidence, an unverifiable format, or an +// unsatisfied policy is a failure. +func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "s3.Repository.Verify")() + + if err := r.checkSubject("verify", subject); err != nil { + return nil, err + } + verifier := r.settings.verifier + if verifier == nil { + return nil, artifact.NewUnsupportedCapability(r.alias, "verify", "evidence/verify", "no signature verifier is configured for this repository") + } + if _, err := r.requireCurrentRecord(ctx, subject.Digest); err != nil { + return nil, err + } + loaded, diags, err := r.loadEvidence(ctx, subject.Digest) + if err != nil { + return nil, err + } + blobs := evidenceBlobs(subject, loaded, &diags) + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed, Diagnostics: diags} + if fingerprint, fingerprintErr := policy.Fingerprint(); fingerprintErr == nil { + result.PolicyFingerprint = fingerprint + } + if len(blobs) == 0 { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-evidence", Message: "no signature evidence is stored for the subject"}) + return result, fmt.Errorf("%w: no signature evidence is stored for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + verified, verifyErr := verifier.Verify(ctx, subject, blobs, policy) + return finishVerification(result, verified, verifyErr) +} + +// evidenceBlobs converts the evidence bound to the subject, reporting any bound to another subject. +func evidenceBlobs(subject artifact.SignatureSubject, loaded []loadedEvidence, diags *[]artifact.Diagnostic) []artifact.EvidenceBlob { + blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) + for index := range loaded { + stored := loaded[index].evidence + if stored.Subject.Kind != subject.Kind || stored.Subject.Digest != subject.Digest { + *diags = append(*diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "subject-mismatch", Message: loaded[index].locator + " is bound to a different subject and was ignored"}) + continue + } + blobs = append(blobs, stored.Blob(loaded[index].locator)) + } + return blobs +} + +// finishVerification merges the verifier's result with the storage diagnostics and turns anything +// but a verified status into an error. +func finishVerification(base, verified *artifact.VerificationResult, verifyErr error) (*artifact.VerificationResult, error) { + result := base + if verified != nil { + merged := *verified + merged.Subject = base.Subject + merged.Diagnostics = append(append([]artifact.Diagnostic(nil), base.Diagnostics...), verified.Diagnostics...) + if merged.PolicyFingerprint == "" { + merged.PolicyFingerprint = base.PolicyFingerprint + } + result = &merged + } + switch { + case verifyErr != nil && errors.Is(verifyErr, errUtils.ErrArtifactVerificationFailed): + return result, verifyErr + case verifyErr != nil: + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) + case result.Status == artifact.VerificationFailed: + return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) + } + return result, nil +} + +// ReadEvidence implements [artifact.EvidenceReader]: it exports the stored evidence blobs of a +// pinned artifact so a copy through the shared staging path can carry them along. The evidence is +// untrusted until verified. +func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "s3.Repository.ReadEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + if _, err := r.requireCurrentRecord(ctx, pinned.RecordDigest); err != nil { + return nil, err + } + loaded, _, err := r.loadEvidence(ctx, pinned.RecordDigest) + if err != nil { + return nil, err + } + blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) + for index := range loaded { + blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) + } + return blobs, nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this +// repository already holds. The evidence must bind to the pinned Record and carry matching bytes; +// importing never changes the Record or the reference. +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "s3.Repository.WriteEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + if _, err := r.requireCurrentRecord(ctx, pinned.RecordDigest); err != nil { + return nil, err + } + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + blobEvidence := &evidence[index] + subject := blobEvidence.Evidence.Subject + if err := r.checkSubject("write evidence", subject); err != nil { + return stored, err + } + if subject.Digest != pinned.RecordDigest { + return stored, fmt.Errorf("%w: evidence is bound to %s, not to the pinned Record %s", errUtils.ErrArtifactRefInvalid, subject.Digest, pinned.RecordDigest) + } + if blobEvidence.Evidence.Descriptor.Digest != digest.FromBytes(blobEvidence.Data) { + return stored, fmt.Errorf("%w: evidence bytes do not match their descriptor %s", errUtils.ErrArtifactIntegrityFailed, blobEvidence.Evidence.Descriptor.Digest) + } + item, err := r.storeEvidence(ctx, subject, &artifact.EvidencePayload{Format: blobEvidence.Evidence.Format, MediaType: blobEvidence.Evidence.Descriptor.MediaType, Data: blobEvidence.Data}) + if err != nil { + return stored, err + } + stored = append(stored, item) + } + return stored, nil +} diff --git a/pkg/artifact/s3/faults.go b/pkg/artifact/s3/faults.go new file mode 100644 index 00000000000..861fca4bfc0 --- /dev/null +++ b/pkg/artifact/s3/faults.go @@ -0,0 +1,62 @@ +package s3 + +import ( + "errors" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// faultPoint names a point in the publication lifecycle at which tests simulate a crash. +type faultPoint string + +// Fault points, in lifecycle order. +const ( + // The before-payload point fires after the pending state was recorded and before any payload + // object is uploaded. + faultBeforePayload faultPoint = "before-payload" + + // The during-payload point fires after the first payload object was uploaded. + faultDuringPayload faultPoint = "during-payload" + + // The after-metadata point fires after the payload, Record, and representation are in place + // and before the reference is completed. + faultAfterMetadata faultPoint = "after-metadata" + + // The after-commit point fires after the reference was completed, before cleanup and before + // the caller sees the response. + faultAfterCommit faultPoint = "after-commit" +) + +// faultHook returns a non-nil error to simulate a crash at the point. +type faultHook func(point faultPoint) error + +// crashError marks an injected crash. A crashed process cannot clean up after itself, so the +// publication leaves its journal and partial objects exactly as a real interruption would. +type crashError struct{ err error } + +func (c *crashError) Error() string { + defer perf.Track(nil, "s3.crashError.Error")() + + return c.err.Error() +} + +func (c *crashError) Unwrap() error { + defer perf.Track(nil, "s3.crashError.Unwrap")() + + return c.err +} + +func (s *settings) fault(point faultPoint) error { + if s.faults == nil { + return nil + } + if err := s.faults(point); err != nil { + return &crashError{err: err} + } + return nil +} + +func isCrash(err error) bool { + var crash *crashError + return errors.As(err, &crash) +} diff --git a/pkg/artifact/s3/helpers_test.go b/pkg/artifact/s3/helpers_test.go new file mode 100644 index 00000000000..97168c4a8b8 --- /dev/null +++ b/pkg/artifact/s3/helpers_test.go @@ -0,0 +1,390 @@ +package s3 + +import ( + "context" + "errors" + "io" + "math/rand" + "sort" + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guards for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Packaging: "tgz", Identity: "id", Options: nil} + +const ( + testName = "team/app" + testVersion = "1.0.0" + + // A part size that lets tests exercise multipart uploads with kilobytes. + smallPart = 16 << 10 +) + +// errCrash is the error the fault hooks simulate a crash with. +var errCrash = errors.New("simulated crash") + +// capturedAt is the fixed preparation time of the test artifacts. +var capturedAt = time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + +// rspec configures a repository under test. +type rspec struct { + packaging artifact.Packaging + prefix string + tags map[string]string + keyPattern string + opts []Option + fake *s3test.Server + storeOpts []blobs3.Option + probe bool +} + +func (s *rspec) clone() *rspec { + c := *s + c.opts = append([]Option(nil), s.opts...) + c.storeOpts = append([]blobs3.Option(nil), s.storeOpts...) + return &c +} + +func (s *rspec) with(fn func(*rspec)) *rspec { + c := s.clone() + fn(c) + return c +} + +// fresh returns a copy of the spec bound to a new fake S3 endpoint, so the test can inspect it. +func (s *rspec) fresh(t *testing.T) *rspec { + t.Helper() + return s.with(func(c *rspec) { c.fake = s3test.NewServer(t) }) +} + +func (s *rspec) withOpts(opts ...Option) *rspec { + return s.with(func(c *rspec) { c.opts = append(c.opts, opts...) }) +} + +func (s *rspec) config(bucket string) *schema.ArtifactRepositoryConfig { + options := map[string]any{"bucket": bucket} + if s.prefix != "" { + options["prefix"] = s.prefix + } + if s.keyPattern != "" { + options["key_pattern"] = s.keyPattern + } + if len(s.tags) > 0 { + tags := map[string]any{} + for k, v := range s.tags { + tags[k] = v + } + options["tags"] = tags + } + cfg := &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Options: options} + if s.packaging != "" { + cfg.Packaging = string(s.packaging) + } + return cfg +} + +// newStoreFor builds the blob store the repository runs over, rooted at the spec's prefix. +func newStoreFor(t *testing.T, spec *rspec) *blobs3.Store { + t.Helper() + opts := []blobs3.Option{blobs3.WithPartSize(smallPart), blobs3.WithPrefix(spec.prefix)} + if spec.probe { + opts = append(opts, blobs3.WithConditionalMode(blobs3.ConditionalProbe)) + } + opts = append(opts, spec.storeOpts...) + store, err := blobs3.NewFromClient(spec.fake.Client(), spec.fake.Bucket(), opts...) + require.NoError(t, err) + return store +} + +// newRepo builds a repository over a fresh fake unless the spec names one. Lock polling is fast so +// contention tests stay quick. +func newRepo(t *testing.T, spec *rspec) *Repository { + t.Helper() + spec = spec.clone() + if spec.fake == nil { + spec.fake = s3test.NewServer(t) + } + opts := append([]Option{WithStore(newStoreFor(t, spec)), withLockPoll(5 * time.Millisecond)}, spec.opts...) + repo, err := newRepository("test", spec.config(spec.fake.Bucket()), opts, &artifact.RepositoryOptions{}) + require.NoError(t, err) + return repo +} + +func withLockPoll(poll time.Duration) Option { + return func(s *settings) { s.lockPoll = poll } +} + +func withFaults(hook faultHook) Option { + return func(s *settings) { s.faults = hook } +} + +// crashAt returns a fault hook that fails once at the point. +func crashAt(point faultPoint) faultHook { + fired := false + return func(p faultPoint) error { + if p == point && !fired { + fired = true + return errCrash + } + return nil + } +} + +// armedFault is a fault hook that does nothing until a test arms it. +type armedFault struct{ hook faultHook } + +func (a *armedFault) option() Option { + return withFaults(func(point faultPoint) error { + if a.hook == nil { + return nil + } + return a.hook(point) + }) +} + +func noneSpec() *rspec { return &rspec{packaging: artifact.PackagingNone} } + +func archiveSpec(packaging artifact.Packaging) *rspec { return &rspec{packaging: packaging} } + +var allSpecs = []*rspec{noneSpec(), archiveSpec(artifact.PackagingTar), archiveSpec(artifact.PackagingTgz), archiveSpec(artifact.PackagingZip)} + +func specName(spec *rspec) string { return string(spec.packaging) } + +// prepareSpec describes the artifact a test prepares with the production files producer. +type prepareSpec struct { + name string + version string + packaging artifact.Packaging + files map[string]string + at time.Time + opID string + labels map[string]string + source string +} + +func (p *prepareSpec) withDefaults(packaging artifact.Packaging) *prepareSpec { + c := *p + if c.name == "" { + c.name = testName + } + if c.version == "" { + c.version = testVersion + } + if c.packaging == "" { + c.packaging = packaging + } + if c.at.IsZero() { + c.at = capturedAt + } + if c.opID == "" { + c.opID = "op-" + c.name + "-" + c.version + } + if c.files == nil { + c.files = map[string]string{"data.txt": "hello\n"} + } + return &c +} + +func prepare(t *testing.T, repo *Repository, spec *prepareSpec) *artifact.PreparedArtifact { + t.Helper() + spec = spec.withDefaults(repo.packaging) + src := spec.source + if src == "" { + src = t.TempDir() + for name, content := range spec.files { + fixtures.WriteFile(t, src, name, content, 0o644) + } + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: spec.name, Version: spec.version, Packaging: spec.packaging, + Metadata: artifact.Metadata{Labels: spec.labels}, + OperationID: spec.opID, PreparedAt: spec.at, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +// publish prepares and publishes, failing the test on error. +func publish(t *testing.T, repo *Repository, spec *prepareSpec, options artifact.PublishOptions) *artifact.PublishResult { + t.Helper() + result, err := repo.Publish(context.Background(), prepare(t, repo, spec), options) + require.NoError(t, err) + return result +} + +func ref(name, version string) artifact.Ref { return artifact.Ref{Name: name, Version: version} } + +func defaultRef() artifact.Ref { return ref(testName, testVersion) } + +func resolve(t *testing.T, repo *Repository, r artifact.Ref) artifact.PinnedReference { + t.Helper() + pinned, err := repo.Resolve(context.Background(), r) + require.NoError(t, err) + return pinned +} + +// readAll drains every stream of the content, returning the first read error. +func readAll(t *testing.T, content *artifact.Content) error { + t.Helper() + for index := range content.Streams { + if _, err := io.Copy(io.Discard, content.Streams[index].Reader); err != nil { + return err + } + } + return nil +} + +// assertIntact reads the artifact end to end and checks that every stream verifies. +func assertIntact(t *testing.T, repo *Repository, r artifact.Ref) { + t.Helper() + pinned := resolve(t, repo, r) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NoError(t, readAll(t, content)) +} + +func listNames(t *testing.T, repo *Repository) []string { + t.Helper() + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + names := make([]string, 0, len(page.Entries)) + for index := range page.Entries { + names = append(names, page.Entries[index].Name+":"+page.Entries[index].Version) + } + return names +} + +// bucketKeys lists the keys of the fake bucket, sorted. +func bucketKeys(fake *s3test.Server) []string { + keys := fake.Keys() + sort.Strings(keys) + return keys +} + +// payloadKeys lists the bucket keys outside the managed namespace. +func payloadKeys(fake *s3test.Server, prefix string) []string { + var out []string + for _, key := range bucketKeys(fake) { + rel := strings.TrimPrefix(strings.TrimPrefix(key, prefix), "/") + if !layout.InManagedNamespace(rel) { + out = append(out, rel) + } + } + return out +} + +// managedKeys lists the bucket keys below a managed directory, relative to the prefix. +func managedKeys(fake *s3test.Server, prefix, dir string) []string { + var out []string + for _, key := range bucketKeys(fake) { + rel := strings.TrimPrefix(strings.TrimPrefix(key, prefix), "/") + if strings.HasPrefix(rel, dir+"/") { + out = append(out, rel) + } + } + return out +} + +func newSigner() *artifacttest.FakeSigner { + return &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} +} + +// storeRequests returns the recorded requests that mutate objects, in arrival order. +func mutations(fake *s3test.Server) []s3test.Request { + var out []s3test.Request + for _, request := range fake.Requests() { + switch request.Op { + case s3test.OpPutObject, s3test.OpDeleteObject, s3test.OpCompleteMultipartUpload: + out = append(out, request) + } + } + return out +} + +var _ blob.Store = (*blobs3.Store)(nil) + +// hookStore decorates a store so a test can run code around writes: before delegating a Put (to +// stage a concurrent writer) or after it (to lose the response). +type hookStore struct { + blob.Store + beforePut func(key string, opts blob.PutOptions) + afterPut func(key string, info *blob.ObjectInfo, err error) (*blob.ObjectInfo, error) +} + +func (h *hookStore) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { + if h.beforePut != nil { + h.beforePut(key, opts) + } + info, err := h.Store.Put(ctx, key, body, opts) + if h.afterPut != nil { + return h.afterPut(key, info, err) + } + return info, err +} + +// refKey is the bucket key of the default artifact's reference object. +func refKey(prefix string) string { + key := layout.ReferencePath(layout.ReferenceKey(testName, testVersion)) + if prefix == "" { + return key + } + return prefix + "/" + key +} + +// dummyLock is a lock handle that lets a test call Publish without taking the reference lock. +type dummyLock struct{} + +func (dummyLock) Resource() string { return "dummy" } +func (dummyLock) Owner() string { return "test" } +func (dummyLock) Scope() artifact.LockScope { return artifact.LockScopeRepository } +func (dummyLock) Token() string { return "dummy-token" } +func (dummyLock) ExpiresAt() (time.Time, bool) { return time.Time{}, false } +func (dummyLock) Renew(context.Context, time.Duration) error { return nil } +func (dummyLock) Release(context.Context) error { return nil } + +// recordSubject builds the signature subject of a pinned artifact's canonical Record. +func recordSubject(t *testing.T, repo *Repository, pinned *artifact.PinnedReference) artifact.SignatureSubject { + t.Helper() + inspection, err := repo.Inspect(context.Background(), *pinned) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + return artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} +} + +// incompressible returns n deterministic pseudo-random bytes as a string, so archives of it stay +// large enough to need multipart uploads however they are compressed. +func incompressible(n int) string { + data := make([]byte, n) + _, _ = rand.New(rand.NewSource(1)).Read(data) + return string(data) +} + +// packageSpec prepares the shared "File package" fixture: nested files, an executable, an empty +// file and directory, a confined symlink, a Unicode name, and distinct README and CHANGELOG. +func packageSpec(t *testing.T) (*prepareSpec, fixtures.FilePackage) { + t.Helper() + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + return &prepareSpec{source: src}, pkg +} + +// refKeyOnly is the hashed reference key of a name and version. +func refKeyOnly(name, version string) string { return layout.ReferenceKey(name, version) } diff --git a/pkg/artifact/s3/identity.go b/pkg/artifact/s3/identity.go new file mode 100644 index 00000000000..7d94e37007a --- /dev/null +++ b/pkg/artifact/s3/identity.go @@ -0,0 +1,37 @@ +package s3 + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/store" +) + +// identityAdapter lets the S3 store resolve an Atmos identity through the repository's lazy +// resolver. The generic resolver returns a backend-specific value; the S3 backend accepts only AWS +// authentication configuration, and anything else fails closed. +type identityAdapter struct { + resolve artifact.IdentityResolver +} + +// ResolveAWSAuthContext implements the store's AuthResolver. +func (a identityAdapter) ResolveAWSAuthContext(ctx context.Context, identity string) (*store.AWSAuthConfig, error) { + defer perf.Track(nil, "s3.identityAdapter.ResolveAWSAuthContext")() + + value, err := a.resolve(ctx, identity) + if err != nil { + return nil, err + } + switch typed := value.(type) { + case *store.AWSAuthConfig: + if typed != nil { + return typed, nil + } + case store.AWSAuthConfig: + return &typed, nil + } + return nil, fmt.Errorf("%w: identity %q did not resolve to AWS credentials (got %T)", errUtils.ErrBlobIdentityUnresolved, identity, value) +} diff --git a/pkg/artifact/s3/identity_test.go b/pkg/artifact/s3/identity_test.go new file mode 100644 index 00000000000..2e31893e0c4 --- /dev/null +++ b/pkg/artifact/s3/identity_test.go @@ -0,0 +1,179 @@ +package s3 + +import ( + "context" + "errors" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/store" +) + +// isolateAWS gives the SDK static environment credentials and no ambient profile, so a test proves +// what the repository does with an identity rather than what the developer machine happens to have. +func isolateAWS(t *testing.T) { + t.Helper() + dir := t.TempDir() + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-test-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-test-secret") + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +// resolverStub records calls and returns a canned value. +type resolverStub struct { + calls atomic.Int32 + values []any + errs []error +} + +func (r *resolverStub) resolve(_ context.Context, _ string) (any, error) { + index := int(r.calls.Add(1)) - 1 + if index >= len(r.values) { + index = len(r.values) - 1 + } + return r.values[index], r.errs[index] +} + +func identityRepo(t *testing.T, fake *s3test.Server, resolver artifact.IdentityResolver, identity string) *Repository { + t.Helper() + cfg := &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindS3, Identity: identity, + Options: map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1"}, + } + var ropts []artifact.RepositoryOption + if resolver != nil { + ropts = append(ropts, artifact.WithIdentityResolver(resolver)) + } + applied := artifact.ApplyRepositoryOptions(ropts...) + repo, err := newRepository("shared", cfg, nil, &applied) + require.NoError(t, err) + return repo +} + +// Constructing, describing, and asking for capabilities resolve no identity and touch no endpoint. +func TestIdentity_IsResolvedLazily(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + stub := &resolverStub{values: []any{&store.AWSAuthConfig{Region: "us-east-1"}}, errs: []error{nil}} + repo := identityRepo(t, fake, stub.resolve, "deploy") + _ = repo.Capabilities() + _ = repo.Location() + _ = Descriptor() + assert.Zero(t, stub.calls.Load(), "no identity is resolved before an operation needs credentials") + assert.Empty(t, fake.Requests()) + + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + _, err = repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Equal(t, int32(1), stub.calls.Load(), "the resolved client is reused") + assert.NotEmpty(t, fake.Requests()) +} + +// An identity that cannot be resolved fails closed: no request is sent, whatever ambient +// credentials the environment offers. +func TestIdentity_FailsClosed(t *testing.T) { + tests := []struct { + name string + resolver func() artifact.IdentityResolver + }{ + {name: "no resolver injected", resolver: func() artifact.IdentityResolver { return nil }}, + {name: "resolver fails", resolver: func() artifact.IdentityResolver { + return (&resolverStub{values: []any{nil}, errs: []error{errors.New("sso session expired")}}).resolve + }}, + {name: "resolver returns nothing", resolver: func() artifact.IdentityResolver { + return (&resolverStub{values: []any{nil}, errs: []error{nil}}).resolve + }}, + {name: "resolver returns another cloud's credentials", resolver: func() artifact.IdentityResolver { + return (&resolverStub{values: []any{"azure-token"}, errs: []error{nil}}).resolve + }}, + {name: "resolver returns a nil AWS config", resolver: func() artifact.IdentityResolver { + var none *store.AWSAuthConfig + return (&resolverStub{values: []any{none}, errs: []error{nil}}).resolve + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + repo := identityRepo(t, fake, tt.resolver(), "deploy") + + for _, operation := range []func() error{ + func() error { _, err := repo.Resolve(context.Background(), defaultRef()); return err }, + func() error { _, err := repo.List(context.Background(), artifact.ListQuery{}); return err }, + func() error { + _, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "x"}) + return err + }, + } { + err := operation() + require.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved) + assert.NotErrorIs(t, err, errUtils.ErrArtifactNotFound, "an unresolved identity is never reported as an empty repository") + } + assert.Empty(t, fake.Requests(), "no request reached the endpoint with ambient credentials") + }) + } +} + +// A transient identity failure is not cached: the next operation resolves again. +func TestIdentity_TransientFailureIsRetried(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + stub := &resolverStub{ + values: []any{nil, &store.AWSAuthConfig{Region: "us-east-1"}}, + errs: []error{errors.New("temporarily unavailable"), nil}, + } + repo := identityRepo(t, fake, stub.resolve, "deploy") + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved) + _, err = repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Equal(t, int32(2), stub.calls.Load()) +} + +// Without an identity the default credential chain applies, as for any AWS SDK user. +func TestIdentity_AbsentUsesTheDefaultChain(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + repo := identityRepo(t, fake, nil, "") + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.NotEmpty(t, fake.Requests()) +} + +// The factory path end to end against the fake: a conditional-write probe, a publish, and a read. +func TestFactoryRepository_PublishesThroughTheSDKClient(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz} { + t.Run(string(packaging), func(t *testing.T) { + cfg := &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindS3, Packaging: string(packaging), + Options: map[string]any{ + "bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1", "prefix": "factory/" + string(packaging), + "tags": map[string]any{"team": "platform"}, + }, + } + repo, err := newRepository("shared", cfg, nil, &artifact.RepositoryOptions{}) + require.NoError(t, err) + result := publish(t, repo, &prepareSpec{files: bigFiles()}, artifact.PublishOptions{}) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + object, ok := fake.Object("factory/" + string(packaging) + "/.atmos-artifacts/v1/refs/" + refKeyOnly(testName, testVersion) + ".json") + require.True(t, ok) + assert.Equal(t, map[string]string{"team": "platform"}, object.Tags, "the production store tags what the repository writes") + }) + } + assert.NotEmpty(t, fake.RequestsFor(s3test.OpPutObject)) +} diff --git a/pkg/artifact/s3/integrity_test.go b/pkg/artifact/s3/integrity_test.go new file mode 100644 index 00000000000..d58904036c0 --- /dev/null +++ b/pkg/artifact/s3/integrity_test.go @@ -0,0 +1,278 @@ +package s3 + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// payloadKey returns the bucket key of the primary payload object of the default artifact. +func payloadKey(packaging artifact.Packaging, file string) string { + location := "team/app/1.0.0" + if isArchive(packaging) { + return location + layout.ArchiveExtension(packaging) + } + return location + "/" + file +} + +func metadataKeyFor(packaging artifact.Packaging) string { + location := "team/app/1.0.0" + if isArchive(packaging) { + return layout.MetadataSidecarPath(location + layout.ArchiveExtension(packaging)) + } + return layout.DistributionPath(location, layout.ManifestName) +} + +func documentKeyFor(packaging artifact.Packaging, name string) string { + location := "team/app/1.0.0" + if isArchive(packaging) { + return layout.DocsSidecarDir(location+layout.ArchiveExtension(packaging)) + "/" + name + } + return layout.DistributionPath(location, name) +} + +// edit rewrites an object in the fake with fn, as a stranger with bucket access could. +func edit(t *testing.T, fake *s3test.Server, key string, fn func([]byte) []byte) { + t.Helper() + object, ok := fake.Object(key) + require.True(t, ok, "object %s exists", key) + fake.Seed(key, fn(object.Data), object.Tags) +} + +// sameLengthEdit changes content without changing its size, so only the digest can catch it. +func sameLengthEdit(data []byte) []byte { + out := append([]byte(nil), data...) + out[0] = 'X' + if data[0] == 'X' { + out[0] = 'Y' + } + return out +} + +func truncate(data []byte) []byte { return data[:len(data)-1] } + +// Every consumed byte is verified against the digest in the Record; ETags prove nothing. +func TestTamperedPayload_FailsAtTheDeclaredDigest(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec)+"/same length", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + edit(t, spec.fake, payloadKey(spec.packaging, "nested/deep/file.txt"), sameLengthEdit) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err, "opening is cheap; verification happens as the bytes are consumed") + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) + }) + t.Run(specName(spec)+"/truncated", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + edit(t, spec.fake, payloadKey(spec.packaging, "nested/deep/file.txt"), truncate) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed, "a size mismatch is caught before any byte is served") + }) + } +} + +func TestMissingObjects_FailExplicitly(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec)+"/payload", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, repo.deleteObject(context.Background(), payloadKey(spec.packaging, "nested/deep/file.txt"))) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed, "a completed reference promised the object exists") + }) + t.Run(specName(spec)+"/metadata", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, repo.deleteObject(context.Background(), metadataKeyFor(spec.packaging))) + _, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run(specName(spec)+"/document", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, repo.deleteObject(context.Background(), documentKeyFor(spec.packaging, "README.md"))) + + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + _, err = io.Copy(io.Discard, stream.Reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.NotErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a lost object is damage, not an absent optional document") + other, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.NoError(t, err, "the other document is unaffected") + _, err = io.Copy(io.Discard, other.Reader) + require.NoError(t, err) + }) + } +} + +func TestTamperedMetadataAndDocuments_FailIntegrity(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingZip)} { + t.Run(specName(spec)+"/metadata copy", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + edit(t, spec.fake, metadataKeyFor(spec.packaging), sameLengthEdit) + _, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run(specName(spec)+"/document bytes", func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + edit(t, spec.fake, documentKeyFor(spec.packaging, "README.md"), sameLengthEdit) + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + _, err = io.Copy(io.Discard, stream.Reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } +} + +// Tampered managed state fails integrity instead of being trusted. +func TestTamperedState_FailsIntegrity(t *testing.T) { + setup := func(t *testing.T) (*Repository, *s3test.Server, artifact.PinnedReference) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + return repo, spec.fake, result.Pinned + } + t.Run("record bytes do not hash to their name", func(t *testing.T) { + repo, fake, pinned := setup(t) + edit(t, fake, layout.RecordPath(pinned.RecordDigest), sameLengthEdit) + _, err := repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("reference is not canonical", func(t *testing.T) { + repo, fake, _ := setup(t) + fake.Seed(layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), []byte(`{"not":"a reference"}`), nil) + _, err := repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("representation is missing", func(t *testing.T) { + repo, _, pinned := setup(t) + require.NoError(t, repo.deleteObject(context.Background(), layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)))) + _, err := repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("an unchanged republish detects a missing representation", func(t *testing.T) { + repo, _, pinned := setup(t) + require.NoError(t, repo.deleteObject(context.Background(), layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)))) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{opID: "op-again"}), artifact.PublishOptions{ExpectedPrior: &pinned}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +// A reader notices when the reference moves while it reads from the mutable payload location, and +// discards the bytes instead of serving them under the old digest. +func TestRead_ReaderReportsTheReferenceMoving(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + base := publish(t, repo, &prepareSpec{labels: map[string]string{"rev": "1"}}, artifact.PublishOptions{}) + content, err := repo.Read(context.Background(), base.Pinned) + require.NoError(t, err) + defer content.Close() + + // Same payload bytes, different Record: only the generation recheck can tell. + publish(t, repo, &prepareSpec{labels: map[string]string{"rev": "2"}}, artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + err = readAll(t, content) + if spec.packaging == artifact.PackagingNone { + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + require.ErrorContains(t, err, "changed while it was being read") + return + } + // An archive embeds the Record, so the replaced bytes already fail their digest. + require.Error(t, err) + assert.True(t, errors.Is(err, errUtils.ErrArtifactPublicationUnavailable) || errors.Is(err, errUtils.ErrArtifactIntegrityFailed), "%v", err) + }) + } +} + +func TestRead_ReaderReportsDeletion(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + base := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + content, err := repo.Read(context.Background(), base.Pinned) + require.NoError(t, err) + defer content.Close() + _, err = repo.Delete(context.Background(), base.Pinned, artifact.DeleteOptions{}) + require.NoError(t, err) + require.Error(t, readAll(t, content), "bytes of a deleted artifact are never reported as valid") +} + +func TestCanceledContextStopsReads(t *testing.T) { + repo := newRepo(t, noneSpec()) + base := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := repo.Resolve(ctx, defaultRef()) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Read(ctx, base.Pinned) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.List(ctx, artifact.ListQuery{}) + require.ErrorIs(t, err, context.Canceled) +} + +// Objects added inside a payload prefix are never adopted into the artifact. +func TestExtraObjectsInsideAPayloadAreNotAdopted(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + spec.fake.Seed("team/app/1.0.0/injected.txt", []byte("not in the inventory"), nil) + content, err := repo.Read(context.Background(), result.Pinned) + require.NoError(t, err) + defer content.Close() + for index := range content.Streams { + assert.NotEqual(t, "injected.txt", content.Streams[index].Path) + } +} + +// A payload that grows past its declared size is rejected as soon as it exceeds it. +func TestRead_PayloadThatGrowsIsRejected(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + edit(t, spec.fake, payloadKey(artifact.PackagingNone, "data.txt"), func(data []byte) []byte { return append(append([]byte(nil), data...), "extra"...) }) + content, err := repo.Read(context.Background(), result.Pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) +} diff --git a/pkg/artifact/s3/internal_test.go b/pkg/artifact/s3/internal_test.go new file mode 100644 index 00000000000..e4600a6ef2f --- /dev/null +++ b/pkg/artifact/s3/internal_test.go @@ -0,0 +1,139 @@ +package s3 + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// A tampered journal or representation can never address an object outside its location. +func TestObjectSetKeys_RejectsUnsafeKeys(t *testing.T) { + for _, file := range []string{"../escape", "a/../../b", "/absolute", ""} { + _, err := (objectSet{files: []string{file}}).keys(artifact.PackagingNone, "team/app/1.0.0") + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, "file %q", file) + } + keys, err := (objectSet{files: []string{"a.txt"}, docs: []string{"README.md"}}).keys(artifact.PackagingTgz, "team/app/1.0.0.tgz") + require.NoError(t, err) + assert.Equal(t, []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json", "team/app/1.0.0.tgz.docs/README.md"}, keys, "an archive owns the object and its sidecars, not per-file keys") + _, err = (objectSet{}).keys(artifact.PackagingNone, "../x") + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestClaimPaths(t *testing.T) { + assert.Equal(t, []string{"a/b"}, claimPaths(artifact.PackagingNone, "a/b")) + assert.Equal(t, []string{"a/b.tgz", "a/b.tgz.metadata.json", "a/b.tgz.docs"}, claimPaths(artifact.PackagingTgz, "a/b.tgz")) +} + +func TestMapErr_KeepsTheCauseAndChoosesTheSentinel(t *testing.T) { + repo := newRepo(t, noneSpec()) + tests := []struct { + name string + err error + is error + }{ + {name: "canceled", err: context.Canceled, is: context.Canceled}, + {name: "precondition", err: blob.NewError(blob.KindPrecondition, blob.OpPut, "k", nil), is: errUtils.ErrArtifactConflict}, + {name: "ambiguous", err: blob.NewError(blob.KindAmbiguous, blob.OpPut, "k", nil), is: errUtils.ErrArtifactPublicationAmbiguous}, + {name: "conditional unsupported", err: blob.NewError(blob.KindUnsupported, blob.OpPut, "k", nil), is: errUtils.ErrArtifactUnsupportedCapability}, + {name: "identity", err: blob.NewError(blob.KindIdentity, "client", "", nil), is: errUtils.ErrBlobIdentityUnresolved}, + {name: "other write", err: errors.New("boom"), is: errUtils.ErrArtifactUploadFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mapped := repo.mapErr(opWrite, "object", tt.err) + require.ErrorIs(t, mapped, tt.is) + require.ErrorIs(t, mapped, tt.err, "the original error stays in the chain") + }) + } + require.NoError(t, repo.mapErr(opWrite, "object", nil)) + assert.ErrorIs(t, repo.mapErr(opRead, "object", errors.New("boom")), errUtils.ErrArtifactDownloadFailed) + assert.ErrorIs(t, repo.mapErr(opDelete, "object", errors.New("boom")), errUtils.ErrArtifactDeleteFailed) +} + +func TestNewJob_RejectsMalformedPreparations(t *testing.T) { + repo := newRepo(t, noneSpec()) + _, err := repo.newJob(nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + + prepared := prepare(t, repo, &prepareSpec{}) + prepared.Packaging = "rar" + _, err = repo.newJob(prepared) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + noOpen := prepare(t, repo, &prepareSpec{}) + noOpen.Blobs[0].Open = nil + _, err = repo.newJob(noOpen) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + + mismatched := prepare(t, repo, &prepareSpec{}) + mismatched.Blobs[0].Descriptor.Size++ + _, err = repo.newJob(mismatched) + require.Error(t, err) + + archive := newRepo(t, archiveSpec(artifact.PackagingTgz)) + doubled := prepare(t, archive, &prepareSpec{}) + doubled.Blobs = append(doubled.Blobs, doubled.Blobs[0]) + _, err = archive.newJob(doubled) + require.Error(t, err) +} + +// An upload that is corrupted on its way out is never committed: the verifying reader fails the put. +func TestPublish_CorruptStagedBlobIsNeverCommitted(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{files: bigFiles()}) + for index := range prepared.Blobs { + original := prepared.Blobs[index].Open + if prepared.Blobs[index].Path != "nested/big.bin" { + continue + } + prepared.Blobs[index].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + return &flippingReader{ReadCloser: reader}, nil + } + } + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the reference never reached completed") + assert.Empty(t, spec.fake.OpenUploads(), "the failed multipart upload was aborted") + _, ok := spec.fake.Object("team/app/1.0.0/nested/big.bin") + assert.False(t, ok, "the corrupted object was never completed") +} + +func TestPublish_ServerErrorsAreUploadFailures(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: "team/app", Status: 403, Code: "AccessDenied", Times: -1}) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the failed upload leaves a pending reference for the retry to repair") +} + +// flippingReader corrupts the first byte it delivers. +type flippingReader struct { + io.ReadCloser + flipped bool +} + +func (f *flippingReader) Read(p []byte) (int, error) { + n, err := f.ReadCloser.Read(p) + if n > 0 && !f.flipped { + p[0] ^= 0xff + f.flipped = true + } + return n, err +} diff --git a/pkg/artifact/s3/job.go b/pkg/artifact/s3/job.go new file mode 100644 index 00000000000..abd4bb002f1 --- /dev/null +++ b/pkg/artifact/s3/job.go @@ -0,0 +1,154 @@ +package s3 + +import ( + "crypto/sha256" + "fmt" + "io" + "sort" + + "github.com/google/uuid" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// publishJob is everything one publication attempt derives from the prepared artifact, computed +// once before any state changes. +type publishJob struct { + prepared *artifact.PreparedArtifact + record *artifact.Record + recordBytes []byte + recordDigest digest.Digest + key string + opID string + location string + payload []ocispec.Descriptor + documents []ocispec.Descriptor +} + +// newJob validates a prepared artifact against the repository and derives the publication inputs. +func (r *Repository) newJob(prepared *artifact.PreparedArtifact) (*publishJob, error) { + if prepared == nil { + return nil, fmt.Errorf("%w: no prepared artifact to publish", errUtils.ErrArtifactConfigInvalid) + } + if !supportedPackaging(prepared.Packaging) { + return nil, artifact.NewUnsupportedCapability(r.alias, "publish", "packaging/"+string(prepared.Packaging), + fmt.Sprintf("%s repositories store %v payloads", artifact.RepositoryKindS3, r.Capabilities().Packagings)) + } + canonical, err := prepared.Record.Canonical() + if err != nil { + return nil, err + } + record := prepared.Record + opID := prepared.OperationID + if opID == "" { + opID = uuid.NewString() + } + job := &publishJob{ + prepared: prepared, record: &record, recordBytes: canonical, recordDigest: digest.FromBytes(canonical), + key: layout.ReferenceKey(record.Name, record.Version), opID: opID, + } + if err := job.describeBlobs(); err != nil { + return nil, err + } + if _, err := job.plan(); err != nil { + return nil, err + } + return job, nil +} + +// describeBlobs derives the representation descriptors from the staged blobs and checks that the +// blobs agree with the Record before anything is written. +func (j *publishJob) describeBlobs() error { + j.payload, j.documents = []ocispec.Descriptor{}, []ocispec.Descriptor{} + docs := map[artifact.DocumentRole]bool{} + for index := range j.prepared.Blobs { + staged := &j.prepared.Blobs[index] + if staged.Open == nil { + return fmt.Errorf("%w: staged blob %q cannot be opened", errUtils.ErrArtifactConfigInvalid, staged.Path) + } + descriptor := staged.Descriptor + if staged.Role == artifact.BlobDocument { + if docs[staged.DocumentRole] { + return fmt.Errorf("%w: duplicate %s document blob", errUtils.ErrArtifactRecordInvalid, staged.DocumentRole) + } + docs[staged.DocumentRole] = true + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: layout.DocumentFileName(staged.DocumentRole)} + j.documents = append(j.documents, descriptor) + continue + } + if staged.Path != "" { + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: staged.Path} + } + j.payload = append(j.payload, descriptor) + } + for index := range j.record.Documents { + if !docs[j.record.Documents[index].Role] { + return fmt.Errorf("%w: the Record lists a %s document with no staged blob", errUtils.ErrArtifactIntegrityFailed, j.record.Documents[index].Role) + } + } + if len(docs) != len(j.record.Documents) { + return fmt.Errorf("%w: %d document blobs but %d documents in the Record", errUtils.ErrArtifactIntegrityFailed, len(docs), len(j.record.Documents)) + } + return nil +} + +// splitBlobs separates the staged document blobs (by role) from the payload blobs. +func (j *publishJob) splitBlobs() (payload []*artifact.StagedBlob, docs map[artifact.DocumentRole]*artifact.StagedBlob) { + docs = map[artifact.DocumentRole]*artifact.StagedBlob{} + for index := range j.prepared.Blobs { + staged := &j.prepared.Blobs[index] + if staged.Role == artifact.BlobDocument { + docs[staged.DocumentRole] = staged + continue + } + payload = append(payload, staged) + } + return payload, docs +} + +// matchDocuments checks every document blob against the Record's descriptor for its role. +func (j *publishJob) matchDocuments(docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + for role, staged := range docs { + doc, ok := documentDescriptor(j.record, role) + if !ok || doc.Digest != staged.Descriptor.Digest || doc.Size != staged.Descriptor.Size { + return fmt.Errorf("%w: the staged %s document does not match the Record", errUtils.ErrArtifactIntegrityFailed, role) + } + } + return nil +} + +// documentDescriptor returns the Record's descriptor for a role. +func documentDescriptor(record *artifact.Record, role artifact.DocumentRole) (*artifact.Document, bool) { + for index := range record.Documents { + if record.Documents[index].Role == role { + return &record.Documents[index], true + } + } + return nil, false +} + +// representationDigest is the checksum of the transferred payload representation: the archive +// digest, or a digest of the sorted per-file payload digests. +func (j *publishJob) representationDigest() digest.Digest { + if isArchive(j.prepared.Packaging) { + if len(j.payload) == 1 { + return j.payload[0].Digest + } + return "" + } + hasher := sha256.New() + files := setOfRecord(j.record).files + sort.Strings(files) + byPath := map[string]digest.Digest{} + for index := range j.record.Inventory { + byPath[j.record.Inventory[index].Path] = j.record.Inventory[index].Digest + } + for _, file := range files { + _, _ = io.WriteString(hasher, file+"\x00"+byPath[file].String()+"\n") + } + return digest.NewDigest(digest.SHA256, hasher) +} diff --git a/pkg/artifact/s3/keys.go b/pkg/artifact/s3/keys.go new file mode 100644 index 00000000000..57ac598026e --- /dev/null +++ b/pkg/artifact/s3/keys.go @@ -0,0 +1,86 @@ +package s3 + +import ( + "fmt" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// objectSet names what a payload consists of, independent of where it lives: the payload file +// paths (empty for an archive) and the distribution document file names. +type objectSet struct { + files []string + docs []string +} + +// setOfRecord derives the object set from a verified Record. +func setOfRecord(record *artifact.Record) objectSet { + var set objectSet + for index := range record.Inventory { + if record.Inventory[index].Type == artifact.EntryFile { + set.files = append(set.files, record.Inventory[index].Path) + } + } + for index := range record.Documents { + set.docs = append(set.docs, layout.DocumentFileName(record.Documents[index].Role)) + } + return set +} + +// setOfDescriptors derives the object set from the descriptors a representation or journal holds. +// Payload descriptors of per-file packaging are titled with their path. +func setOfDescriptors(payload, documents []ocispec.Descriptor) objectSet { + var set objectSet + for index := range payload { + if title := payload[index].Annotations[ocispec.AnnotationTitle]; title != "" { + set.files = append(set.files, title) + } + } + for index := range documents { + if title := documents[index].Annotations[ocispec.AnnotationTitle]; title != "" { + set.docs = append(set.docs, title) + } + } + return set +} + +// keys lists the object keys of a payload at a location, including the adjacent metadata. Every +// key is validated, so a tampered journal can never address a key outside its location. +func (s objectSet) keys(packaging artifact.Packaging, location string) ([]string, error) { + var keys []string + if isArchive(packaging) { + keys = append(keys, location, layout.MetadataSidecarPath(location)) + for _, doc := range s.docs { + keys = append(keys, layout.DocsSidecarDir(location)+"/"+doc) + } + } else { + for _, file := range s.files { + keys = append(keys, location+"/"+file) + } + keys = append(keys, layout.DistributionPath(location, layout.ManifestName)) + for _, doc := range s.docs { + keys = append(keys, layout.DistributionPath(location, doc)) + } + } + for _, key := range keys { + if err := blob.ValidateKey(key); err != nil || layout.InManagedNamespace(key) { + return nil, fmt.Errorf("%w: payload object key %q is not a valid location-relative key", errUtils.ErrArtifactUnsafePath, key) + } + } + return keys, nil +} + +// claimPaths lists the paths a payload at a location owns, for collision detection. An archive owns +// the object, its metadata sidecar, and its documents directory; per-file payloads own the +// location prefix. +func claimPaths(packaging artifact.Packaging, location string) []string { + if isArchive(packaging) { + return []string{location, layout.MetadataSidecarPath(location), layout.DocsSidecarDir(location)} + } + return []string{location} +} diff --git a/pkg/artifact/s3/list_test.go b/pkg/artifact/s3/list_test.go new file mode 100644 index 00000000000..a332717e06a --- /dev/null +++ b/pkg/artifact/s3/list_test.go @@ -0,0 +1,150 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Listing enumerates managed completed references only: never arbitrary user objects, pending +// publications, journals, claims, or evidence. +func TestList_EnumeratesManagedReferencesOnly(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t).with(func(s *rspec) { s.prefix = testPrefix }) + var fault armedFault + repo := newRepo(t, spec.withOpts(fault.option())) + publish(t, repo, &prepareSpec{name: "a/app", version: "1"}, artifact.PublishOptions{}) + publish(t, repo, &prepareSpec{name: "a/app", version: "2"}, artifact.PublishOptions{}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{name: "b/pending", version: "1"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + spec.fake.Seed(testPrefix+"/user/notes.json", []byte(`{"name":"not-an-artifact"}`), nil) + spec.fake.Seed(testPrefix+"/"+layout.ReferencesDir()+"/nested/ignored.json", []byte(`{}`), nil) + spec.fake.Seed(testPrefix+"/"+layout.ReferencesDir()+"/notes.txt", []byte(`x`), nil) + + assert.Equal(t, []string{"a/app:1", "a/app:2"}, listNames(t, repo)) + }) + } +} + +func TestList_PaginatesInLexicalOrderAndFiltersByLabel(t *testing.T) { + repo := newRepo(t, noneSpec()) + for _, item := range []struct{ name, version, tier string }{{"b/app", "1", "gold"}, {"a/app", "2", "gold"}, {"a/app", "1", "silver"}, {"c/app", "1", "gold"}, {"a/app", "10", "gold"}} { + publish(t, repo, &prepareSpec{name: item.name, version: item.version, labels: map[string]string{"tier": item.tier}}, artifact.PublishOptions{}) + } + var seen []string + cursor := "" + for pages := 0; pages < 10; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{Cursor: cursor, PageSize: 2}) + require.NoError(t, err) + assert.LessOrEqual(t, len(page.Entries), 2) + for index := range page.Entries { + seen = append(seen, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + require.NotEqual(t, cursor, page.NextCursor) + cursor = page.NextCursor + } + assert.Equal(t, []string{"a/app:1", "a/app:10", "a/app:2", "b/app:1", "c/app:1"}, seen, "lexical order, every entry exactly once") + + page, err := repo.List(context.Background(), artifact.ListQuery{Labels: []artifact.Label{{Key: "tier", Value: "gold"}}, PageSize: 2}) + require.NoError(t, err) + require.Len(t, page.Entries, 2) + assert.Equal(t, "a/app:10", page.Entries[0].Name+":"+page.Entries[0].Version) + assert.NotEmpty(t, page.NextCursor, "the label filter pages too") + + named, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) + require.NoError(t, err) + assert.Len(t, named.Entries, 3) + + empty, err := newRepo(t, noneSpec()).List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Empty(t, empty.Entries, "an empty bucket lists nothing") +} + +// More references than one S3 listing page hold are all found, because the listing follows the +// continuation token; the fake caps pages well below the client's request. +func TestList_FollowsContinuationAcrossManyReferences(t *testing.T) { + const count, cap = 1010, 100 + spec := noneSpec().with(func(s *rspec) { s.fake = s3test.NewServer(t, s3test.WithPageCap(cap)); s.prefix = testPrefix }) + repo := newRepo(t, spec) + base := publish(t, repo, &prepareSpec{name: "seed/app", version: "0"}, artifact.PublishOptions{}) + template, _, err := repo.loadRecord(context.Background(), base.Pinned.RecordDigest) + require.NoError(t, err) + seedReferences(t, repo, template, count) + spec.fake.ResetRequests() + + var versions []string + cursor := "" + pages := 0 + for ; pages < 20; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{Name: "seed/app", Cursor: cursor, PageSize: 300}) + require.NoError(t, err) + for index := range page.Entries { + versions = append(versions, page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + } + require.Len(t, versions, count+1, "the original and every seeded reference") + assert.Equal(t, "0", versions[0]) + assert.Equal(t, "v0000", versions[1]) + assert.Equal(t, "v1009", versions[len(versions)-1]) + assert.True(t, sort.StringsAreSorted(versions), "sorted lexically across page boundaries") + refListings := 0 + for _, request := range spec.fake.RequestsFor(s3test.OpListObjectsV2) { + if strings.Contains(request.Query.Get("prefix"), "/refs/") { + refListings++ + } + } + assert.GreaterOrEqual(t, refListings, (pages+1)*((count+1)/cap), "every List walked every S3 page of the reference objects") +} + +// seedReferences writes n completed references for versions v0000..v, each with its own +// Record, straight into the bucket. +func seedReferences(t *testing.T, repo *Repository, template *artifact.Record, n int) { + t.Helper() + require.NoError(t, repo.forEach(context.Background(), n, func(ctx context.Context, index int) error { + record := *template + record.Version = fmt.Sprintf("v%04d", index) + if err := record.Seal(); err != nil { + return err + } + raw, err := record.Canonical() + if err != nil { + return err + } + recordDigest := digest.FromBytes(raw) + intent, err := record.IntentDigest() + if err != nil { + return err + } + if err := repo.putBytes(ctx, layout.RecordPath(recordDigest), raw); err != nil { + return err + } + completed := layout.Reference{ + SchemaVersion: layout.SchemaVersion, Name: record.Name, Version: record.Version, State: layout.StateCompleted, + Generation: 2, OperationID: "seed", IntentDigest: intent, RecordDigest: recordDigest, + } + encoded, err := completed.Marshal() + if err != nil { + return err + } + return repo.putBytes(ctx, layout.ReferencePath(completed.Key()), encoded) + })) +} diff --git a/pkg/artifact/s3/location_test.go b/pkg/artifact/s3/location_test.go new file mode 100644 index 00000000000..1d4f3bc65b3 --- /dev/null +++ b/pkg/artifact/s3/location_test.go @@ -0,0 +1,194 @@ +package s3 + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// withPattern returns a spec whose key_pattern is set. +func (s *rspec) withPattern(pattern string) *rspec { + return s.with(func(c *rspec) { c.keyPattern = pattern }) +} + +func TestDatedRoot_LocationIsRenderedOnceAndPersisted(t *testing.T) { + const pattern = `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}` + spec := noneSpec().fresh(t).withPattern(pattern) + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{at: time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC)}, artifact.PublishOptions{}) + + assert.Equal(t, "s3://"+spec.fake.Bucket()+"/2026/10/30/team/app/1.0.0", result.Location) + assert.Contains(t, payloadKeys(spec.fake, ""), "2026/10/30/team/app/1.0.0/data.txt") + rep, err := repo.loadRepresentation(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + assert.Equal(t, "2026/10/30/team/app/1.0.0", rep.Location) + + // A later read uses the persisted location, whatever the pattern now says. + reader := newRepo(t, spec.withPattern("elsewhere/{{ .Name }}")) + assertIntact(t, reader, defaultRef()) +} + +// A force replacement that lands on a new location, even with a different packaging, retires the +// old payload objects and Record, and the old digest reports not found. +func TestForceReplacementAcrossLocationsRetiresTheOldPayload(t *testing.T) { + tests := []struct { + name string + oldSpec *rspec + newSpec *rspec + oldPackaging artifact.Packaging + newPackaging artifact.Packaging + }{ + {name: "files to files", oldSpec: noneSpec().withPattern("old/{{ .Name }}"), newSpec: noneSpec().withPattern("new/{{ .Name }}")}, + {name: "files to archive", oldSpec: noneSpec().withPattern("old/{{ .Name }}"), newSpec: archiveSpec(artifact.PackagingTgz).withPattern("new/{{ .Name }}.tgz")}, + {name: "archive to files", oldSpec: archiveSpec(artifact.PackagingZip).withPattern("old/{{ .Name }}.zip"), newSpec: noneSpec().withPattern("new/{{ .Name }}")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := s3test.NewServer(t) + oldRepo := newRepo(t, tt.oldSpec.with(func(s *rspec) { s.fake = fake })) + newRepoUnderTest := newRepo(t, tt.newSpec.with(func(s *rspec) { s.fake = fake })) + base := publish(t, oldRepo, &prepareSpec{files: map[string]string{"data.txt": "old", "extra.txt": "x"}}, artifact.PublishOptions{}) + oldKeys := payloadKeys(fake, "") + require.NotEmpty(t, oldKeys) + + replaced := publish(t, newRepoUnderTest, &prepareSpec{files: map[string]string{"data.txt": "new"}}, artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + assert.Equal(t, artifact.OutcomeReplaced, replaced.Outcome) + for _, key := range payloadKeys(fake, "") { + assert.NotContains(t, key, "old/", "the old payload objects were retired: %s", key) + } + _, err := newRepoUnderTest.Read(context.Background(), base.Pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assertIntact(t, newRepoUnderTest, defaultRef()) + }) + } +} + +// A second reference whose key_pattern resolves onto an owned location fails with a conflict and is +// never bound to the first reference's payload. +func TestCollidingKeyPattern_SecondReferenceConflicts(t *testing.T) { + tests := []struct { + name string + first *rspec + second *rspec + wantMsg string + }{ + {name: "identical files location", first: noneSpec().withPattern("shared/payload"), second: noneSpec().withPattern("shared/payload")}, + {name: "identical archive location", first: archiveSpec(artifact.PackagingTgz).withPattern("shared/p.tgz"), second: archiveSpec(artifact.PackagingTgz).withPattern("shared/p.tgz")}, + {name: "second is below the first", first: noneSpec().withPattern("shared/payload"), second: noneSpec().withPattern("shared/payload/deeper")}, + {name: "second is above the first", first: noneSpec().withPattern("shared/payload/deeper"), second: noneSpec().withPattern("shared/payload")}, + {name: "case aliases", first: noneSpec().withPattern("shared/Payload"), second: noneSpec().withPattern("shared/payload")}, + {name: "second archive claims the first's metadata sidecar", first: archiveSpec(artifact.PackagingTgz).withPattern("shared/p.tgz"), second: archiveSpec(artifact.PackagingTgz).withPattern("shared/p.tgz.metadata.json")}, + {name: "second archive claims the first's documents directory", first: archiveSpec(artifact.PackagingTgz).withPattern("shared/p.tgz"), second: noneSpec().withPattern("shared/p.tgz.docs")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := s3test.NewServer(t) + first := newRepo(t, tt.first.with(func(s *rspec) { s.fake = fake })) + second := newRepo(t, tt.second.with(func(s *rspec) { s.fake = fake })) + publish(t, first, &prepareSpec{name: "one/app", files: map[string]string{"a.txt": "first"}}, artifact.PublishOptions{}) + before := bucketKeys(fake) + + _, err := second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "two/app", files: map[string]string{"a.txt": "second"}}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, before, bucketKeys(fake), "the failed reservation changed nothing") + assertIntact(t, first, ref("one/app", testVersion)) + _, err = second.Resolve(context.Background(), ref("two/app", testVersion)) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// A pending publication reserves its location too. +func TestCollidingKeyPattern_PendingReservationBlocksOthers(t *testing.T) { + fake := s3test.NewServer(t) + first := newRepo(t, noneSpec().withPattern("shared/payload").with(func(s *rspec) { s.fake = fake }).withOpts(withFaults(crashAt(faultDuringPayload)))) + second := newRepo(t, noneSpec().withPattern("shared/payload").with(func(s *rspec) { s.fake = fake })) + _, err := first.Publish(context.Background(), prepare(t, first, &prepareSpec{name: "one/app"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + _, err = second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "two/app"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "op-one/app", "the pending operation is named") +} + +// Objects the repository does not manage are never overwritten or adopted. +func TestUnmanagedContentAtTheLocationIsNeverOverwritten(t *testing.T) { + tests := []struct { + name string + spec *rspec + seed string + }{ + {name: "file below the payload prefix", spec: noneSpec(), seed: "team/app/1.0.0/user.txt"}, + {name: "object at the payload key itself", spec: noneSpec(), seed: "team/app/1.0.0"}, + {name: "archive object", spec: archiveSpec(artifact.PackagingTgz), seed: "team/app/1.0.0.tgz"}, + {name: "object in the documents directory", spec: archiveSpec(artifact.PackagingTgz), seed: "team/app/1.0.0.tgz.docs/notes.md"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec.fresh(t) + repo := newRepo(t, spec) + spec.fake.Seed(tt.seed, []byte("mine"), nil) + + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + object, ok := spec.fake.Object(tt.seed) + require.True(t, ok) + assert.Equal(t, []byte("mine"), object.Data, "the user's object is untouched") + }) + } +} + +// Writers racing for one location: exactly one reference wins and the rest conflict. +func TestCollidingKeyPattern_ConcurrentWritersExactlyOneWins(t *testing.T) { + const writers = 6 + fake := s3test.NewServer(t) + repos := make([]*Repository, writers) + prepared := make([]*artifact.PreparedArtifact, writers) + for index := range repos { + repos[index] = newRepo(t, noneSpec().withPattern("shared/payload").with(func(s *rspec) { s.fake = fake }).withOpts(WithLockWait(time.Minute))) + prepared[index] = prepare(t, repos[index], &prepareSpec{name: fmt.Sprintf("writer%d/app", index), files: map[string]string{"a.txt": fmt.Sprintf("writer %d", index)}}) + } + errs := make([]error, writers) + var wg sync.WaitGroup + for index := range repos { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[index] = repos[index].Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wg.Wait() + winners := 0 + for _, err := range errs { + if err == nil { + winners++ + continue + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + assert.Equal(t, 1, winners) + assert.Len(t, listNames(t, repos[0]), 1) + assert.Empty(t, managedKeys(fake, "", layout.ManagedRoot+"/locks"), "every lock was released") +} + +// Different key_patterns keep references apart, including a files payload beside an archive. +func TestDistinctLocationsCoexist(t *testing.T) { + fake := s3test.NewServer(t) + files := newRepo(t, noneSpec().withPattern("files/{{ .Name }}").with(func(s *rspec) { s.fake = fake })) + archives := newRepo(t, archiveSpec(artifact.PackagingTgz).withPattern("files/{{ .Name }}.tgz").with(func(s *rspec) { s.fake = fake })) + publish(t, files, &prepareSpec{name: "one/app"}, artifact.PublishOptions{}) + publish(t, archives, &prepareSpec{name: "two/app"}, artifact.PublishOptions{}) + assertIntact(t, files, ref("one/app", testVersion)) + assertIntact(t, archives, ref("two/app", testVersion)) + assert.Equal(t, []string{"one/app:1.0.0", "two/app:1.0.0"}, listNames(t, files), "both repositories share one namespace") +} diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go new file mode 100644 index 00000000000..49a2b7c4c19 --- /dev/null +++ b/pkg/artifact/s3/lock.go @@ -0,0 +1,283 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// opLock names the lock operation in capability errors. +const opLock = "lock" + +// maxClaimBytes bounds a claim object read. +const maxClaimBytes int64 = 64 << 10 + +// claimDoc is the content of a lock claim object. It names the resource, owner, and operation so a +// recovery diagnostic can tell an operator whose claim is blocking. +type claimDoc struct { + SchemaVersion int `json:"schema_version"` + Resource string `json:"resource"` + Owner string `json:"owner"` + OperationID string `json:"operation_id,omitempty"` + Token string `json:"token"` + AcquiredAt string `json:"acquired_at"` +} + +// claimHandle is an acquired lock claim. It implements [artifact.LockHandle]. +type claimHandle struct { + repo *Repository + key string + resource string + owner string + operationID string + token string + + mu sync.Mutex + released bool +} + +var _ artifact.LockHandle = (*claimHandle)(nil) + +// Lock implements [artifact.LockProvider] with repository-scope owner claims. A claim object is +// created with If-None-Match; a competing owner waits until ctx ends or the configured wait elapses +// and then receives a conflict that names the claim. Claims never expire and are never taken over, +// so leases are rejected, and an endpoint that cannot enforce If-None-Match offers no lock at all. +// A canceled acquisition leaks no ownership. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "s3.Repository.Lock")() + + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lease", "S3 claims do not expire automatically, so a lease cannot be fenced") + } + if request.Required != "" && !artifact.LockScopeRepository.Satisfies(request.Required) { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lock/"+string(request.Required), "S3 repositories provide host and repository scope locks") + } + if request.Resource == "" { + return nil, fmt.Errorf("%w: a lock needs a resource", errUtils.ErrArtifactRefInvalid) + } + owner := request.Owner + if owner == "" { + owner = r.settings.owner + } + return r.acquire(ctx, request.Resource, owner, request.OperationID) +} + +// acquire takes the claim for resource, waiting for a competing owner. +func (r *Repository) acquire(ctx context.Context, resource, owner, operationID string) (*claimHandle, error) { + if !r.store.Capabilities().ConditionalPut { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lock/repository", "the S3 endpoint cannot enforce conditional writes, which owner claims require") + } + doc := claimDoc{ + SchemaVersion: layout.SchemaVersion, Resource: resource, Owner: owner, OperationID: operationID, + Token: uuid.NewString(), AcquiredAt: time.Now().UTC().Format(time.RFC3339), + } + raw, err := json.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("%w: encode lock claim: %w", errUtils.ErrArtifactUploadFailed, err) + } + key := layout.LockPath(resource) + deadline := time.Now().Add(r.settings.lockWait) + for { + handle, err := r.tryClaim(ctx, key, &doc, raw) + if handle != nil || err != nil { + return handle, err + } + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, r.heldError(ctx, key, resource, ctxErr) + } + if !time.Now().Before(deadline) { + return nil, r.heldError(ctx, key, resource, nil) + } + if err := sleep(ctx, r.settings.lockPoll); err != nil { + return nil, r.heldError(ctx, key, resource, err) + } + } +} + +// sleep waits for d or until ctx ends. +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// tryClaim makes one attempt to create the claim. It returns the handle on success, (nil, nil) +// when another owner holds the claim, and an error for anything else. A response lost after the +// request was sent is settled by reading the claim back. +func (r *Repository) tryClaim(ctx context.Context, key string, doc *claimDoc, raw []byte) (*claimHandle, error) { + _, err := r.putConditional(ctx, key, raw, contentTypeJSON, blob.IfAbsent()) + switch { + case err == nil: + return r.adopt(ctx, key, doc) + case blob.IsPreconditionFailed(err): + return nil, nil + case blob.IsAmbiguous(err): + return r.settleClaim(ctx, key, doc) + case errors.Is(err, errUtils.ErrArtifactUnsupportedCapability): + unsupported := artifact.NewUnsupportedCapability(r.alias, opLock, "lock/repository", "the S3 endpoint cannot enforce If-None-Match, which owner claims require") + unsupported.Cause = err + return nil, unsupported + default: + return nil, err + } +} + +// settleClaim reads the claim back after an ambiguous create. Finding our own token proves the +// create was applied. +func (r *Repository) settleClaim(ctx context.Context, key string, doc *claimDoc) (*claimHandle, error) { + readCtx, cancel := detached(ctx) + defer cancel() + current, _, err := r.readClaim(readCtx, key) + switch { + case err != nil: + return nil, fmt.Errorf("%w: settle claim %s: %w", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key), err) + case current != nil && current.Token == doc.Token: + return r.adopt(ctx, key, doc) + } + // Another owner's claim, or no claim at all: the create did not take effect. + return nil, nil +} + +// adopt turns a created claim into a handle. A context that ended while the claim was being created +// releases it again at once, so cancellation never leaves ownership behind. +func (r *Repository) adopt(ctx context.Context, key string, doc *claimDoc) (*claimHandle, error) { + handle := &claimHandle{repo: r, key: key, resource: doc.Resource, owner: doc.Owner, operationID: doc.OperationID, token: doc.Token} + if err := ctx.Err(); err != nil { + if releaseErr := handle.Release(ctx); releaseErr != nil { + return nil, fmt.Errorf("%w: and the claim %s it created could not be released: %w", err, r.uri(key), releaseErr) + } + return nil, err + } + return handle, nil +} + +// readClaim reads a claim object. A missing claim returns (nil, "", nil); an unreadable one is +// returned as a claim with no token so diagnostics can still name the object. +func (r *Repository) readClaim(ctx context.Context, key string) (*claimDoc, string, error) { + raw, etag, err := r.readObject(ctx, key, maxClaimBytes) + if err != nil || raw == nil { + return nil, "", err + } + var doc claimDoc + if jsonErr := json.Unmarshal(raw, &doc); jsonErr != nil { + return &claimDoc{Owner: fmt.Sprintf("(unreadable claim: %v)", jsonErr)}, etag, nil + } + return &doc, etag, nil +} + +// heldError builds the recovery diagnostic for a claim that another owner holds. It names the claim +// object, the resource, the owner, and the operation, and says how an operator recovers. +func (r *Repository) heldError(ctx context.Context, key, resource string, cause error) error { + readCtx, cancel := detached(ctx) + defer cancel() + holder, _, _ := r.readClaim(readCtx, key) + message := fmt.Sprintf("lock %q is held through claim object %s", resource, r.uri(key)) + if holder != nil { + message = fmt.Sprintf("lock %q is held by owner %q (operation %q, since %s) through claim object %s", + resource, holder.Owner, holder.OperationID, holder.AcquiredAt, r.uri(key)) + } + message += "; claims never expire and are never taken over automatically. If that writer has stopped, confirm it is no longer running and delete the claim object to recover" + if cause != nil { + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactConflict, message, cause) + } + return fmt.Errorf("%w: %s", errUtils.ErrArtifactConflict, message) +} + +// Resource implements [artifact.LockHandle]. +func (l *claimHandle) Resource() string { + defer perf.Track(nil, "s3.claimHandle.Resource")() + + return l.resource +} + +// Owner implements [artifact.LockHandle]. +func (l *claimHandle) Owner() string { + defer perf.Track(nil, "s3.claimHandle.Owner")() + + return l.owner +} + +// Scope implements [artifact.LockHandle]; claims coordinate writers across hosts. +func (l *claimHandle) Scope() artifact.LockScope { + defer perf.Track(nil, "s3.claimHandle.Scope")() + + return artifact.LockScopeRepository +} + +// Token implements [artifact.LockHandle]. +func (l *claimHandle) Token() string { + defer perf.Track(nil, "s3.claimHandle.Token")() + + return l.token +} + +// ExpiresAt implements [artifact.LockHandle]; claims never expire. +func (l *claimHandle) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "s3.claimHandle.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; claims have no leases. +func (l *claimHandle) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "s3.claimHandle.Renew")() + + return artifact.NewUnsupportedCapability(l.repo.alias, opLock, "lease", "S3 claims do not expire automatically, so there is nothing to renew") +} + +// Release implements [artifact.LockHandle]. It reads the claim, checks that it still carries this +// handle's ownership token, and deletes it with a conditional delete on the observed ETag, so a +// stale or foreign handle can never release another operation's claim. Release proceeds even when +// ctx was canceled: a canceled operation must still let go. +func (l *claimHandle) Release(ctx context.Context) error { + defer perf.Track(nil, "s3.claimHandle.Release")() + + l.mu.Lock() + defer l.mu.Unlock() + if l.released { + return fmt.Errorf("%w: lock %q was already released", errUtils.ErrArtifactConflict, l.resource) + } + releaseCtx, cancel := detached(ctx) + defer cancel() + current, etag, err := l.repo.readClaim(releaseCtx, l.key) + switch { + case err != nil: + return err + case current == nil: + return fmt.Errorf("%w: lock %q is not held: claim object %s is gone", errUtils.ErrArtifactConflict, l.resource, l.repo.uri(l.key)) + case current.Token != l.token: + return fmt.Errorf("%w: lock %q is held by owner %q (operation %q), not by token %s; refusing to release another owner's claim", + errUtils.ErrArtifactConflict, l.resource, current.Owner, current.OperationID, l.token) + } + if err := l.repo.deleteConditional(releaseCtx, l.key, etag); err != nil && !l.goneAfter(releaseCtx, err) { + return fmt.Errorf("release lock %q: %w", l.resource, err) + } + l.released = true + return nil +} + +// goneAfter reports whether a failed conditional delete nevertheless removed the claim: after a +// lost response, an absent claim proves it did. +func (l *claimHandle) goneAfter(ctx context.Context, err error) bool { + if !blob.IsAmbiguous(err) { + return false + } + current, _, readErr := l.repo.readClaim(ctx, l.key) + return readErr == nil && current == nil +} diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go new file mode 100644 index 00000000000..46d2f627f55 --- /dev/null +++ b/pkg/artifact/s3/lock_test.go @@ -0,0 +1,321 @@ +package s3 + +import ( + "context" + "encoding/json" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +const lockResource = "team/app:1.0.0" + +func lockObjects(fake *s3test.Server) []string { + return managedKeys(fake, "", layout.ManagedRoot+"/locks") +} + +// peerOf returns a second repository over the same bucket, the way a second host would see it. +func peerOf(t *testing.T, spec *rspec, opts ...Option) *Repository { + t.Helper() + return newRepo(t, spec.withOpts(opts...)) +} + +func TestLock_HandleContract(t *testing.T) { + spec := noneSpec().fresh(t).with(func(s *rspec) { s.tags = map[string]string{"team": "platform"} }) + repo := newRepo(t, spec) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a", OperationID: "op-a", Required: artifact.LockScopeRepository}) + require.NoError(t, err) + assert.Equal(t, lockResource, handle.Resource()) + assert.Equal(t, "owner-a", handle.Owner()) + assert.NotEmpty(t, handle.Token()) + assert.Equal(t, artifact.LockScopeRepository, handle.Scope(), "the claim coordinates writers across hosts") + _, expires := handle.ExpiresAt() + assert.False(t, expires, "V1 claims never expire") + require.ErrorIs(t, handle.Renew(context.Background(), time.Minute), errUtils.ErrArtifactUnsupportedCapability) + + keys := lockObjects(spec.fake) + require.Len(t, keys, 1) + assert.Equal(t, layout.LockPath(lockResource), keys[0]) + object, ok := spec.fake.Object(keys[0]) + require.True(t, ok) + var doc claimDoc + require.NoError(t, json.Unmarshal(object.Data, &doc)) + assert.Equal(t, claimDoc{SchemaVersion: 1, Resource: lockResource, Owner: "owner-a", OperationID: "op-a", Token: handle.Token(), AcquiredAt: doc.AcquiredAt}, doc) + assert.Equal(t, map[string]string{"team": "platform"}, object.Tags, "claims are tagged like every other object") + creates := 0 + for _, request := range spec.fake.RequestsFor(s3test.OpPutObject) { + if request.Key == keys[0] { + creates++ + assert.Equal(t, "*", request.Header.Get("If-None-Match"), "the claim is created if absent") + } + } + assert.Equal(t, 1, creates) + + require.NoError(t, handle.Release(context.Background())) + assert.Empty(t, lockObjects(spec.fake)) + var release s3test.Request + for _, request := range spec.fake.RequestsFor(s3test.OpDeleteObject) { + release = request + } + assert.NotEmpty(t, release.Header.Get("If-Match"), "release deletes conditionally on the observed claim") +} + +func TestLock_RejectsUnsupportedRequests(t *testing.T) { + repo := newRepo(t, noneSpec()) + tests := []struct { + name string + request artifact.LockRequest + wantErr error + }{ + {name: "lease", request: artifact.LockRequest{Resource: "x", Lease: time.Minute}, wantErr: errUtils.ErrArtifactUnsupportedCapability}, + {name: "unknown scope", request: artifact.LockRequest{Resource: "x", Required: "galaxy"}, wantErr: errUtils.ErrArtifactUnsupportedCapability}, + {name: "no resource", request: artifact.LockRequest{}, wantErr: errUtils.ErrArtifactRefInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handle, err := repo.Lock(context.Background(), tt.request) + require.ErrorIs(t, err, tt.wantErr) + assert.Nil(t, handle) + }) + } + for _, scope := range []artifact.LockScope{"", artifact.LockScopeHost, artifact.LockScopeRepository} { + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "scope-" + string(scope), Required: scope}) + require.NoError(t, err, "scope %q", scope) + require.NoError(t, handle.Release(context.Background())) + } +} + +func TestLock_ReleaseChecksTheOwnershipToken(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + other := peerOf(t, spec) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err) + claim, ok := handle.(*claimHandle) + require.True(t, ok) + + t.Run("a handle with the wrong token cannot release", func(t *testing.T) { + forged := &claimHandle{repo: other, key: claim.key, resource: lockResource, token: "not-the-token"} + require.ErrorIs(t, forged.Release(context.Background()), errUtils.ErrArtifactConflict) + assert.Len(t, lockObjects(spec.fake), 1, "the claim is untouched") + }) + + t.Run("an operator removal and a new owner make the old handle harmless", func(t *testing.T) { + require.NoError(t, repo.store.Delete(context.Background(), claim.key, blob.DeleteOptions{})) + successor, err := other.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + require.NoError(t, err) + err = handle.Release(context.Background()) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "owner-b", "the diagnostic names the real owner") + assert.Len(t, lockObjects(spec.fake), 1, "the successor's claim survives the stale release") + require.NoError(t, successor.Release(context.Background())) + }) + + t.Run("a claim that is gone cannot be released", func(t *testing.T) { + require.ErrorIs(t, handle.Release(context.Background()), errUtils.ErrArtifactConflict) + }) +} + +func TestLock_DoubleReleaseIsRejected(t *testing.T) { + repo := newRepo(t, noneSpec()) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource}) + require.NoError(t, err) + require.NoError(t, handle.Release(context.Background())) + require.ErrorIs(t, handle.Release(context.Background()), errUtils.ErrArtifactConflict) +} + +// A competing owner is rejected with a diagnostic that names the claim object, never taken over. +func TestLock_CompetingOwnerIsRejectedWithARecoveryDiagnostic(t *testing.T) { + spec := noneSpec().fresh(t) + holder := newRepo(t, spec) + waiter := peerOf(t, spec, WithLockWait(40*time.Millisecond)) + held, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "host-a:100", OperationID: "op-holder"}) + require.NoError(t, err) + + handle, err := waiter.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "host-b:200", OperationID: "op-waiter"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Nil(t, handle) + for _, want := range []string{"host-a:100", "op-holder", "claim object s3://" + spec.fake.Bucket() + "/" + layout.LockPath(lockResource), "never expire", "delete the claim object"} { + assert.ErrorContains(t, err, want) + } + assert.Len(t, lockObjects(spec.fake), 1, "the waiter wrote nothing") + + require.NoError(t, held.Release(context.Background())) + handle, err = waiter.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "host-b:200"}) + require.NoError(t, err) + require.NoError(t, handle.Release(context.Background())) +} + +// The context bounds the wait too, and its error stays visible alongside the conflict. +func TestLock_WaitIsBoundedByTheContext(t *testing.T) { + spec := noneSpec().fresh(t) + holder := newRepo(t, spec) + _, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + _, err = peerOf(t, spec).Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) +} + +// A crash leaves the claim; nothing expires or steals it, and removing exactly that object recovers. +func TestLock_CrashLeftClaimBlocksUntilTheOperatorRemovesIt(t *testing.T) { + spec := noneSpec().fresh(t) + crashed := newRepo(t, spec) + _, err := crashed.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "crashed-host:1", OperationID: "op-crashed"}) + require.NoError(t, err) + // The process dies here: no Release. + + survivor := peerOf(t, spec, WithLockWait(20*time.Millisecond)) + for attempt := 0; attempt < 3; attempt++ { + _, err = survivor.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "survivor:2"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "claims never expire, however long they sit") + require.ErrorContains(t, err, "crashed-host:1") + require.ErrorContains(t, err, "op-crashed") + } + + keys := lockObjects(spec.fake) + require.Len(t, keys, 1) + require.NoError(t, survivor.store.Delete(context.Background(), keys[0], blob.DeleteOptions{}), "the operator deletes the one named claim object") + handle, err := survivor.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "survivor:2"}) + require.NoError(t, err) + require.NoError(t, handle.Release(context.Background())) +} + +// Cancellation, in every shape, leaves no claim of ours behind. +func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { + t.Run("canceled while waiting", func(t *testing.T) { + spec := noneSpec().fresh(t) + holder := newRepo(t, spec) + held, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = peerOf(t, spec).Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + require.ErrorIs(t, err, context.Canceled) + require.NoError(t, held.Release(context.Background())) + assert.Empty(t, lockObjects(spec.fake), "only the holder's claim ever existed") + }) + + for _, lostResponse := range []bool{false, true} { + name := "canceled just after the claim was created" + if lostResponse { + name = "canceled with the create response lost" + } + t.Run(name, func(t *testing.T) { + spec := noneSpec().fresh(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store := &hookStore{Store: newStoreFor(t, spec)} + store.afterPut = func(key string, info *blob.ObjectInfo, err error) (*blob.ObjectInfo, error) { + cancel() + if lostResponse { + return nil, blob.NewError(blob.KindAmbiguous, blob.OpPut, key, nil) + } + return info, err + } + repo, err := newRepository("test", spec.config(spec.fake.Bucket()), []Option{WithStore(store)}, &artifact.RepositoryOptions{}) + require.NoError(t, err) + + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, handle) + assert.Empty(t, lockObjects(spec.fake), "the claim that was created is released again") + }) + } +} + +// A lost response to the create or to the release is settled by reading the claim back. +func TestLock_LostResponsesAreReconciled(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + key := layout.LockPath(lockResource) + + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: key, Action: s3test.DropAfterCommit}) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err, "the create was applied, and the claim proves it") + assert.Len(t, lockObjects(spec.fake), 1) + + spec.fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, KeyPrefix: key, Action: s3test.DropAfterCommit}) + require.NoError(t, handle.Release(context.Background()), "an absent claim proves the release was applied") + assert.Empty(t, lockObjects(spec.fake)) +} + +// Concurrent holders of one resource never overlap. +func TestLock_CompetingHandlesAreMutuallyExclusive(t *testing.T) { + spec := noneSpec().fresh(t) + const workers, rounds = 4, 3 + var active, overlaps atomic.Int32 + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + repo := peerOf(t, spec, WithLockWait(time.Minute)) + wg.Add(1) + go func() { + defer wg.Done() + for round := 0; round < rounds; round++ { + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "worker"}) + if !assert.NoError(t, err) { + return + } + if active.Add(1) > 1 { + overlaps.Add(1) + } + time.Sleep(2 * time.Millisecond) + active.Add(-1) + assert.NoError(t, handle.Release(context.Background())) + } + }() + } + wg.Wait() + assert.Zero(t, overlaps.Load(), "two owners held the same claim at once") + assert.Empty(t, lockObjects(spec.fake)) +} + +// Publish honors a reference lock held by another owner and leaks nothing when canceled while +// waiting for it. +func TestPublish_HonorsTheReferenceLock(t *testing.T) { + spec := noneSpec().fresh(t) + holder := newRepo(t, spec) + held, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: testName + ":" + testVersion, Owner: "owner-a"}) + require.NoError(t, err) + + writer := peerOf(t, spec, WithLockWait(30*time.Millisecond)) + _, err = writer.Publish(context.Background(), prepare(t, writer, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Empty(t, managedKeys(spec.fake, "", layout.ReferencesDir()), "nothing was written while the lock was held elsewhere") + assert.Empty(t, payloadKeys(spec.fake, "")) + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err = writer.Publish(canceled, prepare(t, writer, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.Canceled) + + require.NoError(t, held.Release(context.Background())) + publish(t, writer, &prepareSpec{}, artifact.PublishOptions{}) + assert.Empty(t, lockObjects(spec.fake), "the writer released its own lock and the claims lock") +} + +// A publication holding a service lock does not take (and deadlock on) its own. +func TestPublish_UsesTheCallersLock(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: testName + ":" + testVersion, Owner: "service"}) + require.NoError(t, err) + result, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{Lock: handle}) + require.NoError(t, err) + assert.Equal(t, artifact.LockScopeRepository, result.LockScope) + assert.Len(t, lockObjects(spec.fake), 1, "the caller's lock is still held") + require.NoError(t, handle.Release(context.Background())) +} diff --git a/pkg/artifact/s3/objects.go b/pkg/artifact/s3/objects.go new file mode 100644 index 00000000000..ce87da5925e --- /dev/null +++ b/pkg/artifact/s3/objects.go @@ -0,0 +1,166 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "maps" + + "golang.org/x/sync/errgroup" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// Content types of the objects the repository writes. +const ( + contentTypeJSON = "application/json" + contentTypeBinary = "application/octet-stream" + contentTypeMD = "text/markdown" +) + +// putAttempts bounds how often an unconditional write is repeated after an ambiguous outcome. +// Unconditional writes of the same bytes are idempotent, so repeating them is safe. +const putAttempts = 3 + +// uri returns the display locator of a store key. +func (r *Repository) uri(key string) string { + if r.prefix == "" { + return "s3://" + r.bucket + "/" + key + } + return "s3://" + r.bucket + "/" + r.prefix + "/" + key +} + +// putOptions builds the write options of one object. Every object the repository writes carries +// the configured tags; a fresh copy is passed so the store can never mutate the shared map. +func (r *Repository) putOptions(size int64, contentType string, cond blob.Condition) blob.PutOptions { + return blob.PutOptions{Size: size, ContentType: contentType, Tags: maps.Clone(r.tags), Condition: cond} +} + +// readObject reads one managed object, bounded by limit. A missing object returns (nil, "", nil). +func (r *Repository) readObject(ctx context.Context, key string, limit int64) (data []byte, etag string, err error) { + object, err := r.store.Get(ctx, key, blob.GetOptions{}) + if blob.IsNotFound(err) { + return nil, "", nil + } + if err != nil { + return nil, "", r.mapErr(opRead, key, err) + } + defer object.Body.Close() + data, err = io.ReadAll(io.LimitReader(object.Body, limit+1)) + if err != nil { + return nil, "", r.mapErr(opRead, key, err) + } + if int64(len(data)) > limit { + return nil, "", fmt.Errorf("%w: %s exceeds %d bytes", errUtils.ErrArtifactTooLarge, r.uri(key), limit) + } + return data, object.ETag, nil +} + +// putConditional writes bytes under a condition and returns the new ETag. It does not retry: the +// caller decides how to reconcile a lost response because a repeated conditional write could +// conflict with itself. +func (r *Repository) putConditional(ctx context.Context, key string, data []byte, contentType string, cond blob.Condition) (string, error) { + info, err := r.store.Put(ctx, key, bytes.NewReader(data), r.putOptions(int64(len(data)), contentType, cond)) + if err != nil { + return "", r.mapErr(opWrite, key, err) + } + return info.ETag, nil +} + +// putBytes writes a JSON document unconditionally, repeating the idempotent write after an ambiguous outcome. +func (r *Repository) putBytes(ctx context.Context, key string, data []byte) error { + var err error + for range putAttempts { + _, err = r.store.Put(ctx, key, bytes.NewReader(data), r.putOptions(int64(len(data)), contentTypeJSON, blob.Condition{})) + if !blob.IsAmbiguous(err) { + break + } + } + return r.mapErr(opWrite, key, err) +} + +// putImmutable writes content-addressed bytes. An existing object must hold identical bytes: +// different bytes under the same name are corruption, never silently replaced. +func (r *Repository) putImmutable(ctx context.Context, key string, data []byte, contentType string, limit int64) error { + _, err := r.putConditional(ctx, key, data, contentType, blob.IfAbsent()) + switch { + case err == nil: + return nil + case !blob.IsPreconditionFailed(err) && !blob.IsAmbiguous(err): + return err + } + existing, _, readErr := r.readObject(ctx, key, limit) + switch { + case readErr != nil: + return readErr + case existing == nil: + return err + case !bytes.Equal(existing, data): + return fmt.Errorf("%w: %s already exists with different bytes", errUtils.ErrArtifactIntegrityFailed, r.uri(key)) + } + return nil +} + +// deleteObject removes one object; a missing object is not an error. +func (r *Repository) deleteObject(ctx context.Context, key string) error { + return r.mapErr(opDelete, key, r.store.Delete(ctx, key, blob.DeleteOptions{})) +} + +// deleteConditional removes one object only while its ETag is unchanged. +func (r *Repository) deleteConditional(ctx context.Context, key, etag string) error { + return r.mapErr(opDelete, key, r.store.Delete(ctx, key, blob.DeleteOptions{IfMatch: etag})) +} + +// deleteAll removes every key, continuing past failures and returning the first. +func (r *Repository) deleteAll(ctx context.Context, keys []string) error { + var first error + for _, key := range keys { + if err := r.deleteObject(ctx, key); err != nil && first == nil { + first = err + } + } + return first +} + +// stat returns the object attributes, or nil when the object is missing. +func (r *Repository) stat(ctx context.Context, key string) (*blob.ObjectInfo, error) { + info, err := r.store.Stat(ctx, key, blob.StatOptions{}) + if blob.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, r.mapErr(opRead, key, err) + } + return info, nil +} + +// listPrefix returns every object below a prefix, following the pagination cursor. +func (r *Repository) listPrefix(ctx context.Context, prefix string) ([]blob.ObjectInfo, error) { + objects, _, err := blob.ListAll(ctx, r.store, blob.ListOptions{Prefix: prefix}) + if err != nil { + return nil, r.mapErr(opRead, prefix, err) + } + return objects, nil +} + +// hasAny reports whether at least one object exists below a prefix. +func (r *Repository) hasAny(ctx context.Context, prefix string) (bool, error) { + page, err := r.store.List(ctx, blob.ListOptions{Prefix: prefix, Limit: 1}) + if err != nil { + return false, r.mapErr(opRead, prefix, err) + } + return len(page.Objects) > 0, nil +} + +// forEach runs fn for every index below count with at most the configured number of calls in +// flight, stopping at the first failure. +func (r *Repository) forEach(ctx context.Context, count int, fn func(ctx context.Context, index int) error) error { + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(max(r.settings.concurrency, 1)) + for index := range count { + group.Go(func() error { return fn(groupCtx, index) }) + } + return group.Wait() +} diff --git a/pkg/artifact/s3/options.go b/pkg/artifact/s3/options.go new file mode 100644 index 00000000000..bbe432f185c --- /dev/null +++ b/pkg/artifact/s3/options.go @@ -0,0 +1,124 @@ +package s3 + +import ( + "fmt" + "os" + "time" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Defaults of the repository. +const ( + // DefaultConcurrency bounds the objects and multipart parts uploaded at once. + DefaultConcurrency = blobs3.DefaultConcurrency + + // DefaultLockWait is how long Lock waits for a competing owner before it reports the claim. + DefaultLockWait = 30 * time.Second + + // The interval between attempts to take a held claim. + defaultLockPoll = 250 * time.Millisecond +) + +// settings holds the construction options of an S3 repository. +type settings struct { + store blob.Store + verifier artifact.Verifier + owner string + concurrency int + partSize int64 + lockWait time.Duration + lockPoll time.Duration + faults faultHook +} + +// Option configures an S3 repository or its registration. +type Option func(*settings) + +// WithStore injects the object store. Without it the repository builds the AWS SDK store from its +// configuration. The injected store must be rooted at the configured prefix, because the repository +// hands it keys relative to that root. It does not need to apply the configured tags: the +// repository passes them on every write. +func WithStore(store blob.Store) Option { + defer perf.Track(nil, "s3.WithStore")() + + return func(s *settings) { s.store = store } +} + +// WithVerifier injects the cryptographic verifier used by Verify. The formats it understands are +// the evidence formats the repository advertises. Without a verifier the repository stores no +// evidence and Sign and Verify return unsupported-capability errors. +func WithVerifier(verifier artifact.Verifier) Option { + defer perf.Track(nil, "s3.WithVerifier")() + + return func(s *settings) { s.verifier = verifier } +} + +// WithOwner sets the owner recorded in lock claims and operation journals when the caller does not +// supply one. The default is ":". +func WithOwner(owner string) Option { + defer perf.Track(nil, "s3.WithOwner")() + + return func(s *settings) { s.owner = owner } +} + +// WithConcurrency bounds the objects uploaded at once and, for a store built from configuration, +// the multipart parts per object. The default is [DefaultConcurrency]. +func WithConcurrency(workers int) Option { + defer perf.Track(nil, "s3.WithConcurrency")() + + return func(s *settings) { + if workers > 0 { + s.concurrency = workers + } + } +} + +// WithPartSize sets the multipart part size of a store built from configuration. Real S3 requires +// at least 5 MiB for every part but the last; smaller sizes are for emulators. +func WithPartSize(size int64) Option { + defer perf.Track(nil, "s3.WithPartSize")() + + return func(s *settings) { + if size > 0 { + s.partSize = size + } + } +} + +// WithLockWait bounds how long Lock waits for a competing owner. Past it, Lock fails with a +// conflict that names the claim object. Claims never expire, so waiting longer only helps when the +// owner is alive. +func WithLockWait(wait time.Duration) Option { + defer perf.Track(nil, "s3.WithLockWait")() + + return func(s *settings) { + if wait > 0 { + s.lockWait = wait + } + } +} + +func applySettings(opts []Option) settings { + applied := settings{concurrency: DefaultConcurrency, lockWait: DefaultLockWait, lockPoll: defaultLockPoll} + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.owner == "" { + applied.owner = defaultOwner() + } + return applied +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s:%d", host, os.Getpid()) +} diff --git a/pkg/artifact/s3/plan.go b/pkg/artifact/s3/plan.go new file mode 100644 index 00000000000..1e02c6e0cdc --- /dev/null +++ b/pkg/artifact/s3/plan.go @@ -0,0 +1,146 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// objectPlan is one object a publication writes: where it goes (relative to the resolved +// location), where its bytes come from, and what they must verify against. +type objectPlan struct { + place func(location string) string + staged *artifact.StagedBlob + data []byte + size int64 + digest digest.Digest + contentType string + payload bool +} + +// open returns a fresh reader over the object's bytes. +func (o *objectPlan) open(ctx context.Context) (io.ReadCloser, error) { + if o.staged != nil { + return o.staged.Open(ctx) + } + return io.NopCloser(bytes.NewReader(o.data)), nil +} + +// plan validates the staged blobs against the Record and lists the objects to write, payload +// first. It runs before any state changes, so a malformed preparation never leaves a pending +// reference behind. +func (j *publishJob) plan() ([]objectPlan, error) { + payload, docs := j.splitBlobs() + if err := j.matchDocuments(docs); err != nil { + return nil, err + } + if isArchive(j.prepared.Packaging) { + return j.archivePlan(payload, docs) + } + return j.filesPlan(payload, docs) +} + +// sortedDocs returns the document roles in a stable order. +func sortedDocs(docs map[artifact.DocumentRole]*artifact.StagedBlob) []artifact.DocumentRole { + roles := make([]artifact.DocumentRole, 0, len(docs)) + for role := range docs { + roles = append(roles, role) + } + sort.Slice(roles, func(i, k int) bool { return roles[i] < roles[k] }) + return roles +} + +// filesPlan plans per-file packaging: one object per inventory file, the distribution documents, +// and the manifest. +func (j *publishJob) filesPlan(payload []*artifact.StagedBlob, docs map[artifact.DocumentRole]*artifact.StagedBlob) ([]objectPlan, error) { + byPath, err := indexPayload(payload) + if err != nil { + return nil, err + } + items, err := j.fileItems(byPath) + if err != nil { + return nil, err + } + for _, role := range sortedDocs(docs) { + items = append(items, j.documentItem(docs[role], func(location string) string { + return layout.DistributionPath(location, layout.DocumentFileName(role)) + })) + } + return append(items, j.metadataItem(func(location string) string { return layout.DistributionPath(location, layout.ManifestName) })), nil +} + +// indexPayload maps the staged payload blobs by inventory path. +func indexPayload(payload []*artifact.StagedBlob) (map[string]*artifact.StagedBlob, error) { + byPath := map[string]*artifact.StagedBlob{} + for _, staged := range payload { + if _, duplicate := byPath[staged.Path]; duplicate || staged.Path == "" { + return nil, fmt.Errorf("%w: payload blob path %q is missing or duplicated", errUtils.ErrArtifactIntegrityFailed, staged.Path) + } + byPath[staged.Path] = staged + } + return byPath, nil +} + +// fileItems plans one object per inventory file and checks each against its staged blob. +func (j *publishJob) fileItems(byPath map[string]*artifact.StagedBlob) ([]objectPlan, error) { + var items []objectPlan + for index := range j.record.Inventory { + entry := &j.record.Inventory[index] + if entry.Path == layout.DistributionDir || strings.HasPrefix(entry.Path, layout.DistributionDir+"/") { + return nil, fmt.Errorf("%w: inventory path %q is inside the generated %s directory", errUtils.ErrArtifactUnsafePath, entry.Path, layout.DistributionDir) + } + if entry.Type != artifact.EntryFile { + continue + } + staged, ok := byPath[entry.Path] + if !ok || staged.Descriptor.Digest != entry.Digest || staged.Descriptor.Size != entry.Size { + return nil, fmt.Errorf("%w: no staged blob matches inventory file %q", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + path := entry.Path + items = append(items, objectPlan{ + place: func(location string) string { return location + "/" + path }, + staged: staged, size: entry.Size, digest: entry.Digest, contentType: contentTypeBinary, payload: true, + }) + } + if len(items) != len(byPath) { + return nil, fmt.Errorf("%w: %d staged payload blobs but %d inventory files", errUtils.ErrArtifactIntegrityFailed, len(byPath), len(items)) + } + return items, nil +} + +// archivePlan plans archive packaging: one archive object, the document sidecars, and the +// metadata sidecar. +func (j *publishJob) archivePlan(payload []*artifact.StagedBlob, docs map[artifact.DocumentRole]*artifact.StagedBlob) ([]objectPlan, error) { + if len(payload) != 1 { + return nil, fmt.Errorf("%w: an archive publication needs exactly one payload blob", errUtils.ErrArtifactPackagingInvalid) + } + staged := payload[0] + items := []objectPlan{{ + place: func(location string) string { return location }, + staged: staged, size: staged.Descriptor.Size, digest: staged.Descriptor.Digest, contentType: contentTypeBinary, payload: true, + }} + for _, role := range sortedDocs(docs) { + items = append(items, j.documentItem(docs[role], func(location string) string { + return layout.DocumentSidecarPath(location, role) + })) + } + return append(items, j.metadataItem(layout.MetadataSidecarPath)), nil +} + +func (j *publishJob) documentItem(staged *artifact.StagedBlob, place func(string) string) objectPlan { + return objectPlan{place: place, staged: staged, size: staged.Descriptor.Size, digest: staged.Descriptor.Digest, contentType: contentTypeMD} +} + +// metadataItem is the portable copy of the canonical Record that travels beside the payload. +func (j *publishJob) metadataItem(place func(string) string) objectPlan { + return objectPlan{place: place, data: j.recordBytes, size: int64(len(j.recordBytes)), digest: j.recordDigest, contentType: contentTypeJSON} +} diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go new file mode 100644 index 00000000000..cfbea73e25c --- /dev/null +++ b/pkg/artifact/s3/publish.go @@ -0,0 +1,248 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ownedPayload is a payload location a reference already owns, with the packaging that decides +// which object keys belong to it. +type ownedPayload struct { + packaging artifact.Packaging + location string + set objectSet +} + +// publishPlan is what the admission decision and the existing state say about one publication. +type publishPlan struct { + admission layout.Admission + current *refState + prior *layout.Prior + owned []ownedPayload + abandoned *ownedPayload + oldOp *layout.Operation + expected *artifact.PinnedReference + scope artifact.LockScope +} + +// Publish implements [artifact.Repository] with the publication sequence of the contracts: +// +// 1. Hold the reference lock (the service's, or one taken here). +// 2. Read the current reference and admit the publication: conflict, force, unchanged, or resume. +// 3. Reserve the location and record the pending operation journal and pending reference before +// any payload object is written. Reference ownership is a conditional write. +// 4. Upload the payload and documents, verifying every byte, then write the immutable Record and +// the representation. +// 5. Complete the reference last, with a conditional write on the pending state's ETag. +// 6. Clean up only the objects this operation owns. +// +// A failure after step 3 leaves the reference pending: unavailable to readers, repaired by a retry +// with the same intent. A response lost after the request was sent is reconciled by reading the +// reference back. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (result *artifact.PublishResult, err error) { + defer perf.Track(nil, "s3.Repository.Publish")() + + job, err := r.newJob(prepared) + if err != nil { + return nil, err + } + scope := artifact.LockScopeRepository + if options.Lock != nil { + scope = options.Lock.Scope() + } else { + lock, lockErr := r.acquire(ctx, prepared.Record.Name+":"+prepared.Record.Version, r.settings.owner, job.opID) + if lockErr != nil { + return nil, lockErr + } + defer func() { + if releaseErr := lock.Release(ctx); err == nil { + err = releaseErr + } + }() + } + return r.publishLocked(ctx, job, &options, scope) +} + +// publishLocked runs the publication while the reference lock is held. +func (r *Repository) publishLocked(ctx context.Context, job *publishJob, options *artifact.PublishOptions, scope artifact.LockScope) (*artifact.PublishResult, error) { + plan, err := r.admit(ctx, job, options) + if err != nil { + return nil, err + } + plan.scope = scope + if plan.admission == layout.AdmitUnchanged { + return r.unchanged(ctx, job, plan) + } + if err := r.resolveLocation(ctx, job, plan); err != nil { + return nil, err + } + pending, err := r.begin(ctx, job, plan) + if err != nil { + return nil, err + } + return r.build(ctx, job, plan, pending) +} + +// admit reads the current reference and applies the admission rules. +func (r *Repository) admit(ctx context.Context, job *publishJob, options *artifact.PublishOptions) (*publishPlan, error) { + current, err := r.loadReference(ctx, job.key) + if err != nil { + return nil, err + } + input := layout.AdmitInput{ + Current: refOf(current), IncomingRecord: job.recordDigest, IncomingIntent: job.prepared.IntentDigest, + ExpectedPrior: options.ExpectedPrior, Force: options.Force, + } + if input.Current != nil && input.Current.Available() { + if input.CurrentIntent, err = r.completedIntent(ctx, input.Current); err != nil { + return nil, err + } + } + admission, err := layout.Admit(&input) + if err != nil { + return nil, err + } + return &publishPlan{admission: admission, current: current, expected: options.ExpectedPrior}, nil +} + +// refOf returns the reference of a stored state, or nil for none. +func refOf(state *refState) *layout.Reference { + if state == nil { + return nil + } + return state.ref +} + +// completedIntent reads the intent digest of a completed reference's Record. +func (r *Repository) completedIntent(ctx context.Context, current *layout.Reference) (digest.Digest, error) { + record, _, err := r.loadRecord(ctx, current.RecordDigest) + if err != nil { + return "", err + } + return intentOf(record), nil +} + +// intentOf returns the intent digest of a Record. An intent that cannot be computed is empty, which +// never counts as unchanged. +func intentOf(record *artifact.Record) digest.Digest { + intent, err := record.IntentDigest() + if err != nil { + return "" + } + return intent +} + +// unchanged returns the existing artifact for an identical republication and tidies the journal of +// the operation that completed it. +func (r *Repository) unchanged(ctx context.Context, job *publishJob, plan *publishPlan) (*artifact.PublishResult, error) { + current := plan.current.ref + record, _, err := r.loadRecord(ctx, current.RecordDigest) + if err != nil { + return nil, err + } + rep, err := r.loadRepresentation(ctx, job.key) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != current.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is completed but has no matching representation", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) + } + r.tidyOperation(ctx, current.OperationID) + return r.result(record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome(), plan.scope), nil +} + +// tidyOperation removes the journal of a finished operation. It is best-effort: a leftover is +// harmless and is retried by the next publication. +func (r *Repository) tidyOperation(ctx context.Context, operationID string) { + if operationID == "" { + return + } + cleanup, cancel := detached(ctx) + defer cancel() + _ = r.deleteObject(cleanup, layout.OperationPath(operationID)) +} + +// resolveLocation decides where the payload lives. A resumed publication reuses the location in +// its journal; otherwise the key_pattern is rendered exactly once, here, from the Record's +// captured creation time. The result is persisted; it is never rendered again. +func (r *Repository) resolveLocation(ctx context.Context, job *publishJob, plan *publishPlan) error { + rep, err := r.loadRepresentation(ctx, job.key) + if err != nil { + return err + } + if err := r.loadPendingState(ctx, plan, rep); err != nil { + return err + } + if plan.admission.Resuming() && plan.oldOp != nil && plan.oldOp.IntentDigest == job.prepared.IntentDigest && plan.oldOp.Packaging == job.prepared.Packaging { + job.location = plan.oldOp.Location + return nil + } + if plan.oldOp != nil { + plan.abandoned = &ownedPayload{ + packaging: plan.oldOp.Packaging, location: plan.oldOp.Location, + set: setOfDescriptors(plan.oldOp.Payload, plan.oldOp.Documents), + } + } + job.location, err = layout.RenderLocation(r.keyPattern, &layout.PatternInput{ + Name: job.record.Name, Version: job.record.Version, Kind: job.record.Kind, CreatedAt: job.record.CreatedAt.Time(), + }, job.prepared.Packaging) + return err +} + +// loadPendingState records the locations this reference already owns and the content a +// publication replaces. The interrupted operation's journal is read when one is pending. +func (r *Repository) loadPendingState(ctx context.Context, plan *publishPlan, rep *layout.Representation) error { + if rep != nil { + plan.owned = append(plan.owned, ownedPayload{packaging: rep.Packaging, location: rep.Location}) + } + if plan.current != nil && !plan.current.ref.Available() { + oldOp, err := r.loadOperation(ctx, plan.current.ref.OperationID) + if err != nil { + return err + } + plan.oldOp = oldOp + } + plan.prior = priorState(plan, rep) + if plan.oldOp != nil { + plan.owned = append(plan.owned, ownedPayload{packaging: plan.oldOp.Packaging, location: plan.oldOp.Location}) + if prior := plan.oldOp.Prior; prior != nil && prior.Location != "" { + plan.owned = append(plan.owned, ownedPayload{packaging: prior.Packaging, location: prior.Location}) + } + } + return nil +} + +// priorState describes the content a publication replaces, so a later cleanup can retire it. +func priorState(plan *publishPlan, rep *layout.Representation) *layout.Prior { + if plan.admission == layout.AdmitReplace { + return priorFrom(plan.current.ref.RecordDigest, rep) + } + if !plan.admission.Resuming() { + return nil + } + if plan.oldOp != nil && plan.oldOp.Prior != nil { + prior := *plan.oldOp.Prior + return &prior + } + if plan.current.ref.PriorRecordDigest == "" { + return nil + } + return priorFrom(plan.current.ref.PriorRecordDigest, rep) +} + +// priorFrom builds the prior state for a Record digest, taking the payload location from the +// representation when it still describes that Record. +func priorFrom(recordDigest digest.Digest, rep *layout.Representation) *layout.Prior { + prior := &layout.Prior{RecordDigest: recordDigest} + if rep != nil && rep.RecordDigest == recordDigest { + prior.Location, prior.Packaging = rep.Location, rep.Packaging + } + return prior +} diff --git a/pkg/artifact/s3/publish_test.go b/pkg/artifact/s3/publish_test.go new file mode 100644 index 00000000000..38a0eb59b1e --- /dev/null +++ b/pkg/artifact/s3/publish_test.go @@ -0,0 +1,214 @@ +package s3 + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +const testPrefix = "atmos/prod" + +// bigFiles forces the multipart path (the test part size is 16 KiB) and includes an empty file. +func bigFiles() map[string]string { + return map[string]string{ + "data.txt": "hello\n", + "empty.txt": "", + "nested/big.bin": incompressible(48 << 10), + } +} + +// The completed reference is the very last object written: the pending state and the journal come +// first, every payload and metadata object is uploaded in between, and reference writes carry the +// conditional headers that make ownership atomic. +func TestPublish_RequestOrdering(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t).with(func(s *rspec) { s.prefix = testPrefix }) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: bigFiles()}, artifact.PublishOptions{}) + + reference := refKey(testPrefix) + var refWrites []s3test.Request + lastWrite := -1 + writes := mutations(spec.fake) + for index, request := range writes { + if request.Op == s3test.OpDeleteObject { + continue + } + lastWrite = index + if request.Key == reference { + refWrites = append(refWrites, request) + } + } + require.Len(t, refWrites, 2, "one pending write and one completed write") + assert.Equal(t, reference, writes[lastWrite].Key, "the completed reference is written last") + assert.Equal(t, "*", refWrites[0].Header.Get("If-None-Match"), "the pending reference is created with If-None-Match") + assert.Empty(t, refWrites[0].Header.Get("If-Match")) + assert.NotEmpty(t, refWrites[1].Header.Get("If-Match"), "the completed reference replaces the pending ETag with If-Match") + assert.Empty(t, refWrites[1].Header.Get("If-None-Match")) + + journal := testPrefix + "/" + layout.OperationPath("op-"+testName+"-"+testVersion) + journalIndex, pendingIndex := -1, -1 + for index, request := range writes { + switch { + case request.Key == journal && journalIndex < 0 && request.Op == s3test.OpPutObject: + journalIndex = index + case request.Key == reference && pendingIndex < 0: + pendingIndex = index + } + } + require.GreaterOrEqual(t, journalIndex, 0) + assert.Less(t, journalIndex, pendingIndex, "the journal is recorded before the reference turns pending") + for index, request := range writes { + if request.Op == s3test.OpDeleteObject || request.Key == reference || request.Key == journal || strings.Contains(request.Key, "/locks/") { + continue + } + assert.Greater(t, index, pendingIndex, "%s is written only after the reference is pending", request.Key) + } + assertIntact(t, repo, defaultRef()) + }) + } +} + +// Every object of a publication carries the configured tags, including the parts of multipart +// uploads, documents, records, journals, and claims; portable labels never become tags. +func TestPublish_TagsOnEveryObject(t *testing.T) { + tags := map[string]string{"team": "platform", "env": "prod"} + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t).with(func(s *rspec) { s.tags, s.prefix = tags, testPrefix }) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: bigFiles(), labels: map[string]string{"suite": "tags"}}, artifact.PublishOptions{}) + signer := newSigner() + repo = newRepo(t, spec.withOpts(WithVerifier(&artifacttest.FakeVerifier{}))) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo, &pinned) + _, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + keys := bucketKeys(spec.fake) + require.NotEmpty(t, keys) + for _, key := range keys { + object, ok := spec.fake.Object(key) + require.True(t, ok) + assert.Equal(t, tags, object.Tags, "object %s", key) + } + assert.NotEmpty(t, payloadKeys(spec.fake, testPrefix)) + assert.NotEmpty(t, managedKeys(spec.fake, testPrefix, layout.ManagedRoot+"/signatures"), "evidence is tagged like everything else") + for _, request := range spec.fake.Requests() { + if request.Op == s3test.OpPutObject || request.Op == s3test.OpCreateMultipartUpload { + assert.NotEmpty(t, request.Header.Get("X-Amz-Tagging"), "%s %s", request.Op, request.Key) + assert.NotContains(t, request.Header.Get("X-Amz-Tagging"), "suite", "labels are not tags") + } + } + assert.GreaterOrEqual(t, spec.fake.CompletedUploads(), 1, "the large file used multipart") + }) + } +} + +// Two writers racing on one reference: the loser of the conditional write gets a conflict and never +// overwrites the winner. +func TestPublish_ConditionalConflictBetweenTwoWriters(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, winner *Repository) *artifact.PublishOptions + }{ + {name: "creation loses If-None-Match", setup: func(*testing.T, *Repository) *artifact.PublishOptions { + return &artifact.PublishOptions{Lock: dummyLock{}} + }}, + {name: "replacement loses If-Match", setup: func(t *testing.T, winner *Repository) *artifact.PublishOptions { + base := publish(t, winner, &prepareSpec{files: map[string]string{"data.txt": "base"}}, artifact.PublishOptions{}) + return &artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned, Lock: dummyLock{}} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := noneSpec().fresh(t) + winner := newRepo(t, spec) + options := tt.setup(t, winner) + + var finished *artifact.PublishResult + loserStore := &hookStore{Store: newStoreFor(t, spec)} + loserStore.beforePut = func(key string, _ blob.PutOptions) { + if finished != nil || !strings.Contains(key, "/locks/") { + return + } + // The winner publishes completely between the loser's read and its reservation. + won, err := winner.Publish(context.Background(), prepare(t, winner, &prepareSpec{files: map[string]string{"data.txt": "winner"}, opID: "winner-op"}), + artifact.PublishOptions{Force: options.Force, ExpectedPrior: options.ExpectedPrior, Lock: dummyLock{}}) + require.NoError(t, err) + finished = won + } + loser, err := newRepository("loser", spec.config(spec.fake.Bucket()), []Option{WithStore(loserStore), withLockPoll(5 * time.Millisecond)}, &artifact.RepositoryOptions{}) + require.NoError(t, err) + + _, err = loser.Publish(context.Background(), prepare(t, loser, &prepareSpec{files: map[string]string{"data.txt": "loser"}, opID: "loser-op"}), *options) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NotNil(t, finished) + pinned := resolve(t, winner, defaultRef()) + assert.Equal(t, finished.Pinned.RecordDigest, pinned.RecordDigest, "the winner's artifact is untouched") + assertIntact(t, winner, defaultRef()) + }) + } +} + +// An endpoint that cannot enforce a needed condition fails the operation clearly and never +// overwrites: nothing is written to the reference namespace. +func TestPublish_UnsupportedConditionalFailsClearly(t *testing.T) { + tests := []struct { + name string + spec func(*testing.T) *rspec + }{ + {name: "endpoint answers 501", spec: func(t *testing.T) *rspec { + return &rspec{packaging: artifact.PackagingTgz, fake: s3test.NewServer(t, s3test.WithConditionalBehavior(s3test.ConditionalNotImplemented))} + }}, + {name: "endpoint ignores the headers and is probed", spec: func(t *testing.T) *rspec { + return &rspec{packaging: artifact.PackagingTgz, probe: true, fake: s3test.NewServer(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore))} + }}, + {name: "conditional writes disabled", spec: func(t *testing.T) *rspec { + return &rspec{packaging: artifact.PackagingTgz, storeOpts: []blobs3.Option{blobs3.WithConditionalMode(blobs3.ConditionalDisabled)}, fake: s3test.NewServer(t)} + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec(t) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + capability, ok := artifact.AsCapabilityError(err) + require.True(t, ok) + assert.Contains(t, capability.Capability, "lock", "the lock claim is the first thing that needs a condition") + + // Bypassing the lock reaches the reference write itself, which must fail just as clearly. + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{Lock: dummyLock{}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Empty(t, managedKeys(spec.fake, "", layout.ReferencesDir()), "no reference was written") + assert.Empty(t, payloadKeys(spec.fake, ""), "no payload was written") + }) + } +} + +// A conditional-less store advertises no lock scope and no conditional update. +func TestCapabilities_WithoutConditionalWrites(t *testing.T) { + spec := noneSpec().fresh(t) + spec.storeOpts = []blobs3.Option{blobs3.WithConditionalMode(blobs3.ConditionalDisabled)} + caps := newRepo(t, spec).Capabilities() + assert.False(t, caps.ConditionalUpdate) + assert.Empty(t, caps.LockScopes) + _, err := newRepo(t, spec).Lock(context.Background(), artifact.LockRequest{Resource: "x"}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/s3/read.go b/pkg/artifact/s3/read.go new file mode 100644 index 00000000000..a8617c275ae --- /dev/null +++ b/pkg/artifact/s3/read.go @@ -0,0 +1,291 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash" + "io" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// filePayloadMediaType is the media type of one copied file. +const filePayloadMediaType = "application/octet-stream" + +// Read implements [artifact.Repository]. It opens the verified streams of exactly the pinned +// Record. The reference must still name that Record; the streams open their objects lazily, and +// when the caller reads one to its end the reference generation is rechecked, so bytes that changed +// underneath the reader are reported instead of served under the old digest. ETags play no part: +// every stream is verified against the digest in the Record. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "s3.Repository.Read")() + + v, err := r.openView(ctx, &pinned) + if err != nil { + return nil, err + } + if err := r.verifyMetadata(ctx, v); err != nil { + return nil, err + } + streams, err := r.streams(ctx, v, r.guard(ctx, v.ref.ref)) + if err != nil { + closeStreams(streams) + return nil, err + } + return &artifact.Content{Record: v.record, Representation: v.rep.Artifact(r.uri(v.rep.Location)), Streams: streams}, nil +} + +func closeStreams(streams []artifact.ContentStream) { + for index := range streams { + if streams[index].Reader != nil { + _ = streams[index].Reader.Close() + } + } +} + +// metadataKey is the adjacent metadata copy of the Record next to the payload. +func metadataKey(rep *layout.Representation) string { + if isArchive(rep.Packaging) { + return layout.MetadataSidecarPath(rep.Location) + } + return layout.DistributionPath(rep.Location, layout.ManifestName) +} + +// documentKey is the object that holds a distribution document. +func documentKey(rep *layout.Representation, role artifact.DocumentRole) string { + if isArchive(rep.Packaging) { + return layout.DocumentSidecarPath(rep.Location, role) + } + return layout.DistributionPath(rep.Location, layout.DocumentFileName(role)) +} + +// verifyMetadata checks the adjacent metadata copy against the stored Record bytes. +func (r *Repository) verifyMetadata(ctx context.Context, v *view) error { + key := metadataKey(v.rep) + got, _, err := r.readObject(ctx, key, artifact.MaxRecordSize) + switch { + case err != nil: + return err + case got == nil: + return fmt.Errorf("%w: metadata %s is missing", errUtils.ErrArtifactIntegrityFailed, r.uri(key)) + case !bytes.Equal(got, v.recordBytes): + return fmt.Errorf("%w: metadata %s does not match the stored Record", errUtils.ErrArtifactIntegrityFailed, r.uri(key)) + } + return nil +} + +// streams opens the verified payload and document streams of a completed artifact. +func (r *Repository) streams(ctx context.Context, v *view, guard func() error) ([]artifact.ContentStream, error) { + var streams []artifact.ContentStream + if isArchive(v.rep.Packaging) { + if len(v.rep.Payload) != 1 { + return nil, fmt.Errorf("%w: an archive representation must have exactly one payload, found %d", errUtils.ErrArtifactIntegrityFailed, len(v.rep.Payload)) + } + descriptor := v.rep.Payload[0] + streams = append(streams, artifact.ContentStream{ + Descriptor: descriptor, Role: artifact.BlobPayload, + Reader: r.objectReader(ctx, v.rep.Location, descriptor.Digest, descriptor.Size, guard), + }) + } + for index := range v.record.Inventory { + entry := &v.record.Inventory[index] + if entry.Type != artifact.EntryFile || isArchive(v.rep.Packaging) { + continue + } + streams = append(streams, artifact.ContentStream{ + Descriptor: ocispec.Descriptor{MediaType: filePayloadMediaType, Digest: entry.Digest, Size: entry.Size}, + Role: artifact.BlobPayload, + Path: entry.Path, + Reader: r.objectReader(ctx, v.rep.Location+"/"+entry.Path, entry.Digest, entry.Size, guard), + }) + } + for index := range v.record.Documents { + doc := &v.record.Documents[index] + streams = append(streams, artifact.ContentStream{ + Descriptor: ocispec.Descriptor{MediaType: doc.MediaType, Digest: doc.Digest, Size: doc.Size}, + Role: artifact.BlobDocument, + Reader: r.objectReader(ctx, documentKey(v.rep, doc.Role), doc.Digest, doc.Size, guard), + }) + } + return streams, nil +} + +// guard returns the generation recheck for a reader that observed ref. It reads the reference +// again and fails when the generation, state, or Record changed, because the payload location is +// mutable and a changed reference means the bytes may belong to a different version. +func (r *Repository) guard(ctx context.Context, observed *layout.Reference) func() error { + return func() error { + current, err := r.loadReference(ctx, observed.Key()) + switch { + case err != nil: + return err + case current == nil: + return fmt.Errorf("%w: %s:%s was deleted while it was being read", errUtils.ErrArtifactNotFound, observed.Name, observed.Version) + case current.ref.Generation != observed.Generation || !current.ref.Available() || current.ref.RecordDigest != observed.RecordDigest: + return fmt.Errorf("%w: %s:%s changed while it was being read (generation %d to %d); the bytes read are discarded rather than served under the old digest", + errUtils.ErrArtifactPublicationUnavailable, observed.Name, observed.Version, observed.Generation, current.ref.Generation) + } + return nil + } +} + +// objectReader returns a verifying reader over one payload or document object. The object is +// requested when the first byte is read, so opening many streams holds no connections. +func (r *Repository) objectReader(ctx context.Context, key string, want digest.Digest, size int64, guard func() error) io.ReadCloser { + return newVerifyingReader(&lazyObject{repo: r, ctx: ctx, key: key, size: size}, want, size, guard) +} + +// OpenDocument implements [artifact.Repository] without reading the payload. The reader verifies +// the document digest and size and rechecks the reference generation at EOF. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) OpenDocument(ctx context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "s3.Repository.OpenDocument")() + + v, err := r.openView(ctx, &pinned) + if err != nil { + return nil, err + } + doc, ok := documentDescriptor(v.record, role) + if !ok { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDocumentMissing, role) + } + reader := r.objectReader(ctx, documentKey(v.rep, role), doc.Digest, doc.Size, r.guard(ctx, v.ref.ref)) + return &artifact.DocumentStream{Document: *doc, Reader: reader}, nil +} + +// lazyObject requests an object on first read and checks its size against the declared one. +type lazyObject struct { + repo *Repository + ctx context.Context + key string + size int64 + body io.ReadCloser + opened bool + err error +} + +// Read implements io.Reader. +func (l *lazyObject) Read(p []byte) (int, error) { + defer perf.Track(nil, "s3.lazyObject.Read")() + + if !l.opened { + l.opened = true + l.err = l.open() + } + if l.err != nil { + return 0, l.err + } + return l.body.Read(p) +} + +func (l *lazyObject) open() error { + object, err := l.repo.store.Get(l.ctx, l.key, blob.GetOptions{}) + switch { + case blob.IsNotFound(err): + return fmt.Errorf("%w: object %s is missing", errUtils.ErrArtifactIntegrityFailed, l.repo.uri(l.key)) + case err != nil: + return l.repo.mapErr(opRead, l.key, err) + case object.Size != l.size: + _ = object.Body.Close() + return fmt.Errorf("%w: object %s is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, l.repo.uri(l.key), object.Size, l.size) + } + l.body = object.Body + return nil +} + +// Close implements io.Closer. +func (l *lazyObject) Close() error { + defer perf.Track(nil, "s3.lazyObject.Close")() + + if l.body == nil { + return nil + } + return l.body.Close() +} + +// verifyingReader verifies size and digest as the content flows through. At EOF it compares the +// digest and then runs the optional generation recheck, so a clean EOF means the bytes were intact +// and still belonged to the pinned reference. On a mismatch the final read reports the failure +// instead of io.EOF, which also makes it abort an upload that is consuming the reader. +type verifyingReader struct { + source io.ReadCloser + hasher hash.Hash + want digest.Digest + size int64 + read int64 + guard func() error + finished bool + failure error +} + +func newVerifyingReader(source io.ReadCloser, want digest.Digest, size int64, guard func() error) *verifyingReader { + return &verifyingReader{source: source, hasher: want.Algorithm().Hash(), want: want, size: size, guard: guard} +} + +// Read implements io.Reader. +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "s3.verifyingReader.Read")() + + if v.failure != nil { + return 0, v.failure + } + n, err := v.source.Read(p) + v.read += int64(n) + _, _ = v.hasher.Write(p[:n]) + switch { + case v.read > v.size: + v.failure = fmt.Errorf("%w: content is larger than its declared %d bytes", errUtils.ErrArtifactIntegrityFailed, v.size) + return n, v.failure + case errors.Is(err, io.EOF) && !v.finished: + v.finished = true + if verr := v.finish(); verr != nil { + v.failure = verr + return n, verr + } + case err != nil && !errors.Is(err, io.EOF): + v.failure = integrityOrCause(err) + return n, v.failure + } + return n, err +} + +// integrityOrCause keeps typed artifact and context errors and marks anything else as an integrity +// failure of the read. +func integrityOrCause(err error) error { + if isArtifactError(err) || blob.IsNotFound(err) { + return err + } + return fmt.Errorf("%w: %w", errUtils.ErrArtifactIntegrityFailed, err) +} + +func (v *verifyingReader) finish() error { + if v.read != v.size { + return fmt.Errorf("%w: content is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, v.read, v.size) + } + if got := digest.NewDigest(v.want.Algorithm(), v.hasher); got != v.want { + return fmt.Errorf("%w: content digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, v.want) + } + if v.guard != nil { + return v.guard() + } + return nil +} + +// Close implements io.Closer. +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "s3.verifyingReader.Close")() + + return v.source.Close() +} diff --git a/pkg/artifact/s3/recovery_test.go b/pkg/artifact/s3/recovery_test.go new file mode 100644 index 00000000000..8e66050ddc5 --- /dev/null +++ b/pkg/artifact/s3/recovery_test.go @@ -0,0 +1,248 @@ +package s3 + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// An interruption before the commit never advertises partial success: the reference is unavailable, +// invisible to listing, and a same-intent retry repairs it completely. +func TestInterruptedPublication_RetryRepairs(t *testing.T) { + points := []faultPoint{faultBeforePayload, faultDuringPayload, faultAfterMetadata} + for _, spec := range allSpecs { + for _, point := range points { + t.Run(specName(spec)+"/"+string(point), func(t *testing.T) { + spec = spec.fresh(t).withOpts(withFaults(crashAt(point))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{files: bigFiles()}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "a pending reference is unavailable to managed readers") + require.ErrorContains(t, err, prepared.OperationID, "the diagnostic names the owning operation") + assert.Empty(t, listNames(t, repo), "a pending reference is absent from search results") + _, err = repo.Pin(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + recordDigest, err := prepared.Record.RecordDigest() + require.NoError(t, err) + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: recordDigest}) + require.Error(t, err, "even the digest does not expose a pending record") + + retry, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retry.Outcome) + assertIntact(t, repo, defaultRef()) + assert.Equal(t, []string{testName + ":" + testVersion}, listNames(t, repo)) + assert.Empty(t, managedKeys(spec.fake, "", layout.ManagedRoot+"/operations"), "the retry removes the journal") + }) + } + } +} + +// A crash after the commit is already complete; only cleanup is left behind. +func TestInterruptedPublication_AfterCommitIsComplete(t *testing.T) { + spec := noneSpec().fresh(t).withOpts(withFaults(crashAt(faultAfterCommit))) + repo := newRepo(t, spec) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + assertIntact(t, repo, defaultRef()) + assert.Equal(t, []string{testName + ":" + testVersion}, listNames(t, repo)) +} + +// A pending operation of a different intent is never stolen, even under force. +func TestPendingOperation_IsNeverStolen(t *testing.T) { + spec := noneSpec().fresh(t).withOpts(withFaults(crashAt(faultDuringPayload))) + repo := newRepo(t, spec) + first := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "first"}, opID: "op-first"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + other := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "second"}, opID: "op-second"}) + for _, force := range []bool{false, true} { + _, err = repo.Publish(context.Background(), other, artifact.PublishOptions{Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + require.ErrorContains(t, err, "op-first", "the owning operation ID is surfaced for recovery") + } + _, err = repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.NoError(t, err, "the original owner repairs it with the same intent") +} + +// An interrupted force replacement leaves the reference unavailable until the same-intent retry +// repairs it, and never serves the replacement's bytes under the old digest. +func TestInterruptedForceReplacement(t *testing.T) { + var fault armedFault + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec.withOpts(fault.option())) + base := publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "base", "gone.txt": "old"}}, artifact.PublishOptions{}) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "replacement"}, opID: "op-replace"}) + + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + _, err = repo.Read(context.Background(), base.Pinned) + require.Error(t, err, "the old digest is not served while the replacement is unfinished") + + fault.hook = nil + retry, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, retry.Outcome) + assertIntact(t, repo, defaultRef()) + _, err = repo.Read(context.Background(), base.Pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the retired digest reports not found") + if spec.packaging == artifact.PackagingNone { + assert.NotContains(t, payloadKeys(spec.fake, ""), "team/app/1.0.0/gone.txt", "the replaced payload's extra file was retired") + } + }) + } +} + +// A response lost after the request was sent is reconciled by reading the reference back; the +// publication succeeds exactly once and nothing is written twice with different intent. +func TestPublish_ReconcilesLostResponses(t *testing.T) { + reference := refKey("") + tests := []struct { + name string + fault s3test.Fault + }{ + {name: "pending write applied, response lost", fault: s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: reference, Action: s3test.DropAfterCommit}}, + {name: "completed write applied, response lost", fault: s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: reference, Skip: 1, Action: s3test.DropAfterCommit}}, + {name: "completed write never arrived", fault: s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: reference, Skip: 1, Action: s3test.DropBeforeCommit}}, + {name: "pending write never arrived", fault: s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: reference, Action: s3test.DropBeforeCommit}}, + {name: "payload write applied, response lost", fault: s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: "team/app/1.0.0", Action: s3test.DropAfterCommit}}, + } + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + for _, tt := range tests { + t.Run(specName(spec)+"/"+tt.name, func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + spec.fake.Inject(tt.fault) + + result, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + faulted := 0 + for _, request := range spec.fake.Requests() { + if request.Faulted { + faulted++ + } + } + assert.Positive(t, faulted, "the fault actually fired") + assert.Equal(t, []string{testName + ":" + testVersion}, listNames(t, repo)) + }) + } + } +} + +// A conditional write whose response is lost and whose outcome was taken over by another writer is a +// conflict, not a success. +func TestPublish_LostResponseOverwrittenByAnotherWriterIsAConflict(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: refKey(""), Skip: 1, Action: s3test.DropBeforeCommit, Times: -1}) + // The request never arrives, every retry is dropped too, and the reference stays pending. + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.Error(t, err) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "an unreconciled write leaves the reference pending, never half-published") +} + +// Canceling an upload aborts the multipart upload the operation owns, and the same intent resumes. +func TestPublish_CancellationAbortsMultipartAndResumes(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{files: bigFiles()}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for index := range prepared.Blobs { + original := prepared.Blobs[index].Open + if prepared.Blobs[index].Path != "nested/big.bin" { + continue + } + prepared.Blobs[index].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + return &cancelingReader{ReadCloser: reader, after: 20 << 10, cancel: cancel}, nil + } + } + + _, err := repo.Publish(ctx, prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, context.Canceled) + assert.Empty(t, spec.fake.OpenUploads(), "no multipart upload is left open") + assert.GreaterOrEqual(t, spec.fake.AbortedUploads(), 1) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + + prepared = prepare(t, repo, &prepareSpec{files: bigFiles()}) + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err, "the same intent resumes after cancellation") + assertIntact(t, repo, defaultRef()) + assert.Empty(t, managedKeys(spec.fake, "", layout.ManagedRoot+"/locks"), "no lock claim leaked") +} + +// cancelingReader cancels a context once it has delivered a number of bytes. +type cancelingReader struct { + io.ReadCloser + after int + read int + cancel context.CancelFunc +} + +func (c *cancelingReader) Read(p []byte) (int, error) { + n, err := c.ReadCloser.Read(p) + c.read += n + if c.read >= c.after { + c.cancel() + } + return n, err +} + +// An identical republication is a no-op: no payload object is written again. +func TestRepublishIsUnchanged(t *testing.T) { + for _, spec := range []*rspec{noneSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + spec.fake.ResetRequests() + + again, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{opID: "op-again"}), artifact.PublishOptions{ExpectedPrior: &first.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) + assert.Equal(t, first.Pinned.RecordDigest, again.Pinned.RecordDigest) + for _, request := range mutations(spec.fake) { + assert.Contains(t, request.Key, layout.ManagedRoot, "only managed lock claims and journals may change: %s %s", request.Op, request.Key) + assert.NotContains(t, request.Key, "/refs/") + } + }) + } +} + +// A publication rejected before any state change writes nothing. +func TestRejectedPublicationWritesNothing(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + prepared.Blobs = prepared.Blobs[:len(prepared.Blobs)-1] + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.Error(t, err) + assert.Empty(t, spec.fake.Keys()) + assert.Empty(t, spec.fake.Requests()) +} diff --git a/pkg/artifact/s3/register.go b/pkg/artifact/s3/register.go new file mode 100644 index 00000000000..0ec802f2bdd --- /dev/null +++ b/pkg/artifact/s3/register.go @@ -0,0 +1,58 @@ +package s3 + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Register registers the aws/s3 repository kind in the process-wide registry. It is called +// explicitly by a composition root, never from init. Options such as [WithVerifier] apply to every +// repository the registered factory constructs. It does not touch the legacy CI artifact registry, +// whose own "aws/s3" store is a different thing. +func Register(opts ...Option) error { + defer perf.Track(nil, "s3.Register")() + + return RegisterWith(artifact.DefaultRepositoryRegistry(), opts...) +} + +// RegisterWith registers the aws/s3 kind in the given registry. A kind that is already registered +// is an error, so double registration is caught instead of silently replaced. +func RegisterWith(registry *artifact.RepositoryRegistry, opts ...Option) error { + defer perf.Track(nil, "s3.RegisterWith")() + + desc := Descriptor(opts...) + return registry.Register(&desc, factory(opts)) +} + +// Descriptor returns the static descriptor of the aws/s3 kind: the built-in descriptor refined with +// the capabilities the registered backend provides. It needs no I/O. +func Descriptor(opts ...Option) artifact.KindDescriptor { + defer perf.Track(nil, "s3.Descriptor")() + + desc, _ := artifact.BuiltinKind(artifact.RepositoryKindS3) + applied := applySettings(opts) + conditional := applied.store == nil || applied.store.Capabilities().ConditionalPut + desc.Capabilities = staticCapabilities(conditional, applied.verifier) + return *desc +} + +func factory(opts []Option) artifact.RepositoryFactory { + return func(_ context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, ropts ...artifact.RepositoryOption) (artifact.Repository, error) { + applied := artifact.ApplyRepositoryOptions(ropts...) + repo, err := newRepository(alias, cfg, opts, &applied) + if err != nil { + return nil, err + } + return repo, nil + } +} + +// Kinds lists the repository kinds this package registers. +func Kinds() []string { + defer perf.Track(nil, "s3.Kinds")() + + return []string{artifact.RepositoryKindS3} +} diff --git a/pkg/artifact/s3/register_test.go b/pkg/artifact/s3/register_test.go new file mode 100644 index 00000000000..ad610b865bc --- /dev/null +++ b/pkg/artifact/s3/register_test.go @@ -0,0 +1,119 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/schema" +) + +func s3Config(options map[string]any) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Options: options} +} + +func TestRegisterWith_RegistersTheKind(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithVerifier(&artifacttest.FakeVerifier{}))) + assert.Equal(t, []string{artifact.RepositoryKindS3}, Kinds()) + assert.True(t, registry.IsRegistered(artifact.RepositoryKindS3)) + + desc, ok := registry.Lookup(artifact.RepositoryKindS3) + require.True(t, ok) + assert.Equal(t, artifact.DeleteObject, desc.Capabilities.Delete, "the registered descriptor carries the backend's real capabilities") + assert.True(t, desc.Capabilities.ConditionalUpdate) + assert.Equal(t, []artifact.LockScope{artifact.LockScopeHost, artifact.LockScopeRepository}, desc.Capabilities.LockScopes) + assert.Contains(t, desc.Capabilities.Packagings, artifact.PackagingNone) + assert.Equal(t, artifact.PackagingTgz, desc.DefaultPackaging) + assert.Equal(t, []artifact.EvidenceFormat{artifacttest.FakeEvidenceFormat}, desc.Capabilities.EvidenceFormats) + assert.True(t, desc.Capabilities.EnumerateCollection) + + require.ErrorIs(t, RegisterWith(registry), errUtils.ErrArtifactConflict, "double registration is an error, not a silent replacement") +} + +func TestDefaultRegistryRegistration(t *testing.T) { + // The process-wide registry may already hold the kind when other tests registered it first, so + // only the explicit call is checked: it either registers or reports the duplicate. + err := Register() + if err != nil { + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + assert.True(t, artifact.DefaultRepositoryRegistry().IsRegistered(artifact.RepositoryKindS3)) +} + +func TestDescriptor_NeedsNoIO(t *testing.T) { + plain := Descriptor() + assert.Empty(t, plain.Capabilities.EvidenceFormats) + assert.True(t, plain.Capabilities.ConditionalUpdate) + signed := Descriptor(WithVerifier(&artifacttest.FakeVerifier{})) + assert.Len(t, signed.Capabilities.EvidenceFormats, 1) + + spec := noneSpec().fresh(t) + spec.storeOpts = nil + conditionless := Descriptor(WithStore(newStoreFor(t, spec.with(func(s *rspec) {})))) + assert.True(t, conditionless.Capabilities.ConditionalUpdate) + assert.Empty(t, spec.fake.Requests(), "describing a kind touches no endpoint") +} + +func TestRegisteredFactory_ConstructsWithoutIO(t *testing.T) { + fake := s3test.NewServer(t) + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + repo, err := registry.New(context.Background(), "shared", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindS3, Packaging: "zip", + Options: map[string]any{ + "bucket": fake.Bucket(), "prefix": "a/b", "region": "us-east-2", "endpoint": fake.Endpoint(), "path_style": true, + "tags": map[string]any{"team": "platform"}, "key_pattern": "{{ .Name }}/{{ .Version }}.zip", + }, + }) + require.NoError(t, err) + s3repo, ok := repo.(*Repository) + require.True(t, ok) + assert.Equal(t, artifact.PackagingZip, s3repo.packaging) + assert.Equal(t, "s3://"+fake.Bucket()+"/a/b", s3repo.Location()) + assert.Equal(t, map[string]string{"team": "platform"}, s3repo.tags) + caps := repo.Capabilities() + assert.True(t, caps.HasPackaging(artifact.PackagingZip)) + assert.True(t, caps.HasLockScope(artifact.LockScopeRepository)) + assert.Empty(t, fake.Requests(), "construction and capability description touch no endpoint") +} + +func TestNewRepository_RejectsBadInputs(t *testing.T) { + tests := []struct { + name string + cfg *schema.ArtifactRepositoryConfig + }{ + {name: "nil config"}, + {name: "wrong kind", cfg: &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"bucket": "acme-artifacts"}}}, + {name: "missing bucket", cfg: s3Config(map[string]any{})}, + {name: "unknown option", cfg: s3Config(map[string]any{"bucket": "acme-artifacts", "buckt": "typo"})}, + {name: "credential-looking option", cfg: s3Config(map[string]any{"bucket": "acme-artifacts", "secret_key": "x"})}, + {name: "bad endpoint", cfg: s3Config(map[string]any{"bucket": "acme-artifacts", "endpoint": "localhost:4566"})}, + {name: "unsupported packaging", cfg: &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Packaging: "rar", Options: map[string]any{"bucket": "acme-artifacts"}}}, + {name: "permissions are local-only", cfg: &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Permissions: "0600", Options: map[string]any{"bucket": "acme-artifacts"}}}, + {name: "reserved prefix", cfg: s3Config(map[string]any{"bucket": "acme-artifacts", "prefix": "x/.atmos-artifacts"})}, + {name: "too many tags", cfg: s3Config(map[string]any{"bucket": "acme-artifacts", "tags": map[string]any{"1": "a", "2": "a", "3": "a", "4": "a", "5": "a", "6": "a", "7": "a", "8": "a", "9": "a", "10": "a", "11": "a"}})}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo, err := New("bad", tt.cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Nil(t, repo) + }) + } +} + +// The registered factory returns a nil interface, not a typed nil, on failure. +func TestRegisteredFactory_ReturnsNilInterfaceOnFailure(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + repo, err := registry.New(context.Background(), "bad", s3Config(map[string]any{})) + require.Error(t, err) + assert.True(t, repo == nil) +} diff --git a/pkg/artifact/s3/repository.go b/pkg/artifact/s3/repository.go new file mode 100644 index 00000000000..da9655161f1 --- /dev/null +++ b/pkg/artifact/s3/repository.go @@ -0,0 +1,292 @@ +package s3 + +import ( + "context" + "fmt" + "maps" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Repository is an aws/s3 artifact repository. Construct it with [New]. It is safe for concurrent +// use: all coordination happens through conditional writes and claim objects in the bucket. +type Repository struct { + alias string + bucket string + prefix string + keyPattern string + packaging artifact.Packaging + tags map[string]string + store blob.Store + settings settings +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// New returns an aws/s3 repository for the configuration. It performs no network access and +// resolves no credentials: the AWS client, and the configured identity, are built on first use. +func New(alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "s3.New")() + + return newRepository(alias, cfg, opts, &artifact.RepositoryOptions{}) +} + +func newRepository(alias string, cfg *schema.ArtifactRepositoryConfig, opts []Option, ropts *artifact.RepositoryOptions) (*Repository, error) { + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, ok := artifact.BuiltinKind(artifact.RepositoryKindS3) + if !ok { + return nil, fmt.Errorf("%w: unknown repository kind %q", errUtils.ErrArtifactConfigInvalid, artifact.RepositoryKindS3) + } + if cfg.Kind != "" && cfg.Kind != artifact.RepositoryKindS3 { + return nil, fmt.Errorf("%w: configuration kind %q cannot construct a %q repository", errUtils.ErrArtifactConfigInvalid, cfg.Kind, artifact.RepositoryKindS3) + } + checked := *cfg + checked.Kind = artifact.RepositoryKindS3 + if err := artifact.ValidateRepositoryConfig(desc, &checked); err != nil { + return nil, err + } + var options artifact.S3Options + if err := artifact.DecodeOptions(cfg.Options, &options); err != nil { + return nil, err + } + repo := &Repository{ + alias: alias, bucket: options.Bucket, prefix: options.Prefix, keyPattern: options.KeyPattern, + packaging: desc.EffectivePackaging(cfg.Packaging), tags: maps.Clone(options.Tags), settings: applySettings(opts), + } + repo.store = repo.settings.store + if repo.store == nil { + store, err := buildStore(&options, cfg.Identity, ropts, &repo.settings) + if err != nil { + return nil, err + } + repo.store = store + } + return repo, nil +} + +// buildStore constructs the AWS SDK store. It performs no I/O; credentials resolve lazily. An +// endpoint override selects the conditional-write probe, because third-party endpoints can accept +// If-None-Match and silently ignore it. +func buildStore(options *artifact.S3Options, identity string, ropts *artifact.RepositoryOptions, applied *settings) (blob.Store, error) { + storeOpts := []blobs3.Option{ + blobs3.WithPrefix(options.Prefix), + blobs3.WithRegion(options.Region), + blobs3.WithTags(options.Tags), + blobs3.WithConcurrency(applied.concurrency), + } + if applied.partSize > 0 { + storeOpts = append(storeOpts, blobs3.WithPartSize(applied.partSize)) + } + if options.Endpoint != "" { + storeOpts = append(storeOpts, blobs3.WithEndpoint(options.Endpoint), blobs3.WithConditionalMode(blobs3.ConditionalProbe)) + } + if options.PathStyle != nil { + storeOpts = append(storeOpts, blobs3.WithPathStyle(*options.PathStyle)) + } + if identity != "" { + // A nil resolver stays a nil interface so the store fails closed instead of falling back. + var resolver blobs3.AuthResolver + if ropts != nil && ropts.ResolveIdentity != nil { + resolver = identityAdapter{resolve: ropts.ResolveIdentity} + } + storeOpts = append(storeOpts, blobs3.WithIdentity(identity, resolver)) + } + store, err := blobs3.New(options.Bucket, storeOpts...) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactConfigInvalid, err) + } + return store, nil +} + +// Location returns the display locator of the repository: s3://bucket[/prefix]. +func (r *Repository) Location() string { + defer perf.Track(nil, "s3.Repository.Location")() + + if r.prefix == "" { + return "s3://" + r.bucket + } + return "s3://" + r.bucket + "/" + r.prefix +} + +// Capabilities implements [artifact.Repository]. It is computed from configuration alone. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "s3.Repository.Capabilities")() + + return staticCapabilities(r.store.Capabilities().ConditionalPut, r.settings.verifier) +} + +// staticCapabilities describes the kind without constructing a repository. Without conditional +// writes the repository cannot own references or claims, so it offers no lock scope and no +// conditional publication. +func staticCapabilities(conditional bool, verifier artifact.Verifier) artifact.Capabilities { + desc, _ := artifact.BuiltinKind(artifact.RepositoryKindS3) + caps := desc.Capabilities + caps.Read, caps.Write = true, true + caps.EnumerateNamed, caps.EnumerateCollection = true, true + caps.Delete = artifact.DeleteObject + caps.ConditionalUpdate = conditional + caps.PreservesModes = true + caps.SubjectKinds = []artifact.SubjectKind{artifact.SubjectRecord} + caps.LockScopes = nil + if conditional { + caps.LockScopes = []artifact.LockScope{artifact.LockScopeHost, artifact.LockScopeRepository} + } + caps.EvidenceFormats = nil + if verifier != nil { + caps.EvidenceFormats = append(caps.EvidenceFormats, verifier.Formats()...) + } + return caps +} + +// pinned builds the immutable reference for a stored Record. +func (r *Repository) pinned(ref artifact.Ref, record *artifact.Record, recordDigest digest.Digest) artifact.PinnedReference { + return artifact.PinnedReference{ + Ref: ref, + Repository: r.Location(), + ResolvedDigest: recordDigest, + RecordDigest: recordDigest, + ContentDigest: record.ContentDigest, + } +} + +// view is a consistent, verified read of one completed artifact: its reference, Record, and +// representation. +type view struct { + ref *refState + record *artifact.Record + recordBytes []byte + rep *layout.Representation +} + +// unavailable builds the error for a reference that exists but cannot be used right now. +func (r *Repository) unavailable(ref *layout.Reference) error { + return fmt.Errorf("%w: %s:%s has a pending publication by operation %s; retry the publication with the same intent to repair it", + errUtils.ErrArtifactPublicationUnavailable, ref.Name, ref.Version, ref.OperationID) +} + +// resolved is a completed reference together with the verified Record it names. +type resolved struct { + ref *refState + record *artifact.Record + recordBytes []byte +} + +// resolveRef finds the reference and Record a request names. A version reference must be +// completed. A digest reference checks the Record's embedded name and succeeds only while that +// Record is still the current completed one: retired digests are not found, never redirected to +// the current bytes. +func (r *Repository) resolveRef(ctx context.Context, ref artifact.Ref) (*resolved, error) { + if err := ref.Validate(); err != nil { + return nil, err + } + if ref.IsDigest() { + return r.resolveDigest(ctx, ref) + } + stored, err := r.loadReference(ctx, layout.ReferenceKey(ref.Name, ref.Version)) + if err != nil { + return nil, err + } + if stored == nil { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + if !stored.ref.Available() { + return nil, r.unavailable(stored.ref) + } + record, raw, err := r.loadRecord(ctx, stored.ref.RecordDigest) + if err != nil { + return nil, err + } + if err := matchesReference(stored.ref, record); err != nil { + return nil, err + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +func (r *Repository) resolveDigest(ctx context.Context, ref artifact.Ref) (*resolved, error) { + record, raw, err := r.loadRecord(ctx, ref.Digest) + if err != nil { + return nil, err + } + if record.Name != ref.Name { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + stored, err := r.loadReference(ctx, layout.ReferenceKey(record.Name, record.Version)) + if err != nil { + return nil, err + } + switch { + case stored == nil: + return nil, fmt.Errorf("%w: %s (the reference was deleted)", errUtils.ErrArtifactNotFound, ref) + case !stored.ref.Available(): + return nil, r.unavailable(stored.ref) + case stored.ref.RecordDigest != ref.Digest: + return nil, fmt.Errorf("%w: %s is retired; %s:%s now resolves to another record", errUtils.ErrArtifactNotFound, ref, record.Name, record.Version) + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +// matchesReference checks that a Record agrees with the reference that names it. +func matchesReference(ref *layout.Reference, record *artifact.Record) error { + if record.Name != ref.Name || record.Version != ref.Version { + return fmt.Errorf("%w: record names %s:%s but its reference is %s:%s", errUtils.ErrArtifactIntegrityFailed, record.Name, record.Version, ref.Name, ref.Version) + } + return nil +} + +// openView loads the verified view of a pinned artifact. The pinned Record must still be the +// current completed Record of its reference. +func (r *Repository) openView(ctx context.Context, pinned *artifact.PinnedReference) (*view, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + current, err := r.resolveRef(ctx, artifact.Ref{Name: pinned.Ref.Name, Digest: pinned.RecordDigest}) + if err != nil { + return nil, err + } + stored := current.ref.ref + rep, err := r.loadRepresentation(ctx, stored.Key()) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != pinned.RecordDigest || rep.RepositoryKind != artifact.RepositoryKindS3 { + return nil, fmt.Errorf("%w: %s:%s has no valid %s representation for record %s", errUtils.ErrArtifactIntegrityFailed, stored.Name, stored.Version, artifact.RepositoryKindS3, pinned.RecordDigest) + } + if !supportedPackaging(rep.Packaging) { + return nil, fmt.Errorf("%w: %s:%s is stored as %q, which this repository cannot read", errUtils.ErrArtifactPackagingInvalid, stored.Name, stored.Version, rep.Packaging) + } + return &view{ref: current.ref, record: current.record, recordBytes: current.recordBytes, rep: rep}, nil +} + +// supportedPackaging reports whether the repository can store and read the packaging. +func supportedPackaging(packaging artifact.Packaging) bool { + switch packaging { + case artifact.PackagingNone, artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return true + default: + return false + } +} + +// isArchive reports whether the packaging stores one archive object. +func isArchive(packaging artifact.Packaging) bool { + switch packaging { + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return true + default: + return false + } +} diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go new file mode 100644 index 00000000000..9c1e469453f --- /dev/null +++ b/pkg/artifact/s3/reserve.go @@ -0,0 +1,191 @@ +package s3 + +import ( + "context" + "fmt" + "slices" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// claimsResource is the repository-wide lock that serializes location reservations. It starts with +// a NUL byte, which no artifact reference can contain, so it can never collide with a reference +// lock. It is held only while collisions are checked and the pending state is written. +const claimsResource = "\x00claims" + +// begin reserves the location, writes the operation journal, and marks the reference pending +// through a conditional write. It holds the repository-wide claims lock so two references can never +// both reserve overlapping locations, and it never changes a payload object. +func (r *Repository) begin(ctx context.Context, job *publishJob, plan *publishPlan) (*refState, error) { + claims, err := r.acquire(ctx, claimsResource, r.settings.owner, job.opID) + if err != nil { + return nil, err + } + pending, beginErr := r.reserve(ctx, job, plan) + if releaseErr := claims.Release(ctx); beginErr == nil { + beginErr = releaseErr + } + if beginErr != nil { + return nil, beginErr + } + if plan.oldOp != nil && plan.oldOp.OperationID != job.opID { + r.tidyOperation(ctx, plan.oldOp.OperationID) + } + return pending, r.settings.fault(faultBeforePayload) +} + +// reserve checks for location collisions and unmanaged content, then records the journal and the +// pending reference. The pending reference is created with If-None-Match, or replaces the observed +// state with If-Match, so a concurrent writer to the same reference loses with a conflict. +func (r *Repository) reserve(ctx context.Context, job *publishJob, plan *publishPlan) (*refState, error) { + if err := r.checkCollision(ctx, job, plan); err != nil { + return nil, err + } + journal := &layout.Operation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: artifact.RepositoryKindS3, OperationID: job.opID, + Name: job.record.Name, Version: job.record.Version, IntentDigest: job.prepared.IntentDigest, + RecordDigest: job.recordDigest, CapturedAt: job.record.CreatedAt, Packaging: job.prepared.Packaging, + Location: job.location, Prior: plan.prior, Payload: job.payload, Documents: job.documents, Owner: r.settings.owner, + } + if plan.expected != nil { + journal.ExpectedPrior = plan.expected.ResolvedDigest + } + if err := r.writeOperation(ctx, journal); err != nil { + return nil, wrapWrite(fmt.Sprintf("record operation %s", job.opID), err) + } + var prior *layout.Reference + cond := blob.IfAbsent() + if plan.current != nil { + prior, cond = plan.current.ref, blob.IfMatch(plan.current.etag) + } + pending := layout.NewPendingReference(job.record.Name, job.record.Version, job.opID, job.prepared.IntentDigest, prior) + etag, err := r.writeReference(ctx, &pending, cond) + if err != nil { + return nil, wrapWrite(fmt.Sprintf("mark %s:%s pending", job.record.Name, job.record.Version), err) + } + return &refState{ref: &pending, etag: etag}, nil +} + +// checkCollision rejects a location that another reference owns or that holds objects this +// repository does not manage. A second reference with a colliding key_pattern therefore fails with +// a conflict and is never bound to the first reference's payload, and a user's own objects at the +// location are never overwritten. The caller holds the claims lock, so no other publication can +// reserve between the check and the journal write. +func (r *Repository) checkCollision(ctx context.Context, job *publishJob, plan *publishPlan) error { + claims, err := r.collectClaims(ctx, job.key) + if err != nil { + return err + } + candidate := layout.Claim{ + ReferenceKey: job.key, Name: job.record.Name, Version: job.record.Version, + OperationID: job.opID, Paths: claimPaths(job.prepared.Packaging, job.location), + } + if other := layout.FindConflict(&candidate, claims); other != nil { + return fmt.Errorf("%w: location %q for %s:%s is already claimed by %s:%s%s; give the repository a key_pattern that keeps references apart", + errUtils.ErrArtifactConflict, job.location, job.record.Name, job.record.Version, other.Name, other.Version, pendingSuffix(other)) + } + return r.checkUnmanaged(ctx, &candidate, plan) +} + +func pendingSuffix(claim *layout.Claim) string { + if claim.OperationID == "" { + return "" + } + return " (pending operation " + claim.OperationID + ")" +} + +// collectClaims lists the paths owned by every other reference: the representation of each +// completed reference and the journal of each pending one. +func (r *Repository) collectClaims(ctx context.Context, ownKey string) ([]layout.Claim, error) { + refs, err := r.listReferences(ctx) + if err != nil { + return nil, err + } + var others []*layout.Reference + for _, state := range refs { + if state.ref.Key() != ownKey { + others = append(others, state.ref) + } + } + found := make([]*layout.Claim, len(others)) + err = r.forEach(ctx, len(others), func(ctx context.Context, index int) error { + claim, claimErr := r.claimOf(ctx, others[index]) + found[index] = claim + return claimErr + }) + if err != nil { + return nil, err + } + var claims []layout.Claim + for _, claim := range found { + if claim != nil { + claims = append(claims, *claim) + } + } + return claims, nil +} + +// claimOf returns the claim a reference holds: its representation when completed, its journal when +// pending. A reference whose state is missing holds no claim. +func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layout.Claim, error) { + base := layout.Claim{ReferenceKey: ref.Key(), Name: ref.Name, Version: ref.Version} + if ref.Available() { + rep, err := r.loadRepresentation(ctx, ref.Key()) + if err != nil || rep == nil { + return nil, err + } + base.Paths = claimPaths(rep.Packaging, rep.Location) + return &base, nil + } + op, err := r.loadOperation(ctx, ref.OperationID) + if err != nil || op == nil { + return nil, err + } + base.OperationID = op.OperationID + base.Paths = claimPaths(op.Packaging, op.Location) + return &base, nil +} + +// checkUnmanaged fails when a path the publication would write already holds objects and this +// reference does not own it. +func (r *Repository) checkUnmanaged(ctx context.Context, candidate *layout.Claim, plan *publishPlan) error { + var owned []string + for index := range plan.owned { + for _, path := range claimPaths(plan.owned[index].packaging, plan.owned[index].location) { + owned = append(owned, layout.FoldPath(path)) + } + // A prior written before packaging was recorded may be either shape. + if plan.owned[index].packaging == "" { + for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz} { + for _, path := range claimPaths(packaging, plan.owned[index].location) { + owned = append(owned, layout.FoldPath(path)) + } + } + } + } + for _, path := range candidate.Paths { + if slices.Contains(owned, layout.FoldPath(path)) { + continue + } + occupied, err := r.occupied(ctx, path) + if err != nil { + return err + } + if occupied { + return fmt.Errorf("%w: %q already holds objects that are not managed by this repository; remove them or choose another key_pattern", errUtils.ErrArtifactConflict, path) + } + } + return nil +} + +// occupied reports whether an object, or any object below the path as a prefix, exists. +func (r *Repository) occupied(ctx context.Context, path string) (bool, error) { + info, err := r.stat(ctx, path) + if err != nil || info != nil { + return info != nil, err + } + return r.hasAny(ctx, path+"/") +} diff --git a/pkg/artifact/s3/resolve.go b/pkg/artifact/s3/resolve.go new file mode 100644 index 00000000000..3d6b4e08bd3 --- /dev/null +++ b/pkg/artifact/s3/resolve.go @@ -0,0 +1,200 @@ +package s3 + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// defaultPageSize is the list page size when the query does not set one. +const defaultPageSize = 100 + +// Resolve implements [artifact.Repository]. A version reference resolves once to the digest of +// its immutable Record, which every later operation uses. A pending reference is unavailable; a +// digest reference checks the Record's embedded name and names only the current Record. +func (r *Repository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "s3.Repository.Resolve")() + + if err := ctx.Err(); err != nil { + return artifact.PinnedReference{}, err + } + current, err := r.resolveRef(ctx, ref) + if err != nil { + return artifact.PinnedReference{}, err + } + return r.pinned(ref, current.record, current.ref.ref.RecordDigest), nil +} + +// Pin implements [artifact.Repository]. It resolves the reference and lists the evidence +// descriptors known now. It takes no lock and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "s3.Repository.Pin")() + + pinned, err := r.Resolve(ctx, ref) + if err != nil { + return pinned, err + } + loaded, _, err := r.loadEvidence(ctx, pinned.RecordDigest) + if err != nil { + return artifact.PinnedReference{}, err + } + pinned.Evidence = descriptorsOf(r.evidenceRefs(loaded)) + return pinned, nil +} + +// Inspect implements [artifact.Repository]. It reads only managed records; it never opens the +// payload or its documents. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Inspect(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "s3.Repository.Inspect")() + + v, err := r.openView(ctx, &pinned) + if err != nil { + return nil, err + } + record := *v.record + loaded, _, err := r.loadEvidence(ctx, v.ref.ref.RecordDigest) + if err != nil { + return nil, err + } + return &artifact.Inspection{ + Pinned: r.pinned(pinned.Ref, &record, v.ref.ref.RecordDigest), + Record: &record, + Representation: v.rep.Artifact(r.uri(v.rep.Location)), + Evidence: r.evidenceRefs(loaded), + }, nil +} + +// listCandidate is one completed reference awaiting its Record. +type listCandidate struct { + name string + version string + digest digest.Digest +} + +func (c *listCandidate) less(other *listCandidate) bool { + if c.name != other.name { + return c.name < other.name + } + return c.version < other.version +} + +// List implements [artifact.Repository]. It enumerates managed completed references only: never +// arbitrary user objects, pending publications, journals, or claims. The reference objects are +// listed with the store's pagination, so any number of references is found; entries are returned +// sorted by name then version, which the hashed reference keys cannot provide on their own. +func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "s3.Repository.List")() + + refs, err := r.listReferences(ctx) + if err != nil { + return nil, err + } + candidates := make([]listCandidate, 0, len(refs)) + for _, state := range refs { + if state.ref.Available() && (query.Name == "" || state.ref.Name == query.Name) { + candidates = append(candidates, listCandidate{name: state.ref.Name, version: state.ref.Version, digest: state.ref.RecordDigest}) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].less(&candidates[j]) }) + return r.page(ctx, candidates, &query) +} + +// page loads the Records for one page. Records are read a page at a time, concurrently; label +// selectors may skip entries, in which case the next batch is read. +func (r *Repository) page(ctx context.Context, candidates []listCandidate, query *artifact.ListQuery) (*artifact.ListPage, error) { + size := query.PageSize + if size <= 0 { + size = defaultPageSize + } + if query.Cursor != "" { + start := sort.Search(len(candidates), func(i int) bool { return cursorAfter(&candidates[i], query.Cursor) }) + candidates = candidates[start:] + } + page := &artifact.ListPage{} + for len(candidates) > 0 { + batch := candidates[:min(len(candidates), size+1)] + candidates = candidates[len(batch):] + entries, matched, err := r.loadEntries(ctx, batch, query.Labels) + if err != nil { + return nil, err + } + for index := range entries { + if !matched[index] { + continue + } + if len(page.Entries) == size { + last := page.Entries[len(page.Entries)-1] + page.NextCursor = last.Name + ":" + last.Version + return page, nil + } + page.Entries = append(page.Entries, entries[index]) + } + } + return page, nil +} + +// loadEntries reads the Records of a batch of candidates concurrently. +func (r *Repository) loadEntries(ctx context.Context, batch []listCandidate, labels []artifact.Label) ([]artifact.ListEntry, []bool, error) { + entries := make([]artifact.ListEntry, len(batch)) + matched := make([]bool, len(batch)) + err := r.forEach(ctx, len(batch), func(ctx context.Context, index int) error { + entry, ok, err := r.listEntry(ctx, &batch[index], labels) + entries[index], matched[index] = entry, ok + return err + }) + return entries, matched, err +} + +// cursorAfter reports whether the candidate sorts strictly after the cursor's name and version. +func cursorAfter(candidate *listCandidate, cursor string) bool { + index := strings.LastIndex(cursor, ":") + if index < 0 { + return true + } + after := listCandidate{name: cursor[:index], version: cursor[index+1:]} + return after.less(candidate) +} + +func (r *Repository) listEntry(ctx context.Context, candidate *listCandidate, labels []artifact.Label) (artifact.ListEntry, bool, error) { + record, _, err := r.loadRecord(ctx, candidate.digest) + if err != nil { + return artifact.ListEntry{}, false, err + } + if record.Name != candidate.name || record.Version != candidate.version { + return artifact.ListEntry{}, false, fmt.Errorf("%w: record %s describes %s:%s but its reference is %s:%s", + errUtils.ErrArtifactIntegrityFailed, candidate.digest, record.Name, record.Version, candidate.name, candidate.version) + } + for _, label := range labels { + if value, ok := record.Metadata.Labels[label.Key]; !ok || value != label.Value { + return artifact.ListEntry{}, false, nil + } + } + return artifact.ListEntry{ + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, candidate.digest), + Name: record.Name, + Version: record.Version, + Kind: record.Kind, + Description: record.Metadata.Description, + Labels: record.Metadata.Labels, + Documents: record.Documents, + }, true, nil +} + +// descriptorsOf lists evidence descriptors. +func descriptorsOf(evidence []artifact.SignatureEvidence) []ocispec.Descriptor { + out := make([]ocispec.Descriptor, 0, len(evidence)) + for index := range evidence { + out = append(out, evidence[index].Descriptor) + } + return out +} diff --git a/pkg/artifact/s3/roundtrip_test.go b/pkg/artifact/s3/roundtrip_test.go new file mode 100644 index 00000000000..8e1052f4ef4 --- /dev/null +++ b/pkg/artifact/s3/roundtrip_test.go @@ -0,0 +1,130 @@ +package s3 + +import ( + "context" + "io" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/materialize" +) + +// A published package materializes back with every inventory entry, mode, empty file and +// directory, symlink, and Unicode name intact, in every packaging. +func TestRoundTrip_EveryPackaging(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, pkg := packageSpec(t) + result := publish(t, repo, prep, artifact.PublishOptions{}) + content, err := repo.Read(context.Background(), result.Pinned) + require.NoError(t, err) + defer content.Close() + + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), inventoryPaths(content.Record)) + }) + } +} + +func inventoryPaths(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for index := range record.Inventory { + paths = append(paths, record.Inventory[index].Path) + } + return paths +} + +// Per-file packaging is plain objects below the location with the distribution beside them; an +// archive is one object with metadata and document objects beside it. +func TestObjectLayout(t *testing.T) { + t.Run("files", func(t *testing.T) { + spec := noneSpec().fresh(t).with(func(s *rspec) { s.prefix = testPrefix }) + repo := newRepo(t, spec) + prep, pkg := packageSpec(t) + result := publish(t, repo, prep, artifact.PublishOptions{}) + assert.Equal(t, "s3://"+spec.fake.Bucket()+"/"+testPrefix+"/team/app/1.0.0", result.Location) + + want := []string{ + "team/app/1.0.0/.atmos/artifact/CHANGELOG.md", + "team/app/1.0.0/.atmos/artifact/README.md", + "team/app/1.0.0/.atmos/artifact/manifest.json", + "team/app/1.0.0/README.md", + "team/app/1.0.0/bin/helper.sh", + "team/app/1.0.0/empty.txt", + "team/app/1.0.0/nested/deep/file.txt", + "team/app/1.0.0/" + fixtures.UnicodeName, + } + assert.Equal(t, want, payloadKeys(spec.fake, testPrefix), "directories and symlinks live in the Record, not as objects (symlinks: %v)", pkg.Symlinks) + empty, ok := spec.fake.Object(testPrefix + "/team/app/1.0.0/empty.txt") + require.True(t, ok) + assert.Zero(t, empty.Size, "zero-length files are stored as zero-length objects") + }) + t.Run("archive", func(t *testing.T) { + spec := archiveSpec(artifact.PackagingTgz).fresh(t).with(func(s *rspec) { s.prefix = testPrefix }) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + result := publish(t, repo, prep, artifact.PublishOptions{}) + assert.Equal(t, "s3://"+spec.fake.Bucket()+"/"+testPrefix+"/team/app/1.0.0.tgz", result.Location) + assert.Equal(t, []string{ + "team/app/1.0.0.tgz", + "team/app/1.0.0.tgz.docs/CHANGELOG.md", + "team/app/1.0.0.tgz.docs/README.md", + "team/app/1.0.0.tgz.metadata.json", + }, payloadKeys(spec.fake, testPrefix)) + stored, ok := spec.fake.Object(testPrefix + "/team/app/1.0.0.tgz") + require.True(t, ok) + assert.Equal(t, result.Representation.Payload[0].Digest, digest.FromBytes(stored.Data), "the representation records the archive digest, not the ETag") + assert.NotEqual(t, stored.ETag, result.Representation.Payload[0].Digest.String()) + }) +} + +// Documents are readable without opening the payload. +func TestDocumentsAreReadWithoutThePayload(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.fresh(t) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + result := publish(t, repo, prep, artifact.PublishOptions{}) + spec.fake.ResetRequests() + + stream, err := repo.OpenDocument(context.Background(), result.Pinned, artifact.DocumentChangelog) + require.NoError(t, err) + data, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + assert.Equal(t, fixtures.PackageChangelog, string(data)) + for _, request := range spec.fake.Requests() { + managed := layout.InManagedNamespace(request.Key) + assert.True(t, managed || request.Key == documentKeyFor(spec.packaging, "CHANGELOG.md"), + "only the document object and managed records are requested: %s %s", request.Op, request.Key) + } + inspection, err := repo.Inspect(context.Background(), result.Pinned) + require.NoError(t, err) + assert.Equal(t, spec.packaging, inspection.Representation.Packaging) + }) + } +} + +// Multipart transfers keep memory bounded and honor the configured part size and concurrency. +func TestPublish_LargePayloadUsesBoundedMultipart(t *testing.T) { + spec := noneSpec().fresh(t).withOpts(WithConcurrency(2)) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"big.bin": incompressible(256 << 10)}}, artifact.PublishOptions{}) + assert.GreaterOrEqual(t, spec.fake.CompletedUploads(), 1) + assert.Empty(t, spec.fake.OpenUploads()) + assert.LessOrEqual(t, spec.fake.MaxConcurrentParts(), 4, "uploads stay bounded") + assertIntact(t, repo, defaultRef()) +} diff --git a/pkg/artifact/s3/service_test.go b/pkg/artifact/s3/service_test.go new file mode 100644 index 00000000000..0f6d731af63 --- /dev/null +++ b/pkg/artifact/s3/service_test.go @@ -0,0 +1,158 @@ +package s3 + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// newConfiguredService wires the local kinds and the S3 kind through the configuration-driven +// repository table, the way a composition root does. The S3 repositories run through the real SDK +// client against the fake endpoint. +func newConfiguredService(t *testing.T, fake *s3test.Server, extra map[string]schema.ArtifactRepositoryConfig) *service.Service { + t.Helper() + isolateAWS(t) + root := t.TempDir() + registry := artifact.NewRepositoryRegistry() + require.NoError(t, local.RegisterWith(registry)) + require.NoError(t, RegisterWith(registry)) + repositories := map[string]schema.ArtifactRepositoryConfig{ + "files": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "files")}}, + "packed": {Kind: artifact.RepositoryKindLocalArchive, Packaging: "tgz", Options: map[string]any{"path": filepath.Join(root, "packed")}}, + "bucket-files": { + Kind: artifact.RepositoryKindS3, Packaging: "none", + Options: map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1", "prefix": "files", "tags": map[string]any{"team": "platform"}}, + }, + "bucket-archives": { + Kind: artifact.RepositoryKindS3, Packaging: "tgz", + Options: map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1", "prefix": "archives"}, + }, + } + for alias := range extra { + repositories[alias] = extra[alias] + } + svc, err := service.New(service.FromConfig(&schema.ArtifactConfig{Repositories: repositories}, registry), nil, + service.WithStagingRoot(t.TempDir()), service.WithRetry(2, time.Millisecond), + service.WithSleep(func(context.Context, time.Duration) error { return nil })) + require.NoError(t, err) + return svc +} + +// An artifact moves between local and S3 repositories in both packagings, in both directions, +// without its Record identity changing. +func TestService_CopiesBetweenLocalAndS3(t *testing.T) { + fake := s3test.NewServer(t) + svc := newConfiguredService(t, fake, nil) + ctx := context.Background() + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + + published, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + require.Equal(t, artifact.OutcomeCreated, published.Outcome) + + hops := []struct{ from, to string }{ + {"files", "bucket-files"}, + {"bucket-files", "bucket-archives"}, + {"bucket-archives", "packed"}, + {"packed", "bucket-files"}, + } + for index, hop := range hops { + result, err := svc.Copy(ctx, artifact.CopyRequest{Ref: defaultRef(), From: hop.from, To: hop.to, Force: index == len(hops)-1}) + if index == len(hops)-1 { + require.NoError(t, err, "%s to %s", hop.from, hop.to) + assert.Equal(t, artifact.OutcomeUnchanged, result.Outcome, "copying the same Record again is a no-op") + continue + } + require.NoError(t, err, "%s to %s", hop.from, hop.to) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome, "%s to %s", hop.from, hop.to) + } + + listed, err := svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + require.Len(t, listed.Artifacts, 4) + for index := range listed.Artifacts { + assert.Equal(t, published.RecordDigest, listed.Artifacts[index].Pinned.RecordDigest, "%s holds the same Record", listed.Artifacts[index].Repository) + } + + for _, alias := range []string{"bucket-files", "bucket-archives"} { + out := filepath.Join(t.TempDir(), alias) + _, err = svc.Pull(ctx, artifact.PullRequest{Repository: alias, Ref: defaultRef(), Destination: out}) + require.NoError(t, err, alias) + inspection, err := svc.Inspect(ctx, artifact.InspectRequest{Repository: alias, Ref: defaultRef()}) + require.NoError(t, err) + state, err := materialize.InspectDestination(out, inspection.Inspection.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, alias) + document, err := svc.ShowDocument(ctx, artifact.ShowDocumentRequest{Repository: alias, Ref: defaultRef(), Role: artifact.DocumentReadme}) + require.NoError(t, err) + assert.Equal(t, fixtures.PackageReadme, string(document.Content)) + } + + pin, err := svc.Pin(ctx, artifact.PinRequest{Repository: "bucket-archives", Ref: defaultRef()}) + require.NoError(t, err) + assert.Equal(t, published.RecordDigest, pin.Pinned.RecordDigest, "the pin is the canonical Record digest") + + _, err = svc.Delete(ctx, artifact.DeleteRequest{Repository: "bucket-files", Ref: defaultRef()}) + require.NoError(t, err) + listed, err = svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + assert.Len(t, listed.Artifacts, 3) +} + +// A repository that requires repository-scope coordination reports it, and the service's lock is +// the S3 claim. (Routing a delegating repository's lock to an S3 repository is the service's job; +// this repository only has to be a valid delegate: it advertises the repository scope.) +func TestService_UsesRepositoryScopeClaims(t *testing.T) { + fake := s3test.NewServer(t) + svc := newConfiguredService(t, fake, map[string]schema.ArtifactRepositoryConfig{ + "coordinated": { + Kind: artifact.RepositoryKindS3, Packaging: "tgz", Locking: &schema.ArtifactLockingConfig{RequiredScope: "repository"}, + Options: map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1", "prefix": "coordinated"}, + }, + }) + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "coordinated", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.LockScopeRepository, result.LockScope) + claimed := false + for _, request := range fake.RequestsFor(s3test.OpPutObject) { + claimed = claimed || (request.Header.Get("If-None-Match") == "*" && containsLockSegment(request.Key)) + } + assert.True(t, claimed, "the service lock is an S3 owner claim") + + assert.Empty(t, lockObjects(fake), "and released") +} + +func containsLockSegment(key string) bool { return strings.Contains(key, "/locks/") } + +// The service reconciles a commit whose response was lost by reading the completed reference. +func TestService_ReconcilesALostResponseAfterCommit(t *testing.T) { + fake := s3test.NewServer(t) + svc := newConfiguredService(t, fake, nil) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload\n", 0o644) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: "files/" + refKey(""), Skip: 1, Action: s3test.DropAfterCommit}) + + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "bucket-files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + pulled, err := svc.Pull(context.Background(), artifact.PullRequest{Repository: "bucket-files", Ref: defaultRef(), Destination: filepath.Join(t.TempDir(), "out")}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, pulled.Outcome) +} diff --git a/pkg/artifact/s3/signature_test.go b/pkg/artifact/s3/signature_test.go new file mode 100644 index 00000000000..48bd7905448 --- /dev/null +++ b/pkg/artifact/s3/signature_test.go @@ -0,0 +1,224 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +func signedSpec(t *testing.T, spec *rspec) *rspec { + t.Helper() + return spec.fresh(t).withOpts(WithVerifier(&artifacttest.FakeVerifier{})) +} + +// objectSnapshot copies every object of the fake for before/after comparison. +func objectSnapshot(t *testing.T, fake *s3test.Server) map[string]string { + t.Helper() + out := map[string]string{} + for _, key := range fake.Keys() { + object, ok := fake.Object(key) + require.True(t, ok) + out[key] = string(object.Data) + } + return out +} + +func isEvidenceKey(key string) bool { + return len(key) > len(layout.ManagedRoot+"/signatures") && key[:len(layout.ManagedRoot+"/signatures")] == layout.ManagedRoot+"/signatures" +} + +// Evidence is a digest-named object beside the Record; signing changes no other object. +func TestSignature_StorageNeverMutatesTheSignedRecord(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = signedSpec(t, spec) + repo := newRepo(t, spec) + prep, _ := packageSpec(t) + result := publish(t, repo, prep, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo, &pinned) + assert.Equal(t, pinned.RecordDigest, subject.Digest) + before := objectSnapshot(t, spec.fake) + + signer := newSigner() + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) + require.Len(t, evidence, 1) + evidenceKey := layout.SignaturePath(artifact.SubjectRecord, subject.Digest, evidence[0].Descriptor.Digest) + assert.Equal(t, "s3://"+spec.fake.Bucket()+"/"+evidenceKey, evidence[0].Locator) + + after := objectSnapshot(t, spec.fake) + added := 0 + for key, content := range after { + if isEvidenceKey(key) { + added++ + continue + } + assert.Equal(t, before[key], content, "%s must be byte-identical after signing", key) + } + assert.Equal(t, 1, added, "exactly one detached evidence object was added") + assert.Len(t, after, len(before)+1) + + again := resolve(t, repo, defaultRef()) + assert.Equal(t, result.Pinned.RecordDigest, again.RecordDigest, "the signed Record, and so its digest, did not change") + assertIntact(t, repo, defaultRef()) + + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.Len(t, inspection.Evidence, 1) + pin, err := repo.Pin(context.Background(), defaultRef()) + require.NoError(t, err) + require.Len(t, pin.Evidence, 1, "a pin carries the evidence descriptors known now") + assert.Equal(t, pinned.RecordDigest, pin.RecordDigest, "the pin is the canonical Record digest") + + verified, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + _, err = repo.Verify(context.Background(), subject, signer.UntrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + } +} + +// Verification fails closed: a store that cannot be listed is an error, never "no evidence". +func TestSignature_VerifyFailsClosed(t *testing.T) { + spec := signedSpec(t, noneSpec()) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + signer := newSigner() + _, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + t.Run("listing failure", func(t *testing.T) { + spec.fake.Inject(s3test.Fault{Op: s3test.OpListObjectsV2, Status: 500, Times: -1}) + defer spec.fake.ClearFaults() + _, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.Error(t, err) + assert.NotErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "a store failure is reported as such, and verification did not pass") + _, err = repo.Pin(context.Background(), defaultRef()) + require.Error(t, err) + }) + t.Run("malformed and foreign evidence is reported and ignored", func(t *testing.T) { + dir := layout.SignatureDir(artifact.SubjectRecord, subject.Digest) + spec.fake.Seed(dir+"/"+"0000000000000000000000000000000000000000000000000000000000000000.json", []byte("not json"), nil) + result, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err, "valid evidence still verifies") + var malformed bool + for _, diagnostic := range result.Diagnostics { + malformed = malformed || diagnostic.Code == "malformed-evidence" + } + assert.True(t, malformed, "the malformed object is reported: %v", result.Diagnostics) + }) + t.Run("no evidence never verifies", func(t *testing.T) { + bare := newRepo(t, signedSpec(t, noneSpec())) + pinned := publish(t, bare, &prepareSpec{}, artifact.PublishOptions{}).Pinned + _, err := bare.Verify(context.Background(), recordSubject(t, bare, &pinned), signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) +} + +func TestSignature_WithoutVerifierIsACapabilityError(t *testing.T) { + repo := newRepo(t, noneSpec().fresh(t)) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + _, err := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, err = repo.Verify(context.Background(), subject, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Empty(t, repo.Capabilities().EvidenceFormats) +} + +func TestSignature_SubjectBinding(t *testing.T) { + repo := newRepo(t, signedSpec(t, noneSpec())) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + + wrongSize := subject + wrongSize.Size++ + _, err := repo.Sign(context.Background(), wrongSize, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + + badKind := subject + badKind.Kind = artifact.SubjectOCIRoot + _, err = repo.Sign(context.Background(), badKind, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + _, err = repo.Sign(context.Background(), subject, nil, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +// Evidence exported from one S3 repository imports into another that holds the same Record. +func TestSignature_ExportAndImportEvidenceAcrossRepositories(t *testing.T) { + source := newRepo(t, signedSpec(t, archiveSpec(artifact.PackagingTgz))) + publish(t, source, &prepareSpec{}, artifact.PublishOptions{}) + sourcePinned := resolve(t, source, defaultRef()) + subject := recordSubject(t, source, &sourcePinned) + signer := newSigner() + _, err := source.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + blobs, err := source.ReadEvidence(context.Background(), &sourcePinned) + require.NoError(t, err) + require.Len(t, blobs, 1) + + destination := newRepo(t, signedSpec(t, noneSpec())) + publish(t, destination, &prepareSpec{}, artifact.PublishOptions{}) + destPinned := resolve(t, destination, defaultRef()) + assert.Equal(t, subject.Digest, destPinned.RecordDigest, "packaging does not change Record identity") + _, err = destination.WriteEvidence(context.Background(), &destPinned, blobs, nil) + require.NoError(t, err) + verified, err := destination.Verify(context.Background(), recordSubject(t, destination, &destPinned), signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + + different := newRepo(t, signedSpec(t, noneSpec())) + publish(t, different, &prepareSpec{files: map[string]string{"data.txt": "other content"}}, artifact.PublishOptions{}) + differentPinned := resolve(t, different, defaultRef()) + _, err = different.WriteEvidence(context.Background(), &differentPinned, blobs, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid, "evidence for another Record is never imported") + + imported, err := source.WriteEvidence(context.Background(), &sourcePinned, blobs, nil) + require.NoError(t, err) + require.Len(t, imported, 1) + assert.Equal(t, blobs[0].Evidence.Descriptor.Digest, imported[0].Descriptor.Digest, "re-import is idempotent") + + corrupted := blobs[0] + corrupted.Data = append([]byte(nil), corrupted.Data...) + corrupted.Data[0] ^= 0xff + _, err = source.WriteEvidence(context.Background(), &sourcePinned, []artifact.EvidenceBlob{corrupted}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestSignature_EvidenceFollowsTheRecordLifecycle(t *testing.T) { + spec := signedSpec(t, noneSpec()) + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + oldSubject := recordSubject(t, repo, &first.Pinned) + _, err := repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + oldDir := layout.SignatureDir(artifact.SubjectRecord, oldSubject.Digest) + assert.NotEmpty(t, managedKeys(spec.fake, "", oldDir)) + + second := publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &first.Pinned}) + assert.Empty(t, managedKeys(spec.fake, "", oldDir), "evidence of a retired Record is retired with it") + _, err = repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a retired Record cannot be signed") + + currentSubject := recordSubject(t, repo, &second.Pinned) + _, err = repo.Sign(context.Background(), currentSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + currentDir := layout.SignatureDir(artifact.SubjectRecord, currentSubject.Digest) + assert.NotEmpty(t, managedKeys(spec.fake, "", currentDir)) + _, err = repo.Delete(context.Background(), second.Pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.Empty(t, managedKeys(spec.fake, "", currentDir), "delete removes the evidence the artifact exclusively owns") +} diff --git a/pkg/artifact/s3/state.go b/pkg/artifact/s3/state.go new file mode 100644 index 00000000000..4f307fe54ac --- /dev/null +++ b/pkg/artifact/s3/state.go @@ -0,0 +1,217 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// reconcileTimeout bounds the reads that settle an ambiguous write after the caller's context ended. +const reconcileTimeout = 30 * time.Second + +// refState is a stored reference together with the ETag that guards its next conditional write. +type refState struct { + ref *layout.Reference + etag string +} + +// detached returns a context that ignores the caller's cancellation, for reconciliation and cleanup +// that must complete even when the caller gave up. +func detached(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), reconcileTimeout) +} + +// loadReference returns the stored reference, or nil when it does not exist. +func (r *Repository) loadReference(ctx context.Context, key string) (*refState, error) { + raw, etag, err := r.readObject(ctx, layout.ReferencePath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + ref, err := layout.DecodeReference(raw) + if err != nil { + return nil, fmt.Errorf("reference %s: %w", key, err) + } + if ref.Key() != key { + return nil, fmt.Errorf("%w: reference object %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, ref.Name, ref.Version) + } + return &refState{ref: ref, etag: etag}, nil +} + +// loadRepresentation returns the stored representation, or nil when it does not exist. +func (r *Repository) loadRepresentation(ctx context.Context, key string) (*layout.Representation, error) { + raw, _, err := r.readObject(ctx, layout.RepresentationPath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + rep, err := layout.DecodeRepresentation(raw) + if err != nil { + return nil, fmt.Errorf("representation %s: %w", key, err) + } + if layout.ReferenceKey(rep.Name, rep.Version) != key { + return nil, fmt.Errorf("%w: representation object %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, rep.Name, rep.Version) + } + return rep, nil +} + +// loadOperation returns the stored journal, or nil when it does not exist. +func (r *Repository) loadOperation(ctx context.Context, operationID string) (*layout.Operation, error) { + raw, _, err := r.readObject(ctx, layout.OperationPath(operationID), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + op, err := layout.DecodeOperation(raw) + if err != nil { + return nil, fmt.Errorf("operation %s: %w", operationID, err) + } + if op.OperationID != operationID { + return nil, fmt.Errorf("%w: operation journal for %q describes %q", errUtils.ErrArtifactIntegrityFailed, operationID, op.OperationID) + } + return op, nil +} + +// loadRecord reads and verifies the immutable Record with the given digest. A missing Record is +// not found; a Record whose bytes do not hash to its object name, or that is not canonical, fails +// integrity. +func (r *Repository) loadRecord(ctx context.Context, recordDigest digest.Digest) (*artifact.Record, []byte, error) { + if err := artifact.ValidateDigest(recordDigest); err != nil { + return nil, nil, err + } + raw, _, err := r.readObject(ctx, layout.RecordPath(recordDigest), artifact.MaxRecordSize) + if err != nil { + return nil, nil, err + } + if raw == nil { + return nil, nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, recordDigest) + } + if got := digest.FromBytes(raw); got != recordDigest { + return nil, nil, fmt.Errorf("%w: record object %s holds bytes that hash to %s", errUtils.ErrArtifactIntegrityFailed, recordDigest, got) + } + record, err := artifact.DecodeRecord(raw) + if err != nil { + return nil, nil, fmt.Errorf("%w: record %s: %w", errUtils.ErrArtifactIntegrityFailed, recordDigest, err) + } + return record, raw, nil +} + +// writeRecord stores the immutable canonical Record. +func (r *Repository) writeRecord(ctx context.Context, recordDigest digest.Digest, canonical []byte) error { + return r.putImmutable(ctx, layout.RecordPath(recordDigest), canonical, contentTypeJSON, artifact.MaxRecordSize) +} + +// writeRepresentation replaces the representation record. +func (r *Repository) writeRepresentation(ctx context.Context, rep *layout.Representation) error { + raw, err := rep.Marshal() + if err != nil { + return err + } + return r.putBytes(ctx, layout.RepresentationPath(layout.ReferenceKey(rep.Name, rep.Version)), raw) +} + +// writeOperation replaces the operation journal. +func (r *Repository) writeOperation(ctx context.Context, op *layout.Operation) error { + raw, err := op.Marshal() + if err != nil { + return err + } + return r.putBytes(ctx, layout.OperationPath(op.OperationID), raw) +} + +// writeReference replaces the reference record under a condition and returns the new ETag. The +// condition is If-None-Match for a first write and If-Match with the observed ETag afterwards. +// A response that was lost after the request was sent is reconciled by reading the reference back: +// an identical reference proves the write was applied, an untouched one lets the idempotent +// conditional write be repeated, and anything else is another writer's change and a conflict. +func (r *Repository) writeReference(ctx context.Context, ref *layout.Reference, cond blob.Condition) (string, error) { + raw, err := ref.Marshal() + if err != nil { + return "", err + } + key := layout.ReferencePath(ref.Key()) + var lastErr error + for range putAttempts { + etag, putErr := r.putConditional(ctx, key, raw, contentTypeJSON, cond) + if putErr == nil { + return etag, nil + } + if !blob.IsAmbiguous(putErr) { + return "", putErr + } + lastErr = putErr + applied, retry, reconcileErr := r.reconcileReference(ctx, ref, cond) + switch { + case reconcileErr != nil: + return "", errors.Join(putErr, reconcileErr) + case applied != nil: + return applied.etag, nil + case !retry: + return "", fmt.Errorf("%w: %s:%s was changed by another writer while this write's outcome was unknown", errUtils.ErrArtifactConflict, ref.Name, ref.Version) + } + } + return "", lastErr +} + +// reconcileReference reads the reference back after an ambiguous write. It reports the applied +// state when the stored reference equals want, and whether the write provably did not happen. +func (r *Repository) reconcileReference(ctx context.Context, want *layout.Reference, cond blob.Condition) (applied *refState, retry bool, err error) { + readCtx, cancel := detached(ctx) + defer cancel() + current, err := r.loadReference(readCtx, want.Key()) + switch { + case err != nil: + return nil, false, err + case current != nil && *current.ref == *want: + return current, false, nil + case cond.Kind == blob.ConditionIfAbsent && current == nil: + return nil, true, nil + case cond.Kind == blob.ConditionIfMatch && current != nil && current.etag == cond.ETag: + return nil, true, nil + } + return nil, false, nil +} + +// referenceDirPrefix is the listing prefix of the reference objects. +func referenceDirPrefix() string { + return layout.ReferencesDir() + "/" +} + +// listReferences returns every stored reference, sorted by object key, failing on a corrupt one. +// Only objects directly below the references directory are read: journals, staging leftovers, +// claims, and user payloads are never interpreted. +func (r *Repository) listReferences(ctx context.Context) ([]*refState, error) { + objects, err := r.listPrefix(ctx, referenceDirPrefix()) + if err != nil { + return nil, err + } + var keys []string + for index := range objects { + name := strings.TrimPrefix(objects[index].Key, referenceDirPrefix()) + if key, ok := strings.CutSuffix(name, ".json"); ok && !strings.Contains(key, "/") { + keys = append(keys, key) + } + } + states := make([]*refState, len(keys)) + err = r.forEach(ctx, len(keys), func(ctx context.Context, index int) error { + state, loadErr := r.loadReference(ctx, keys[index]) + states[index] = state + return loadErr + }) + if err != nil { + return nil, err + } + refs := states[:0] + for _, state := range states { + if state != nil { + refs = append(refs, state) + } + } + return refs, nil +} diff --git a/pkg/artifact/s3/upload.go b/pkg/artifact/s3/upload.go new file mode 100644 index 00000000000..46866eed0bd --- /dev/null +++ b/pkg/artifact/s3/upload.go @@ -0,0 +1,61 @@ +package s3 + +import ( + "context" + "sync/atomic" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// uploadAll writes the payload objects with bounded concurrency, then the documents and metadata. +// The reservation already made the location exclusively this reference's, so the writes are +// unconditional and idempotent: a retry simply writes the same bytes again. +func (r *Repository) uploadAll(ctx context.Context, job *publishJob, items []objectPlan) error { + var payload, rest []*objectPlan + for index := range items { + if items[index].payload { + payload = append(payload, &items[index]) + } else { + rest = append(rest, &items[index]) + } + } + var started atomic.Bool + err := r.forEach(ctx, len(payload), func(ctx context.Context, index int) error { + if err := r.uploadObject(ctx, payload[index].place(job.location), payload[index]); err != nil { + return err + } + if started.CompareAndSwap(false, true) { + return r.settings.fault(faultDuringPayload) + } + return nil + }) + if err != nil { + return err + } + return r.forEach(ctx, len(rest), func(ctx context.Context, index int) error { + return r.uploadObject(ctx, rest[index].place(job.location), rest[index]) + }) +} + +// uploadObject streams one object, verifying size and digest while it goes. An ambiguous outcome is +// repeated: the write is unconditional and the bytes are the same. +func (r *Repository) uploadObject(ctx context.Context, key string, item *objectPlan) error { + var err error + for range putAttempts { + err = r.uploadOnce(ctx, key, item) + if !blob.IsAmbiguous(err) { + break + } + } + return r.mapErr(opWrite, key, err) +} + +func (r *Repository) uploadOnce(ctx context.Context, key string, item *objectPlan) error { + source, err := item.open(ctx) + if err != nil { + return err + } + defer source.Close() + _, err = r.store.Put(ctx, key, newVerifyingReader(source, item.digest, item.size, nil), r.putOptions(item.size, item.contentType, blob.Condition{})) + return err +} From 47e6954ea3acfd77b64f3779853e8a44687704df Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 11:01:02 -0500 Subject: [PATCH 02/53] fix(artifacts): settle an S3 lock claim when the context ends mid-request A request cut short by the context may already have created the claim, so read it back instead of returning only the context error. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/lock.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 49a2b7c4c19..6103231a074 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -127,7 +127,8 @@ func (r *Repository) tryClaim(ctx context.Context, key string, doc *claimDoc, ra return r.adopt(ctx, key, doc) case blob.IsPreconditionFailed(err): return nil, nil - case blob.IsAmbiguous(err): + case blob.IsAmbiguous(err) || ctx.Err() != nil: + // A request cut short by the context may still have created the claim, so read it back. return r.settleClaim(ctx, key, doc) case errors.Is(err, errUtils.ErrArtifactUnsupportedCapability): unsupported := artifact.NewUnsupportedCapability(r.alias, opLock, "lock/repository", "the S3 endpoint cannot enforce If-None-Match, which owner claims require") From ae6de69c55d5736ab8eedabf620d9bbcb95213b3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 11:01:47 -0500 Subject: [PATCH 03/53] feat(artifacts): add OCI registry and layout repositories with sigstore signing (A06) Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/assemble.go | 202 +++++++++ pkg/artifact/oci/assemble_test.go | 161 ++++++++ pkg/artifact/oci/attach.go | 243 +++++++++++ pkg/artifact/oci/backend.go | 317 +++++++++++++++ pkg/artifact/oci/budget_test.go | 82 ++++ pkg/artifact/oci/classify.go | 202 +++++++++ pkg/artifact/oci/conformance_test.go | 94 +++++ pkg/artifact/oci/copy.go | 206 ++++++++++ pkg/artifact/oci/delete.go | 131 ++++++ pkg/artifact/oci/delete_test.go | 128 ++++++ pkg/artifact/oci/doc.go | 30 ++ pkg/artifact/oci/edge_test.go | 156 +++++++ pkg/artifact/oci/fetch.go | 197 +++++++++ pkg/artifact/oci/foreignfixtures_test.go | 73 ++++ pkg/artifact/oci/helpers_test.go | 226 +++++++++++ pkg/artifact/oci/inspect.go | 215 ++++++++++ pkg/artifact/oci/list.go | 170 ++++++++ pkg/artifact/oci/list_test.go | 131 ++++++ pkg/artifact/oci/lock.go | 228 +++++++++++ pkg/artifact/oci/lock_test.go | 173 ++++++++ pkg/artifact/oci/names.go | 108 +++++ pkg/artifact/oci/options.go | 126 ++++++ pkg/artifact/oci/plugin.go | 85 ++++ pkg/artifact/oci/plugin_test.go | 272 +++++++++++++ pkg/artifact/oci/portability_test.go | 161 ++++++++ pkg/artifact/oci/publish.go | 244 +++++++++++ pkg/artifact/oci/register.go | 168 ++++++++ pkg/artifact/oci/register_test.go | 208 ++++++++++ pkg/artifact/oci/repository.go | 200 +++++++++ pkg/artifact/oci/resolve.go | 241 +++++++++++ pkg/artifact/oci/roundtrip_test.go | 156 +++++++ pkg/artifact/oci/sign.go | 369 +++++++++++++++++ pkg/artifact/oci/sign_test.go | 495 +++++++++++++++++++++++ pkg/artifact/oci/testutil_test.go | 32 ++ pkg/artifact/oci/transfer_test.go | 255 ++++++++++++ pkg/artifact/sigstore/doc.go | 38 ++ pkg/artifact/sigstore/helpers_test.go | 124 ++++++ pkg/artifact/sigstore/keys.go | 92 +++++ pkg/artifact/sigstore/signer.go | 59 +++ pkg/artifact/sigstore/signer_test.go | 189 +++++++++ pkg/artifact/sigstore/statement.go | 105 +++++ pkg/artifact/sigstore/verifier.go | 402 ++++++++++++++++++ pkg/artifact/sigstore/verifier_test.go | 466 +++++++++++++++++++++ pkg/oci/ocitest/registry.go | 81 +++- pkg/oci/status.go | 8 + pkg/oci/transport_catalog.go | 58 +++ pkg/oci/transport_catalog_test.go | 147 +++++++ pkg/oci/transport_layout.go | 20 + 48 files changed, 8270 insertions(+), 4 deletions(-) create mode 100644 pkg/artifact/oci/assemble.go create mode 100644 pkg/artifact/oci/assemble_test.go create mode 100644 pkg/artifact/oci/attach.go create mode 100644 pkg/artifact/oci/backend.go create mode 100644 pkg/artifact/oci/budget_test.go create mode 100644 pkg/artifact/oci/classify.go create mode 100644 pkg/artifact/oci/conformance_test.go create mode 100644 pkg/artifact/oci/copy.go create mode 100644 pkg/artifact/oci/delete.go create mode 100644 pkg/artifact/oci/delete_test.go create mode 100644 pkg/artifact/oci/doc.go create mode 100644 pkg/artifact/oci/edge_test.go create mode 100644 pkg/artifact/oci/fetch.go create mode 100644 pkg/artifact/oci/foreignfixtures_test.go create mode 100644 pkg/artifact/oci/helpers_test.go create mode 100644 pkg/artifact/oci/inspect.go create mode 100644 pkg/artifact/oci/list.go create mode 100644 pkg/artifact/oci/list_test.go create mode 100644 pkg/artifact/oci/lock.go create mode 100644 pkg/artifact/oci/lock_test.go create mode 100644 pkg/artifact/oci/names.go create mode 100644 pkg/artifact/oci/options.go create mode 100644 pkg/artifact/oci/plugin.go create mode 100644 pkg/artifact/oci/plugin_test.go create mode 100644 pkg/artifact/oci/portability_test.go create mode 100644 pkg/artifact/oci/publish.go create mode 100644 pkg/artifact/oci/register.go create mode 100644 pkg/artifact/oci/register_test.go create mode 100644 pkg/artifact/oci/repository.go create mode 100644 pkg/artifact/oci/resolve.go create mode 100644 pkg/artifact/oci/roundtrip_test.go create mode 100644 pkg/artifact/oci/sign.go create mode 100644 pkg/artifact/oci/sign_test.go create mode 100644 pkg/artifact/oci/testutil_test.go create mode 100644 pkg/artifact/oci/transfer_test.go create mode 100644 pkg/artifact/sigstore/doc.go create mode 100644 pkg/artifact/sigstore/helpers_test.go create mode 100644 pkg/artifact/sigstore/keys.go create mode 100644 pkg/artifact/sigstore/signer.go create mode 100644 pkg/artifact/sigstore/signer_test.go create mode 100644 pkg/artifact/sigstore/statement.go create mode 100644 pkg/artifact/sigstore/verifier.go create mode 100644 pkg/artifact/sigstore/verifier_test.go create mode 100644 pkg/oci/transport_catalog.go create mode 100644 pkg/oci/transport_catalog_test.go diff --git a/pkg/artifact/oci/assemble.go b/pkg/artifact/oci/assemble.go new file mode 100644 index 00000000000..42ba552fbac --- /dev/null +++ b/pkg/artifact/oci/assemble.go @@ -0,0 +1,202 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "encoding/json" + "fmt" + "path" + "time" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// ociSchemaVersion is the only OCI manifest schema version. +const ociSchemaVersion = 2 + +// Package is an assembled Atmos package ready to upload: the config blob (the canonical Record +// bytes), the deterministic manifest, and the staged blobs in manifest layer order. +type Package struct { + // Config describes the config blob. Its digest is the Record digest. + Config ocispec.Descriptor + + // ConfigBytes are the canonical Record bytes. + ConfigBytes []byte + + // Manifest describes the manifest. + Manifest ocispec.Descriptor + + // ManifestBytes are the exact manifest bytes whose digest is Manifest.Digest. + ManifestBytes []byte + + // Blobs are the payload and document blobs in manifest layer order. + Blobs []artifact.StagedBlob +} + +// Assemble builds the manifest of a new Atmos package from a prepared artifact. +// +// The config blob is the canonical Record, byte for byte what local and S3 records store. The +// layers are the single archive payload followed by the README and CHANGELOG documents as +// separate Markdown blobs with explicit roles and titles. The creation time is the Record's +// captured time, so assembly reads no clock and identical input yields an identical manifest +// digest. +func Assemble(prepared *artifact.PreparedArtifact) (*Package, error) { + defer perf.Track(nil, "oci.Assemble")() + + if prepared == nil { + return nil, fmt.Errorf("%w: no prepared artifact", errUtils.ErrArtifactPackagingInvalid) + } + record := &prepared.Record + configBytes, err := record.Canonical() + if err != nil { + return nil, err + } + payload, documents, err := stagedLayers(prepared) + if err != nil { + return nil, err + } + pkg := &Package{ + Config: ocispec.Descriptor{MediaType: artifact.MediaTypeConfig, Digest: digest.FromBytes(configBytes), Size: int64(len(configBytes))}, + ConfigBytes: configBytes, + } + layers := []ocispec.Descriptor{payload.layer} + pkg.Blobs = append(pkg.Blobs, payload.blob) + for _, doc := range documents { + layers = append(layers, doc.layer) + pkg.Blobs = append(pkg.Blobs, doc.blob) + } + manifest := ocispec.Manifest{ + Versioned: specVersion(), + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: artifactTypeFor(record.Kind), + Config: pkg.Config, + Layers: layers, + Annotations: manifestAnnotations(record), + } + pkg.ManifestBytes, err = json.Marshal(manifest) + if err != nil { + return nil, fmt.Errorf("%w: manifest: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + pkg.Manifest = ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(pkg.ManifestBytes), Size: int64(len(pkg.ManifestBytes))} + return pkg, nil +} + +// stagedBlob pairs a manifest layer with the staged blob that uploads it. +type stagedBlob struct { + layer ocispec.Descriptor + blob artifact.StagedBlob +} + +// stagedLayers selects the one archive payload and the document blobs the Record describes, +// checking each against the Record and the declared packaging. +func stagedLayers(prepared *artifact.PreparedArtifact) (stagedBlob, []stagedBlob, error) { + mediaType, ok := mediaTypeOf(prepared.Packaging) + if !ok { + return stagedBlob{}, nil, fmt.Errorf("%w: OCI repositories store tar, tgz, or zip archives, not %q", errUtils.ErrArtifactPackagingInvalid, prepared.Packaging) + } + var ( + payload *stagedBlob + documents = map[artifact.DocumentRole]stagedBlob{} + ) + for index := range prepared.Blobs { + blob := prepared.Blobs[index] + switch blob.Role { + case artifact.BlobPayload: + if payload != nil { + return stagedBlob{}, nil, fmt.Errorf("%w: an OCI package has exactly one archive payload blob", errUtils.ErrArtifactPackagingInvalid) + } + blob.Descriptor.MediaType = mediaType + payload = &stagedBlob{blob: blob, layer: layerOf(blob.Descriptor, payloadTitle(prepared.Packaging), "")} + case artifact.BlobDocument: + doc, err := documentLayer(&prepared.Record, blob) + if err != nil { + return stagedBlob{}, nil, err + } + documents[blob.DocumentRole] = doc + } + } + if payload == nil { + return stagedBlob{}, nil, fmt.Errorf("%w: the prepared artifact has no payload blob", errUtils.ErrArtifactPackagingInvalid) + } + ordered, err := orderedDocuments(&prepared.Record, documents) + return *payload, ordered, err +} + +// documentLayer builds the layer of one staged document, requiring it to match the Record. +func documentLayer(record *artifact.Record, blob artifact.StagedBlob) (stagedBlob, error) { + for index := range record.Documents { + doc := record.Documents[index] + if doc.Role != blob.DocumentRole { + continue + } + if doc.Digest != blob.Descriptor.Digest || doc.Size != blob.Descriptor.Size { + return stagedBlob{}, fmt.Errorf("%w: the staged %s differs from the Record's descriptor", errUtils.ErrArtifactPackagingInvalid, doc.Role) + } + blob.Descriptor.MediaType = doc.MediaType + return stagedBlob{blob: blob, layer: layerOf(blob.Descriptor, path.Base(doc.SourcePath), doc.Role)}, nil + } + return stagedBlob{}, fmt.Errorf("%w: a staged %s document is not described by the Record", errUtils.ErrArtifactPackagingInvalid, blob.DocumentRole) +} + +// orderedDocuments returns the documents in the fixed README then CHANGELOG order and requires +// every document the Record describes to have been staged. +func orderedDocuments(record *artifact.Record, staged map[artifact.DocumentRole]stagedBlob) ([]stagedBlob, error) { + var ordered []stagedBlob + for _, role := range []artifact.DocumentRole{artifact.DocumentReadme, artifact.DocumentChangelog} { + doc, present := staged[role] + described := false + for index := range record.Documents { + described = described || record.Documents[index].Role == role + } + switch { + case present: + ordered = append(ordered, doc) + case described: + return nil, fmt.Errorf("%w: the Record describes a %s that was not staged", errUtils.ErrArtifactPackagingInvalid, role) + } + } + return ordered, nil +} + +func layerOf(desc ocispec.Descriptor, title string, role artifact.DocumentRole) ocispec.Descriptor { + annotations := map[string]string{annotationTitle: title} + if role != "" { + annotations[annotationDocumentRole] = string(role) + } + return ocispec.Descriptor{MediaType: desc.MediaType, Digest: desc.Digest, Size: desc.Size, Annotations: annotations} +} + +func payloadTitle(packaging artifact.Packaging) string { + switch packaging { + case artifact.PackagingTar: + return "content.tar" + case artifact.PackagingZip: + return "content.zip" + default: + return "content.tar.gz" + } +} + +// manifestAnnotations carries the standard OCI fields for registry-side browsing. They are +// derived from the Record only, never from the clock. +func manifestAnnotations(record *artifact.Record) map[string]string { + annotations := map[string]string{ + annotationCreated: record.CreatedAt.Time().UTC().Format(time.RFC3339), + annotationTitle: record.Name, + annotationVersion: record.Version, + annotationKind: record.Kind, + } + if record.Metadata.Description != "" { + annotations[annotationDescription] = record.Metadata.Description + } + return annotations +} + +func specVersion() specs.Versioned { + return specs.Versioned{SchemaVersion: ociSchemaVersion} +} diff --git a/pkg/artifact/oci/assemble_test.go b/pkg/artifact/oci/assemble_test.go new file mode 100644 index 00000000000..34e8d11bded --- /dev/null +++ b/pkg/artifact/oci/assemble_test.go @@ -0,0 +1,161 @@ +package oci + +import ( + "encoding/json" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +func parseManifest(t *testing.T, raw []byte) ocispec.Manifest { + t.Helper() + var manifest ocispec.Manifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + return manifest +} + +func TestAssemble_ManifestShape(t *testing.T) { + prepared := prepare(t, publishSpec{content: "shape", labels: map[string]string{"team": "platform"}}) + pkg, err := Assemble(prepared) + require.NoError(t, err) + + canonical, err := prepared.Record.Canonical() + require.NoError(t, err) + assert.Equal(t, canonical, pkg.ConfigBytes, "the config blob is the canonical Record bytes") + recordDigest, err := prepared.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, recordDigest, pkg.Config.Digest) + assert.Equal(t, artifact.MediaTypeConfig, pkg.Config.MediaType) + + manifest := parseManifest(t, pkg.ManifestBytes) + assert.Equal(t, ocispec.MediaTypeImageManifest, manifest.MediaType) + assert.Equal(t, artifact.MediaTypeArtifact, manifest.ArtifactType) + assert.Equal(t, pkg.Config, manifest.Config) + assert.Equal(t, preparedAt.Format(time.RFC3339), manifest.Annotations[ocispec.AnnotationCreated], "the captured creation time, not the clock") + assert.Equal(t, appName, manifest.Annotations[ocispec.AnnotationTitle]) + assert.Equal(t, appVersion, manifest.Annotations[ocispec.AnnotationVersion]) + assert.Equal(t, "a test package", manifest.Annotations[ocispec.AnnotationDescription]) + + require.Len(t, manifest.Layers, 3, "one archive payload plus README and CHANGELOG as separate blobs") + assert.Equal(t, artifact.MediaTypeTarGzip, manifest.Layers[0].MediaType) + assert.Equal(t, "content.tar.gz", manifest.Layers[0].Annotations[ocispec.AnnotationTitle]) + assert.Equal(t, artifact.DocumentMediaType, manifest.Layers[1].MediaType) + assert.Equal(t, string(artifact.DocumentReadme), manifest.Layers[1].Annotations[annotationDocumentRole]) + assert.Equal(t, "README.md", manifest.Layers[1].Annotations[ocispec.AnnotationTitle]) + assert.Equal(t, string(artifact.DocumentChangelog), manifest.Layers[2].Annotations[annotationDocumentRole]) + assert.Equal(t, "CHANGELOG.md", manifest.Layers[2].Annotations[ocispec.AnnotationTitle]) + for _, doc := range prepared.Record.Documents { + var found bool + for _, layer := range manifest.Layers[1:] { + found = found || (layer.Digest == doc.Digest && layer.Size == doc.Size) + } + assert.True(t, found, "%s layer matches the Record's descriptor", doc.Role) + } + assert.Len(t, pkg.Blobs, 3) + assert.Equal(t, pkg.Manifest.Digest.String(), digestOf(pkg.ManifestBytes)) +} + +func digestOf(raw []byte) string { return digest.FromBytes(raw).String() } + +func TestAssemble_IdenticalInputYieldsIdenticalManifestDigest(t *testing.T) { + first, err := Assemble(prepare(t, publishSpec{content: "same"})) + require.NoError(t, err) + second, err := Assemble(prepare(t, publishSpec{content: "same"})) + require.NoError(t, err) + assert.Equal(t, first.Manifest.Digest, second.Manifest.Digest) + assert.Equal(t, first.ManifestBytes, second.ManifestBytes) + + later, err := Assemble(prepare(t, publishSpec{content: "same", at: preparedAt.Add(time.Hour)})) + require.NoError(t, err) + assert.NotEqual(t, first.Manifest.Digest, later.Manifest.Digest, "the creation time is part of the Record") + assert.NotEqual(t, first.Config.Digest, later.Config.Digest) + + changed, err := Assemble(prepare(t, publishSpec{content: "different"})) + require.NoError(t, err) + assert.NotEqual(t, first.Manifest.Digest, changed.Manifest.Digest) +} + +func TestAssemble_PackagingChangesEnvelopeNotRecord(t *testing.T) { + tgz, err := Assemble(prepare(t, publishSpec{content: "pack", packaging: artifact.PackagingTgz})) + require.NoError(t, err) + tar, err := Assemble(prepare(t, publishSpec{content: "pack", packaging: artifact.PackagingTar})) + require.NoError(t, err) + zip, err := Assemble(prepare(t, publishSpec{content: "pack", packaging: artifact.PackagingZip})) + require.NoError(t, err) + + assert.Equal(t, tgz.Config.Digest, tar.Config.Digest, "Record and content identity do not depend on packaging") + assert.Equal(t, tgz.Config.Digest, zip.Config.Digest) + assert.NotEqual(t, tgz.Manifest.Digest, tar.Manifest.Digest, "the envelope digest does") + assert.NotEqual(t, tgz.Manifest.Digest, zip.Manifest.Digest) + assert.Equal(t, artifact.MediaTypeTar, parseManifest(t, tar.ManifestBytes).Layers[0].MediaType) + assert.Equal(t, artifact.MediaTypeZip, parseManifest(t, zip.ManifestBytes).Layers[0].MediaType) +} + +func TestAssemble_WithoutDocumentsHasOnlyThePayloadLayer(t *testing.T) { + pkg, err := Assemble(prepare(t, publishSpec{content: "bare", noDocs: true})) + require.NoError(t, err) + assert.Len(t, parseManifest(t, pkg.ManifestBytes).Layers, 1) +} + +func TestAssemble_Rejections(t *testing.T) { + tests := []struct { + name string + mutate func(*artifact.PreparedArtifact) *artifact.PreparedArtifact + }{ + {"nil", func(*artifact.PreparedArtifact) *artifact.PreparedArtifact { return nil }}, + {"unarchived packaging", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Packaging = artifact.PackagingNone + return p + }}, + {"no payload", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs = p.Blobs[1:] + return p + }}, + {"two payloads", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs = append(p.Blobs, p.Blobs[0]) + return p + }}, + {"document the Record does not describe", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs[1].DocumentRole = artifact.DocumentChangelog + p.Record.Documents = p.Record.Documents[:1] + return p + }}, + {"missing staged document", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs = p.Blobs[:2] + return p + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prepared := prepare(t, publishSpec{content: "reject"}) + _, err := Assemble(tt.mutate(prepared)) + require.Error(t, err) + }) + } + _, err := Assemble(nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) +} + +func TestArtifactTypeFor(t *testing.T) { + tests := map[string]string{ + "files": artifact.MediaTypeArtifact, + "scaffold": artifact.MediaTypeScaffold, + "scaffold/simple": artifact.MediaTypeScaffold, + "component": artifact.MediaTypeComponent, + "component/helm": artifact.MediaTypeComponent, + "toolchain/package": artifact.MediaTypeToolchainPackage, + "catalog": artifact.MediaTypeCatalog, + "scaffolding": artifact.MediaTypeArtifact, + "other/kind": artifact.MediaTypeArtifact, + } + for kind, want := range tests { + assert.Equal(t, want, artifactTypeFor(kind), kind) + } +} diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go new file mode 100644 index 00000000000..318796ae47d --- /dev/null +++ b/pkg/artifact/oci/attach.go @@ -0,0 +1,243 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" +) + +// attachment is one signature referrer of a root: a manifest whose subject is the root and whose +// single layer is a Sigstore bundle. It is evidence candidate only; nothing here is trusted. +type attachment struct { + manifestDesc ocispec.Descriptor + layer ocispec.Descriptor + kind artifact.SubjectKind + subject digest.Digest + configDigest digest.Digest +} + +// attachmentSet is the attachments found for a root, split by whether they are well formed. +// Malformed and over-limit referrers are reported, never silently dropped and never able to +// override valid matching evidence. +type attachmentSet struct { + valid []attachment + diagnostics []artifact.Diagnostic +} + +// attachments lists the Sigstore bundle referrers of a root, in digest order, inspecting at most +// [maxAttachments] of them. Referrers of other types (SBOMs, attestations) and manifests that +// merely contain the root as a child are not evidence and are skipped. A failure to list +// referrers is an error: required signature material must never be silently omitted. +func (r *Repository) attachments(ctx context.Context, st store, name string, rootDesc ocispec.Descriptor) (*attachmentSet, error) { + predecessors, err := st.Predecessors(ctx, rootDesc) + if err != nil { + if isNotFound(err) { + return &attachmentSet{}, nil + } + return nil, fmt.Errorf("%w: list signature referrers of %s: %w", errUtils.ErrArtifactDownloadFailed, rootDesc.Digest, err) + } + sort.Slice(predecessors, func(i, j int) bool { return predecessors[i].Digest < predecessors[j].Digest }) + set := &attachmentSet{} + inspected := 0 + for index := range predecessors { + if !isManifestMediaType(predecessors[index].MediaType) { + continue + } + if inspected >= maxAttachments { + set.warn("evidence-truncated", fmt.Sprintf("more than %d referrers of %s; the rest were not inspected", maxAttachments, rootDesc.Digest)) + break + } + inspected++ + item, problem, err := r.readAttachment(ctx, st, name, rootDesc, predecessors[index]) + switch { + case err != nil: + return nil, err + case problem != "": + set.warn("malformed-evidence", problem) + case item != nil: + set.valid = append(set.valid, *item) + } + } + return set, nil +} + +func (a *attachmentSet) warn(code, message string) { + a.diagnostics = append(a.diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) +} + +// readAttachment parses one referrer. It returns a nil attachment and no problem for a referrer +// that is simply not signature evidence, and a problem string for a malformed bundle referrer. +func (r *Repository) readAttachment(ctx context.Context, st store, name string, rootDesc, desc ocispec.Descriptor) (*attachment, string, error) { + raw, err := r.cachedBytes(ctx, st, name, desc) + if err != nil { + if ctx.Err() != nil { + return nil, "", ctx.Err() + } + return nil, fmt.Sprintf("referrer %s could not be read: %v", desc.Digest, err), nil + } + var manifest ocispec.Manifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Sprintf("referrer %s is not a manifest: %v", desc.Digest, err), nil + } + if !isBundleReferrer(&manifest, rootDesc.Digest) { + return nil, "", nil + } + if len(manifest.Layers) != 1 { + return nil, fmt.Sprintf("signature referrer %s has %d layers; a bundle attachment has exactly one", desc.Digest, len(manifest.Layers)), nil + } + layer := manifest.Layers[0] + if layer.Size <= 0 || layer.Size > artifact.MaxEvidenceSize { + return nil, fmt.Sprintf("signature referrer %s has a %d byte bundle; the limit is %d", desc.Digest, layer.Size, artifact.MaxEvidenceSize), nil + } + return newAttachment(&manifest, desc, layer, rootDesc.Digest), "", nil +} + +// isBundleReferrer reports whether a manifest is a Sigstore bundle attached to the subject digest. +// Referrers of other types and manifests that merely contain the root as a child are not evidence. +func isBundleReferrer(manifest *ocispec.Manifest, subject digest.Digest) bool { + if manifest.Subject == nil || manifest.Subject.Digest != subject { + return false + } + artifactType := manifest.ArtifactType + if artifactType == "" { + artifactType = manifest.Config.MediaType + } + return strings.HasPrefix(artifactType, bundleMediaTypePrefix) +} + +// newAttachment reads the Record binding from the attachment's annotations. An attachment without +// one covers the root itself, like a signature made by any other Sigstore tool. +func newAttachment(manifest *ocispec.Manifest, desc, layer ocispec.Descriptor, rootDigest digest.Digest) *attachment { + found := &attachment{manifestDesc: desc, layer: layer, kind: artifact.SubjectOCIRoot, subject: rootDigest} + if manifest.Annotations[annotationSubjectKind] == string(artifact.SubjectRecord) { + found.kind = artifact.SubjectRecord + found.subject = digest.Digest(manifest.Annotations[annotationSubjectDigest]) + found.configDigest = digest.Digest(manifest.Annotations[annotationConfigDigest]) + } + return found +} + +// cachedBytes reads a small manifest through the verified cache. +func (r *Repository) cachedBytes(ctx context.Context, st store, name string, desc ocispec.Descriptor) ([]byte, error) { + if raw, ok := r.cached(name, desc); ok { + return raw, nil + } + raw, err := fetchBounded(ctx, st, desc, maxManifestSize) + if err != nil { + return nil, err + } + r.remember(name, desc, raw) + return raw, nil +} + +// bindsTo reports whether the attachment covers the subject: a Record signature must name the +// Record and the config descriptor of the addressed manifest; an image signature must carry no +// Record binding and cover the manifest or index digest itself. +func (a *attachment) bindsTo(subject artifact.SignatureSubject, loaded *root) bool { + switch subject.Kind { + case artifact.SubjectRecord: + return a.kind == artifact.SubjectRecord && loaded.class == classAtmos && + a.subject == subject.Digest && a.configDigest == subject.Digest && loaded.manifest.Config.Digest == subject.Digest + case artifact.SubjectOCIRoot: + return a.kind == artifact.SubjectOCIRoot && loaded.desc.Digest == subject.Digest + default: + return false + } +} + +// bundleFormat classifies a bundle by its verification material: a bare public key means a +// public-key bundle, everything else is treated as a certificate bundle. The verifier still +// enforces the structure; this only routes the evidence. +func bundleFormat(data []byte) artifact.EvidenceFormat { + var probe struct { + VerificationMaterial struct { + PublicKey json.RawMessage `json:"publicKey"` + } `json:"verificationMaterial"` + } + if json.Unmarshal(data, &probe) == nil && len(probe.VerificationMaterial.PublicKey) > 0 { + return artifact.EvidencePublicKeyBundle + } + return artifact.EvidenceSigstoreBundle +} + +// attachRequest is one signature bundle to store as a referrer of a root. +type attachRequest struct { + root ocispec.Descriptor + subject artifact.SignatureSubject + payload *artifact.EvidencePayload + configDigest digest.Digest +} + +// storeAttachment writes a signature bundle as an artifact manifest whose subject is the root, +// so registries with the referrers API list it and layouts keep the relationship. A Record +// signature additionally carries the Record digest and the config descriptor digest of the +// addressed manifest. Writing is idempotent: identical evidence yields an identical manifest. +func (r *Repository) storeAttachment(ctx context.Context, st store, request *attachRequest) (artifact.SignatureEvidence, string, error) { + rootDesc, subject, payload, configDigest := request.root, request.subject, request.payload, request.configDigest + layer := ocispec.Descriptor{ + MediaType: payload.MediaType, + Digest: digest.FromBytes(payload.Data), + Size: int64(len(payload.Data)), + Annotations: map[string]string{annotationTitle: bundleFileTitle}, + } + annotations := map[string]string{ + annotationSubjectKind: string(subject.Kind), + annotationSubjectDigest: subject.Digest.String(), + annotationEvidenceFormat: string(payload.Format), + } + if subject.Kind == artifact.SubjectRecord { + annotations[annotationConfigDigest] = configDigest.String() + } + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: ociSchemaVersion}, + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: payload.MediaType, + Config: ocispec.DescriptorEmptyJSON, + Layers: []ocispec.Descriptor{layer}, + Subject: &ocispec.Descriptor{MediaType: rootDesc.MediaType, Digest: rootDesc.Digest, Size: rootDesc.Size}, + Annotations: annotations, + } + raw, err := json.Marshal(manifest) + if err != nil { + return artifact.SignatureEvidence{}, "", fmt.Errorf("%w: signature manifest: %w", errUtils.ErrArtifactPackagingInvalid, err) + } + desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(raw), Size: int64(len(raw))} + for _, blob := range []struct { + desc ocispec.Descriptor + data []byte + }{{layer, payload.Data}, {ocispec.DescriptorEmptyJSON, []byte("{}")}, {desc, raw}} { + if err := pushIfAbsent(ctx, st, blob.desc, blob.data); err != nil { + return artifact.SignatureEvidence{}, "", fmt.Errorf("%w: store signature evidence: %w", errUtils.ErrArtifactUploadFailed, err) + } + } + evidence := artifact.SignatureEvidence{Subject: subject, Format: payload.Format, Descriptor: layer} + return evidence, desc.Digest.String(), nil +} + +// pushIfAbsent uploads a blob unless it is already present; a concurrent identical upload is +// success because content addressing makes it the same object. +func pushIfAbsent(ctx context.Context, st store, desc ocispec.Descriptor, data []byte) error { + exists, err := st.Exists(ctx, desc) + if err != nil { + return err + } + if exists { + return nil + } + err = st.Push(ctx, desc, bytesReader(data)) + if err != nil && !atmosoci.IsAlreadyExists(err) { + return err + } + return nil +} diff --git a/pkg/artifact/oci/backend.go b/pkg/artifact/oci/backend.go new file mode 100644 index 00000000000..fbbef01f2f8 --- /dev/null +++ b/pkg/artifact/oci/backend.go @@ -0,0 +1,317 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" +) + +// store is the content-addressed graph store of one OCI repository or layout. +type store = atmosoci.GraphStore + +// backend is what differs between the registry and the layout kinds. The shared core in +// repository.go does everything else. +type backend interface { + // kind returns the repository kind. + kind() string + + // locator returns the human-readable location of a name: registry/repository or layout path. + locator(name string) string + + // open returns the content store holding a name. write reports that the caller will write; a + // read of an absent layout returns a not-found error instead of creating it. + open(ctx context.Context, name string, write bool) (store, error) + + // reference returns the store reference that names a version of a name. + reference(name, version string) string + + // versions lists the versions (tags) published for a name, unsorted. + versions(ctx context.Context, name string) ([]string, error) + + // names enumerates the artifact names of the collection, or returns an unsupported-capability + // error when the backend cannot enumerate. + names(ctx context.Context) ([]string, error) + + // deleteScope describes what deleting a reference removes in this backend. + deleteScope() artifact.DeleteSemantics + + // remove deletes a reference. For a registry it deletes the manifest by digest, which removes + // every tag of it; for a layout it removes only the named reference. + remove(ctx context.Context, name, version string, root ocispec.Descriptor) error + + // enumeratesCollection reports whether collection enumeration is available. + enumeratesCollection() bool +} + +// registryBackend maps names onto repositories of one registry below a collection prefix. +type registryBackend struct { + registry string + prefix string + transport atmosoci.Transport + noCatalog bool + mu sync.Mutex + stores map[string]atmosoci.RepositoryStore +} + +var _ backend = (*registryBackend)(nil) + +func newRegistryBackend(registry, prefix string, transport atmosoci.Transport, noCatalog bool) *registryBackend { + return ®istryBackend{registry: registry, prefix: prefix, transport: transport, noCatalog: noCatalog, stores: map[string]atmosoci.RepositoryStore{}} +} + +func (b *registryBackend) kind() string { return artifact.RepositoryKindOCIRegistry } + +// repositoryPath maps a name onto the OCI repository path below the collection prefix. +func (b *registryBackend) repositoryPath(name string) string { + if b.prefix == "" { + return name + } + return b.prefix + "/" + name +} + +func (b *registryBackend) locator(name string) string { + return b.registry + "/" + b.repositoryPath(name) +} + +func (b *registryBackend) open(ctx context.Context, name string, _ bool) (store, error) { + return b.repository(ctx, name) +} + +// repository opens (once per name) the registry repository. Credentials are resolved by the +// transport here, not at construction, so describing configuration never authenticates. +func (b *registryBackend) repository(ctx context.Context, name string) (atmosoci.RepositoryStore, error) { + b.mu.Lock() + defer b.mu.Unlock() + if opened, ok := b.stores[name]; ok { + return opened, nil + } + opened, err := b.transport.Repository(ctx, atmosoci.Repository{Registry: b.registry, Path: b.repositoryPath(name)}) + if err != nil { + return nil, fmt.Errorf("%w: open %s: %w", errUtils.ErrArtifactDownloadFailed, b.locator(name), err) + } + b.stores[name] = opened + return opened, nil +} + +func (b *registryBackend) reference(_, version string) string { return version } + +func (b *registryBackend) versions(ctx context.Context, name string) ([]string, error) { + repository, err := b.repository(ctx, name) + if err != nil { + return nil, err + } + tags, err := repository.Tags(ctx) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("%w: list tags of %s: %w", errUtils.ErrArtifactListFailed, b.locator(name), err) + } + return tags, nil +} + +func (b *registryBackend) deleteScope() artifact.DeleteSemantics { return artifact.DeleteManifest } + +func (b *registryBackend) remove(ctx context.Context, name, _ string, root ocispec.Descriptor) error { + repository, err := b.repository(ctx, name) + if err != nil { + return err + } + if err := repository.Delete(ctx, root); err != nil { + return fmt.Errorf("%w: delete manifest %s: %w", errUtils.ErrArtifactDeleteFailed, root.Digest, err) + } + return nil +} + +func (b *registryBackend) enumeratesCollection() bool { + if b.noCatalog { + return false + } + _, ok := b.transport.(atmosoci.RegistryCatalog) + return ok +} + +// names enumerates the registry catalog, scoped to the collection prefix. A registry that does +// not provide a catalog is reported as an unsupported capability, never as an empty collection. +func (b *registryBackend) names(ctx context.Context) ([]string, error) { + catalog, ok := b.transport.(atmosoci.RegistryCatalog) + if !ok || b.noCatalog { + return nil, b.unsupportedEnumeration("this repository is configured without registry enumeration") + } + repositories, err := catalog.Repositories(ctx, b.registry) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, b.unsupportedEnumeration(fmt.Sprintf("the registry did not provide a repository catalog (%v)", err)) + } + scope := "" + if b.prefix != "" { + scope = b.prefix + "/" + } + var names []string + for _, repository := range repositories { + name, found := strings.CutPrefix(repository, scope) + if !found || name == "" || artifact.ValidateName(name) != nil { + continue + } + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +func (b *registryBackend) unsupportedEnumeration(reason string) error { + return artifact.NewUnsupportedCapability(b.registry+"/"+b.prefix, "list", "enumerate/collection", + reason+"; listing the versions of a named artifact and every other direct named operation still works without a catalog, or configure an artifact catalog for discovery") +} + +// layoutBackend keeps every name in one shared OCI layout; a name and version are one named root. +type layoutBackend struct { + root string +} + +var _ backend = (*layoutBackend)(nil) + +func newLayoutBackend(root string) *layoutBackend { return &layoutBackend{root: root} } + +func (b *layoutBackend) kind() string { return artifact.RepositoryKindOCILayout } + +func (b *layoutBackend) locator(string) string { return b.root } + +func (b *layoutBackend) reference(name, version string) string { return name + ":" + version } + +func (b *layoutBackend) deleteScope() artifact.DeleteSemantics { return artifact.DeleteReference } + +func (b *layoutBackend) enumeratesCollection() bool { return true } + +// layoutPathLocks serializes in-process writers to one layout; cross-process writers are excluded +// by the host guard the service takes. +var layoutPathLocks sync.Map + +func lockLayout(root string) func() { + value, _ := layoutPathLocks.LoadOrStore(root, &sync.Mutex{}) + mu, _ := value.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +// exists reports whether the layout has been created. +func (b *layoutBackend) exists() bool { + info, err := os.Stat(filepath.Join(b.root, ocispec.ImageIndexFile)) + return err == nil && info.Mode().IsRegular() +} + +// open returns a fresh handle on the layout so a long-lived Repository never serves a stale +// reference table. A read of an absent layout is not found; only a write creates it. +func (b *layoutBackend) open(ctx context.Context, _ string, write bool) (store, error) { + return b.layout(ctx, write) +} + +func (b *layoutBackend) layout(ctx context.Context, write bool) (*atmosoci.Layout, error) { + if !write && !b.exists() { + return nil, fmt.Errorf("%w: the OCI layout %q does not exist", errUtils.ErrArtifactNotFound, b.root) + } + opened, err := atmosoci.OpenLayout(ctx, b.root) + if err != nil { + return nil, fmt.Errorf("%w: open layout %q: %w", errUtils.ErrArtifactDownloadFailed, b.root, err) + } + return opened, nil +} + +// references returns every named root as name and version, skipping references that are not +// artifact references (for example a layout shared with other tools). +func (b *layoutBackend) references(ctx context.Context) ([]artifact.Ref, error) { + if !b.exists() { + return nil, nil + } + opened, err := b.layout(ctx, false) + if err != nil { + return nil, err + } + var refs []artifact.Ref + for _, reference := range opened.References() { + index := strings.LastIndex(reference, ":") + if index <= 0 { + continue + } + ref := artifact.Ref{Name: reference[:index], Version: reference[index+1:]} + if ref.Validate() != nil { + continue + } + refs = append(refs, ref) + } + return refs, nil +} + +func (b *layoutBackend) versions(ctx context.Context, name string) ([]string, error) { + refs, err := b.references(ctx) + if err != nil { + return nil, err + } + var versions []string + for _, ref := range refs { + if ref.Name == name { + versions = append(versions, ref.Version) + } + } + return versions, nil +} + +func (b *layoutBackend) names(ctx context.Context) ([]string, error) { + refs, err := b.references(ctx) + if err != nil { + return nil, err + } + seen := map[string]bool{} + var names []string + for _, ref := range refs { + if !seen[ref.Name] { + seen[ref.Name] = true + names = append(names, ref.Name) + } + } + sort.Strings(names) + return names, nil +} + +// remove untags one reference. Blobs stay: they may be shared with other references. The caller +// holds the in-process layout lock. +func (b *layoutBackend) remove(ctx context.Context, name, version string, _ ocispec.Descriptor) error { + opened, err := b.layout(ctx, true) + if err != nil { + return err + } + if err := opened.Untag(ctx, b.reference(name, version)); err != nil { + return fmt.Errorf("%w: remove reference %s:%s: %w", errUtils.ErrArtifactDeleteFailed, name, version, err) + } + return nil +} + +// isNotFound reports whether a store error means the content or repository is absent. +func isNotFound(err error) bool { + return err != nil && (atmosoci.IsNotFound(err) || errors.Is(err, os.ErrNotExist)) +} + +// locatorDigest returns "locator@digest" for a root. +func locatorDigest(locator string, root ocispec.Descriptor) string { + return locator + "@" + root.Digest.String() +} + +// layoutLocator returns the representation locator of a layout reference. +func layoutLocator(root string, ref artifact.Ref) string { + return path.Clean(filepath.ToSlash(root)) + "#" + ref.Name + ":" + ref.Version +} diff --git a/pkg/artifact/oci/budget_test.go b/pkg/artifact/oci/budget_test.go new file mode 100644 index 00000000000..dbc8208f946 --- /dev/null +++ b/pkg/artifact/oci/budget_test.go @@ -0,0 +1,82 @@ +package oci + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Inspecting, pinning, listing, and showing a document must never fetch a payload layer. A fresh +// repository (empty caches) makes every request visible to the recorder. +func TestMetadataAndDocumentReadsNeverFetchThePayload(t *testing.T) { + reg := newRegistry(t, nil) + writer := newRegistryRepo(t, reg) + result := publish(t, writer, publishSpec{content: "budget payload"}) + payload := payloadDigest(t, result) + + reader := newRegistryRepo(t, reg) + reg.recorder.reset() + ctx := context.Background() + + pinned, err := reader.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + inspection, err := reader.Inspect(ctx, pinned) + require.NoError(t, err) + require.NotNil(t, inspection.Record) + stream, err := reader.OpenDocument(ctx, pinned, artifact.DocumentReadme) + require.NoError(t, err) + readme, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + require.NoError(t, stream.Reader.Close()) + assert.Equal(t, "readme for "+appName+"\n", string(readme)) + _, err = reader.Pin(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + page, err := reader.List(ctx, artifact.ListQuery{Name: appName}) + require.NoError(t, err) + require.Len(t, page.Entries, 1) + + assert.Zero(t, reg.recorder.count("GET", payload.Encoded()), "no GET of the payload layer") + assert.Zero(t, reg.recorder.count("HEAD", payload.Encoded()), "no existence probe of the payload layer") + assert.Equal(t, 1, reg.recorder.count("GET", "/blobs/"+inspection.Representation.Documents[0].Digest.String()), "the selected document is fetched once") + assert.Zero(t, reg.recorder.count("GET", "/blobs/"+inspection.Representation.Documents[1].Digest.String()), "the unselected document is not fetched") + assert.Equal(t, 1, reg.recorder.count("GET", "/blobs/"+pinned.RecordDigest.String()), "the Record config is fetched once and then cached") + t.Logf("requests for resolve, inspect, show, pin, and list: %d", reg.recorder.total()) + + reg.recorder.reset() + content, err := reader.Read(ctx, pinned) + require.NoError(t, err) + assert.Zero(t, reg.recorder.count("GET", payload.Encoded()), "opening streams is lazy") + for index := range content.Streams { + if content.Streams[index].Role == artifact.BlobPayload { + _, err = io.Copy(io.Discard, content.Streams[index].Reader) + require.NoError(t, err) + } + } + require.NoError(t, content.Close()) + assert.Equal(t, 1, reg.recorder.count("GET", payload.Encoded()), "reading the payload fetches it") +} + +func TestForeignInspectionNeverFetchesLayers(t *testing.T) { + reg := newRegistry(t, nil) + image := pushImage(t, reg, testPrefix+"/tools/img", "v1") + repo := newRegistryRepo(t, reg) + reg.recorder.reset() + + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: "tools/img", Version: "v1"}) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + assert.Nil(t, inspection.Record, "a foreign image has no Atmos config blob") + require.NotNil(t, inspection.Representation.OriginalRoot) + assert.Equal(t, image.manifest, inspection.Representation.OriginalRoot.Digest) + assert.Equal(t, artifact.PackagingOCIGraph, inspection.Representation.Packaging) + for _, layer := range image.layers { + assert.Zero(t, reg.recorder.count("GET", layer.Encoded()), "layers are not fetched to inspect") + } + assert.Zero(t, reg.recorder.count("GET", image.config.Encoded()), "a foreign config blob is not read either") +} diff --git a/pkg/artifact/oci/classify.go b/pkg/artifact/oci/classify.go new file mode 100644 index 00000000000..1c35c4220e6 --- /dev/null +++ b/pkg/artifact/oci/classify.go @@ -0,0 +1,202 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" +) + +// rootClass is what a resolved root is. The classes never overlap. +type rootClass int + +const ( + // Atmos package whose config blob is the canonical Record. + classAtmos rootClass = iota + + // Legacy Atmos plugin release, immutable and never a generic artifact. + classPlugin + + // Foreign single image or artifact manifest. + classImage + + // Foreign image index, for example a multi-platform image. + classIndex +) + +// String names the class for diagnostics. +func (c rootClass) String() string { + defer perf.Track(nil, "oci.rootClass.String")() + + switch c { + case classAtmos: + return "atmos-package" + case classPlugin: + return "plugin-release" + case classImage: + return "foreign-image" + case classIndex: + return "foreign-index" + default: + return "unknown" + } +} + +// A root is one resolved and classified manifest or index. The raw field holds the exact bytes whose digest +// is desc.Digest, so a foreign root is always handled byte-identically. +type root struct { + class rootClass + desc ocispec.Descriptor + raw []byte + manifest ocispec.Manifest + index ocispec.Index +} + +// original returns the descriptor of a foreign root with the root-level facts a policy needs +// (artifact type and annotations), so inspection carries them without an Atmos config blob. +func (r *root) original() *ocispec.Descriptor { + desc := r.desc + switch { + case r.isIndex(): + desc.Annotations = r.index.Annotations + desc.ArtifactType = r.index.ArtifactType + default: + desc.Annotations = r.manifest.Annotations + desc.ArtifactType = r.manifest.ArtifactType + } + return &desc +} + +// isIndex reports whether the root is an index or manifest list. +func (r *root) isIndex() bool { return isIndexMediaType(r.desc.MediaType) } + +func isIndexMediaType(mediaType string) bool { + return mediaType == ocispec.MediaTypeImageIndex || strings.HasSuffix(mediaType, ".manifest.list.v2+json") +} + +func isManifestMediaType(mediaType string) bool { + return mediaType == ocispec.MediaTypeImageManifest || strings.HasSuffix(mediaType, ".manifest.v2+json") +} + +// loadRoot fetches a root by its resolved descriptor with a bounded, digest-verified read and +// classifies it. The class is derived only from the root itself: the config media type of an +// image manifest, or the release annotations of an index. +func loadRoot(ctx context.Context, st atmosoci.Fetcher, desc ocispec.Descriptor) (*root, error) { + raw, err := atmosoci.FetchBytes(ctx, st, desc, maxManifestSize) + if err != nil { + return nil, fmt.Errorf("%w: read manifest %s: %w", errUtils.ErrArtifactDownloadFailed, desc.Digest, err) + } + return classify(desc, raw) +} + +func classify(desc ocispec.Descriptor, raw []byte) (*root, error) { + loaded := &root{desc: desc, raw: raw} + switch { + case isIndexMediaType(desc.MediaType): + if err := json.Unmarshal(raw, &loaded.index); err != nil { + return nil, fmt.Errorf("%w: index %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) + } + loaded.class = classIndex + if hasPluginAnnotations(loaded.index.Annotations) { + loaded.class = classPlugin + } + case isManifestMediaType(desc.MediaType): + if err := json.Unmarshal(raw, &loaded.manifest); err != nil { + return nil, fmt.Errorf("%w: manifest %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) + } + loaded.class = classImage + switch loaded.manifest.Config.MediaType { + case artifact.MediaTypeConfig: + loaded.class = classAtmos + case pluginConfigMediaType: + loaded.class = classPlugin + } + default: + return nil, fmt.Errorf("%w: %s has unsupported media type %q", errUtils.ErrArtifactRecordInvalid, desc.Digest, desc.MediaType) + } + return loaded, nil +} + +// hasPluginAnnotations reports whether annotations identify a legacy plugin release. +func hasPluginAnnotations(annotations map[string]string) bool { + if annotations[pluginAnnotationName] != "" { + return true + } + for key := range annotations { + if strings.HasPrefix(key, pluginPrefix) { + return true + } + } + return false +} + +// packagingOf maps a payload layer media type onto the packaging it implies. +func packagingOf(mediaType string) (artifact.Packaging, bool) { + switch mediaType { + case artifact.MediaTypeTar: + return artifact.PackagingTar, true + case artifact.MediaTypeTarGzip: + return artifact.PackagingTgz, true + case artifact.MediaTypeZip: + return artifact.PackagingZip, true + default: + return "", false + } +} + +// mediaTypeOf maps a packaging onto its PRD payload media type. +func mediaTypeOf(packaging artifact.Packaging) (string, bool) { + switch packaging { + case artifact.PackagingTar: + return artifact.MediaTypeTar, true + case artifact.PackagingTgz: + return artifact.MediaTypeTarGzip, true + case artifact.PackagingZip: + return artifact.MediaTypeZip, true + default: + return "", false + } +} + +// atmosLayers splits an Atmos package's layers into payload and document layers and checks that +// they form exactly the shape this package writes: one archive payload plus role-tagged +// Markdown documents. Anything else is an integrity failure, never a guess. +type atmosLayers struct { + packaging artifact.Packaging + payload []ocispec.Descriptor + documents map[artifact.DocumentRole]ocispec.Descriptor +} + +func splitAtmosLayers(manifest *ocispec.Manifest) (*atmosLayers, error) { + out := &atmosLayers{documents: map[artifact.DocumentRole]ocispec.Descriptor{}} + for _, layer := range manifest.Layers { + if packaging, ok := packagingOf(layer.MediaType); ok { + if len(out.payload) > 0 { + return nil, fmt.Errorf("%w: an Atmos package has exactly one archive payload layer", errUtils.ErrArtifactIntegrityFailed) + } + out.packaging = packaging + out.payload = append(out.payload, layer) + continue + } + role := artifact.DocumentRole(layer.Annotations[annotationDocumentRole]) + if layer.MediaType != artifact.DocumentMediaType || (role != artifact.DocumentReadme && role != artifact.DocumentChangelog) { + return nil, fmt.Errorf("%w: layer %s (%s) is neither the archive payload nor a README or CHANGELOG document", errUtils.ErrArtifactIntegrityFailed, layer.Digest, layer.MediaType) + } + if _, duplicate := out.documents[role]; duplicate { + return nil, fmt.Errorf("%w: duplicate %s document layer", errUtils.ErrArtifactIntegrityFailed, role) + } + out.documents[role] = layer + } + if len(out.payload) != 1 { + return nil, fmt.Errorf("%w: an Atmos package has exactly one archive payload layer, found %d", errUtils.ErrArtifactIntegrityFailed, len(out.payload)) + } + return out, nil +} diff --git a/pkg/artifact/oci/conformance_test.go b/pkg/artifact/oci/conformance_test.go new file mode 100644 index 00000000000..3fe087a5215 --- /dev/null +++ b/pkg/artifact/oci/conformance_test.go @@ -0,0 +1,94 @@ +package oci + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +// bundleSigner wraps the conformance fake signer so its evidence carries a Sigstore bundle media +// type, the only attachment type OCI repositories store. +type bundleSigner struct{ *artifacttest.FakeSigner } + +func (s bundleSigner) Sign(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + payload, err := s.FakeSigner.Sign(ctx, subject) + if payload != nil { + payload.MediaType = bundleMediaTypePrefix + "v0.3+json" + } + return payload, err +} + +func conformanceSigner() (bundleSigner, *artifacttest.FakeSigner) { + fake := &artifacttest.FakeSigner{Identity: signerID, Issuer: signerIssue} + return bundleSigner{fake}, fake +} + +func TestRegistryRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingTar, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer, fake := conformanceSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRegistryRepo(t, newRegistry(t, nil), WithVerifier(&artifacttest.FakeVerifier{})) + }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: fake.TrustedPolicy(), + UntrustedPolicy: fake.UntrustedPolicy(), + }) + }) + } +} + +func TestLayoutRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer, fake := conformanceSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newLayoutRepo(t, WithVerifier(&artifacttest.FakeVerifier{})) + }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: fake.TrustedPolicy(), + UntrustedPolicy: fake.UntrustedPolicy(), + }) + }) + } +} + +// Without a verifier the repositories advertise no evidence formats, and Sign returns a +// capability error instead of a successful no-op. +func TestRepositoryConformance_WithoutVerifier(t *testing.T) { + t.Run("registry", func(t *testing.T) { + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { return newRegistryRepo(t, newRegistry(t, nil)) }, + }) + }) + t.Run("layout", func(t *testing.T) { + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { return newLayoutRepo(t) }, + }) + }) +} + +func TestCapabilitiesAreHonestPerBackend(t *testing.T) { + registryCaps := newRegistryRepo(t, newRegistry(t, nil)).Capabilities() + layoutCaps := newLayoutRepo(t).Capabilities() + + require.Equal(t, artifact.DeleteManifest, registryCaps.Delete, "a registry deletes manifests, never a lone tag") + require.Equal(t, artifact.DeleteReference, layoutCaps.Delete) + for _, caps := range []artifact.Capabilities{registryCaps, layoutCaps} { + require.False(t, caps.ConditionalUpdate, "OCI tags have no compare-and-swap") + require.True(t, caps.HasLockScope(artifact.LockScopeHost)) + require.False(t, caps.HasLockScope(artifact.LockScopeRepository), "a tag or host guard is never advertised as a distributed lock") + require.True(t, caps.EnumerateNamed) + require.Empty(t, caps.EvidenceFormats, "without a verifier no evidence format is claimed") + } + require.True(t, registryCaps.EnumerateCollection, "the default client supports the repository catalog") + require.True(t, layoutCaps.EnumerateCollection) +} diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go new file mode 100644 index 00000000000..c1179dd24fa --- /dev/null +++ b/pkg/artifact/oci/copy.go @@ -0,0 +1,206 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" +) + +// CanCopyFrom implements [artifact.Copier]. Another OCI repository from this package can always +// transfer a graph natively: every original manifest, blob, and referrer is copied byte for byte, +// so the resolved digest and the signatures survive. +func (r *Repository) CanCopyFrom(source artifact.Repository) bool { + defer perf.Track(nil, "oci.Repository.CanCopyFrom")() + + other, ok := source.(*Repository) + return ok && other != r +} + +// CopyFrom implements [artifact.Copier]. It copies the pinned root with its whole graph and every +// referrer (signatures, SBOMs) to this repository through the policy-aware transport. Nothing is +// repackaged: an Atmos package and a foreign image alike keep their original raw manifests, so +// the destination resolves to the source digest. A registry that cannot list or copy the +// required referrers fails the copy; signature material is never silently omitted. +func (r *Repository) CopyFrom(ctx context.Context, source artifact.Repository, pinned artifact.PinnedReference, options artifact.CopyFromOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "oci.Repository.CopyFrom")() + + other, ok := source.(*Repository) + if !ok { + return nil, artifact.NewUnsupportedCapability(r.alias, "copy", "transfer/native", "only OCI repositories transfer graphs natively") + } + if pinned.Platform != nil { + return nil, artifact.NewUnsupportedCapability(r.alias, "copy", "transfer/platform", "a native copy transfers the whole image index; narrowing to one platform is available through Read's graph source") + } + st, loaded, err := other.rootOf(ctx, &pinned) + if err != nil { + return nil, err + } + ref, err := other.destinationRef(ctx, st, loaded, &pinned) + if err != nil { + return nil, err + } + return r.transfer(ctx, st, loaded, ref, artifact.PublishOptions(options)) +} + +// destinationRef names the reference a graph is published under: the Record's name and version for +// an Atmos package, or the requested name:version for a foreign graph. +func (r *Repository) destinationRef(ctx context.Context, st store, loaded *root, pinned *artifact.PinnedReference) (artifact.Ref, error) { + if loaded.class == classAtmos { + record, err := r.readRecord(ctx, st, pinned.Ref.Name, loaded) + if err != nil { + return artifact.Ref{}, err + } + return artifact.Ref{Name: record.Name, Version: record.Version}, nil + } + if pinned.Ref.Version == "" { + return artifact.Ref{}, fmt.Errorf("%w: copying a foreign graph needs a name:version reference to publish under", errUtils.ErrArtifactRefInvalid) + } + return artifact.Ref{Name: pinned.Ref.Name, Version: pinned.Ref.Version}, nil +} + +// transfer copies a root and its referrers from src into this repository under ref after the +// reference contract (expected prior, force, plugin immutability) allows it. +func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphStore, loaded *root, ref artifact.Ref, options artifact.PublishOptions) (*artifact.PublishResult, error) { + if err := ref.Validate(); err != nil { + return nil, err + } + if layout, ok := r.be.(*layoutBackend); ok { + defer lockLayout(layout.root)() + } + dst, err := r.be.open(ctx, ref.Name, true) + if err != nil { + return nil, err + } + reference := r.be.reference(ref.Name, ref.Version) + state, err := r.readTag(ctx, dst, ref, reference) + if err != nil { + return nil, err + } + outcome, err := decidePublish(ref, &state, &incomingRoot{digest: loaded.desc.Digest, class: loaded.class}, options) + if err != nil { + return nil, err + } + if outcome != artifact.OutcomeUnchanged { + _, err = atmosoci.ExtendedCopy(ctx, + atmosoci.GraphSource{Store: src, Reference: loaded.desc.Digest.String()}, + atmosoci.Destination{Store: dst, Reference: reference}, + atmosoci.WithCopyConcurrency(r.settings.concurrency)) + if err != nil { + return nil, fmt.Errorf("%w: copy %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) + } + } + return r.transferred(ctx, ref, outcome, options) +} + +// transferred builds the result by inspecting what the destination now holds. +func (r *Repository) transferred(ctx context.Context, ref artifact.Ref, outcome artifact.Outcome, options artifact.PublishOptions) (*artifact.PublishResult, error) { + pinned, err := r.Pin(ctx, ref) + if err != nil { + return nil, err + } + inspection, err := r.Inspect(ctx, pinned) + if err != nil { + return nil, err + } + result := &artifact.PublishResult{ + Outcome: outcome, Pinned: inspection.Pinned, Representation: inspection.Representation, Location: r.referenceLocation(ref), + } + if options.Lock != nil { + result.LockScope = options.Lock.Scope() + } + return result, nil +} + +// ImportOptions control importing a graph from an OCI layout payload. +type ImportOptions struct { + // Select picks the root inside the payload by its reference name or digest. Empty requires + // the payload to hold exactly one root. + Select string + + // ExpectedPrior is the reference the caller observed at the destination; nil means none. + ExpectedPrior *artifact.PinnedReference + + // Force permits replacing a conflicting reference. It never replaces a plugin release. + Force bool + + // Lock is the writer lock already held by the caller. + Lock artifact.LockHandle +} + +// ImportLayout imports an OCI content graph from a standard OCI image layout directory (the +// carrier used for plugin releases, offline media, and graph payloads) under ref. The layout is +// opened read-only. The original manifests, blobs, and referrers are copied unchanged, so the +// imported root keeps its digest and signatures; a foreign graph is never repackaged. +func (r *Repository) ImportLayout(ctx context.Context, directory string, ref artifact.Ref, options ImportOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "oci.Repository.ImportLayout")() + + desc, err := layoutRoot(directory, options.Select) + if err != nil { + return nil, err + } + src, err := atmosoci.OpenLayoutReadOnly(ctx, directory) + if err != nil { + return nil, fmt.Errorf("%w: open layout %q: %w", errUtils.ErrArtifactDownloadFailed, directory, err) + } + loaded, err := loadRoot(ctx, src, desc) + if err != nil { + return nil, err + } + return r.transfer(ctx, src, loaded, ref, artifact.PublishOptions{ExpectedPrior: options.ExpectedPrior, Force: options.Force, Lock: options.Lock}) +} + +// ExportLayout exports the pinned graph, with every referrer, to a standard OCI image layout +// directory holding exactly one root, so it can travel as a payload and be imported again with its +// digest intact. It returns the exported root descriptor. +func (r *Repository) ExportLayout(ctx context.Context, pinned artifact.PinnedReference, directory string) (ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Repository.ExportLayout")() + + st, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return ocispec.Descriptor{}, err + } + destination, err := atmosoci.OpenLayout(ctx, directory) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("%w: open layout %q: %w", errUtils.ErrArtifactUploadFailed, directory, err) + } + reference := pinned.Ref.String() + if _, err := atmosoci.ExtendedCopy(ctx, + atmosoci.GraphSource{Store: st, Reference: loaded.desc.Digest.String()}, + atmosoci.Destination{Store: destination, Reference: reference}, + atmosoci.WithCopyConcurrency(r.settings.concurrency)); err != nil { + return ocispec.Descriptor{}, fmt.Errorf("%w: export %s: %w", errUtils.ErrArtifactUploadFailed, pinned.Ref, err) + } + return loaded.desc, nil +} + +// layoutRoot reads a layout's index.json and picks the root to import. +func layoutRoot(directory, selector string) (ocispec.Descriptor, error) { + raw, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("%w: read the layout index of %q: %w", errUtils.ErrArtifactNotFound, directory, err) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return ocispec.Descriptor{}, fmt.Errorf("%w: layout index of %q: %w", errUtils.ErrArtifactIntegrityFailed, directory, err) + } + var matches []ocispec.Descriptor + for _, entry := range index.Manifests { + if selector == "" || entry.Digest.String() == selector || entry.Annotations[ocispec.AnnotationRefName] == selector { + matches = append(matches, entry) + } + } + if len(matches) != 1 { + return ocispec.Descriptor{}, fmt.Errorf("%w: %d roots in %q match %q; select exactly one by reference name or digest", errUtils.ErrArtifactRefInvalid, len(matches), directory, selector) + } + return ocispec.Descriptor{MediaType: matches[0].MediaType, Digest: matches[0].Digest, Size: matches[0].Size}, nil +} diff --git a/pkg/artifact/oci/delete.go b/pkg/artifact/oci/delete.go new file mode 100644 index 00000000000..4af28ee5628 --- /dev/null +++ b/pkg/artifact/oci/delete.go @@ -0,0 +1,131 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Repository] with tag-safe semantics. +// +// Deleting by tag inspects every other tag of the name that resolves to the same digest. A +// registry can only delete a manifest, which removes all of its tags, so a tag delete that would +// also remove a sibling reference is rejected as an unsupported tag-only deletion. A layout +// removes just the named reference. Deleting by digest is explicitly manifest-scoped and reports +// every reference that stopped resolving. Blobs are never deleted directly, because other +// manifests may share them. +func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + defer perf.Track(nil, "oci.Repository.Delete")() + + name := pinned.Ref.Name + if err := artifact.ValidateName(name); err != nil { + return nil, err + } + if layout, ok := r.be.(*layoutBackend); ok { + defer lockLayout(layout.root)() + } + st, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return nil, err + } + if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != loaded.desc.Digest { + return nil, fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, pinned.Ref, loaded.desc.Digest, options.ExpectedCurrent.ResolvedDigest) + } + if loaded.class == classPlugin { + return nil, fmt.Errorf("%w: %s is a legacy plugin release; it is immutable and is never deleted by generic artifact operations", errUtils.ErrArtifactConflict, pinned.Ref) + } + holders, err := r.referencesTo(ctx, st, name, loaded.desc.Digest) + if err != nil { + return nil, err + } + if pinned.Ref.IsDigest() { + return r.deleteDigest(ctx, loaded, name, holders) + } + return r.deleteTag(ctx, loaded, pinned.Ref, holders) +} + +// deleteTag removes one tag, refusing when that would also remove a sibling reference. +func (r *Repository) deleteTag(ctx context.Context, loaded *root, ref artifact.Ref, holders []artifact.Ref) (*artifact.DeleteResult, error) { + var siblings []artifact.Ref + named := false + for _, holder := range holders { + if holder.Version != ref.Version { + siblings = append(siblings, holder) + } else { + named = true + } + } + if !named { + return nil, fmt.Errorf("%w: %s no longer resolves to %s, so it is not deleted", errUtils.ErrArtifactConflict, ref, loaded.desc.Digest) + } + if r.be.deleteScope() != artifact.DeleteReference && len(siblings) > 0 { + return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/tag", + fmt.Sprintf("the registry can only delete a manifest, which would also remove %v; delete by digest to remove them together, or retag them first", siblings)) + } + if err := r.be.remove(ctx, ref.Name, ref.Version, loaded.desc); err != nil { + return nil, err + } + return &artifact.DeleteResult{Deleted: []string{r.referenceLocation(ref)}, Digest: loaded.desc.Digest}, nil +} + +// deleteDigest removes the manifest and every reference to it, reporting each affected one. +func (r *Repository) deleteDigest(ctx context.Context, loaded *root, name string, holders []artifact.Ref) (*artifact.DeleteResult, error) { + result := &artifact.DeleteResult{Digest: loaded.desc.Digest} + if r.be.deleteScope() == artifact.DeleteReference { + if len(holders) == 0 { + return nil, fmt.Errorf("%w: no reference of %s resolves to %s", errUtils.ErrArtifactNotFound, name, loaded.desc.Digest) + } + for _, holder := range holders { + if err := r.be.remove(ctx, holder.Name, holder.Version, loaded.desc); err != nil { + return result, err + } + result.Deleted = append(result.Deleted, r.referenceLocation(holder)) + result.Affected = append(result.Affected, holder) + } + return result, nil + } + if err := r.be.remove(ctx, name, "", loaded.desc); err != nil { + return nil, err + } + result.Deleted = []string{locatorDigest(r.be.locator(name), loaded.desc)} + result.Affected = holders + return result, nil +} + +// referencesTo lists, in version order, the tags of a name that currently resolve to a digest. +// The scan is bounded; a name with more tags than the bound cannot be deleted safely and fails. +func (r *Repository) referencesTo(ctx context.Context, st store, name string, target digest.Digest) ([]artifact.Ref, error) { + versions, err := r.be.versions(ctx, name) + if err != nil { + return nil, err + } + sort.Strings(versions) + if len(versions) > maxTagsScanned { + return nil, fmt.Errorf("%w: %s has %d tags; the sibling check inspects at most %d, so it cannot prove the delete is safe", errUtils.ErrArtifactDeleteFailed, name, len(versions), maxTagsScanned) + } + var holders []artifact.Ref + for _, version := range versions { + desc, err := st.Resolve(ctx, r.be.reference(name, version)) + if err != nil { + if isNotFound(err) { + continue + } + return nil, fmt.Errorf("%w: resolve %s:%s: %w", errUtils.ErrArtifactDeleteFailed, name, version, err) + } + if desc.Digest == target { + holders = append(holders, artifact.Ref{Name: name, Version: version}) + } + } + return holders, nil +} + +// errorsIsNotFound reports whether err is the artifact not-found sentinel. +func errorsIsNotFound(err error) bool { return errors.Is(err, errUtils.ErrArtifactNotFound) } diff --git a/pkg/artifact/oci/delete_test.go b/pkg/artifact/oci/delete_test.go new file mode 100644 index 00000000000..5588711a351 --- /dev/null +++ b/pkg/artifact/oci/delete_test.go @@ -0,0 +1,128 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// alias tags an existing manifest with another version, creating a sibling reference. +func alias(t *testing.T, repo *Repository, pinned artifact.PinnedReference, version string) { + t.Helper() + st, err := repo.be.open(context.Background(), pinned.Ref.Name, true) + require.NoError(t, err) + desc, ok := repo.descriptorFor(pinned.Ref.Name, pinned.ResolvedDigest) + require.True(t, ok) + require.NoError(t, st.Tag(context.Background(), desc, repo.be.reference(pinned.Ref.Name, version))) +} + +func resolves(repo *Repository, name, version string) bool { + _, err := repo.Resolve(context.Background(), artifact.Ref{Name: name, Version: version}) + return err == nil +} + +func TestRegistryTagDeleteRefusesWhenItWouldRemoveASiblingTag(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + first := publish(t, repo, publishSpec{content: "shared manifest"}) + alias(t, repo, first.Pinned, "stable") + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteManifest}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "only manifest deletion exists and it would remove stable") + assert.Contains(t, err.Error(), "stable") + assert.True(t, resolves(repo, appName, appVersion)) + assert.True(t, resolves(repo, appName, "stable")) +} + +func TestRegistryDigestDeleteIsManifestScopedAndReportsAffectedReferences(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + first := publish(t, repo, publishSpec{content: "shared manifest"}) + alias(t, repo, first.Pinned, "stable") + publish(t, repo, publishSpec{version: "2.0.0", content: "unrelated"}) + + byDigest := artifact.PinnedReference{Ref: artifact.Ref{Name: appName, Digest: first.Pinned.ResolvedDigest}, ResolvedDigest: first.Pinned.ResolvedDigest} + result, err := repo.Delete(context.Background(), byDigest, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + assert.Equal(t, first.Pinned.ResolvedDigest, result.Digest) + assert.ElementsMatch(t, []artifact.Ref{{Name: appName, Version: appVersion}, {Name: appName, Version: "stable"}}, result.Affected) + assert.False(t, resolves(repo, appName, appVersion)) + assert.False(t, resolves(repo, appName, "stable")) + assert.True(t, resolves(repo, appName, "2.0.0"), "deletion is scoped to the selected manifest") +} + +func TestRegistryTagDeleteWithoutSiblingsKeepsBlobsAndOtherVersions(t *testing.T) { + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg) + drop := publish(t, repo, publishSpec{content: "drop"}) + publish(t, repo, publishSpec{version: "2.0.0", content: "keep"}) + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + + reg.recorder.reset() + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteManifest}) + require.NoError(t, err) + assert.Empty(t, result.Affected) + assert.NotEmpty(t, result.Deleted) + assert.Zero(t, reg.recorder.count("DELETE", "/blobs/"), "shared blobs are never deleted directly") + assert.False(t, resolves(repo, appName, appVersion)) + assert.True(t, resolves(repo, appName, "2.0.0")) + assert.NotEqual(t, drop.Pinned.ResolvedDigest, mustResolve(t, repo, "2.0.0").ResolvedDigest) +} + +func mustResolve(t *testing.T, repo *Repository, version string) artifact.PinnedReference { + t.Helper() + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: version}) + require.NoError(t, err) + return pinned +} + +func TestLayoutTagDeleteRemovesOnlyTheNamedReference(t *testing.T) { + repo := newLayoutRepo(t) + first := publish(t, repo, publishSpec{content: "shared"}) + alias(t, repo, first.Pinned, "stable") + pinned := mustResolve(t, repo, appVersion) + + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteReference}) + require.NoError(t, err) + assert.Empty(t, result.Affected) + assert.False(t, resolves(repo, appName, appVersion)) + assert.True(t, resolves(repo, appName, "stable"), "the sibling reference and the shared blobs survive") + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err, "the pinned manifest and its blobs are still readable") + require.NoError(t, content.Close()) + + byDigest := artifact.PinnedReference{Ref: artifact.Ref{Name: appName, Digest: first.Pinned.ResolvedDigest}, ResolvedDigest: first.Pinned.ResolvedDigest} + result, err = repo.Delete(context.Background(), byDigest, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.NoError(t, err) + assert.Equal(t, []artifact.Ref{{Name: appName, Version: "stable"}}, result.Affected) + assert.False(t, resolves(repo, appName, "stable")) +} + +func TestDeleteRejectsAStaleExpectedCurrent(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + publish(t, repo, publishSpec{content: "current"}) + pinned := mustResolve(t, repo, appVersion) + stale := pinned + stale.ResolvedDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &stale, Scope: repo.Capabilities().Delete}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.True(t, resolves(repo, appName, appVersion)) + }) +} + +func TestDeleteRefusesAMovedTag(t *testing.T) { + repo := newLayoutRepo(t) + first := publish(t, repo, publishSpec{content: "one"}) + old := mustResolve(t, repo, appVersion) + _, err := repo.Publish(context.Background(), prepare(t, publishSpec{content: "two"}), artifact.PublishOptions{ExpectedPrior: &first.Pinned, Force: true}) + require.NoError(t, err) + _, err = repo.Delete(context.Background(), old, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the tag no longer names the pinned manifest") +} diff --git a/pkg/artifact/oci/doc.go b/pkg/artifact/oci/doc.go new file mode 100644 index 00000000000..f4d80ce1ebf --- /dev/null +++ b/pkg/artifact/oci/doc.go @@ -0,0 +1,30 @@ +// Package oci implements the oci/registry and oci/layout artifact repositories on the pkg/oci +// transport boundary. +// +// Both kinds share one implementation core. A backend supplies the content store for an artifact +// name (an ORAS registry repository, or the single shared OCI layout), how versions are named, +// and how references are removed; everything else is common: classification of a resolved root, +// manifest assembly for new Atmos packages, metadata-only inspection, verified reads, guarded +// publication, name-scoped and collection listing, tag-safe deletion, signature attachments as +// OCI referrers, host and delegated write locks, and byte-identical graph transfer. +// +// A resolved root is exactly one of four things, and the package never confuses them: +// +// - an Atmos package, whose config blob is the canonical portable Record; +// - a legacy Atmos plugin release (immutable; never listed as a generic artifact, never +// overwritten, even with force); +// - a foreign image or artifact manifest; +// - a foreign image index (for example a multi-platform image). +// +// Foreign graphs are inspected without pretending they have an Atmos config blob, are never +// repackaged, and are transferred with every original manifest, blob, and referrer unchanged, so +// their resolved digests and signatures survive registry, layout, and second-registry hops. +// +// Signature evidence is stored as Sigstore bundle referrers. A Record signature binds the Record +// to the addressed manifest's config descriptor; an image signature binds to the original +// manifest or index digest. Cryptography is delegated to an injected [artifact.Verifier]; this +// package neither imports Sigstore libraries nor implements cryptography. +// +// The package registers nothing by itself. A composition root calls [Register] explicitly, and the +// go-containerregistry and ORAS types never appear in this package's API. +package oci diff --git a/pkg/artifact/oci/edge_test.go b/pkg/artifact/oci/edge_test.go new file mode 100644 index 00000000000..7c524afcb2f --- /dev/null +++ b/pkg/artifact/oci/edge_test.go @@ -0,0 +1,156 @@ +//nolint:gocritic // OCI descriptors are passed by value, mirroring the ORAS content contracts. +package oci + +import ( + "context" + "errors" + "io" + "testing" + "time" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" +) + +// flakyTags wraps a registry transport so the tag write reports failure; when lands is true the +// write actually reaches the registry first, like a response lost after a server-side commit. +type flakyTags struct { + atmosoci.Transport + lands bool +} + +func (f flakyTags) Repository(ctx context.Context, repository atmosoci.Repository) (atmosoci.RepositoryStore, error) { + inner, err := f.Transport.Repository(ctx, repository) + if err != nil { + return nil, err + } + return flakyStore{RepositoryStore: inner, lands: f.lands}, nil +} + +type flakyStore struct { + atmosoci.RepositoryStore + lands bool +} + +func (f flakyStore) PushReference(ctx context.Context, expected ocispec.Descriptor, content io.Reader, reference string) error { + if f.lands { + if err := f.RepositoryStore.PushReference(ctx, expected, content, reference); err != nil { + return err + } + } + return errors.New("connection reset by peer") +} + +func TestUnknownTagWriteOutcomeIsReconciledByResolving(t *testing.T) { + ctx := context.Background() + reg := newRegistry(t, nil) + + landed, err := NewRepository(ctx, "landed", registryConfig(reg.server.Host), WithTransport(flakyTags{reg.transport, true}), WithLockDir(t.TempDir())) + require.NoError(t, err) + result, err := landed.Publish(ctx, prepare(t, publishSpec{content: "lost response"}), artifact.PublishOptions{}) + require.NoError(t, err, "the tag is at the new manifest, so the publication completed") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + + lost, err := NewRepository(ctx, "lost", registryConfig(reg.server.Host), WithTransport(flakyTags{reg.transport, false}), WithLockDir(t.TempDir())) + require.NoError(t, err) + _, err = lost.Publish(ctx, prepare(t, publishSpec{name: "other/app", content: "never landed"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + _, err = newRegistryRepo(t, reg).Resolve(ctx, artifact.Ref{Name: "other/app", Version: appVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a failed write is not advertised as published") +} + +func TestStoredEvidenceIsValidatedBeforeItIsWritten(t *testing.T) { + tr := newTrust(t) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + valid, err := tr.keyless.Sign(context.Background(), env.record) + require.NoError(t, err) + tests := map[string]*artifact.EvidencePayload{ + "nil": nil, + "unknown format": {Format: "cosign-tag", MediaType: bundleMediaTypePrefix + "v0.3+json", Data: valid.Data}, + "not a bundle": {Format: valid.Format, MediaType: "application/json", Data: valid.Data}, + "empty": {Format: valid.Format, MediaType: valid.MediaType}, + "over the bound": {Format: valid.Format, MediaType: valid.MediaType, Data: make([]byte, artifact.MaxEvidenceSize+1)}, + } + for name, payload := range tests { + t.Run(name, func(t *testing.T) { + require.Error(t, env.repo.checkPayload(payload)) + }) + } + require.NoError(t, env.repo.checkPayload(valid)) + }) +} + +func TestSigningNeedsAVerifierAndAKnownSubject(t *testing.T) { + tr := newTrust(t) + plain := newLayoutRepo(t) + result := publish(t, plain, publishSpec{content: "no verifier"}) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: result.Pinned.RecordDigest, MediaType: artifact.MediaTypeConfig} + _, err := plain.Sign(context.Background(), subject, tr.keyless, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, err = plain.Verify(context.Background(), subject, identityPolicy(trustedIdentity, trustedIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + _, err := env.repo.Sign(context.Background(), env.record, nil, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + wrongSize := env.record + wrongSize.Size++ + _, err = env.repo.Sign(context.Background(), wrongSize, tr.keyless, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + bad := env.record + bad.Kind = "unknown" + _, err = env.repo.Sign(context.Background(), bad, tr.keyless, artifact.SignOptions{}) + require.Error(t, err) + }) +} + +func TestMissingDocumentLayerAndHostGuardLeases(t *testing.T) { + repo := newLayoutRepo(t) + publish(t, repo, publishSpec{content: "bare", noDocs: true}) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: appName, Version: appVersion}) + for _, role := range []artifact.DocumentRole{artifact.DocumentReadme, artifact.DocumentChangelog} { + _, err := repo.OpenDocument(context.Background(), pinned, role) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, string(role)) + } + + handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "x:1", Owner: "a"}) + require.NoError(t, err) + require.ErrorIs(t, handle.Renew(context.Background(), time.Minute), errUtils.ErrArtifactUnsupportedCapability) + require.NoError(t, handle.Release(context.Background())) +} + +func TestOptionsApplyDefaultsAndOverrides(t *testing.T) { + defaults := applySettings(nil) + assert.Equal(t, artifact.DefaultConcurrency, defaults.concurrency) + assert.Equal(t, defaultPageSize, defaults.pageSize) + assert.NotEmpty(t, defaults.owner) + custom := applySettings([]Option{WithConcurrency(7), WithPageSize(3), WithOwner("me"), nil}) + assert.Equal(t, 7, custom.concurrency) + assert.Equal(t, 3, custom.pageSize) + assert.Equal(t, "me", custom.owner) + + repo := newLayoutRepo(t, WithPageSize(1)) + publish(t, repo, publishSpec{version: "1.0.0", content: "a"}) + publish(t, repo, publishSpec{version: "2.0.0", content: "b"}) + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Len(t, page.Entries, 1, "the configured page size applies when the query has none") + assert.NotEmpty(t, page.NextCursor) +} + +func TestReaderClosedBeforeReadingFetchesNothing(t *testing.T) { + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg) + result := publish(t, repo, publishSpec{content: "unread"}) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: appName, Version: appVersion}) + reg.recorder.reset() + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + require.NoError(t, content.Close()) + assert.Zero(t, reg.recorder.count("GET", payloadDigest(t, result).Encoded())) +} diff --git a/pkg/artifact/oci/fetch.go b/pkg/artifact/oci/fetch.go new file mode 100644 index 00000000000..f94f667b427 --- /dev/null +++ b/pkg/artifact/oci/fetch.go @@ -0,0 +1,197 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash" + "io" + "sync" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Format of an error that names a digest and keeps the cause. +const errFormatDigestCause = "%w: %s: %w" + +// fetchBounded reads one small blob with the declared size and digest verified; a larger or +// tampered blob never reaches the caller. Oversize content is reported as too large, everything +// else that fails verification as an integrity failure. +func fetchBounded(ctx context.Context, st store, desc ocispec.Descriptor, limit int64) ([]byte, error) { + if desc.Size > limit || desc.Size < 0 { + return nil, fmt.Errorf("%w: %s declares %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, desc.Digest, desc.Size, limit) + } + raw, err := atmosoci.FetchBytes(ctx, st, desc, limit) + switch { + case err == nil: + return raw, nil + case errors.Is(err, errUtils.ErrOCIIntegrity): + return nil, fmt.Errorf(errFormatDigestCause, errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) + case isNotFound(err): + return nil, fmt.Errorf("%w: %s is missing: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) + default: + return nil, fmt.Errorf(errFormatDigestCause, errUtils.ErrArtifactDownloadFailed, desc.Digest, err) + } +} + +// lazyReader opens a blob on first read and verifies size and digest as it streams: a clean EOF +// means the bytes matched their descriptor. Nothing is fetched for a stream nobody reads. +type lazyReader struct { + ctx context.Context + st store + desc ocispec.Descriptor + once sync.Once + source io.ReadCloser + hasher hash.Hash + read int64 + err error + done bool +} + +func newLazyReader(ctx context.Context, st store, desc ocispec.Descriptor) *lazyReader { + return &lazyReader{ctx: ctx, st: st, desc: desc} +} + +// Read implements io.Reader. +func (l *lazyReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "oci.lazyReader.Read")() + + l.once.Do(l.open) + if l.err != nil { + return 0, l.err + } + if err := l.ctx.Err(); err != nil { + l.err = err + return 0, err + } + n, err := l.source.Read(p) + l.read += int64(n) + _, _ = l.hasher.Write(p[:n]) + switch { + case l.read > l.desc.Size: + l.err = fmt.Errorf("%w: %s is larger than its declared %d bytes", errUtils.ErrArtifactIntegrityFailed, l.desc.Digest, l.desc.Size) + return n, l.err + case errors.Is(err, io.EOF): + l.done = true + if finishErr := l.finish(); finishErr != nil { + l.err = finishErr + return n, finishErr + } + return n, io.EOF + case err != nil: + l.err = fmt.Errorf(errFormatDigestCause, errUtils.ErrArtifactDownloadFailed, l.desc.Digest, err) + return n, l.err + } + return n, nil +} + +func (l *lazyReader) open() { + l.hasher = l.desc.Digest.Algorithm().Hash() + source, err := l.st.Fetch(l.ctx, l.desc) + if err != nil { + if isNotFound(err) { + l.err = fmt.Errorf("%w: %s is missing: %w", errUtils.ErrArtifactIntegrityFailed, l.desc.Digest, err) + } else { + l.err = fmt.Errorf(errFormatDigestCause, errUtils.ErrArtifactDownloadFailed, l.desc.Digest, err) + } + return + } + l.source = source +} + +func (l *lazyReader) finish() error { + if l.read != l.desc.Size { + return fmt.Errorf("%w: %s is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, l.desc.Digest, l.read, l.desc.Size) + } + if got := digest.NewDigest(l.desc.Digest.Algorithm(), l.hasher); got != l.desc.Digest { + return fmt.Errorf("%w: content digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, l.desc.Digest) + } + return nil +} + +// Close implements io.Closer. +func (l *lazyReader) Close() error { + defer perf.Track(nil, "oci.lazyReader.Close")() + + l.once.Do(func() { l.err = io.ErrClosedPipe }) + if l.source == nil { + return nil + } + return l.source.Close() +} + +// graphSource exposes a foreign OCI graph for transfer. Every node read is verified against its +// descriptor, and the original root bytes are never rewritten. +type graphSource struct { + st store + root ocispec.Descriptor + // only, when set, narrows the traversal below an index to one selected child. + only *ocispec.Descriptor +} + +var _ artifact.GraphSource = (*graphSource)(nil) + +// Root implements [artifact.GraphSource]. +func (g *graphSource) Root() ocispec.Descriptor { + defer perf.Track(nil, "oci.graphSource.Root")() + + return g.root +} + +// Fetch implements [artifact.GraphSource]. +func (g *graphSource) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + defer perf.Track(nil, "oci.graphSource.Fetch")() + + return newLazyReader(ctx, g.st, desc), nil +} + +// Successors implements [artifact.GraphSource]: the config, layers, and children a manifest or +// index references. Other blobs have none. +func (g *graphSource) Successors(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.graphSource.Successors")() + + if !isManifestMediaType(desc.MediaType) && !isIndexMediaType(desc.MediaType) { + return nil, nil + } + raw, err := fetchBounded(ctx, g.st, desc, maxManifestSize) + if err != nil { + return nil, err + } + successors, err := successorsOf(desc, raw) + if err != nil { + return nil, err + } + if g.only == nil || desc.Digest != g.root.Digest || !isIndexMediaType(desc.MediaType) { + return successors, nil + } + for _, child := range successors { + if child.Digest == g.only.Digest { + return []ocispec.Descriptor{child}, nil + } + } + return nil, fmt.Errorf("%w: the selected platform is not a child of %s", errUtils.ErrArtifactNotFound, desc.Digest) +} + +// successorsOf lists the descriptors a manifest or index references. +func successorsOf(desc ocispec.Descriptor, raw []byte) ([]ocispec.Descriptor, error) { + loaded, err := classify(desc, raw) + if err != nil { + return nil, err + } + if loaded.isIndex() { + return append([]ocispec.Descriptor(nil), loaded.index.Manifests...), nil + } + successors := []ocispec.Descriptor{loaded.manifest.Config} + return append(successors, loaded.manifest.Layers...), nil +} + +// bytesReader returns a reader over data. +func bytesReader(data []byte) io.Reader { return bytes.NewReader(data) } diff --git a/pkg/artifact/oci/foreignfixtures_test.go b/pkg/artifact/oci/foreignfixtures_test.go new file mode 100644 index 00000000000..4094520556f --- /dev/null +++ b/pkg/artifact/oci/foreignfixtures_test.go @@ -0,0 +1,73 @@ +package oci + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +// foreignImage describes a native image or index pushed with a third-party client. +type foreignImage struct { + manifest digest.Digest + config digest.Digest + layers []digest.Digest + children []digest.Digest +} + +// pushImage pushes a random single-layer image. +func pushImage(t *testing.T, reg *registry, repository, tag string) foreignImage { + t.Helper() + image, err := random.Image(256, 1) + require.NoError(t, err) + reference, err := name.ParseReference(reg.server.Host + "/" + repository + ":" + tag) + require.NoError(t, err) + require.NoError(t, remote.Write(reference, image, remote.WithTransport(reg.recorder), remote.WithAuth(reg.auth()))) + return describeImage(t, image) +} + +func describeImage(t *testing.T, image v1.Image) foreignImage { + t.Helper() + described := foreignImage{} + manifestDigest, err := image.Digest() + require.NoError(t, err) + described.manifest = digest.Digest(manifestDigest.String()) + config, err := image.ConfigName() + require.NoError(t, err) + described.config = digest.Digest(config.String()) + layers, err := image.Layers() + require.NoError(t, err) + for _, layer := range layers { + layerDigest, err := layer.Digest() + require.NoError(t, err) + described.layers = append(described.layers, digest.Digest(layerDigest.String())) + } + return described +} + +// pushIndex pushes a multi-platform index of one random image per platform. +func pushIndex(t *testing.T, reg *registry, repository, tag string, platforms ...v1.Platform) foreignImage { + t.Helper() + var index v1.ImageIndex = empty.Index + var children []digest.Digest + for _, platform := range platforms { + image, err := random.Image(256, 2) + require.NoError(t, err) + index = mutate.AppendManifests(index, mutate.IndexAddendum{Add: image, Descriptor: v1.Descriptor{Platform: &platform}}) + imageDigest, err := image.Digest() + require.NoError(t, err) + children = append(children, digest.Digest(imageDigest.String())) + } + reference, err := name.ParseReference(reg.server.Host + "/" + repository + ":" + tag) + require.NoError(t, err) + require.NoError(t, remote.WriteIndex(reference, index, remote.WithTransport(reg.recorder), remote.WithAuth(reg.auth()))) + indexDigest, err := index.Digest() + require.NoError(t, err) + return foreignImage{manifest: digest.Digest(indexDigest.String()), children: children} +} diff --git a/pkg/artifact/oci/helpers_test.go b/pkg/artifact/oci/helpers_test.go new file mode 100644 index 00000000000..70109966c23 --- /dev/null +++ b/pkg/artifact/oci/helpers_test.go @@ -0,0 +1,226 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Fixed values shared by the tests. +const ( + testPrefix = "acme/atmos" + filePerm = 0o644 + execPerm = 0o755 + testOwner = "test-owner" + appName = "team/app" + appVersion = "1.0.0" + signerID = "release@example.com" + signerIssue = "https://issuer.example.com" +) + +var preparedAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + +// recorder counts the HTTP requests a client makes, by method and path. +type recorder struct { + mu sync.Mutex + requests []string + fail error +} + +func (r *recorder) RoundTrip(request *http.Request) (*http.Response, error) { + r.mu.Lock() + r.requests = append(r.requests, request.Method+" "+request.URL.Path) + fail := r.fail + r.mu.Unlock() + if fail != nil { + return nil, fail + } + return http.DefaultTransport.RoundTrip(request) +} + +// reset forgets the recorded requests. +func (r *recorder) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.requests = nil +} + +// count returns how many recorded requests start with the method and contain the path fragment. +func (r *recorder) count(method, fragment string) int { + r.mu.Lock() + defer r.mu.Unlock() + total := 0 + for _, request := range r.requests { + if strings.HasPrefix(request, method+" ") && strings.Contains(request, fragment) { + total++ + } + } + return total +} + +// total returns the number of recorded requests. +func (r *recorder) total() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.requests) +} + +// failAll makes every later request fail, like a transport with no network. +func (r *recorder) failAll(err error) { + r.mu.Lock() + defer r.mu.Unlock() + r.fail = err +} + +// registry is one in-process registry with its own credentials and recording client. +type registry struct { + server *ocitest.Server + recorder *recorder + transport atmosoci.Transport + credential *ocitest.Credential +} + +// auth returns the authenticator third-party clients use against the registry. +func (r *registry) auth() authn.Authenticator { + if r.credential == nil { + return authn.Anonymous + } + return &authn.Basic{Username: r.credential.Username, Password: r.credential.Password} +} + +// credentialsKeychain resolves only the registries it was given, so one registry's credentials are +// never presented to another and nothing falls back to the developer's Docker configuration. +type credentialsKeychain map[string]authn.Authenticator + +func (k credentialsKeychain) Resolve(resource authn.Resource) (authn.Authenticator, error) { + if auth, ok := k[resource.RegistryStr()]; ok { + return auth, nil + } + return authn.Anonymous, nil +} + +// newRegistry starts a registry, optionally behind Basic credentials. +func newRegistry(t *testing.T, credential *ocitest.Credential) *registry { + t.Helper() + server := ocitest.NewServerWithOptions(t, credential, ocitest.WithTagsFollowManifestDelete()) + rec := &recorder{} + keychain := credentialsKeychain{} + if credential != nil { + keychain[server.Host] = &authn.Basic{Username: credential.Username, Password: credential.Password} + } + client := atmosoci.NewClient(nil, atmosoci.WithKeychain(keychain), atmosoci.WithTransport(rec)) + return ®istry{server: server, recorder: rec, transport: client, credential: credential} +} + +// options returns the repository options for a registry-backed repository. +func (r *registry) options(extra ...Option) []Option { + return append([]Option{WithTransport(r.transport), WithOwner(testOwner)}, extra...) +} + +func registryConfig(host string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindOCIRegistry, + Options: map[string]any{"registry": host, "prefix": testPrefix}, + } +} + +func layoutConfig(path string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCILayout, Options: map[string]any{"path": path}} +} + +// newRegistryRepo builds a registry repository against r. +func newRegistryRepo(t *testing.T, r *registry, extra ...Option) *Repository { + t.Helper() + opts := append(r.options(WithLockDir(t.TempDir())), extra...) + repo, err := NewRepository(context.Background(), "registry", registryConfig(r.server.Host), opts...) + require.NoError(t, err) + return repo +} + +// newLayoutRepo builds a layout repository in a fresh directory. +func newLayoutRepo(t *testing.T, extra ...Option) *Repository { + t.Helper() + return newLayoutRepoAt(t, filepath.Join(t.TempDir(), "layout"), extra...) +} + +func newLayoutRepoAt(t *testing.T, path string, extra ...Option) *Repository { + t.Helper() + opts := append([]Option{WithOwner(testOwner), WithLockDir(t.TempDir())}, extra...) + repo, err := NewRepository(context.Background(), "layout", layoutConfig(path), opts...) + require.NoError(t, err) + return repo +} + +// publishSpec describes one package the tests publish. +type publishSpec struct { + name string + version string + content string + packaging artifact.Packaging + labels map[string]string + noDocs bool + at time.Time +} + +// prepare builds a deterministic prepared artifact. +func prepare(t *testing.T, spec publishSpec) *artifact.PreparedArtifact { + t.Helper() + if spec.name == "" { + spec.name = appName + } + if spec.version == "" { + spec.version = appVersion + } + if spec.packaging == "" { + spec.packaging = artifact.PackagingTgz + } + if spec.at.IsZero() { + spec.at = preparedAt + } + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", spec.content, filePerm) + fixtures.WriteFile(t, src, "bin/run.sh", fixtures.HelperScript, execPerm) + if !spec.noDocs { + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "readme for "+spec.name+"\n", filePerm) + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/CHANGELOG.md", "changes for "+spec.name+"\n", filePerm) + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: spec.name, Version: spec.version, Packaging: spec.packaging, + Metadata: artifact.Metadata{Description: "a test package", Labels: spec.labels}, + OperationID: "op-" + spec.name + "-" + spec.version, PreparedAt: spec.at, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +// publish prepares and publishes a package, replacing nothing. +func publish(t *testing.T, repo *Repository, spec publishSpec) *artifact.PublishResult { + t.Helper() + result, err := repo.Publish(context.Background(), prepare(t, spec), artifact.PublishOptions{}) + require.NoError(t, err) + return result +} + +// payloadDigest returns the digest of the single payload layer of a published package. +func payloadDigest(t *testing.T, result *artifact.PublishResult) digest.Digest { + t.Helper() + require.Len(t, result.Representation.Payload, 1) + return result.Representation.Payload[0].Digest +} diff --git a/pkg/artifact/oci/inspect.go b/pkg/artifact/oci/inspect.go new file mode 100644 index 00000000000..4e62d8ba68a --- /dev/null +++ b/pkg/artifact/oci/inspect.go @@ -0,0 +1,215 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "fmt" + "io" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Inspect implements [artifact.Repository]. It reads the manifest and, for an Atmos package, the +// Record config blob; it never fetches a payload layer or a document. A foreign image, index, or +// plugin release is described from its own manifest without pretending it has an Atmos config +// blob: the inspection has no Record and carries the original root descriptor with its +// annotations and artifact type, which is what immutability and signing decisions need. +func (r *Repository) Inspect(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "oci.Repository.Inspect")() + + st, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return nil, err + } + inspection := &artifact.Inspection{Pinned: pinned} + inspection.Pinned.Repository = r.be.locator(pinned.Ref.Name) + if loaded.class == classAtmos { + if err := r.describeAtmos(ctx, st, loaded, inspection); err != nil { + return nil, err + } + } else { + r.describeForeign(loaded, &pinned, inspection) + } + // Evidence is an unverified convenience here; a listing failure must not block inspection. + // Verification and pinning list the referrers strictly. + if set, listErr := r.attachments(ctx, st, pinned.Ref.Name, loaded.desc); listErr == nil { + for index := range set.valid { + inspection.Evidence = append(inspection.Evidence, evidenceFor(&set.valid[index], loaded, r.be.locator(pinned.Ref.Name))) + } + } + return inspection, nil +} + +// evidenceFor describes an attachment as unverified evidence of the subject it binds. +func evidenceFor(att *attachment, loaded *root, locator string) artifact.SignatureEvidence { + subjects := subjectsOf(loaded) + subject := subjects[len(subjects)-1] + for _, candidate := range subjects { + if att.bindsTo(candidate, loaded) { + subject = candidate + } + } + return artifact.SignatureEvidence{Subject: subject, Format: artifact.EvidenceSigstoreBundle, Descriptor: att.layer, Locator: locatorDigest(locator, att.manifestDesc)} +} + +// describeAtmos fills the inspection of an Atmos package. +func (r *Repository) describeAtmos(ctx context.Context, st store, loaded *root, inspection *artifact.Inspection) error { + record, err := r.readRecord(ctx, st, inspection.Pinned.Ref.Name, loaded) + if err != nil { + return err + } + if err := requireSubjectOf(record, inspection.Pinned.Ref); err != nil { + return err + } + layers, err := splitAtmosLayers(&loaded.manifest) + if err != nil { + return err + } + if err := checkDocuments(record, layers); err != nil { + return err + } + inspection.Record = record + inspection.Pinned.RecordDigest = loaded.manifest.Config.Digest + inspection.Pinned.ContentDigest = record.ContentDigest + inspection.Representation = artifact.Representation{ + Packaging: layers.packaging, + Payload: layers.payload, + Locator: r.representationLocator(inspection.Pinned.Ref, loaded), + RecordDigest: loaded.manifest.Config.Digest, + RepresentationDigest: layers.payload[0].Digest, + } + for _, role := range []artifact.DocumentRole{artifact.DocumentReadme, artifact.DocumentChangelog} { + if doc, ok := layers.documents[role]; ok { + inspection.Representation.Documents = append(inspection.Representation.Documents, doc) + } + } + return nil +} + +// checkDocuments requires the manifest's document layers to be exactly the documents the Record +// describes, by role, digest, and size. +func checkDocuments(record *artifact.Record, layers *atmosLayers) error { + if len(record.Documents) != len(layers.documents) { + return fmt.Errorf("%w: the Record describes %d documents but the manifest carries %d", errUtils.ErrArtifactIntegrityFailed, len(record.Documents), len(layers.documents)) + } + for index := range record.Documents { + doc := &record.Documents[index] + layer, ok := layers.documents[doc.Role] + if !ok || layer.Digest != doc.Digest || layer.Size != doc.Size { + return fmt.Errorf("%w: the %s layer does not match the Record's descriptor", errUtils.ErrArtifactIntegrityFailed, doc.Role) + } + } + return nil +} + +// describeForeign fills the inspection of a foreign graph or plugin release. +func (r *Repository) describeForeign(loaded *root, pinned *artifact.PinnedReference, inspection *artifact.Inspection) { + original := loaded.original() + inspection.Representation = artifact.Representation{ + Packaging: artifact.PackagingOCIGraph, + Locator: r.representationLocator(pinned.Ref, loaded), + OriginalRoot: original, + } + switch { + case pinned.Platform != nil && loaded.isIndex(): + for index := range loaded.index.Manifests { + if child := loaded.index.Manifests[index]; child.Platform != nil && platformMatches(pinned.Platform, child.Platform) { + inspection.Representation.Payload = append(inspection.Representation.Payload, child) + } + } + case loaded.isIndex(): + inspection.Representation.Payload = append(inspection.Representation.Payload, loaded.index.Manifests...) + default: + inspection.Representation.Payload = append(inspection.Representation.Payload, loaded.manifest.Layers...) + } +} + +// representationLocator returns where the root physically lives. +func (r *Repository) representationLocator(ref artifact.Ref, loaded *root) string { + if _, isLayout := r.be.(*layoutBackend); isLayout { + return r.be.locator(ref.Name) + "#" + loaded.desc.Digest.String() + } + return locatorDigest(r.be.locator(ref.Name), loaded.desc) +} + +// OpenDocument implements [artifact.Repository]. It reads the manifest and the Record, then +// streams only the selected document layer, bounded and verified against the Record. The payload +// is never downloaded. A foreign object has no documents. +func (r *Repository) OpenDocument(ctx context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "oci.Repository.OpenDocument")() + + st, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return nil, err + } + if loaded.class != classAtmos { + return nil, fmt.Errorf("%w: %s is a %s with no Atmos package documents", errUtils.ErrArtifactDocumentMissing, pinned.ResolvedDigest, loaded.class) + } + record, err := r.readRecord(ctx, st, pinned.Ref.Name, loaded) + if err != nil { + return nil, err + } + if err := requireSubjectOf(record, pinned.Ref); err != nil { + return nil, err + } + layers, err := splitAtmosLayers(&loaded.manifest) + if err != nil { + return nil, err + } + if err := checkDocuments(record, layers); err != nil { + return nil, err + } + for index := range record.Documents { + doc := record.Documents[index] + if doc.Role != role { + continue + } + var reader io.ReadCloser = newLazyReader(ctx, st, layers.documents[role]) + return &artifact.DocumentStream{Document: doc, Reader: reader}, nil + } + return nil, fmt.Errorf("%w: %s has no %s", errUtils.ErrArtifactDocumentMissing, pinned.Ref, role) +} + +// Read implements [artifact.Repository]. An Atmos package yields verified payload and document +// streams; a foreign graph or plugin release yields a graph source rooted at the original +// manifest or index, so it can be transferred byte-identically. A pinned platform narrows the +// graph to that child below the unchanged original index. +func (r *Repository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "oci.Repository.Read")() + + st, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return nil, err + } + inspection := &artifact.Inspection{Pinned: pinned} + if loaded.class != classAtmos { + r.describeForeign(loaded, &pinned, inspection) + source := &graphSource{st: st, root: loaded.desc} + if pinned.Platform != nil { + child, selectErr := childFor(loaded, pinned.Platform) + if selectErr != nil { + return nil, selectErr + } + source.only = &child + } + return &artifact.Content{Representation: inspection.Representation, Graph: source}, nil + } + if err := r.describeAtmos(ctx, st, loaded, inspection); err != nil { + return nil, err + } + return &artifact.Content{Record: inspection.Record, Representation: inspection.Representation, Streams: streamsOf(ctx, st, inspection)}, nil +} + +// streamsOf opens the lazy, verifying streams of an Atmos package's payload and documents. +func streamsOf(ctx context.Context, st store, inspection *artifact.Inspection) []artifact.ContentStream { + var streams []artifact.ContentStream + for _, layer := range inspection.Representation.Payload { + streams = append(streams, artifact.ContentStream{Descriptor: layer, Role: artifact.BlobPayload, Reader: newLazyReader(ctx, st, layer)}) + } + for _, layer := range inspection.Representation.Documents { + streams = append(streams, artifact.ContentStream{Descriptor: layer, Role: artifact.BlobDocument, Reader: newLazyReader(ctx, st, layer)}) + } + return streams +} diff --git a/pkg/artifact/oci/list.go b/pkg/artifact/oci/list.go new file mode 100644 index 00000000000..330cd55b867 --- /dev/null +++ b/pkg/artifact/oci/list.go @@ -0,0 +1,170 @@ +package oci + +import ( + "context" + "sort" + "strings" + + "golang.org/x/sync/errgroup" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// cursorSeparator splits the name from the version in an opaque list cursor. Neither may contain +// it, so the pair is unambiguous. +const cursorSeparator = "|" + +// List implements [artifact.Repository]. A name-scoped listing enumerates the name's tags and +// works without any catalog. A collection listing uses the registry's repository listing scoped +// to the collection prefix, or the layout's references; a registry that cannot enumerate +// returns an unsupported-capability error that says direct named operations still work, never an +// empty page. Only Atmos packages are listed: legacy plugin releases and foreign images that +// share the prefix are never presented as generic artifacts. +func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "oci.Repository.List")() + + size := query.PageSize + if size < 1 { + size = r.settings.pageSize + } + names, err := r.namesFor(ctx, query.Name) + if err != nil { + return nil, err + } + cursor := parseCursor(query.Cursor) + var entries []artifact.ListEntry + for _, name := range names { + if cursor.skipsName(name) { + continue + } + items, err := r.entriesOf(ctx, name, query.Labels) + if err != nil { + return nil, err + } + entries = append(entries, cursor.remaining(name, items)...) + if len(entries) > size { + last := entries[size-1] + return &artifact.ListPage{Entries: entries[:size], NextCursor: last.Name + cursorSeparator + last.Version}, nil + } + } + return &artifact.ListPage{Entries: entries}, nil +} + +// listCursor is a decoded page cursor: the last name and version already returned. +type listCursor struct { + name string + version string + set bool +} + +func parseCursor(raw string) listCursor { + name, version, _ := strings.Cut(raw, cursorSeparator) + return listCursor{name: name, version: version, set: raw != ""} +} + +// skipsName reports whether every entry of name precedes the cursor. +func (c listCursor) skipsName(name string) bool { return c.set && name < c.name } + +// remaining returns the entries of name that follow the cursor. +func (c listCursor) remaining(name string, items []artifact.ListEntry) []artifact.ListEntry { + if !c.set || name != c.name { + return items + } + var rest []artifact.ListEntry + for index := range items { + if items[index].Version > c.version { + rest = append(rest, items[index]) + } + } + return rest +} + +// namesFor returns the names to list: the requested one, or the enumerated collection. +func (r *Repository) namesFor(ctx context.Context, name string) ([]string, error) { + if name != "" { + if err := artifact.ValidateName(name); err != nil { + return nil, err + } + return []string{name}, nil + } + return r.be.names(ctx) +} + +// entriesOf lists the Atmos package versions of one name, sorted by version, that match labels. +// Each tag costs a manifest read and a Record read, bounded by the configured concurrency. +func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifact.Label) ([]artifact.ListEntry, error) { + versions, err := r.be.versions(ctx, name) + if err != nil { + return nil, err + } + sort.Strings(versions) + found := make([]*artifact.ListEntry, len(versions)) + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(r.settings.concurrency) + for index, version := range versions { + if artifact.ValidateVersion(version) != nil { + continue + } + group.Go(func() error { + entry, err := r.entryOf(groupCtx, artifact.Ref{Name: name, Version: version}) + found[index] = entry + return err + }) + } + if err := group.Wait(); err != nil { + return nil, err + } + var entries []artifact.ListEntry + for _, entry := range found { + if entry != nil && matchesLabels(entry.Labels, labels) { + entries = append(entries, *entry) + } + } + return entries, nil +} + +// entryOf builds the list entry of one tag, or nil when the tag is not an Atmos package. +func (r *Repository) entryOf(ctx context.Context, ref artifact.Ref) (*artifact.ListEntry, error) { + st, loaded, err := r.resolveRoot(ctx, ref) + if err != nil { + if errorsIsNotFound(err) { + return nil, nil + } + return nil, err + } + if loaded.class != classAtmos { + return nil, nil + } + record, err := r.readRecord(ctx, st, ref.Name, loaded) + if err != nil { + return nil, err + } + if !recordMatches(record, ref) { + // A tag serving another version's Record is an alias (or tampering); the Record is listed + // once, under its own tag, so this tag is not an entry. + return nil, nil + } + pinned := r.pinnedFor(ref, loaded) + pinned.ContentDigest = record.ContentDigest + return &artifact.ListEntry{ + Pinned: pinned, Name: record.Name, Version: record.Version, Kind: record.Kind, + Description: record.Metadata.Description, Labels: record.Metadata.Labels, + Documents: record.Documents, Repository: r.alias, + }, nil +} + +// matchesLabels reports whether labels hold every selector with an exactly equal value. +func matchesLabels(labels map[string]string, selectors []artifact.Label) bool { + for _, selector := range selectors { + if value, ok := labels[selector.Key]; !ok || value != selector.Value { + return false + } + } + return true +} + +// recordMatches reports whether a Record is the one its reference names. +func recordMatches(record *artifact.Record, ref artifact.Ref) bool { + return requireSubjectOf(record, ref) == nil +} diff --git a/pkg/artifact/oci/list_test.go b/pkg/artifact/oci/list_test.go new file mode 100644 index 00000000000..c6f88884686 --- /dev/null +++ b/pkg/artifact/oci/list_test.go @@ -0,0 +1,131 @@ +package oci + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" +) + +// withoutCatalog hides the repository catalog of a transport, like a registry that has none. +type withoutCatalog struct{ atmosoci.Transport } + +// failingCatalog implements the catalog but the registry rejects it. +type failingCatalog struct { + atmosoci.Transport + err error +} + +func (f failingCatalog) Repositories(context.Context, string) ([]string, error) { return nil, f.err } + +func listAll(t *testing.T, repo *Repository, query artifact.ListQuery) []string { + t.Helper() + var seen []string + for pages := 0; pages < 50; pages++ { + page, err := repo.List(context.Background(), query) + require.NoError(t, err) + for index := range page.Entries { + seen = append(seen, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + return seen + } + require.NotEqual(t, query.Cursor, page.NextCursor) + query.Cursor = page.NextCursor + } + t.Fatal("pagination did not terminate") + return nil +} + +func TestCollectionListingIsScopedSortedAndPaged(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}, {"a/other", "3"}} { + publish(t, repo, publishSpec{name: item.name, version: item.version, content: item.name + item.version}) + } + want := []string{"a/app:1", "a/app:2", "a/other:3", "b/app:1"} + assert.Equal(t, want, listAll(t, repo, artifact.ListQuery{PageSize: 100})) + assert.Equal(t, want, listAll(t, repo, artifact.ListQuery{PageSize: 1}), "every entry exactly once across pages") + assert.Equal(t, []string{"a/app:1", "a/app:2"}, listAll(t, repo, artifact.ListQuery{Name: "a/app", PageSize: 1})) + }) +} + +func TestRegistryListingStaysInsideThePrefix(t *testing.T) { + reg := newRegistry(t, nil) + inside := newRegistryRepo(t, reg) + publish(t, inside, publishSpec{name: "mine/app", content: "inside"}) + + otherConfig := registryConfig(reg.server.Host) + otherConfig.Options["prefix"] = "somebody/else" + outside, err := NewRepository(context.Background(), "outside", otherConfig, reg.options(WithLockDir(t.TempDir()))...) + require.NoError(t, err) + publish(t, outside, publishSpec{name: "theirs/app", content: "outside"}) + + assert.Equal(t, []string{"mine/app:1.0.0"}, listAll(t, inside, artifact.ListQuery{})) + assert.Equal(t, []string{"theirs/app:1.0.0"}, listAll(t, outside, artifact.ListQuery{})) +} + +func TestLabelFilterUsesExactPairsWithAndSemantics(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + publish(t, repo, publishSpec{name: "a/one", content: "1", labels: map[string]string{"team": "platform", "purpose": "backup"}}) + publish(t, repo, publishSpec{name: "a/two", content: "2", labels: map[string]string{"team": "platform"}}) + publish(t, repo, publishSpec{name: "a/three", content: "3", labels: map[string]string{"team": "data", "purpose": "backup"}}) + + assert.Equal(t, []string{"a/one:1.0.0", "a/two:1.0.0"}, listAll(t, repo, artifact.ListQuery{Labels: []artifact.Label{{Key: "team", Value: "platform"}}})) + assert.Equal(t, []string{"a/one:1.0.0"}, listAll(t, repo, artifact.ListQuery{Labels: []artifact.Label{{Key: "team", Value: "platform"}, {Key: "purpose", Value: "backup"}}})) + assert.Empty(t, listAll(t, repo, artifact.ListQuery{Labels: []artifact.Label{{Key: "team", Value: "plat"}}}), "values match exactly") + }) +} + +// A registry without a catalog still lists a named artifact's tags; the collection scan is a +// capability error that says so, never an empty successful result. +func TestNamedListingWorksWithoutACatalogAndCollectionListingFailsLoudly(t *testing.T) { + reg := newRegistry(t, nil) + writer := newRegistryRepo(t, reg) + publish(t, writer, publishSpec{content: "one"}) + publish(t, writer, publishSpec{version: "2.0.0", content: "two"}) + + tests := map[string]atmosoci.Transport{ + "transport without a catalog": withoutCatalog{reg.transport}, + "registry rejects the catalog": failingCatalog{ + Transport: reg.transport, err: errors.New("GET _catalog: response status code 404"), + }, + } + for name, transport := range tests { + t.Run(name, func(t *testing.T) { + repo, err := NewRepository(context.Background(), "registry", registryConfig(reg.server.Host), WithTransport(transport), WithLockDir(t.TempDir())) + require.NoError(t, err) + assert.Equal(t, []string{"team/app:1.0.0", "team/app:2.0.0"}, listAll(t, repo, artifact.ListQuery{Name: appName})) + + _, err = repo.List(context.Background(), artifact.ListQuery{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Contains(t, err.Error(), "direct named operation") + }) + } + caps := mustCaps(t, reg, WithoutCollectionEnumeration()) + assert.False(t, caps.EnumerateCollection) + assert.True(t, caps.EnumerateNamed) +} + +func mustCaps(t *testing.T, reg *registry, opts ...Option) artifact.Capabilities { + t.Helper() + repo, err := NewRepository(context.Background(), "r", registryConfig(reg.server.Host), append(reg.options(WithLockDir(t.TempDir())), opts...)...) + require.NoError(t, err) + return repo.Capabilities() +} + +func TestListingNeverPresentsForeignImagesOrPluginsAsArtifacts(t *testing.T) { + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg) + publish(t, repo, publishSpec{name: "shared/thing", version: "1.0.0", content: "generic"}) + pushImage(t, reg, testPrefix+"/shared/thing", "image") + importPlugin(t, repo, artifact.Ref{Name: "shared/thing", Version: "plugin"}) + + assert.Equal(t, []string{"shared/thing:1.0.0"}, listAll(t, repo, artifact.ListQuery{Name: "shared/thing"})) + assert.Equal(t, []string{"shared/thing:1.0.0"}, listAll(t, repo, artifact.ListQuery{})) +} diff --git a/pkg/artifact/oci/lock.go b/pkg/artifact/oci/lock.go new file mode 100644 index 00000000000..4ed4b2c9613 --- /dev/null +++ b/pkg/artifact/oci/lock.go @@ -0,0 +1,228 @@ +package oci + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/filelock" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/xdg" +) + +// Operation name used in capability errors. +const opLock = "lock" + +const ( + lockDirMode = 0o700 + lockSubdir = "artifacts/oci-locks" +) + +// heldLocks tracks the host guards this process holds, keyed by lock file path. The OS lock +// excludes other processes; this table supplies the ownership-token check on release. +var ( + heldMu sync.Mutex + heldLocks = map[string]*hostLock{} +) + +// hostLock is an OS file lock held until Release. It implements [artifact.LockHandle]. +type hostLock struct { + path string + resource string + owner string + token string + release chan struct{} + done chan error + released bool +} + +var _ artifact.LockHandle = (*hostLock)(nil) + +// Lock implements [artifact.LockProvider]. +// +// By default it is a host-scope guard: an OS file lock keyed by registry, repository, and tag in +// the user data directory (or the configured lock directory). That excludes other Atmos +// processes on this host only; a registry tag is not a distributed lease, so repository scope is +// never claimed here. When the repository configures `locking.repository`, every lock is taken +// from that delegate, which keeps its own identity and storage root and reports its own scope. A +// canceled wait leaks no ownership, and no lock is ever stolen. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "oci.Repository.Lock")() + + if request.Resource == "" { + return nil, fmt.Errorf("%w: a lock needs a resource", errUtils.ErrArtifactRefInvalid) + } + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lease", "OCI host guards do not expire, so a lease cannot be fenced") + } + if r.delegate != "" { + return r.delegateLock(ctx, request) + } + if request.Required != "" && !artifact.LockScopeHost.Satisfies(request.Required) { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lock/"+string(request.Required), + "an OCI tag is not a distributed lease, so this repository provides host-scope locks only; set locking.repository to delegate to a repository that can provide repository scope") + } + owner := request.Owner + if owner == "" { + owner = r.settings.owner + } + return r.acquireHost(ctx, request.Resource, owner) +} + +// delegateLock takes the lock from the configured delegate repository. The resource is scoped by +// this repository's location, so unrelated registries sharing one coordinator never contend. +func (r *Repository) delegateLock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + if r.lookup == nil { + return nil, fmt.Errorf("%w: repository %q delegates locking to %q but no repository lookup was supplied", errUtils.ErrArtifactConfigInvalid, r.alias, r.delegate) + } + if r.delegate == r.alias { + return nil, fmt.Errorf("%w: repository %q cannot delegate locking to itself", errUtils.ErrArtifactConfigInvalid, r.alias) + } + delegate, err := r.lookup(r.delegate) + if err != nil { + return nil, fmt.Errorf("lock delegate %q of repository %q: %w", r.delegate, r.alias, err) + } + caps := delegate.Capabilities() + required := request.Required + if required == "" { + required = artifact.LockScopeHost + } + if !caps.HasLockScope(required) { + return nil, artifact.NewUnsupportedCapability(r.delegate, opLock, "lock/"+string(required), + fmt.Sprintf("the lock delegate of repository %q cannot provide %s-scope coordination", r.alias, required)) + } + request.Resource = r.lockNamespace() + "/" + request.Resource + handle, err := delegate.Lock(ctx, request) + if err != nil { + return nil, fmt.Errorf("lock delegate %q of repository %q: %w", r.delegate, r.alias, err) + } + return handle, nil +} + +// lockNamespace identifies this repository's collection for lock keys. +func (r *Repository) lockNamespace() string { + switch be := r.be.(type) { + case *registryBackend: + return be.registry + "/" + be.prefix + case *layoutBackend: + return filepath.ToSlash(be.root) + default: + return r.alias + } +} + +// lockPath returns the lock file of a resource. The key is a digest of the location and +// resource, so no reference text reaches the file system. +func (r *Repository) lockPath(resource string) (string, error) { + dir := r.settings.lockDir + if dir == "" { + resolved, err := xdg.GetXDGDataDir(lockSubdir, lockDirMode) + if err != nil { + return "", fmt.Errorf("%w: lock directory: %w", errUtils.ErrArtifactConfigInvalid, err) + } + dir = resolved + } + if err := os.MkdirAll(dir, lockDirMode); err != nil { + return "", fmt.Errorf("%w: create lock directory %q: %w", errUtils.ErrArtifactConfigInvalid, dir, err) + } + sum := sha256.Sum256([]byte(r.lockNamespace() + "\x00" + resource)) + return filepath.Join(dir, hex.EncodeToString(sum[:])+".lock"), nil +} + +func (r *Repository) acquireHost(ctx context.Context, resource, owner string) (*hostLock, error) { + path, err := r.lockPath(resource) + if err != nil { + return nil, err + } + held := make(chan struct{}) + lock := &hostLock{path: path, resource: resource, owner: owner, token: uuid.NewString(), release: make(chan struct{}), done: make(chan error, 1)} + go func() { + lock.done <- filelock.New(path).WithExclusive(ctx, func() error { + close(held) + <-lock.release + return nil + }) + }() + select { + case <-held: + heldMu.Lock() + heldLocks[path] = lock + heldMu.Unlock() + return lock, nil + case err := <-lock.done: + return nil, fmt.Errorf("waiting for lock %q in repository %q: %w", resource, r.alias, err) + } +} + +// Resource implements [artifact.LockHandle]. +func (l *hostLock) Resource() string { + defer perf.Track(nil, "oci.hostLock.Resource")() + + return l.resource +} + +// Owner implements [artifact.LockHandle]. +func (l *hostLock) Owner() string { + defer perf.Track(nil, "oci.hostLock.Owner")() + + return l.owner +} + +// Scope implements [artifact.LockHandle]; the OCI guard is host scope. +func (l *hostLock) Scope() artifact.LockScope { + defer perf.Track(nil, "oci.hostLock.Scope")() + + return artifact.LockScopeHost +} + +// Token implements [artifact.LockHandle]. +func (l *hostLock) Token() string { + defer perf.Track(nil, "oci.hostLock.Token")() + + return l.token +} + +// ExpiresAt implements [artifact.LockHandle]; host guards never expire. +func (l *hostLock) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "oci.hostLock.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; host guards have no leases. +func (l *hostLock) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "oci.hostLock.Renew")() + + return artifact.NewUnsupportedCapability("", opLock, "lease", "OCI host guards do not expire") +} + +// Release implements [artifact.LockHandle]. It checks the ownership token against the guard this +// process holds, so a stale or foreign handle can never release another operation's lock. The OS +// also releases the lock if its owner exits. +func (l *hostLock) Release(context.Context) error { + defer perf.Track(nil, "oci.hostLock.Release")() + + heldMu.Lock() + current, owned := heldLocks[l.path] + if !owned || current != l || l.released || current.token != l.token { + heldMu.Unlock() + return fmt.Errorf("%w: lock %q is not held by token %s", errUtils.ErrArtifactConflict, l.resource, l.token) + } + delete(heldLocks, l.path) + l.released = true + heldMu.Unlock() + close(l.release) + if err := <-l.done; err != nil { + return fmt.Errorf("release lock %q: %w", l.resource, err) + } + return nil +} diff --git a/pkg/artifact/oci/lock_test.go b/pkg/artifact/oci/lock_test.go new file mode 100644 index 00000000000..b0de392da7c --- /dev/null +++ b/pkg/artifact/oci/lock_test.go @@ -0,0 +1,173 @@ +package oci + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestHostGuardIsKeyedByRepositoryAndResourceAndNeverClaimsDistributedScope(t *testing.T) { + ctx := context.Background() + shared := t.TempDir() + reg := newRegistry(t, nil) + first := newRegistryRepo(t, reg, WithLockDir(shared)) + otherConfig := registryConfig(reg.server.Host) + otherConfig.Options["prefix"] = "another/collection" + second, err := NewRepository(ctx, "second", otherConfig, reg.options(WithLockDir(shared))...) + require.NoError(t, err) + + request := artifact.LockRequest{Resource: "team/app:1.0.0", Owner: "a", OperationID: "op-a", Required: artifact.LockScopeHost} + held, err := first.Lock(ctx, request) + require.NoError(t, err) + assert.Equal(t, artifact.LockScopeHost, held.Scope(), "an OCI guard reports the host scope it really provides") + _, expires := held.ExpiresAt() + assert.False(t, expires) + + different := request + different.Resource = "team/app:2.0.0" + other, err := first.Lock(ctx, different) + require.NoError(t, err, "a different tag does not contend") + require.NoError(t, other.Release(ctx)) + + elsewhere, err := second.Lock(ctx, request) + require.NoError(t, err, "the same tag in another collection does not contend") + require.NoError(t, elsewhere.Release(ctx)) + + blocked, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + _, err = first.Lock(blocked, artifact.LockRequest{Resource: request.Resource, Owner: "b"}) + require.ErrorIs(t, err, context.DeadlineExceeded, "a competing writer waits and is bounded by its context") + + require.NoError(t, held.Release(ctx)) + require.Error(t, held.Release(ctx), "a released handle cannot release again") + again, err := first.Lock(ctx, artifact.LockRequest{Resource: request.Resource, Owner: "b"}) + require.NoError(t, err, "the canceled acquisition leaked no ownership") + require.NoError(t, again.Release(ctx)) +} + +func TestHostGuardRejectsWhatItCannotProvide(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + ctx := context.Background() + _, err := repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a tag is not a distributed lease") + assert.Contains(t, err.Error(), "locking.repository") + _, err = repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Lease: time.Minute}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a lease cannot be fenced") + _, err = repo.Lock(ctx, artifact.LockRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +// delegateEnv wires an OCI repository whose locks come from a configured coordinator. +type delegateEnv struct { + repo *Repository + coordinator *artifacttest.Repository +} + +func newDelegateEnv(t *testing.T, delegate string, scopes ...artifact.LockScope) *delegateEnv { + t.Helper() + caps := artifacttest.DefaultCapabilities() + caps.LockScopes = scopes + coordinator := artifacttest.New(artifacttest.Options{Alias: "coordination", Capabilities: &caps}) + lookup := func(alias string) (artifact.Repository, error) { + if alias != "coordination" { + return nil, errUtils.ErrArtifactConfigInvalid + } + return coordinator, nil + } + cfg := registryConfig("registry.example.com") + cfg.Locking = &schema.ArtifactLockingConfig{RequiredScope: string(artifact.LockScopeRepository), Repository: delegate} + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithTransport(newRegistry(t, nil).transport), WithLockDir(t.TempDir()))) + opened, err := registry.New(context.Background(), "shared", cfg, artifact.WithRepositoryLookup(lookup)) + require.NoError(t, err) + repo, ok := opened.(*Repository) + require.True(t, ok) + return &delegateEnv{repo: repo, coordinator: coordinator} +} + +func TestRepositoryScopeLockIsDelegatedToTheConfiguredCoordinator(t *testing.T) { + ctx := context.Background() + env := newDelegateEnv(t, "coordination", artifact.LockScopeHost, artifact.LockScopeRepository) + handle, err := env.repo.Lock(ctx, artifact.LockRequest{Resource: "team/app:1.0.0", Owner: "writer", Required: artifact.LockScopeRepository}) + require.NoError(t, err) + assert.Equal(t, artifact.LockScopeRepository, handle.Scope(), "the delegate's real scope is reported") + held := env.coordinator.HeldLocks() + require.Len(t, held, 1) + assert.Contains(t, held[0], "registry.example.com/acme/atmos/team/app:1.0.0", "the coordinator key is scoped to this collection") + + require.NoError(t, handle.Release(ctx)) + assert.Empty(t, env.coordinator.HeldLocks()) +} + +func TestDelegationFailsClosed(t *testing.T) { + ctx := context.Background() + t.Run("delegate lacks the required scope", func(t *testing.T) { + env := newDelegateEnv(t, "coordination", artifact.LockScopeHost) + _, err := env.repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Empty(t, env.coordinator.HeldLocks()) + }) + t.Run("unknown delegate", func(t *testing.T) { + env := newDelegateEnv(t, "missing", artifact.LockScopeRepository) + _, err := env.repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("no lookup supplied", func(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + repo.delegate = "coordination" + _, err := repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("self delegation", func(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + repo.delegate = repo.alias + repo.lookup = func(string) (artifact.Repository, error) { return repo, nil } + _, err := repo.Lock(ctx, artifact.LockRequest{Resource: "x:1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("a canceled wait on the delegate leaks nothing", func(t *testing.T) { + env := newDelegateEnv(t, "coordination", artifact.LockScopeRepository) + first, err := env.repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Owner: "a", Required: artifact.LockScopeRepository}) + require.NoError(t, err) + canceled, cancel := context.WithTimeout(ctx, 30*time.Millisecond) + defer cancel() + _, err = env.repo.Lock(canceled, artifact.LockRequest{Resource: "x:1", Owner: "b", Required: artifact.LockScopeRepository}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, first.Release(ctx)) + again, err := env.repo.Lock(ctx, artifact.LockRequest{Resource: "x:1", Owner: "b", Required: artifact.LockScopeRepository}) + require.NoError(t, err) + require.NoError(t, again.Release(ctx)) + }) +} + +// Through the service, a repository that requires repository scope publishes under the delegate's +// lock and reports the scope that actually coordinated the write. +func TestServicePublishesUnderTheDelegatedLockAndReportsItsScope(t *testing.T) { + ctx := context.Background() + env := newDelegateEnv(t, "coordination", artifact.LockScopeHost, artifact.LockScopeRepository) + layoutRepo := newLayoutRepo(t) + layoutRepo.delegate, layoutRepo.lookup = "coordination", func(string) (artifact.Repository, error) { return env.coordinator, nil } + caps := layoutRepo.Capabilities() + caps.LockScopes = append(caps.LockScopes, artifact.LockScopeRepository) + svc, err := service.New(&service.StaticRepositories{ + Repositories: map[string]artifact.Repository{"shared": layoutRepo}, + Info: map[string]service.RepositoryInfo{"shared": {Capabilities: caps, RequiredLockScope: artifact.LockScopeRepository}}, + }, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + source := t.TempDir() + writeSource(t, source) + outcome, err := svc.PublishDetailed(ctx, artifact.PublishRequest{Repository: "shared", Source: source, Name: appName, Version: appVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.LockScopeRepository, outcome.Operation.LockScope) + assert.Empty(t, env.coordinator.HeldLocks(), "the delegated lock was released") +} diff --git a/pkg/artifact/oci/names.go b/pkg/artifact/oci/names.go new file mode 100644 index 00000000000..50375992128 --- /dev/null +++ b/pkg/artifact/oci/names.go @@ -0,0 +1,108 @@ +package oci + +import ( + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Annotation keys written by this package. They select and describe evidence and documents; they +// never establish trust, which always comes from cryptographic verification. +const ( + // Annotation naming the signature subject kind (record or oci-root) of an attachment. + annotationSubjectKind = "dev.atmos.artifact.signature.subject-kind" + + // Annotation carrying the digest of the signed subject. + annotationSubjectDigest = "dev.atmos.artifact.signature.subject-digest" + + // Annotation binding a Record signature to the addressed manifest's config descriptor. + annotationConfigDigest = "dev.atmos.artifact.signature.config-digest" + + // Annotation naming the evidence format of an attachment. + annotationEvidenceFormat = "dev.atmos.artifact.signature.format" + + // Annotation naming the role (readme or changelog) of a document layer. + annotationDocumentRole = "dev.atmos.artifact.document.role" + + // Annotation recording the artifact kind on the manifest for registry-side browsing. + annotationKind = "dev.atmos.artifact.kind" +) + +// Legacy Atmos plugin release format, as written by pkg/plugin/oci. The constants are repeated +// here, not imported, so generic OCI repositories stay independent of plugin packages; a drift +// test compares them with the plugin package. +const ( + // Config media type of a plugin platform image. + pluginConfigMediaType = "application/vnd.atmos.plugin.config.v1+json" + + // Layer media type of a plugin platform bundle. + pluginBundleMediaType = "application/vnd.atmos.plugin.bundle.v1+tar" + + // Release annotation naming the plugin. + pluginAnnotationName = "dev.atmos.plugin.name" + + // Release annotation carrying the plugin version. + pluginAnnotationVersion = "dev.atmos.plugin.version" + + // Release annotation carrying the manifest digest. + pluginAnnotationManifest = "dev.atmos.plugin.manifest.digest" + + // Namespace shared by every plugin-format annotation and label. + pluginPrefix = "dev.atmos.plugin." +) + +// Sigstore bundle attachments. +const ( + // Media type prefix of Sigstore bundles. + bundleMediaTypePrefix = "application/vnd.dev.sigstore.bundle." + + // Layer title of a stored bundle. + bundleFileTitle = "signature.sigstore.json" +) + +// Bounds on what one operation reads or scans. +const ( + // Bound on one manifest or index read. + maxManifestSize int64 = 4 << 20 + + // Bound on how many referrer manifests one verification or export inspects. + maxAttachments = 32 + + // Bound on how many tags a by-record-digest lookup or sibling check inspects. + maxTagsScanned = 512 + + // List page size when the query does not choose one. + defaultPageSize = 50 + + // Bound on the per-repository cache of verified manifests and configs. + manifestCacheEntries = 256 +) + +// Layer annotation keys from the OCI image spec. +const ( + annotationTitle = ocispec.AnnotationTitle + annotationCreated = ocispec.AnnotationCreated + annotationVersion = ocispec.AnnotationVersion + annotationDescription = ocispec.AnnotationDescription +) + +// artifactTypeFor maps an artifact kind onto the PRD artifact type of its manifest. Unlisted kinds +// use the generic artifact type; the component subtype stays in the portable Record. +func artifactTypeFor(kind string) string { + switch { + case kind == "scaffold" || hasKindPrefix(kind, "scaffold"): + return artifact.MediaTypeScaffold + case kind == "component" || hasKindPrefix(kind, "component"): + return artifact.MediaTypeComponent + case kind == "toolchain/package" || hasKindPrefix(kind, "toolchain/package"): + return artifact.MediaTypeToolchainPackage + case kind == artifact.KindCatalog: + return artifact.MediaTypeCatalog + default: + return artifact.MediaTypeArtifact + } +} + +func hasKindPrefix(kind, prefix string) bool { + return len(kind) > len(prefix) && kind[:len(prefix)] == prefix && kind[len(prefix)] == '/' +} diff --git a/pkg/artifact/oci/options.go b/pkg/artifact/oci/options.go new file mode 100644 index 00000000000..bcf89ea3766 --- /dev/null +++ b/pkg/artifact/oci/options.go @@ -0,0 +1,126 @@ +package oci + +import ( + "context" + "os" + "strconv" + + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// TransportFactory builds the registry transport of one configured repository. The resolve +// argument is the composition root's identity resolver; an implementation binds the repository's identity to the +// transport it returns, so the credentials of one repository are never presented for another. +// Credentials are resolved when a repository is opened for an operation, not here. +type TransportFactory func(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, resolve artifact.IdentityResolver) (atmosoci.Transport, error) + +// settings holds the construction options of an OCI repository or its registration. +type settings struct { + transport atmosoci.Transport + factory TransportFactory + verifier artifact.Verifier + owner string + lockDir string + noCatalog bool + concurrency int + pageSize int +} + +// Option configures an OCI repository or its registration. +type Option func(*settings) + +// WithTransport supplies the registry transport used by every oci/registry repository the +// registered factory constructs. Tests inject an in-process registry client; a transport that +// also implements [atmosoci.RegistryCatalog] enables collection enumeration. +func WithTransport(transport atmosoci.Transport) Option { + defer perf.Track(nil, "oci.WithTransport")() + + return func(s *settings) { s.transport = transport } +} + +// WithTransportFactory supplies a per-repository transport factory that binds the repository's +// identity. It takes precedence over [WithTransport]. A repository that configures an identity +// requires a factory, so an identity is never silently ignored. +func WithTransportFactory(factory TransportFactory) Option { + defer perf.Track(nil, "oci.WithTransportFactory")() + + return func(s *settings) { s.factory = factory } +} + +// WithVerifier injects the cryptographic verifier used by Verify. The formats it understands are +// the evidence formats the repository advertises. Without a verifier the repository stores no +// evidence, and Sign and Verify return unsupported-capability errors. +func WithVerifier(verifier artifact.Verifier) Option { + defer perf.Track(nil, "oci.WithVerifier")() + + return func(s *settings) { s.verifier = verifier } +} + +// WithOwner sets the owner recorded in lock requests when the caller does not supply one. The +// default is ":". +func WithOwner(owner string) Option { + defer perf.Track(nil, "oci.WithOwner")() + + return func(s *settings) { s.owner = owner } +} + +// WithLockDir sets the directory holding the host-scope write guards. The default is a directory +// below the user's data directory, so the guard is shared by every process on the host. +func WithLockDir(dir string) Option { + defer perf.Track(nil, "oci.WithLockDir")() + + return func(s *settings) { s.lockDir = dir } +} + +// WithoutCollectionEnumeration stops an oci/registry repository from advertising or attempting +// registry-wide repository enumeration, for registries known not to support it. +func WithoutCollectionEnumeration() Option { + defer perf.Track(nil, "oci.WithoutCollectionEnumeration")() + + return func(s *settings) { s.noCatalog = true } +} + +// WithConcurrency bounds how many manifests listing inspects at once. The default is +// [artifact.DefaultConcurrency]. +func WithConcurrency(n int) Option { + defer perf.Track(nil, "oci.WithConcurrency")() + + return func(s *settings) { s.concurrency = n } +} + +// WithPageSize sets the list page size used when a query does not choose one. +func WithPageSize(n int) Option { + defer perf.Track(nil, "oci.WithPageSize")() + + return func(s *settings) { s.pageSize = n } +} + +func applySettings(opts []Option) settings { + var applied settings + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.owner == "" { + applied.owner = defaultOwner() + } + if applied.concurrency < 1 { + applied.concurrency = artifact.DefaultConcurrency + } + if applied.pageSize < 1 { + applied.pageSize = defaultPageSize + } + return applied +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return host + ":" + strconv.Itoa(os.Getpid()) +} diff --git a/pkg/artifact/oci/plugin.go b/pkg/artifact/oci/plugin.go new file mode 100644 index 00000000000..920b6d5fb0f --- /dev/null +++ b/pkg/artifact/oci/plugin.go @@ -0,0 +1,85 @@ +package oci + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// pluginPolicy is the plugin-format immutability policy. +type pluginPolicy struct{} + +var _ service.ImmutabilityPolicy = pluginPolicy{} + +// PluginImmutability returns the [service.ImmutabilityPolicy] that keeps legacy Atmos plugin +// releases immutable. Register it with service.WithImmutabilityPolicy. The service consults it for +// the stored artifact and for the incoming one, so a generic publication or copy can neither +// overwrite a plugin release nor turn an existing generic artifact into one, with or without +// force. +// +// A candidate is a plugin release when its original OCI root carries the plugin release +// annotations, when a layer or config uses the plugin media types, or when an incoming prepared +// artifact stages a plugin bundle or labels itself with the plugin namespace. The repositories +// enforce the same rule on their own writes, so this seam and the storage layer agree. +func PluginImmutability() service.ImmutabilityPolicy { + defer perf.Track(nil, "oci.PluginImmutability")() + + return pluginPolicy{} +} + +// Immutable implements [service.ImmutabilityPolicy]. +func (pluginPolicy) Immutable(candidate service.Candidate) (bool, string) { + defer perf.Track(nil, "oci.pluginPolicy.Immutable")() + + if candidate.Inspection != nil { + if immutable, why := inspectionIsPlugin(candidate.Inspection); immutable { + return true, why + } + } + if candidate.Prepared != nil { + return preparedIsPlugin(candidate.Prepared) + } + return false, "" +} + +// inspectionIsPlugin checks an inspected artifact. +func inspectionIsPlugin(inspection *artifact.Inspection) (bool, string) { + if original := inspection.Representation.OriginalRoot; original != nil { + if hasPluginAnnotations(original.Annotations) { + return true, "legacy plugin release " + original.Annotations[pluginAnnotationName] + " " + original.Annotations[pluginAnnotationVersion] + } + } + for index := range inspection.Representation.Payload { + if inspection.Representation.Payload[index].MediaType == pluginBundleMediaType { + return true, "legacy plugin release (plugin bundle layer)" + } + } + if record := inspection.Record; record != nil && hasPluginLabel(record.Metadata.Labels) { + return true, "plugin-format artifact (plugin labels)" + } + return false, "" +} + +// preparedIsPlugin checks an incoming prepared artifact. +func preparedIsPlugin(prepared *artifact.PreparedArtifact) (bool, string) { + for index := range prepared.Blobs { + if prepared.Blobs[index].Descriptor.MediaType == pluginBundleMediaType { + return true, "plugin-format artifact (plugin bundle blob)" + } + } + if hasPluginLabel(prepared.Record.Metadata.Labels) { + return true, "plugin-format artifact (plugin labels)" + } + return false, "" +} + +func hasPluginLabel(labels map[string]string) bool { + for key := range labels { + if strings.HasPrefix(key, pluginPrefix) { + return true + } + } + return false +} diff --git a/pkg/artifact/oci/plugin_test.go b/pkg/artifact/oci/plugin_test.go new file mode 100644 index 00000000000..e8b5030292a --- /dev/null +++ b/pkg/artifact/oci/plugin_test.go @@ -0,0 +1,272 @@ +package oci + +import ( + "context" + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// pluginFixture is the committed plugin release layout captured by the compatibility suite. +var pluginFixture = filepath.Join("..", "compat", "testdata", "plugin-release") + +// pluginRoot is the digest of the fixture's release index. +const pluginRoot digest.Digest = "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + +// copyFixture copies the committed layout so no test ever writes to the fixture. +func copyFixture(t *testing.T) string { + t.Helper() + destination := filepath.Join(t.TempDir(), "plugin") + require.NoError(t, filepath.WalkDir(pluginFixture, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(pluginFixture, path) + if err != nil { + return err + } + target := filepath.Join(destination, relative) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, raw, 0o644) + })) + return destination +} + +// importPlugin imports the plugin release fixture under ref. +func importPlugin(t *testing.T, repo *Repository, ref artifact.Ref) *artifact.PublishResult { + t.Helper() + result, err := repo.ImportLayout(context.Background(), copyFixture(t), ref, ImportOptions{}) + require.NoError(t, err) + return result +} + +func TestPluginReleaseIsImportedByteIdenticallyAndInspectedWithoutAConfigBlob(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + result := importPlugin(t, repo, ref) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, pluginRoot, result.Pinned.ResolvedDigest, "the release keeps its original index digest") + assert.Empty(t, result.Pinned.RecordDigest, "a plugin release has no Atmos Record") + + pinned, err := repo.Resolve(context.Background(), ref) + require.NoError(t, err) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + assert.Nil(t, inspection.Record) + require.NotNil(t, inspection.Representation.OriginalRoot) + assert.Equal(t, "example/fixture", inspection.Representation.OriginalRoot.Annotations[pluginAnnotationName]) + assert.Equal(t, artifact.PackagingOCIGraph, inspection.Representation.Packaging) + + _, err = repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a foreign object has no Atmos documents") + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NotNil(t, content.Graph) + assert.Nil(t, content.Record) + assert.Equal(t, pluginRoot, content.Graph.Root().Digest) + nodes := walkGraph(t, content.Graph) + assert.GreaterOrEqual(t, nodes, 7, "the index, both platform manifests, configs and bundles all transfer") + }) +} + +// walkGraph reads every node below the root with verification and returns how many it saw. +func walkGraph(t *testing.T, graph artifact.GraphSource) int { + t.Helper() + seen := map[digest.Digest]bool{} + var visit func(desc ocispec.Descriptor) + visit = func(desc ocispec.Descriptor) { + if seen[desc.Digest] { + return + } + seen[desc.Digest] = true + reader, err := graph.Fetch(context.Background(), desc) + require.NoError(t, err) + _, err = io.Copy(io.Discard, reader) + require.NoError(t, err, "node %s verifies against its descriptor", desc.Digest) + require.NoError(t, reader.Close()) + successors, err := graph.Successors(context.Background(), desc) + require.NoError(t, err) + for _, successor := range successors { + visit(successor) + } + } + visit(graph.Root()) + return len(seen) +} + +func TestPluginReleaseExportsToAStandardLayoutAndImportsAgain(t *testing.T) { + source := newRegistryRepo(t, newRegistry(t, nil)) + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + importPlugin(t, source, ref) + pinned, err := source.Resolve(context.Background(), ref) + require.NoError(t, err) + + exported := filepath.Join(t.TempDir(), "export") + root, err := source.ExportLayout(context.Background(), pinned, exported) + require.NoError(t, err) + assert.Equal(t, pluginRoot, root.Digest) + + target := newLayoutRepo(t) + result, err := target.ImportLayout(context.Background(), exported, ref, ImportOptions{}) + require.NoError(t, err) + assert.Equal(t, pluginRoot, result.Pinned.ResolvedDigest) + + _, err = target.ImportLayout(context.Background(), exported, ref, ImportOptions{Select: "no-such-root"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestPluginReleaseNeverChangesAndIsNeverOverwritten(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ctx := context.Background() + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + result := importPlugin(t, repo, ref) + + again, err := repo.ImportLayout(ctx, copyFixture(t), ref, ImportOptions{ExpectedPrior: &result.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "an identical re-import is a no-op") + + generic := prepare(t, publishSpec{name: ref.Name, version: ref.Version, content: "generic"}) + for name, options := range map[string]artifact.PublishOptions{ + "no force": {ExpectedPrior: &result.Pinned}, + "with force": {ExpectedPrior: &result.Pinned, Force: true}, + "no expected": {Force: true}, + } { + _, err := repo.Publish(ctx, generic, options) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, name) + assert.Contains(t, err.Error(), "immutable", name) + } + pinned, err := repo.Resolve(ctx, ref) + require.NoError(t, err) + assert.Equal(t, pluginRoot, pinned.ResolvedDigest, "the release was never mutated") + + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Scope: repo.Capabilities().Delete}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "generic deletion never removes a plugin release") + assert.True(t, resolves(repo, ref.Name, ref.Version)) + }) +} + +func TestIncomingPluginReleaseNeverReplacesAGenericArtifact(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + generic := publish(t, repo, publishSpec{name: ref.Name, version: ref.Version, content: "generic"}) + + for _, force := range []bool{false, true} { + _, err := repo.ImportLayout(context.Background(), copyFixture(t), ref, ImportOptions{ExpectedPrior: &generic.Pinned, Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + pinned, err := repo.Resolve(context.Background(), ref) + require.NoError(t, err) + assert.Equal(t, generic.Pinned.ResolvedDigest, pinned.ResolvedDigest) + }) +} + +func TestGenericArtifactsAndPluginsCoexistUnderOnePrefix(t *testing.T) { + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg) + plugin := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + importPlugin(t, repo, plugin) + publish(t, repo, publishSpec{name: plugin.Name, version: "2.0.0", content: "generic sibling"}) + publish(t, repo, publishSpec{name: "plugins/other", version: "1.0.0", content: "generic neighbor"}) + + assert.Equal(t, []string{"plugins/example:2.0.0", "plugins/other:1.0.0"}, listAll(t, repo, artifact.ListQuery{}), "legacy plugin tags are never listed as generic artifacts") + pinned, err := repo.Resolve(context.Background(), plugin) + require.NoError(t, err) + assert.Equal(t, pluginRoot, pinned.ResolvedDigest, "generic publication beside a plugin leaves it untouched") + + removed := mustResolveRef(t, repo, artifact.Ref{Name: "plugins/example", Version: "2.0.0"}) + _, err = repo.Delete(context.Background(), removed, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + assert.True(t, resolves(repo, plugin.Name, plugin.Version), "deleting a generic sibling never touches the plugin tag") +} + +func mustResolveRef(t *testing.T, repo *Repository, ref artifact.Ref) artifact.PinnedReference { + t.Helper() + pinned, err := repo.Resolve(context.Background(), ref) + require.NoError(t, err) + return pinned +} + +func TestPluginImmutabilityPolicy(t *testing.T) { + plugin := &artifact.Inspection{Representation: artifact.Representation{ + OriginalRoot: &ocispec.Descriptor{Annotations: map[string]string{pluginAnnotationName: "x/y", pluginAnnotationVersion: "1"}}, + }} + bundle := &artifact.Inspection{Representation: artifact.Representation{Payload: []ocispec.Descriptor{{MediaType: pluginBundleMediaType}}}} + labeled := &artifact.Inspection{Record: &artifact.Record{Metadata: artifact.Metadata{Labels: map[string]string{pluginPrefix + "name": "x"}}}} + generic := &artifact.Inspection{Representation: artifact.Representation{OriginalRoot: &ocispec.Descriptor{Annotations: map[string]string{"org.example": "x"}}}} + stagedPlugin := &artifact.PreparedArtifact{Blobs: []artifact.StagedBlob{{Descriptor: ocispec.Descriptor{MediaType: pluginBundleMediaType}}}} + stagedGeneric := &artifact.PreparedArtifact{Blobs: []artifact.StagedBlob{{Descriptor: ocispec.Descriptor{MediaType: artifact.MediaTypeTar}}}} + + tests := []struct { + name string + candidate service.Candidate + want bool + }{ + {"existing release annotations", service.Candidate{Existing: true, Inspection: plugin}, true}, + {"copy source release", service.Candidate{Inspection: plugin}, true}, + {"bundle layer", service.Candidate{Existing: true, Inspection: bundle}, true}, + {"plugin labels", service.Candidate{Existing: true, Inspection: labeled}, true}, + {"incoming plugin bundle", service.Candidate{Prepared: stagedPlugin}, true}, + {"foreign image", service.Candidate{Existing: true, Inspection: generic}, false}, + {"incoming generic", service.Candidate{Prepared: stagedGeneric}, false}, + {"empty", service.Candidate{}, false}, + } + policy := PluginImmutability() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + immutable, why := policy.Immutable(tt.candidate) + assert.Equal(t, tt.want, immutable) + assert.Equal(t, tt.want, why != "", "an immutable verdict explains itself") + }) + } +} + +// With the service in front, immutability is enforced in both directions, with and without force. +func TestServiceKeepsPluginReleasesImmutableInBothDirections(t *testing.T) { + ctx := context.Background() + registryRepo := newRegistryRepo(t, newRegistry(t, nil)) + layoutRepo := newLayoutRepo(t) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"reg": registryRepo, "lay": layoutRepo}}, nil, + service.WithImmutabilityPolicy(PluginImmutability()), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + importPlugin(t, registryRepo, ref) + source := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(source, "a.txt"), []byte("generic"), 0o644)) + + // Existing plugin, incoming generic publication. + for _, force := range []bool{false, true} { + _, err := svc.PublishDetailed(ctx, artifact.PublishRequest{Repository: "reg", Source: source, Name: ref.Name, Version: ref.Version, Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + assert.Equal(t, pluginRoot, mustResolveRef(t, registryRepo, ref).ResolvedDigest) + + // Existing generic, incoming plugin release copied from the registry. + _, err = svc.PublishDetailed(ctx, artifact.PublishRequest{Repository: "lay", Source: source, Name: ref.Name, Version: ref.Version}) + require.NoError(t, err) + for _, force := range []bool{false, true} { + _, err := svc.Copy(ctx, artifact.CopyRequest{Ref: ref, From: "reg", To: "lay", Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Contains(t, err.Error(), "immutable") + } + assert.NotEqual(t, pluginRoot, mustResolveRef(t, layoutRepo, ref).ResolvedDigest) +} diff --git a/pkg/artifact/oci/portability_test.go b/pkg/artifact/oci/portability_test.go new file mode 100644 index 00000000000..375246c1065 --- /dev/null +++ b/pkg/artifact/oci/portability_test.go @@ -0,0 +1,161 @@ +package oci + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// newLocalDir builds a local/dir repository that shares the verifier. +func newLocalDir(t *testing.T, verifier artifact.Verifier) artifact.Repository { + t.Helper() + registry := artifact.NewRepositoryRegistry() + require.NoError(t, local.RegisterWith(registry, local.WithVerifier(verifier))) + repo, err := registry.New(context.Background(), "local", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": t.TempDir()}, + }) + require.NoError(t, err) + return repo +} + +// A Record signed in a local repository is restored into an OCI package of the same Record and +// verifies there: the evidence follows the portable Record, not the representation. +func TestRecordSignatureSurvivesLocalToOCIRestore(t *testing.T) { + ctx := context.Background() + tr := newTrust(t) + verifier := tr.verifier() + localRepo := newLocalDir(t, verifier) + localPrepared := prepare(t, publishSpec{content: "signed", packaging: artifact.PackagingNone}) + _, err := localRepo.Publish(ctx, localPrepared, artifact.PublishOptions{}) + require.NoError(t, err) + localPinned, err := localRepo.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + inspection, err := localRepo.Inspect(ctx, localPinned) + require.NoError(t, err) + subjects := subjectsOfRecord(t, inspection) + _, err = localRepo.Sign(ctx, subjects, tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + + reader, ok := localRepo.(artifact.EvidenceReader) + require.True(t, ok) + blobs, err := reader.ReadEvidence(ctx, &localPinned) + require.NoError(t, err) + require.NotEmpty(t, blobs) + + withVerifier(t, verifier, func(t *testing.T, env *signedEnv) { + pinned := mustResolveRef(t, env.repo, artifact.Ref{Name: appName, Version: appVersion}) + // Same Record bytes under a different packaging and envelope. + assert.Equal(t, localPinned.RecordDigest, pinned.RecordDigest) + assert.NotEqual(t, localPinned.ResolvedDigest, pinned.ResolvedDigest, "the OCI envelope digest is its own") + env.requireRejected(t, env.record, identityPolicy(trustedIdentity, trustedIssuer)) + + stored, err := env.repo.WriteEvidence(ctx, &pinned, blobs, nil) + require.NoError(t, err) + assert.Len(t, stored, len(blobs)) + env.requireVerified(t, env.record, identityPolicy(trustedIdentity, trustedIssuer)) + + again, err := env.repo.WriteEvidence(ctx, &pinned, blobs, nil) + require.NoError(t, err) + assert.Len(t, again, len(blobs)) + read, err := env.repo.ReadEvidence(ctx, &pinned) + require.NoError(t, err) + assert.Len(t, read, len(blobs), "importing the same evidence twice does not duplicate it") + + wrong := append([]artifact.EvidenceBlob(nil), blobs...) + wrong[0].Evidence.Subject.Digest = pinned.ResolvedDigest + _, err = env.repo.WriteEvidence(ctx, &pinned, wrong, nil) + require.Error(t, err, "evidence bound to another subject is refused") + wrong[0] = blobs[0] + wrong[0].Data = []byte("changed bytes") + _, err = env.repo.WriteEvidence(ctx, &pinned, wrong, nil) + require.Error(t, err, "evidence bytes must match their descriptor") + }) +} + +func subjectsOfRecord(t *testing.T, inspection *artifact.Inspection) artifact.SignatureSubject { + t.Helper() + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + recordDigest, err := inspection.Record.RecordDigest() + require.NoError(t, err) + return artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: recordDigest, MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} +} + +// Through the service, repacking from a local directory keeps Record and content identity but +// builds a new OCI envelope; a native OCI-to-OCI copy keeps the envelope digest too. +func TestServiceCopyIdentityLocalToOCIAndOCIToOCI(t *testing.T) { + ctx := context.Background() + localRepo := newLocalDir(t, nil) + registryRepo := newRegistryRepo(t, newRegistry(t, nil)) + layoutRepo := newLayoutRepo(t) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"local": localRepo, "reg": registryRepo, "lay": layoutRepo}}, nil, + service.WithImmutabilityPolicy(PluginImmutability()), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + source := t.TempDir() + writeSource(t, source) + _, err = svc.PublishDetailed(ctx, artifact.PublishRequest{Repository: "local", Source: source, Name: appName, Version: appVersion}) + require.NoError(t, err) + ref := artifact.Ref{Name: appName, Version: appVersion} + + _, err = svc.Copy(ctx, artifact.CopyRequest{Ref: ref, From: "local", To: "reg"}) + require.NoError(t, err) + fromLocal, err := localRepo.Resolve(ctx, ref) + require.NoError(t, err) + inRegistry := mustResolveRef(t, registryRepo, ref) + assert.Equal(t, fromLocal.RecordDigest, inRegistry.RecordDigest, "Record identity survives repackaging") + inspection, err := registryRepo.Inspect(ctx, inRegistry) + require.NoError(t, err) + assert.Equal(t, fromLocal.ContentDigest, inspection.Pinned.ContentDigest, "content identity survives repackaging") + assert.NotEqual(t, fromLocal.ResolvedDigest, inRegistry.ResolvedDigest, "the envelope digest is the OCI manifest, not the Record") + + _, err = svc.Copy(ctx, artifact.CopyRequest{Ref: ref, From: "reg", To: "lay"}) + require.NoError(t, err) + inLayout := mustResolveRef(t, layoutRepo, ref) + assert.Equal(t, inRegistry.ResolvedDigest, inLayout.ResolvedDigest, "a native OCI copy keeps the envelope digest") + assert.Equal(t, inRegistry.RecordDigest, inLayout.RecordDigest) + + again, err := svc.Copy(ctx, artifact.CopyRequest{Ref: ref, From: "reg", To: "lay"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) +} + +// A layout is usable with no network at all: every operation works with a transport that fails +// every request, because a layout never opens one. +func TestLayoutOperatesOfflineWithATransportThatFailsEveryRequest(t *testing.T) { + ctx := context.Background() + offline := &recorder{} + offline.failAll(context.DeadlineExceeded) + failing := newRegistry(t, nil) + failing.recorder = offline + tr := newTrust(t) + repo := newLayoutRepo(t, WithTransport(failing.transport), WithVerifier(tr.verifier())) + + result := publish(t, repo, publishSpec{content: "offline"}) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: appName, Version: appVersion}) + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + _, err = repo.Sign(ctx, subjectsOfRecord(t, inspection), tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + _, err = repo.Verify(ctx, subjectsOfRecord(t, inspection), identityPolicy(trustedIdentity, trustedIssuer)) + require.NoError(t, err) + assert.Equal(t, []string{"team/app:1.0.0"}, listAll(t, repo, artifact.ListQuery{})) + content, err := repo.Read(ctx, pinned) + require.NoError(t, err) + require.NoError(t, content.Close()) + + carrier := newLayoutRepo(t, WithTransport(failing.transport)) + _, err = carrier.CopyFrom(ctx, repo, pinned, artifact.CopyFromOptions{}) + require.NoError(t, err) + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteReference}) + require.NoError(t, err) + assert.NotEmpty(t, result.Location) + assert.Zero(t, offline.total(), "no network request was attempted") +} diff --git a/pkg/artifact/oci/publish.go b/pkg/artifact/oci/publish.go new file mode 100644 index 00000000000..ef0c889bdd8 --- /dev/null +++ b/pkg/artifact/oci/publish.go @@ -0,0 +1,244 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" +) + +// tagState is what a reference holds when a write begins. +type tagState struct { + exists bool + desc ocispec.Descriptor + loaded *root +} + +// Publish implements [artifact.Repository]. The service has already compared intent and applied +// the force policy under the writer lock; this method still enforces the reference contract +// itself: the expected prior reference must still be current, replacement needs force, and a +// legacy plugin release is never replaced, with or without force. +// +// OCI tags have no compare-and-swap, so the prior reference is checked immediately before the +// tag moves, and the repository does not advertise conditional updates. A write whose outcome is +// unknown is reconciled by resolving the tag; there are no blind retries. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "oci.Repository.Publish")() + + pkg, err := Assemble(prepared) + if err != nil { + return nil, err + } + ref := artifact.Ref{Name: prepared.Record.Name, Version: prepared.Record.Version} + if err := ref.Validate(); err != nil { + return nil, err + } + if layout, ok := r.be.(*layoutBackend); ok { + defer lockLayout(layout.root)() + } + st, err := r.be.open(ctx, ref.Name, true) + if err != nil { + return nil, err + } + reference := r.be.reference(ref.Name, ref.Version) + state, err := r.readTag(ctx, st, ref, reference) + if err != nil { + return nil, err + } + outcome, err := decidePublish(ref, &state, &incomingRoot{digest: pkg.Manifest.Digest, class: classAtmos}, options) + if err != nil { + return nil, err + } + if outcome != artifact.OutcomeUnchanged { + if err := r.upload(ctx, st, &uploadJob{pkg: pkg, ref: ref, reference: reference, before: state}); err != nil { + return nil, err + } + } + return r.published(ref, pkg, prepared, outcome, options), nil +} + +// readTag resolves and classifies the current holder of a reference, if any. +func (r *Repository) readTag(ctx context.Context, st store, ref artifact.Ref, reference string) (tagState, error) { + desc, err := st.Resolve(ctx, reference) + switch { + case err == nil: + case isNotFound(err): + return tagState{}, nil + default: + return tagState{}, fmt.Errorf("%w: resolve %s: %w", errUtils.ErrArtifactDownloadFailed, ref, err) + } + loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) + if err != nil { + return tagState{}, err + } + return tagState{exists: true, desc: desc, loaded: loaded}, nil +} + +// incomingRoot is what is about to be written under a reference. +type incomingRoot struct { + digest digest.Digest + class rootClass +} + +// decidePublish applies the reference contract to the observed state. +func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, options artifact.PublishOptions) (artifact.Outcome, error) { + if !state.exists { + if options.ExpectedPrior != nil { + return "", fmt.Errorf("%w: %s no longer exists but the caller expected %s", errUtils.ErrArtifactConflict, ref, options.ExpectedPrior.ResolvedDigest) + } + return artifact.OutcomeCreated, nil + } + if state.desc.Digest == incoming.digest { + return artifact.OutcomeUnchanged, nil + } + if state.loaded.class == classPlugin { + return "", fmt.Errorf("%w: %s is a legacy plugin release and stays immutable even with force", errUtils.ErrArtifactConflict, ref) + } + if incoming.class == classPlugin { + return "", fmt.Errorf("%w: the incoming release for %s is a legacy plugin release and may not replace %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest) + } + if options.ExpectedPrior == nil { + return "", fmt.Errorf("%w: %s already exists (%s) but the caller expected it not to", errUtils.ErrArtifactConflict, ref, state.desc.Digest) + } + if options.ExpectedPrior.ResolvedDigest != state.desc.Digest { + return "", fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest, options.ExpectedPrior.ResolvedDigest) + } + if !options.Force { + return "", fmt.Errorf("%w: %s already exists with different content; use force to replace it", errUtils.ErrArtifactConflict, ref) + } + return artifact.OutcomeReplaced, nil +} + +// uploadJob is one package upload and the tag state it started from. +type uploadJob struct { + pkg *Package + ref artifact.Ref + reference string + before tagState +} + +// upload writes the blobs, then the manifest under its reference. +func (r *Repository) upload(ctx context.Context, st store, job *uploadJob) error { + pkg, ref, reference, before := job.pkg, job.ref, job.reference, job.before + for index := range pkg.Blobs { + if err := r.uploadBlob(ctx, st, &pkg.Blobs[index]); err != nil { + return err + } + } + if err := pushIfAbsent(ctx, st, pkg.Config, pkg.ConfigBytes); err != nil { + return fmt.Errorf("%w: upload the Record config of %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) + } + if err := ctx.Err(); err != nil { + return err + } + current, err := r.readTag(ctx, st, ref, reference) + if err != nil { + return err + } + if current.exists != before.exists || current.desc.Digest != before.desc.Digest { + return fmt.Errorf("%w: %s changed while it was being published", errUtils.ErrArtifactConflict, ref) + } + return r.moveTag(ctx, st, pkg, ref, reference) +} + +func (r *Repository) uploadBlob(ctx context.Context, st store, blob *artifact.StagedBlob) error { + if err := ctx.Err(); err != nil { + return err + } + exists, err := st.Exists(ctx, blob.Descriptor) + if err != nil { + return fmt.Errorf("%w: check %s: %w", errUtils.ErrArtifactUploadFailed, blob.Descriptor.Digest, err) + } + if exists { + return nil + } + reader, err := blob.Open(ctx) + if err != nil { + return fmt.Errorf("%w: open staged blob %s: %w", errUtils.ErrArtifactUploadFailed, blob.Descriptor.Digest, err) + } + defer reader.Close() + if err := st.Push(ctx, blob.Descriptor, reader); err != nil && !atmosoci.IsAlreadyExists(err) { + return fmt.Errorf("%w: upload %s: %w", errUtils.ErrArtifactUploadFailed, blob.Descriptor.Digest, err) + } + return nil +} + +// moveTag publishes the manifest under its reference and reconciles an unknown outcome. +func (r *Repository) moveTag(ctx context.Context, st store, pkg *Package, ref artifact.Ref, reference string) error { + err := publishManifest(ctx, st, pkg.Manifest, pkg.ManifestBytes, reference) + if err == nil || atmosoci.IsAlreadyExists(err) { + return nil + } + if ctx.Err() != nil { + return err + } + resolved, resolveErr := st.Resolve(ctx, reference) + switch { + case resolveErr == nil && resolved.Digest == pkg.Manifest.Digest: + return nil + case resolveErr == nil || isNotFound(resolveErr): + return fmt.Errorf("%w: publish %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) + default: + return errors.Join(errUtils.ErrArtifactPublicationAmbiguous, fmt.Errorf("publish %s failed (%w) and the tag could not be checked: %w", ref, err, resolveErr)) + } +} + +// publishManifest writes a manifest under a reference, directly when the store supports it. +func publishManifest(ctx context.Context, st store, desc ocispec.Descriptor, raw []byte, reference string) error { + if pusher, ok := st.(atmosoci.ReferencePusher); ok { + return pusher.PushReference(ctx, desc, bytesReader(raw), reference) + } + if err := st.Push(ctx, desc, bytesReader(raw)); err != nil && !atmosoci.IsAlreadyExists(err) { + return err + } + return st.Tag(ctx, desc, reference) +} + +// published builds the result and remembers the new root so it can be signed right away. +func (r *Repository) published(ref artifact.Ref, pkg *Package, prepared *artifact.PreparedArtifact, outcome artifact.Outcome, options artifact.PublishOptions) *artifact.PublishResult { + if loaded, err := classify(pkg.Manifest, pkg.ManifestBytes); err == nil { + r.remember(ref.Name, pkg.Manifest, pkg.ManifestBytes) + r.remember(ref.Name, pkg.Config, pkg.ConfigBytes) + r.track(ref.Name, loaded) + } + recordDigest := pkg.Config.Digest + representation := artifact.Representation{ + Packaging: prepared.Packaging, + Locator: locatorDigest(r.be.locator(ref.Name), pkg.Manifest), + RecordDigest: recordDigest, + RepresentationDigest: pkg.Blobs[0].Descriptor.Digest, + Payload: []ocispec.Descriptor{pkg.Blobs[0].Descriptor}, + } + for index := 1; index < len(pkg.Blobs); index++ { + representation.Documents = append(representation.Documents, pkg.Blobs[index].Descriptor) + } + result := &artifact.PublishResult{ + Outcome: outcome, + Pinned: artifact.PinnedReference{ + Ref: ref, Repository: r.be.locator(ref.Name), ResolvedDigest: pkg.Manifest.Digest, + RecordDigest: recordDigest, ContentDigest: prepared.Record.ContentDigest, + }, + Representation: representation, + Location: r.referenceLocation(ref), + } + if options.Lock != nil { + result.LockScope = options.Lock.Scope() + } + return result +} + +// referenceLocation is the human-readable location of a name and version. +func (r *Repository) referenceLocation(ref artifact.Ref) string { + if layout, ok := r.be.(*layoutBackend); ok { + return layoutLocator(layout.root, ref) + } + return r.be.locator(ref.Name) + ":" + ref.Version +} diff --git a/pkg/artifact/oci/register.go b/pkg/artifact/oci/register.go new file mode 100644 index 00000000000..857f3b97700 --- /dev/null +++ b/pkg/artifact/oci/register.go @@ -0,0 +1,168 @@ +package oci + +import ( + "context" + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Register registers the oci/registry and oci/layout repository kinds in the process-wide +// registry. A composition root calls it explicitly; nothing registers from init. Options such as +// [WithTransport] and [WithVerifier] apply to every repository the registered factories build. +func Register(opts ...Option) error { + defer perf.Track(nil, "oci.Register")() + + return RegisterWith(artifact.DefaultRepositoryRegistry(), opts...) +} + +// RegisterWith registers the OCI kinds in the given registry. A kind that is already registered +// is an error, so double registration is caught instead of silently replaced. +func RegisterWith(registry *artifact.RepositoryRegistry, opts ...Option) error { + defer perf.Track(nil, "oci.RegisterWith")() + + for _, kind := range Kinds() { + desc := Descriptor(kind, opts...) + if err := registry.Register(&desc, factoryFor(kind, opts)); err != nil { + return err + } + } + return nil +} + +// Kinds lists the repository kinds this package registers. +func Kinds() []string { + defer perf.Track(nil, "oci.Kinds")() + + return []string{artifact.RepositoryKindOCIRegistry, artifact.RepositoryKindOCILayout} +} + +// Descriptor returns the static descriptor of an OCI kind: the built-in descriptor refined with +// the capabilities the registered backend actually provides (delete scope, enumeration, evidence +// formats of the verifier). It needs no I/O. +func Descriptor(kind string, opts ...Option) artifact.KindDescriptor { + defer perf.Track(nil, "oci.Descriptor")() + + desc, _ := artifact.BuiltinKind(kind) + applied := applySettings(opts) + enumerates := kind == artifact.RepositoryKindOCILayout + if kind == artifact.RepositoryKindOCIRegistry { + enumerates = !applied.noCatalog && transportEnumerates(&applied) + } + desc.Capabilities = staticCapabilities(kind, applied.verifier, enumerates) + return *desc +} + +// transportEnumerates reports whether the configured transport can list registry repositories. +// A factory builds its transport later, so it is assumed capable; listing reports the truth. +func transportEnumerates(applied *settings) bool { + if applied.factory != nil || applied.transport == nil { + return true + } + _, ok := applied.transport.(atmosoci.RegistryCatalog) + return ok +} + +func factoryFor(kind string, opts []Option) artifact.RepositoryFactory { + return func(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, ropts ...artifact.RepositoryOption) (artifact.Repository, error) { + return newRepository(ctx, &construction{alias: alias, kind: kind, cfg: cfg, applied: applySettings(opts), ropts: artifact.ApplyRepositoryOptions(ropts...)}) + } +} + +// NewRepository builds one repository from its configuration without going through a registry. +func NewRepository(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "oci.NewRepository")() + + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + } + return newRepository(ctx, &construction{alias: alias, kind: cfg.Kind, cfg: cfg, applied: applySettings(opts)}) +} + +// construction is everything needed to build one repository. +type construction struct { + alias string + kind string + cfg *schema.ArtifactRepositoryConfig + applied settings + ropts artifact.RepositoryOptions +} + +func newRepository(ctx context.Context, build *construction) (*Repository, error) { + alias, kind, cfg, applied, ropts := build.alias, build.kind, build.cfg, build.applied, build.ropts + if cfg.Kind != "" && cfg.Kind != kind { + return nil, fmt.Errorf("%w: configuration kind %q cannot construct a %q repository", errUtils.ErrArtifactConfigInvalid, cfg.Kind, kind) + } + var be backend + var err error + switch kind { + case artifact.RepositoryKindOCIRegistry: + be, err = registryBackendFor(ctx, alias, cfg, &applied, ropts) + case artifact.RepositoryKindOCILayout: + be, err = layoutBackendFor(cfg) + default: + err = fmt.Errorf("%w: %q is not an OCI repository kind", errUtils.ErrArtifactConfigInvalid, kind) + } + if err != nil { + return nil, err + } + delegate := "" + if cfg.Locking != nil { + delegate = cfg.Locking.Repository + } + return newRepositoryCore(alias, be, applied, delegate, ropts.LookupRepository), nil +} + +func registryBackendFor(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, applied *settings, ropts artifact.RepositoryOptions) (backend, error) { + var options artifact.OCIRegistryOptions + if err := artifact.DecodeOptions(cfg.Options, &options); err != nil { + return nil, err + } + if err := options.Validate(); err != nil { + return nil, err + } + transport, err := transportFor(ctx, alias, cfg, applied, ropts) + if err != nil { + return nil, err + } + return newRegistryBackend(options.Registry, options.Prefix, transport, applied.noCatalog), nil +} + +// transportFor picks the transport: the identity-binding factory, the injected transport, or the +// default client. A configured identity without a factory is an error, so it is never ignored. +func transportFor(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, applied *settings, ropts artifact.RepositoryOptions) (atmosoci.Transport, error) { + switch { + case applied.factory != nil: + transport, err := applied.factory(ctx, alias, cfg, ropts.ResolveIdentity) + if err != nil { + return nil, fmt.Errorf("repository %q transport: %w", alias, err) + } + return transport, nil + case cfg.Identity != "": + return nil, fmt.Errorf("%w: repository %q binds identity %q but no transport factory was registered to apply it; registering the OCI kinds needs WithTransportFactory", errUtils.ErrArtifactConfigInvalid, alias, cfg.Identity) + case applied.transport != nil: + return applied.transport, nil + default: + return atmosoci.NewClient(nil), nil + } +} + +func layoutBackendFor(cfg *schema.ArtifactRepositoryConfig) (backend, error) { + var options artifact.OCILayoutOptions + if err := artifact.DecodeOptions(cfg.Options, &options); err != nil { + return nil, err + } + if err := options.Validate(); err != nil { + return nil, err + } + root, err := filepath.Abs(options.Path) + if err != nil { + return nil, fmt.Errorf("%w: layout path %q: %w", errUtils.ErrArtifactConfigInvalid, options.Path, err) + } + return newLayoutBackend(root), nil +} diff --git a/pkg/artifact/oci/register_test.go b/pkg/artifact/oci/register_test.go new file mode 100644 index 00000000000..992b5cd8274 --- /dev/null +++ b/pkg/artifact/oci/register_test.go @@ -0,0 +1,208 @@ +package oci + +import ( + "context" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guards for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Identity: "id", Options: nil} + +func TestRegisterAddsBothKindsWithRefinedDescriptors(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithTransport(newRegistry(t, nil).transport))) + assert.True(t, registry.IsRegistered(artifact.RepositoryKindOCIRegistry)) + assert.True(t, registry.IsRegistered(artifact.RepositoryKindOCILayout)) + assert.Equal(t, []string{artifact.RepositoryKindOCIRegistry, artifact.RepositoryKindOCILayout}, Kinds()) + + err := RegisterWith(registry) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "double registration is caught, not replaced") + + registered, ok := registry.Lookup(artifact.RepositoryKindOCIRegistry) + require.True(t, ok) + assert.Equal(t, artifact.DeleteManifest, registered.Capabilities.Delete) + assert.True(t, registered.AllowsCatalog) + layout, ok := registry.Lookup(artifact.RepositoryKindOCILayout) + require.True(t, ok) + assert.Equal(t, artifact.DeleteReference, layout.Capabilities.Delete) + assert.True(t, layout.UsesLocalPath(artifact.OptionPath)) + + withVerifier := Descriptor(artifact.RepositoryKindOCILayout, WithVerifier(&fakeFormats{})) + assert.Equal(t, []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}, withVerifier.Capabilities.EvidenceFormats) + assert.Empty(t, Descriptor(artifact.RepositoryKindOCILayout).Capabilities.EvidenceFormats) + assert.False(t, Descriptor(artifact.RepositoryKindOCIRegistry, WithoutCollectionEnumeration()).Capabilities.EnumerateCollection) + assert.False(t, Descriptor(artifact.RepositoryKindOCIRegistry, WithTransport(withoutCatalog{})).Capabilities.EnumerateCollection) +} + +// fakeFormats is a verifier that only declares a format. +type fakeFormats struct{ artifact.Verifier } + +func (*fakeFormats) Formats() []artifact.EvidenceFormat { + return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle} +} + +func TestDefaultRegistryRegistrationIsExplicit(t *testing.T) { + artifact.ResetRepositoryKindsForTesting() + t.Cleanup(artifact.ResetRepositoryKindsForTesting) + assert.False(t, artifact.DefaultRepositoryRegistry().IsRegistered(artifact.RepositoryKindOCILayout), "nothing registers from init") + require.NoError(t, Register()) + assert.True(t, artifact.DefaultRepositoryRegistry().IsRegistered(artifact.RepositoryKindOCILayout)) +} + +func TestFactoryValidatesConfigurationBeforeAnyIO(t *testing.T) { + ctx := context.Background() + reg := newRegistry(t, nil) + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithTransport(reg.transport))) + + tests := []struct { + name string + cfg *schema.ArtifactRepositoryConfig + }{ + {"unknown option", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "ghcr.io", "regstry": "typo"}}}, + {"missing registry", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"prefix": "acme"}}}, + {"registry with a scheme", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "https://ghcr.io"}}}, + {"unsafe prefix", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "ghcr.io", "prefix": "../escape"}}}, + {"layout without a path", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCILayout, Options: map[string]any{}}}, + {"unsupported packaging", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCILayout, Packaging: "none", Options: map[string]any{"path": t.TempDir()}}}, + {"permissions apply to local archives only", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCILayout, Permissions: "0600", Options: map[string]any{"path": t.TempDir()}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := registry.New(ctx, "bad", tt.cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + } + + _, err := NewRepository(ctx, "wrong", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir}, WithOwner("x")) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a kind the package does not implement") + _, err = NewRepository(ctx, "nil", nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Zero(t, reg.recorder.total(), "validation touched no network") +} + +func TestIdentityNeedsATransportFactoryAndIsHandedTheResolver(t *testing.T) { + ctx := context.Background() + cfg := registryConfig("registry.example.com") + cfg.Identity = "registry-publisher" + + _, err := NewRepository(ctx, "bound", cfg, WithTransport(newRegistry(t, nil).transport)) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "an identity is never silently ignored") + assert.Contains(t, err.Error(), "WithTransportFactory") + + reg := newRegistry(t, nil) + var gotAlias, gotIdentity string + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithTransportFactory(func(_ context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, resolve artifact.IdentityResolver) (atmosoci.Transport, error) { + gotAlias, gotIdentity = alias, cfg.Identity + require.NotNil(t, resolve) + return reg.transport, nil + }))) + resolver := func(context.Context, string) (any, error) { return nil, nil } + opened, err := registry.New(ctx, "bound", cfg, artifact.WithIdentityResolver(resolver)) + require.NoError(t, err) + assert.Equal(t, "bound", gotAlias) + assert.Equal(t, "registry-publisher", gotIdentity) + assert.Zero(t, reg.recorder.total(), "constructing a bound repository resolves no credentials and makes no request") + assert.NotNil(t, opened) +} + +func TestLayoutDirectoryIsNotCreatedByReadsOnly(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "never", "created") + repo := newLayoutRepoAt(t, path) + _, err := repo.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + page, err := repo.List(ctx, artifact.ListQuery{}) + require.NoError(t, err) + assert.Empty(t, page.Entries, "a layout with no references lists nothing, honestly") + _, err = os.Stat(path) + require.ErrorIs(t, err, os.ErrNotExist, "reads and listing create no destination directory") + + publish(t, repo, publishSpec{content: "created on write"}) + _, err = os.Stat(filepath.Join(path, "index.json")) + require.NoError(t, err) +} + +func TestRelativeLayoutPathsBecomeAbsolute(t *testing.T) { + repo, err := NewRepository(context.Background(), "rel", layoutConfig("relative/layout"), WithLockDir(t.TempDir())) + require.NoError(t, err) + assert.True(t, filepath.IsAbs(repo.be.(*layoutBackend).root)) +} + +// The package keeps the registry client library and Sigstore out of its own imports: it speaks to +// registries only through pkg/oci and to cryptography only through the injected verifier. +func TestPackageDependencyBoundary(t *testing.T) { + forbidden := []string{ + "github.com/google/go-containerregistry", + "oras.land/oras-go", + "github.com/sigstore", + "github.com/aws/aws-sdk-go-v2", + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/ci", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} + +// The plugin-format constants are repeated here to avoid importing plugin packages; they must not +// drift from the ones the plugin package writes. +func TestPluginFormatConstantsMatchThePluginPackage(t *testing.T) { + assert.Equal(t, string(pluginoci.ConfigMediaType), pluginConfigMediaType) + assert.Equal(t, string(pluginoci.BundleMediaType), pluginBundleMediaType) + + root, err := layoutRoot(pluginFixture, "") + require.NoError(t, err) + assert.Equal(t, pluginRoot, root.Digest) + annotations := fixtureIndexAnnotations(t) + for _, key := range []string{pluginAnnotationName, pluginAnnotationVersion, pluginAnnotationManifest} { + assert.Contains(t, annotations, key, "the committed release carries %s", key) + assert.True(t, strings.HasPrefix(key, pluginPrefix)) + } +} + +// fixtureIndexAnnotations reads the release annotations of the committed plugin index. +func fixtureIndexAnnotations(t *testing.T) map[string]string { + t.Helper() + store, err := atmosoci.OpenLayoutReadOnly(context.Background(), pluginFixture) + require.NoError(t, err) + desc, err := store.Resolve(context.Background(), pluginRoot.String()) + require.NoError(t, err) + loaded, err := loadRoot(context.Background(), store, desc) + require.NoError(t, err) + require.Equal(t, classPlugin, loaded.class, "the committed release classifies as a plugin") + return loaded.index.Annotations +} diff --git a/pkg/artifact/oci/repository.go b/pkg/artifact/oci/repository.go new file mode 100644 index 00000000000..72c0abd25d3 --- /dev/null +++ b/pkg/artifact/oci/repository.go @@ -0,0 +1,200 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "fmt" + "sync" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Repository is an OCI-backed [artifact.Repository]. One implementation serves both oci/registry +// and oci/layout; the backend decides where content lives and how versions are named. +// +// A Repository is safe for concurrent use. It keeps two small in-memory tables, both derived +// only from verified content: a bounded cache of manifests and configs addressed by name and +// digest, and the roots it has resolved, which locate the manifest a signature subject is +// attached to. +type Repository struct { + alias string + be backend + settings settings + delegate string + lookup artifact.RepositoryLookup + + mu sync.Mutex + cache map[string][]byte + order []string + roots map[digest.Digest][]located + pinned map[string]ocispec.Descriptor +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.Copier = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// located is a root the repository resolved: the artifact name it was found under and its +// descriptor. Signature subjects carry only a digest, so this is how Sign and Verify find the +// manifest an attachment belongs to. +type located struct { + name string + desc ocispec.Descriptor +} + +func newRepositoryCore(alias string, be backend, applied settings, delegate string, lookup artifact.RepositoryLookup) *Repository { + return &Repository{ + alias: alias, be: be, settings: applied, delegate: delegate, lookup: lookup, + cache: map[string][]byte{}, roots: map[digest.Digest][]located{}, pinned: map[string]ocispec.Descriptor{}, + } +} + +// Capabilities implements [artifact.Repository]. It needs no network access. Repository-scope +// locks are not advertised here: an OCI tag is not a distributed lease, so that scope exists only +// through a configured lock delegate, which the service adds when it describes the repository. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "oci.Repository.Capabilities")() + + return staticCapabilities(r.be.kind(), r.settings.verifier, r.be.enumeratesCollection()) +} + +// staticCapabilities computes the capabilities of a kind without I/O. Evidence formats come from +// the verifier: a repository that cannot verify a format does not claim to store it. +func staticCapabilities(kind string, verifier artifact.Verifier, enumerates bool) artifact.Capabilities { + caps := artifact.Capabilities{ + Read: true, + Write: true, + EnumerateNamed: true, + EnumerateCollection: enumerates, + Delete: artifact.DeleteManifest, + ConditionalUpdate: false, + Packagings: []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip}, + PreservesModes: true, + LockScopes: []artifact.LockScope{artifact.LockScopeHost}, + } + if kind == artifact.RepositoryKindOCILayout { + caps.Delete = artifact.DeleteReference + } + if verifier != nil { + caps.SubjectKinds = []artifact.SubjectKind{artifact.SubjectRecord, artifact.SubjectOCIRoot} + caps.EvidenceFormats = append([]artifact.EvidenceFormat(nil), verifier.Formats()...) + } + return caps +} + +// cacheKey scopes a cached object to the artifact name it was read under, so content proven to +// exist in one repository path is never assumed to exist in another. +func cacheKey(name string, desc ocispec.Descriptor) string { + return name + "@" + desc.Digest.String() +} + +// cached returns a cached verified blob. +func (r *Repository) cached(name string, desc ocispec.Descriptor) ([]byte, bool) { + r.mu.Lock() + defer r.mu.Unlock() + raw, ok := r.cache[cacheKey(name, desc)] + return raw, ok +} + +// remember stores a verified blob, evicting the oldest entry beyond the bound. +func (r *Repository) remember(name string, desc ocispec.Descriptor, raw []byte) { + r.mu.Lock() + defer r.mu.Unlock() + key := cacheKey(name, desc) + if _, present := r.cache[key]; !present { + r.order = append(r.order, key) + if len(r.order) > manifestCacheEntries { + delete(r.cache, r.order[0]) + r.order = r.order[1:] + } + } + r.cache[key] = raw +} + +// track records a resolved root so signature subjects can be located later. +func (r *Repository) track(name string, loaded *root) { + r.mu.Lock() + defer r.mu.Unlock() + keys := []digest.Digest{loaded.desc.Digest} + if loaded.class == classAtmos { + keys = append(keys, loaded.manifest.Config.Digest) + } + for _, key := range keys { + known := r.roots[key] + duplicate := false + for _, item := range known { + duplicate = duplicate || (item.name == name && item.desc.Digest == loaded.desc.Digest) + } + if !duplicate { + r.roots[key] = append(known, located{name: name, desc: loaded.desc}) + } + } + r.pinned[cacheKey(name, loaded.desc)] = loaded.desc +} + +// descriptorFor returns the descriptor remembered for a resolved digest of a name. +func (r *Repository) descriptorFor(name string, dgst digest.Digest) (ocispec.Descriptor, bool) { + r.mu.Lock() + defer r.mu.Unlock() + desc, ok := r.pinned[name+"@"+dgst.String()] + return desc, ok +} + +// locations returns every root known for a subject digest. +func (r *Repository) locations(dgst digest.Digest) []located { + r.mu.Lock() + defer r.mu.Unlock() + return append([]located(nil), r.roots[dgst]...) +} + +// loadCachedRoot loads a root through the verified cache. +func (r *Repository) loadCachedRoot(ctx context.Context, st store, name string, desc ocispec.Descriptor) (*root, error) { + if raw, ok := r.cached(name, desc); ok { + return classify(desc, raw) + } + loaded, err := loadRoot(ctx, st, desc) + if err != nil { + return nil, err + } + r.remember(name, desc, loaded.raw) + r.track(name, loaded) + return loaded, nil +} + +// readRecord fetches the Record config blob of an Atmos package with a bounded, verified read. +func (r *Repository) readRecord(ctx context.Context, st store, name string, loaded *root) (*artifact.Record, error) { + config := loaded.manifest.Config + if raw, ok := r.cached(name, config); ok { + return artifact.DecodeRecord(raw) + } + raw, err := fetchBounded(ctx, st, config, artifact.MaxRecordSize) + if err != nil { + return nil, fmt.Errorf("read the Record of %s: %w", loaded.desc.Digest, err) + } + record, err := artifact.DecodeRecord(raw) + if err != nil { + return nil, err + } + r.remember(name, config, raw) + return record, nil +} + +// requireSubjectOf checks a Record against the reference that led to it: the Record is bound to +// its own name and version, so a tag or repository path serving another Record is tampering. +func requireSubjectOf(record *artifact.Record, ref artifact.Ref) error { + if record.Name != ref.Name { + return fmt.Errorf("%w: %s holds the Record of %q", errUtils.ErrArtifactIntegrityFailed, ref, record.Name) + } + if ref.Version != "" && record.Version != ref.Version { + return fmt.Errorf("%w: %s holds the Record of version %q", errUtils.ErrArtifactIntegrityFailed, ref, record.Version) + } + return nil +} diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go new file mode 100644 index 00000000000..4d8be652b54 --- /dev/null +++ b/pkg/artifact/oci/resolve.go @@ -0,0 +1,241 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Resolve implements [artifact.Repository]. A tag or digest becomes a pinned handle; every +// later read uses the resolved digest, so a tag moved mid-operation is never followed. A digest +// reference selects the OCI manifest or index digest; for an Atmos package the portable Record +// digest, which is the config descriptor digest, also names the artifact. +func (r *Repository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "oci.Repository.Resolve")() + + _, loaded, err := r.resolveRoot(ctx, ref) + if err != nil { + return artifact.PinnedReference{}, err + } + return r.pinnedFor(ref, loaded), nil +} + +// Pin implements [artifact.Repository]. It returns the immutable reference for consumer +// lockfiles: the original manifest or index digest, plus the portable Record and content digests +// bound through the config descriptor and the evidence descriptors known now. It takes no lock +// and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "oci.Repository.Pin")() + + st, loaded, err := r.resolveRoot(ctx, ref) + if err != nil { + return artifact.PinnedReference{}, err + } + pinned := r.pinnedFor(ref, loaded) + if loaded.class == classAtmos { + record, recordErr := r.readRecord(ctx, st, ref.Name, loaded) + if recordErr != nil { + return artifact.PinnedReference{}, recordErr + } + if recordErr = requireSubjectOf(record, ref); recordErr != nil { + return artifact.PinnedReference{}, recordErr + } + pinned.ContentDigest = record.ContentDigest + } + attachments, err := r.attachments(ctx, st, ref.Name, loaded.desc) + if err != nil { + return artifact.PinnedReference{}, err + } + for index := range attachments.valid { + pinned.Evidence = append(pinned.Evidence, attachments.valid[index].manifestDesc) + } + return pinned, nil +} + +// pinnedFor builds the cheap pinned handle of a resolved root: the resolved digest, and for an +// Atmos package the Record digest read from the config descriptor without fetching the config. +func (r *Repository) pinnedFor(ref artifact.Ref, loaded *root) artifact.PinnedReference { + pinned := artifact.PinnedReference{Ref: ref, Repository: r.be.locator(ref.Name), ResolvedDigest: loaded.desc.Digest} + if loaded.class == classAtmos { + pinned.RecordDigest = loaded.manifest.Config.Digest + } + return pinned +} + +// resolveRoot resolves a reference once and classifies the root it names. +func (r *Repository) resolveRoot(ctx context.Context, ref artifact.Ref) (store, *root, error) { + if err := ref.Validate(); err != nil { + return nil, nil, err + } + if err := ctx.Err(); err != nil { + return nil, nil, err + } + st, err := r.be.open(ctx, ref.Name, false) + if err != nil { + return nil, nil, err + } + var desc ocispec.Descriptor + if ref.IsDigest() { + desc, err = r.resolveDigest(ctx, st, ref) + } else { + desc, err = st.Resolve(ctx, r.be.reference(ref.Name, ref.Version)) + } + if err != nil { + return nil, nil, resolveError(ref, err) + } + loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) + if err != nil { + return nil, nil, err + } + return st, loaded, nil +} + +// rootOf re-reads the root of an already pinned reference by its resolved digest. +func (r *Repository) rootOf(ctx context.Context, pinned *artifact.PinnedReference) (store, *root, error) { + if err := artifact.ValidateName(pinned.Ref.Name); err != nil { + return nil, nil, err + } + if err := artifact.ValidateDigest(pinned.ResolvedDigest); err != nil { + return nil, nil, err + } + if err := ctx.Err(); err != nil { + return nil, nil, err + } + st, err := r.be.open(ctx, pinned.Ref.Name, false) + if err != nil { + return nil, nil, err + } + desc, known := r.descriptorFor(pinned.Ref.Name, pinned.ResolvedDigest) + if !known { + desc, err = st.Resolve(ctx, pinned.ResolvedDigest.String()) + if err != nil { + return nil, nil, resolveError(pinned.Ref, err) + } + } + loaded, err := r.loadCachedRoot(ctx, st, pinned.Ref.Name, desc) + if err != nil { + return nil, nil, err + } + return st, loaded, nil +} + +// resolveDigest resolves name@digest. A registry answers a direct manifest lookup in its +// repository path; a layout also requires a live reference to the digest, so a deleted artifact +// does not stay addressable. When the digest is not a manifest, it may be the Record digest of +// an Atmos package, found through the config descriptors of the name's tags. +func (r *Repository) resolveDigest(ctx context.Context, st store, ref artifact.Ref) (ocispec.Descriptor, error) { + if _, isLayout := r.be.(*layoutBackend); !isLayout { + desc, err := st.Resolve(ctx, ref.Digest.String()) + if err == nil || !isNotFound(err) { + return desc, err + } + } + return r.scanForDigest(ctx, st, ref) +} + +// scanForDigest looks through the name's tags for a root whose manifest digest or Record digest is +// the requested digest. The scan is bounded; a name with more tags than the bound is not guessed. +func (r *Repository) scanForDigest(ctx context.Context, st store, ref artifact.Ref) (ocispec.Descriptor, error) { + versions, err := r.be.versions(ctx, ref.Name) + if err != nil { + return ocispec.Descriptor{}, err + } + sort.Strings(versions) + if len(versions) > maxTagsScanned { + versions = versions[:maxTagsScanned] + } + for _, version := range versions { + desc, found, err := r.digestAt(ctx, st, ref, version) + if err != nil { + return ocispec.Descriptor{}, err + } + if found { + return desc, nil + } + } + return ocispec.Descriptor{}, fmt.Errorf("%w: no root of %s has digest %s", errUtils.ErrArtifactNotFound, ref.Name, ref.Digest) +} + +// digestAt reports whether the root at one version is the manifest, or the Record, that ref names. +func (r *Repository) digestAt(ctx context.Context, st store, ref artifact.Ref, version string) (ocispec.Descriptor, bool, error) { + desc, err := st.Resolve(ctx, r.be.reference(ref.Name, version)) + switch { + case isNotFound(err): + return ocispec.Descriptor{}, false, nil + case err != nil: + return ocispec.Descriptor{}, false, err + case desc.Digest == ref.Digest: + return desc, true, nil + } + loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) + if err != nil { + if ctx.Err() != nil { + return ocispec.Descriptor{}, false, err + } + return ocispec.Descriptor{}, false, nil + } + return desc, loaded.class == classAtmos && loaded.manifest.Config.Digest == ref.Digest, nil +} + +// resolveError classifies a store failure: absent content is not found, a canceled context is +// passed through, anything else is a download failure that keeps its cause. +func resolveError(ref artifact.Ref, err error) error { + switch { + case errors.Is(err, errUtils.ErrArtifactNotFound): + return err + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return err + case isNotFound(err): + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactNotFound, ref, err) + default: + return fmt.Errorf("%w: resolve %s: %w", errUtils.ErrArtifactDownloadFailed, ref, err) + } +} + +// SelectPlatform narrows a pinned multi-platform image to one platform without rewriting its +// root. The returned reference keeps the original index digest and evidence, so the original +// signature keeps covering what is verified and transferred; reading it yields a graph whose +// root is the unchanged index and whose only child is the selected platform. A reconstructed +// single-platform index would have a new digest and no signature, and this never builds one. +func (r *Repository) SelectPlatform(ctx context.Context, pinned artifact.PinnedReference, platform ocispec.Platform) (artifact.PinnedReference, ocispec.Descriptor, error) { + defer perf.Track(nil, "oci.Repository.SelectPlatform")() + + _, loaded, err := r.rootOf(ctx, &pinned) + if err != nil { + return artifact.PinnedReference{}, ocispec.Descriptor{}, err + } + if !loaded.isIndex() { + return artifact.PinnedReference{}, ocispec.Descriptor{}, fmt.Errorf("%w: %s is not an image index, so it has no platforms to select", errUtils.ErrArtifactRefInvalid, pinned.ResolvedDigest) + } + child, err := childFor(loaded, &platform) + if err != nil { + return artifact.PinnedReference{}, ocispec.Descriptor{}, err + } + selected := pinned + selected.Platform = &platform + return selected, child, nil +} + +// childFor returns the child of an index that matches the platform. +func childFor(loaded *root, platform *ocispec.Platform) (ocispec.Descriptor, error) { + index := slices.IndexFunc(loaded.index.Manifests, func(child ocispec.Descriptor) bool { + return child.Platform != nil && platformMatches(platform, child.Platform) + }) + if !loaded.isIndex() || index < 0 { + return ocispec.Descriptor{}, fmt.Errorf("%w: platform %s/%s is not in %s", errUtils.ErrArtifactNotFound, platform.OS, platform.Architecture, loaded.desc.Digest) + } + return loaded.index.Manifests[index], nil +} + +func platformMatches(want, have *ocispec.Platform) bool { + return want.OS == have.OS && want.Architecture == have.Architecture && (want.Variant == "" || want.Variant == have.Variant) +} diff --git a/pkg/artifact/oci/roundtrip_test.go b/pkg/artifact/oci/roundtrip_test.go new file mode 100644 index 00000000000..5a39cb42082 --- /dev/null +++ b/pkg/artifact/oci/roundtrip_test.go @@ -0,0 +1,156 @@ +package oci + +import ( + "context" + "io" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/materialize" +) + +// backends runs a test against the registry and the layout repository kinds. +func backends(t *testing.T, run func(t *testing.T, repo *Repository)) { + t.Helper() + t.Run("registry", func(t *testing.T) { run(t, newRegistryRepo(t, newRegistry(t, nil))) }) + t.Run("layout", func(t *testing.T) { run(t, newLayoutRepo(t)) }) +} + +func TestAtmosPackageRoundTrip(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ctx := context.Background() + result := publish(t, repo, publishSpec{content: "round trip"}) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.NotEmpty(t, result.Location) + + pinned, err := repo.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + assert.Equal(t, result.Pinned.ResolvedDigest, pinned.ResolvedDigest) + assert.Equal(t, result.Pinned.RecordDigest, pinned.RecordDigest) + + inspection, err := repo.Inspect(ctx, pinned) + require.NoError(t, err) + require.NotNil(t, inspection.Record) + assert.Nil(t, inspection.Representation.OriginalRoot, "an Atmos package is signed through its Record, not as a native root") + assert.Equal(t, artifact.PackagingTgz, inspection.Representation.Packaging) + assert.Equal(t, result.Pinned.ContentDigest, inspection.Pinned.ContentDigest) + assert.Len(t, inspection.Representation.Documents, 2) + + content, err := repo.Read(ctx, pinned) + require.NoError(t, err) + defer content.Close() + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(ctx, content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + }) +} + +func TestPublishIsIdempotentAndGuardsTheReference(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ctx := context.Background() + first := publish(t, repo, publishSpec{content: "base"}) + + again, err := repo.Publish(ctx, prepare(t, publishSpec{content: "base"}), artifact.PublishOptions{ExpectedPrior: &first.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) + assert.Equal(t, first.Pinned.ResolvedDigest, again.Pinned.ResolvedDigest) + + changed := prepare(t, publishSpec{content: "changed"}) + _, err = repo.Publish(ctx, changed, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "an existing reference expected to be absent") + _, err = repo.Publish(ctx, changed, artifact.PublishOptions{ExpectedPrior: &first.Pinned}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "replacement needs force") + stale := first.Pinned + stale.ResolvedDigest = digest.FromString("stale") + _, err = repo.Publish(ctx, changed, artifact.PublishOptions{ExpectedPrior: &stale, Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected prior is a conflict even with force") + + replaced, err := repo.Publish(ctx, changed, artifact.PublishOptions{ExpectedPrior: &first.Pinned, Force: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, replaced.Outcome) + assert.NotEqual(t, first.Pinned.ResolvedDigest, replaced.Pinned.ResolvedDigest) + + missing := artifact.PinnedReference{ResolvedDigest: digest.FromString("gone")} + _, err = repo.Publish(ctx, prepare(t, publishSpec{name: "other/name", content: "x"}), artifact.PublishOptions{ExpectedPrior: &missing}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "an expected prior that no longer exists") + }) +} + +func TestMovedTagDoesNotChangeAPinnedRead(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + ctx := context.Background() + first := publish(t, repo, publishSpec{content: "first"}) + pinned, err := repo.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + + _, err = repo.Publish(ctx, prepare(t, publishSpec{content: "second"}), artifact.PublishOptions{ExpectedPrior: &first.Pinned, Force: true}) + require.NoError(t, err) + + content, err := repo.Read(ctx, pinned) + require.NoError(t, err) + defer content.Close() + digestOfRecord, err := content.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, pinned.RecordDigest, digestOfRecord, "the read follows the pinned subject, not the moved tag") + }) +} + +func TestReadVerifiesPayloadAgainstItsDescriptor(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + result := publish(t, repo, publishSpec{content: "tamper"}) + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + corruptBlob(t, repo, payloadDigest(t, result)) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + var payload *artifact.ContentStream + for index := range content.Streams { + if content.Streams[index].Role == artifact.BlobPayload { + payload = &content.Streams[index] + } + } + require.NotNil(t, payload) + _, err = io.ReadAll(payload.Reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +func TestCancellationLeavesNoReference(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err := repo.Publish(canceled, prepare(t, publishSpec{content: "never"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Resolve(canceled, artifact.Ref{Name: appName, Version: appVersion}) + require.ErrorIs(t, err, context.Canceled) + + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: appVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a canceled publish created nothing") + }) +} + +func TestRejectsMismatchedRecordForItsTag(t *testing.T) { + repo := newLayoutRepo(t) + result := publish(t, repo, publishSpec{content: "mismatch"}) + // Serve the 1.0.0 package under another version's tag, as a malicious writer could. + st, err := repo.be.open(context.Background(), appName, true) + require.NoError(t, err) + desc, ok := repo.descriptorFor(appName, result.Pinned.ResolvedDigest) + require.True(t, ok) + require.NoError(t, st.Tag(context.Background(), desc, repo.be.reference(appName, "9.9.9"))) + + pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: "9.9.9"}) + require.NoError(t, err) + _, err = repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a tag serving another version's Record is tampering") +} diff --git a/pkg/artifact/oci/sign.go b/pkg/artifact/oci/sign.go new file mode 100644 index 00000000000..d28b0c0c5bf --- /dev/null +++ b/pkg/artifact/oci/sign.go @@ -0,0 +1,369 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// subjectRoot is a manifest or index a signature subject is attached to. +type subjectRoot struct { + name string + st store + loaded *root +} + +// locateSubject finds the roots a subject digest names. Signature subjects carry only a digest, +// so the repository uses the roots it has resolved or inspected in this process; a layout, which +// holds every name in one store, can also search its own references. Anything else needs the +// artifact resolved first, which is what the service always does before signing or verifying. +func (r *Repository) locateSubject(ctx context.Context, subject artifact.SignatureSubject) ([]subjectRoot, error) { + var roots []subjectRoot + for _, known := range r.locations(subject.Digest) { + st, err := r.be.open(ctx, known.name, false) + if err != nil { + return nil, err + } + loaded, err := r.loadCachedRoot(ctx, st, known.name, known.desc) + if err != nil { + return nil, err + } + roots = append(roots, subjectRoot{name: known.name, st: st, loaded: loaded}) + } + if len(roots) == 0 { + if layout, ok := r.be.(*layoutBackend); ok { + return r.scanLayoutSubject(ctx, layout, subject) + } + return nil, fmt.Errorf("%w: subject %s is not a root this repository has resolved; resolve or inspect the artifact first", errUtils.ErrArtifactNotFound, subject.Digest) + } + return roots, nil +} + +// scanLayoutSubject searches the named roots of a layout for the subject. +func (r *Repository) scanLayoutSubject(ctx context.Context, layout *layoutBackend, subject artifact.SignatureSubject) ([]subjectRoot, error) { + refs, err := layout.references(ctx) + if err != nil { + return nil, err + } + var roots []subjectRoot + for _, ref := range refs { + _, loaded, err := r.resolveRoot(ctx, ref) + if err != nil { + if errors.Is(err, errUtils.ErrArtifactNotFound) { + continue + } + return nil, err + } + if loaded.desc.Digest == subject.Digest || (loaded.class == classAtmos && loaded.manifest.Config.Digest == subject.Digest) { + st, openErr := r.be.open(ctx, ref.Name, false) + if openErr != nil { + return nil, openErr + } + roots = append(roots, subjectRoot{name: ref.Name, st: st, loaded: loaded}) + } + } + if len(roots) == 0 { + return nil, fmt.Errorf("%w: no reference in the layout has subject %s", errUtils.ErrArtifactNotFound, subject.Digest) + } + return roots, nil +} + +// checkSubject validates a subject and that the repository can bind and verify evidence for it. +func (r *Repository) checkSubject(operation string, subject artifact.SignatureSubject) error { + if err := subject.Validate(); err != nil { + return err + } + caps := r.Capabilities() + if !caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.alias, operation, "evidence/verify", "no signature verifier is configured, so this repository neither stores nor verifies evidence") + } + return nil +} + +// bindSubject requires the located root to be what the subject says it is. +func bindSubject(subject artifact.SignatureSubject, loaded *root) error { + switch subject.Kind { + case artifact.SubjectRecord: + if loaded.class != classAtmos || loaded.manifest.Config.Digest != subject.Digest { + return fmt.Errorf("%w: %s is not the Record of an Atmos package in this repository", errUtils.ErrArtifactRefInvalid, subject.Digest) + } + if subject.Size != 0 && subject.Size != loaded.manifest.Config.Size { + return fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, loaded.manifest.Config.Size) + } + case artifact.SubjectOCIRoot: + if loaded.desc.Digest != subject.Digest { + return fmt.Errorf("%w: %s is not the manifest or index digest of the located root", errUtils.ErrArtifactRefInvalid, subject.Digest) + } + default: + return fmt.Errorf("%w: unsupported subject kind %q", errUtils.ErrArtifactRefInvalid, subject.Kind) + } + return nil +} + +// Sign implements [artifact.Repository]. The signer produces evidence over the subject; the +// repository stores it as a referrer of the addressed manifest. A Record signature is bound to +// the manifest's config descriptor; an image or plugin signature is bound to the original +// manifest or index digest. Signing never changes the Record, the manifest, or any tag. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "oci.Repository.Sign")() + + if err := r.checkSubject("sign", subject); err != nil { + return nil, err + } + if signer == nil { + return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) + } + roots, err := r.locateSubject(ctx, subject) + if err != nil { + return nil, err + } + for index := range roots { + if err := bindSubject(subject, roots[index].loaded); err != nil { + return nil, err + } + } + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, err + } + if err := r.checkPayload(payload); err != nil { + return nil, err + } + var stored []artifact.SignatureEvidence + for index := range roots { + target := &roots[index] + evidence, manifestDigest, storeErr := r.storeAttachment(ctx, target.st, &attachRequest{root: target.loaded.desc, subject: subject, payload: payload, configDigest: target.loaded.manifest.Config.Digest}) + if storeErr != nil { + return stored, storeErr + } + evidence.Locator = locatorDigest(r.be.locator(target.name), target.loaded.desc) + "#" + manifestDigest + stored = append(stored, evidence) + } + return stored, nil +} + +// checkPayload validates fresh evidence before it is stored. +func (r *Repository) checkPayload(payload *artifact.EvidencePayload) error { + caps := r.Capabilities() + switch { + case payload == nil: + return fmt.Errorf("%w: the signer returned no evidence", errUtils.ErrArtifactVerificationFailed) + case !caps.HasEvidenceFormat(payload.Format): + return artifact.NewUnsupportedCapability(r.alias, "sign", "evidence/"+string(payload.Format), + fmt.Sprintf("the repository stores and verifies %v evidence", caps.EvidenceFormats)) + case !strings.HasPrefix(payload.MediaType, bundleMediaTypePrefix): + return artifact.NewUnsupportedCapability(r.alias, "sign", "evidence/"+payload.MediaType, + "only Sigstore bundle attachments are stored; other formats need a dedicated tested provider") + case len(payload.Data) == 0 || int64(len(payload.Data)) > artifact.MaxEvidenceSize: + return fmt.Errorf("%w: evidence is %d bytes; it must be between 1 and %d", errUtils.ErrArtifactTooLarge, len(payload.Data), artifact.MaxEvidenceSize) + } + return nil +} + +// Verify implements [artifact.Repository]. It collects the subject's bundle referrers, drops +// any not bound to the subject (reporting them), and hands the rest to the injected verifier +// together with the policy. Invalid or unrelated attachments cannot override valid matching +// evidence; malformed ones are bounded and reported. No evidence, an unverifiable format, or an +// unsatisfied policy is a failure. +func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "oci.Repository.Verify")() + + if err := r.checkSubject("verify", subject); err != nil { + return nil, err + } + roots, err := r.locateSubject(ctx, subject) + if err != nil { + return nil, err + } + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, fingerprintErr := policy.Fingerprint(); fingerprintErr == nil { + result.PolicyFingerprint = fingerprint + } + blobs, diagnostics, err := r.collectEvidence(ctx, subject, roots) + result.Diagnostics = diagnostics + if err != nil { + return result, err + } + if len(blobs) == 0 { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-evidence", Message: "no signature evidence is attached to the subject"}) + return result, fmt.Errorf("%w: no signature evidence is attached to %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + verified, verifyErr := r.settings.verifier.Verify(ctx, subject, blobs, policy) + return finishVerification(result, verified, verifyErr) +} + +// collectEvidence gathers the evidence bound to a subject across its located roots. +func (r *Repository) collectEvidence(ctx context.Context, subject artifact.SignatureSubject, roots []subjectRoot) ([]artifact.EvidenceBlob, []artifact.Diagnostic, error) { + var ( + blobs []artifact.EvidenceBlob + diagnostics []artifact.Diagnostic + seen = map[digest.Digest]bool{} + ) + for index := range roots { + target := &roots[index] + set, err := r.attachments(ctx, target.st, target.name, target.loaded.desc) + if err != nil { + return nil, diagnostics, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, err) + } + diagnostics = append(diagnostics, set.diagnostics...) + for item := range set.valid { + att := &set.valid[item] + if !att.bindsTo(subject, target.loaded) { + diagnostics = append(diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "subject-mismatch", Message: fmt.Sprintf("referrer %s is bound to a different subject and was ignored", att.manifestDesc.Digest)}) + continue + } + if seen[att.layer.Digest] { + continue + } + data, fetchErr := fetchBounded(ctx, target.st, att.layer, artifact.MaxEvidenceSize) + if fetchErr != nil { + if ctx.Err() != nil { + return nil, diagnostics, ctx.Err() + } + diagnostics = append(diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("referrer %s was ignored: %v", att.manifestDesc.Digest, fetchErr)}) + continue + } + seen[att.layer.Digest] = true + blobs = append(blobs, artifact.EvidenceBlob{ + Evidence: artifact.SignatureEvidence{ + Subject: subject, Format: bundleFormat(data), Descriptor: att.layer, + Locator: locatorDigest(r.be.locator(target.name), att.manifestDesc), + }, + Data: data, + }) + } + } + return blobs, diagnostics, nil +} + +// finishVerification merges the verifier's result with the storage diagnostics and turns anything +// but a verified status into an error. +func finishVerification(base, verified *artifact.VerificationResult, verifyErr error) (*artifact.VerificationResult, error) { + result := base + if verified != nil { + merged := *verified + merged.Subject = base.Subject + merged.Diagnostics = append(append([]artifact.Diagnostic(nil), base.Diagnostics...), verified.Diagnostics...) + if merged.PolicyFingerprint == "" { + merged.PolicyFingerprint = base.PolicyFingerprint + } + result = &merged + } + switch { + case verifyErr != nil && errors.Is(verifyErr, errUtils.ErrArtifactVerificationFailed): + return result, verifyErr + case verifyErr != nil: + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) + case result.Status == artifact.VerificationFailed: + return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) + } + return result, nil +} + +// subjectsOf returns the signature subjects an Atmos package or foreign root offers. +func subjectsOf(loaded *root) []artifact.SignatureSubject { + subjects := []artifact.SignatureSubject{{Kind: artifact.SubjectOCIRoot, Digest: loaded.desc.Digest, MediaType: loaded.desc.MediaType, Size: loaded.desc.Size}} + if loaded.class == classAtmos { + config := loaded.manifest.Config + subjects = append([]artifact.SignatureSubject{{Kind: artifact.SubjectRecord, Digest: config.Digest, MediaType: artifact.MediaTypeConfig, Size: config.Size}}, subjects...) + } + return subjects +} + +// ReadEvidence implements [artifact.EvidenceReader]: it exports the bundle referrers bound to the +// pinned artifact, so a copy through the shared staging path can carry them along. The evidence +// is untrusted until verified. +func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "oci.Repository.ReadEvidence")() + + st, loaded, err := r.rootOf(ctx, pinned) + if err != nil { + return nil, err + } + set, err := r.attachments(ctx, st, pinned.Ref.Name, loaded.desc) + if err != nil { + return nil, err + } + var blobs []artifact.EvidenceBlob + for item := range set.valid { + att := &set.valid[item] + for _, subject := range subjectsOf(loaded) { + if !att.bindsTo(subject, loaded) { + continue + } + data, fetchErr := fetchBounded(ctx, st, att.layer, artifact.MaxEvidenceSize) + if fetchErr != nil { + return blobs, fetchErr + } + blobs = append(blobs, artifact.EvidenceBlob{ + Evidence: artifact.SignatureEvidence{Subject: subject, Format: bundleFormat(data), Descriptor: att.layer, Locator: locatorDigest(r.be.locator(pinned.Ref.Name), att.manifestDesc)}, + Data: data, + }) + } + } + return blobs, nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this +// repository already holds. Each item must bind to the pinned manifest (a Record signature to its +// config descriptor, an image signature to its digest) and carry bytes matching their descriptor. +// Evidence the manifest already carries is not duplicated. +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "oci.Repository.WriteEvidence")() + + st, loaded, err := r.rootOf(ctx, pinned) + if err != nil { + return nil, err + } + existing, err := r.attachments(ctx, st, pinned.Ref.Name, loaded.desc) + if err != nil { + return nil, err + } + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + blob := &evidence[index] + subject := blob.Evidence.Subject + if err := r.checkSubject("write evidence", subject); err != nil { + return stored, err + } + if err := bindSubject(subject, loaded); err != nil { + return stored, err + } + if blob.Evidence.Descriptor.Digest != digest.FromBytes(blob.Data) { + return stored, fmt.Errorf("%w: evidence bytes do not match their descriptor %s", errUtils.ErrArtifactIntegrityFailed, blob.Evidence.Descriptor.Digest) + } + if holds(existing, subject, loaded, blob.Evidence.Descriptor.Digest) { + stored = append(stored, blob.Evidence) + continue + } + payload := &artifact.EvidencePayload{Format: blob.Evidence.Format, MediaType: blob.Evidence.Descriptor.MediaType, Data: blob.Data} + if err := r.checkPayload(payload); err != nil { + return stored, err + } + item, manifestDigest, storeErr := r.storeAttachment(ctx, st, &attachRequest{root: loaded.desc, subject: subject, payload: payload, configDigest: loaded.manifest.Config.Digest}) + if storeErr != nil { + return stored, storeErr + } + item.Locator = locatorDigest(r.be.locator(pinned.Ref.Name), loaded.desc) + "#" + manifestDigest + stored = append(stored, item) + } + return stored, nil +} + +// holds reports whether the set already carries the bundle bound to the subject. +func holds(set *attachmentSet, subject artifact.SignatureSubject, loaded *root, bundle digest.Digest) bool { + for index := range set.valid { + if set.valid[index].layer.Digest == bundle && set.valid[index].bindsTo(subject, loaded) { + return true + } + } + return false +} diff --git a/pkg/artifact/oci/sign_test.go b/pkg/artifact/oci/sign_test.go new file mode 100644 index 00000000000..284cf4d7ead --- /dev/null +++ b/pkg/artifact/oci/sign_test.go @@ -0,0 +1,495 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "crypto" + "encoding/base64" + "encoding/json" + "sync" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + sigroot "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/sigstore" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +const ( + trustedIdentity = "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main" + trustedIssuer = "https://token.actions.githubusercontent.com" +) + +// keylessBlob signs with a certificate issued by the isolated test authority. +type keylessBlob struct { + t *testing.T + auth *signaturetest.Authority + identity string + issuer string +} + +func (k keylessBlob) Sign(_ context.Context, data []byte) ([]byte, error) { + return k.auth.Sign(k.t, k.identity, k.issuer, data), nil +} + +// keyBlob signs with a bare public key and remembers it so the verifier can be told to trust it. +type keyBlob struct { + t *testing.T + auth *signaturetest.Authority + mu sync.Mutex + last signaturetest.KeySignature +} + +func (k *keyBlob) Sign(_ context.Context, data []byte) ([]byte, error) { + signed := k.auth.SignWithKey(k.t, data) + k.mu.Lock() + k.last = signed + k.mu.Unlock() + return signed.Bundle, nil +} + +func (k *keyBlob) publicKey() crypto.PublicKey { + k.mu.Lock() + defer k.mu.Unlock() + return k.last.PublicKey +} + +// trust is one isolated Sigstore trust domain plus the signers and verifiers built on it. +type trust struct { + t *testing.T + auth *signaturetest.Authority + keyless *sigstore.Signer + keyed *sigstore.Signer + keyBlob *keyBlob + identity string +} + +func newTrust(t *testing.T) *trust { + t.Helper() + auth := signaturetest.New(t) + blob := &keyBlob{t: t, auth: auth} + return &trust{ + t: t, auth: auth, keyBlob: blob, identity: trustedIdentity, + keyless: sigstore.NewSigner(keylessBlob{t: t, auth: auth, identity: trustedIdentity, issuer: trustedIssuer}), + keyed: sigstore.NewSigner(blob), + } +} + +func (tr *trust) loader(material sigroot.TrustedMaterial) sigstore.VerifierOption { + return sigstore.WithRootLoader(func(context.Context, bool) (sigroot.TrustedMaterial, error) { return material, nil }) +} + +// verifier trusts the authority's roots and, optionally, named public keys. +func (tr *trust) verifier(opts ...sigstore.VerifierOption) *sigstore.Verifier { + return sigstore.NewVerifier(append([]sigstore.VerifierOption{tr.loader(tr.auth)}, opts...)...) +} + +func identityPolicy(identity, issuer string) artifact.VerificationPolicy { + return artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: identity, Issuer: issuer}}}}, + } +} + +func keyPolicy(keyName string) artifact.VerificationPolicy { + return artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: []string{keyName}}}} +} + +// signedEnv publishes one package to a repository built with the verifier. +type signedEnv struct { + repo *Repository + result *artifact.PublishResult + record artifact.SignatureSubject + rootSub artifact.SignatureSubject +} + +func newSignedEnv(t *testing.T, repo *Repository) *signedEnv { + t.Helper() + result := publish(t, repo, publishSpec{content: "signed"}) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: appName, Version: appVersion}) + _, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + subjects := subjectsOf(loaded) + return &signedEnv{repo: repo, result: result, record: subjects[0], rootSub: subjects[1]} +} + +// withVerifier runs a test on a registry and a layout repository that use the verifier. +func withVerifier(t *testing.T, verifier artifact.Verifier, run func(t *testing.T, env *signedEnv)) { + t.Helper() + t.Run("registry", func(t *testing.T) { + run(t, newSignedEnv(t, newRegistryRepo(t, newRegistry(t, nil), WithVerifier(verifier)))) + }) + t.Run("layout", func(t *testing.T) { + run(t, newSignedEnv(t, newLayoutRepo(t, WithVerifier(verifier)))) + }) +} + +func (e *signedEnv) sign(t *testing.T, subject artifact.SignatureSubject, signer artifact.Signer) []artifact.SignatureEvidence { + t.Helper() + evidence, err := e.repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + require.NotEmpty(t, evidence) + return evidence +} + +func (e *signedEnv) verify(subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + return e.repo.Verify(context.Background(), subject, policy) +} + +func (e *signedEnv) requireVerified(t *testing.T, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) *artifact.VerificationResult { + t.Helper() + result, err := e.verify(subject, policy) + require.NoError(t, err) + require.Equal(t, artifact.VerificationVerified, result.Status) + return result +} + +func (e *signedEnv) requireRejected(t *testing.T, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) *artifact.VerificationResult { + t.Helper() + result, err := e.verify(subject, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.NotNil(t, result) + require.NotEqual(t, artifact.VerificationVerified, result.Status) + return result +} + +func TestKeylessRecordSignatureIsBoundToTheConfigDescriptor(t *testing.T) { + tr := newTrust(t) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + policy := identityPolicy(trustedIdentity, trustedIssuer) + env.requireRejected(t, env.record, policy) // unsigned + env.sign(t, env.record, tr.keyless) + + result := env.requireVerified(t, env.record, policy) + assert.Equal(t, []artifact.SignerClaim{{Identity: trustedIdentity, Issuer: trustedIssuer}}, result.Claims) + assert.NotEmpty(t, result.EvidenceDigests) + assert.NotEmpty(t, result.PolicyFingerprint) + assert.Equal(t, env.result.Pinned.RecordDigest, env.record.Digest, "the signed Record digest is the config descriptor digest") + + // A signed Record is not a signed native envelope. + env.requireRejected(t, env.rootSub, policy) + env.sign(t, env.rootSub, tr.keyless) + env.requireVerified(t, env.rootSub, policy) + + pinned := mustResolveRef(t, env.repo, artifact.Ref{Name: appName, Version: appVersion}) + pin, err := env.repo.Pin(context.Background(), pinned.Ref) + require.NoError(t, err) + assert.Len(t, pin.Evidence, 2, "pins persist the evidence descriptors, not trust") + inspection, err := env.repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + assert.Len(t, inspection.Evidence, 2) + }) +} + +func TestVerificationFailsForTheWrongSignerOrPolicy(t *testing.T) { + tr := newTrust(t) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + env.sign(t, env.record, tr.keyless) + tests := map[string]artifact.VerificationPolicy{ + "wrong identity": identityPolicy("https://github.com/acme/other/.github/workflows/release.yml@refs/heads/main", trustedIssuer), + "wrong issuer": identityPolicy(trustedIdentity, "https://accounts.example.com"), + "untrusted key": keyPolicy("not-configured"), + "two sets, one unmet": {RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: trustedIdentity, Issuer: trustedIssuer}}}, + {Identities: []artifact.TrustedIdentity{{Identity: "someone@example.com", Issuer: trustedIssuer}}}, + }}, + } + for name, policy := range tests { + t.Run(name, func(t *testing.T) { env.requireRejected(t, env.record, policy) }) + } + wildcard := identityPolicy("https://github.com/acme/images/*", trustedIssuer) + env.requireVerified(t, env.record, wildcard) + }) +} + +func TestConfiguredPublicKeyVerifiesButNeverSatisfiesAnIdentityPolicy(t *testing.T) { + tr := newTrust(t) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + // Sign first so the operator-configured key can be learned, then trust exactly that key. + repo := env.repo + _, err := repo.Sign(context.Background(), env.record, tr.keyed, artifact.SignOptions{}) + require.NoError(t, err) + + trusting := tr.verifier(sigstore.WithTrustedKey(signaturetest.KeyHint, tr.keyBlob.publicKey())) + repo.settings.verifier = trusting + env.requireVerified(t, env.record, keyPolicy(signaturetest.KeyHint)) + env.requireRejected(t, env.record, identityPolicy(trustedIdentity, trustedIssuer)) + env.requireRejected(t, env.record, artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: trustedIdentity, Issuer: trustedIssuer}}}, + {Keys: []string{signaturetest.KeyHint}}, + }}) + + // A different key is not the configured one. + other := sigstore.NewVerifier(tr.loader(tr.auth), sigstore.WithTrustedKey(signaturetest.KeyHint, newOtherKey(t, tr))) + repo.settings.verifier = other + env.requireRejected(t, env.record, keyPolicy(signaturetest.KeyHint)) + + // A keyless signature never satisfies a key policy either. + repo.settings.verifier = trusting + _, err = repo.Sign(context.Background(), env.rootSub, tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + env.requireRejected(t, env.rootSub, keyPolicy(signaturetest.KeyHint)) + }) +} + +// newOtherKey returns the public key of an unrelated signer. +func newOtherKey(t *testing.T, tr *trust) crypto.PublicKey { + t.Helper() + return tr.auth.SignWithKey(t, []byte("unrelated")).PublicKey +} + +func TestMissingOrExpiredTrustFailsPolicyOffline(t *testing.T) { + tr := newTrust(t) + cases := map[string]sigstore.VerifierOption{ + "offline with no cached roots": sigstore.WithRootLoader(func(_ context.Context, offline bool) (sigroot.TrustedMaterial, error) { + require.True(t, offline, "an offline policy asks the loader for offline trust") + return nil, errUtils.ErrSignatureOffline + }), + "roots of another authority": tr.loader(signaturetest.New(t)), + "loader failure": sigstore.WithRootLoader(func(context.Context, bool) (sigroot.TrustedMaterial, error) { + return nil, errUtils.ErrSignatureInvalid + }), + } + for name, option := range cases { + t.Run(name, func(t *testing.T) { + withVerifier(t, sigstore.NewVerifier(option), func(t *testing.T, env *signedEnv) { + // Sign with an unrestricted verifier so the evidence exists, then verify offline. + env.repo.settings.verifier = tr.verifier() + env.sign(t, env.record, tr.keyless) + env.repo.settings.verifier = sigstore.NewVerifier(option) + policy := identityPolicy(trustedIdentity, trustedIssuer) + policy.Offline = true + env.requireRejected(t, env.record, policy) + }) + }) + } +} + +// attach stores bundle bytes as evidence bound to the subject, bypassing the signer. +func (e *signedEnv) attach(t *testing.T, subject artifact.SignatureSubject, data []byte) { + t.Helper() + pinned := mustResolveRef(t, e.repo, artifact.Ref{Name: appName, Version: appVersion}) + st, loaded, err := e.repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: bundleMediaTypePrefix + "v0.3+json", Data: data} + _, _, err = e.repo.storeAttachment(context.Background(), st, &attachRequest{root: loaded.desc, subject: subject, payload: payload, configDigest: loaded.manifest.Config.Digest}) + require.NoError(t, err) +} + +func TestTamperedWrongSubjectAndMalformedEvidence(t *testing.T) { + tr := newTrust(t) + policy := identityPolicy(trustedIdentity, trustedIssuer) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + valid, err := tr.keyless.Sign(context.Background(), env.record) + require.NoError(t, err) + + t.Run("tampered signature", func(t *testing.T) { + var bundle map[string]any + require.NoError(t, json.Unmarshal(valid.Data, &bundle)) + message, ok := bundle["messageSignature"].(map[string]any) + require.True(t, ok) + message["signature"] = base64.StdEncoding.EncodeToString([]byte("forged")) + tampered, err := json.Marshal(bundle) + require.NoError(t, err) + env.attach(t, env.record, tampered) + env.requireRejected(t, env.record, policy) + }) + t.Run("signature made for another subject", func(t *testing.T) { + other := env.record + other.Digest = digest.FromString("another record") + foreign, err := tr.keyless.Sign(context.Background(), other) + require.NoError(t, err) + env.attach(t, env.record, foreign.Data) + env.requireRejected(t, env.record, policy) + }) + t.Run("unrelated invalid evidence never overrides a valid signature", func(t *testing.T) { + env.attach(t, env.record, []byte("not a bundle at all")) + env.attach(t, env.record, []byte(`{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json"}`)) + env.requireRejected(t, env.record, policy) + env.attach(t, env.record, valid.Data) + result := env.requireVerified(t, env.record, policy) + codes := map[string]bool{} + for _, diagnostic := range result.Diagnostics { + codes[diagnostic.Code] = true + } + assert.True(t, codes["malformed-evidence"], "malformed attachments are reported, not hidden: %v", result.Diagnostics) + assert.LessOrEqual(t, len(result.Diagnostics), 17, "reporting is bounded") + }) + }) +} + +func TestSignatureBoundToAnotherManifestIsIgnored(t *testing.T) { + tr := newTrust(t) + withVerifier(t, tr.verifier(), func(t *testing.T, env *signedEnv) { + other := publish(t, env.repo, publishSpec{name: "other/app", content: "other"}) + otherPinned := mustResolveRef(t, env.repo, artifact.Ref{Name: "other/app", Version: appVersion}) + _, err := env.repo.Sign(context.Background(), artifact.SignatureSubject{ + Kind: artifact.SubjectRecord, Digest: other.Pinned.RecordDigest, MediaType: artifact.MediaTypeConfig, Size: otherPinnedSize(t, env.repo, otherPinned), + }, tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + + // The other package's valid signature is not evidence for this one. + result := env.requireRejected(t, env.record, identityPolicy(trustedIdentity, trustedIssuer)) + assert.NotEmpty(t, result.Diagnostics) + }) +} + +func otherPinnedSize(t *testing.T, repo *Repository, pinned artifact.PinnedReference) int64 { + t.Helper() + _, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + return loaded.manifest.Config.Size +} + +func TestSubjectsMustBeResolvedBeforeTheyCanBeSignedOrVerified(t *testing.T) { + tr := newTrust(t) + reg := newRegistry(t, nil) + writer := newRegistryRepo(t, reg, WithVerifier(tr.verifier())) + result := publish(t, writer, publishSpec{content: "fresh"}) + fresh := newRegistryRepo(t, reg, WithVerifier(tr.verifier())) + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: result.Pinned.RecordDigest, MediaType: artifact.MediaTypeConfig} + + _, err := fresh.Verify(context.Background(), subject, identityPolicy(trustedIdentity, trustedIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Contains(t, err.Error(), "resolve") + mustResolveRef(t, fresh, artifact.Ref{Name: appName, Digest: result.Pinned.RecordDigest}) + _, err = fresh.Verify(context.Background(), subject, identityPolicy(trustedIdentity, trustedIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "resolved, so it is checked and found unsigned") +} + +func TestNativeImagePluginAndLegacyCosignSignatures(t *testing.T) { + tr := newTrust(t) + ctx := context.Background() + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg, WithVerifier(tr.verifier())) + policy := identityPolicy(trustedIdentity, trustedIssuer) + + t.Run("image signed through the signer", func(t *testing.T) { + image := pushImage(t, reg, testPrefix+"/tools/img", "v1") + pinned := mustResolveRef(t, repo, artifact.Ref{Name: "tools/img", Version: "v1"}) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: "application/vnd.oci.image.manifest.v1+json"} + _, err := repo.Verify(ctx, subject, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + _, err = repo.Sign(ctx, subject, tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + result, err := repo.Verify(ctx, subject, policy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + _, err = repo.Sign(ctx, artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: image.manifest}, tr.keyless, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid, "a foreign image has no Record to sign") + assert.NotEmpty(t, pinned.ResolvedDigest) + }) + + t.Run("plugin release signed over its raw index like sign-blob", func(t *testing.T) { + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + importPlugin(t, repo, ref) + pinned := mustResolveRef(t, repo, ref) + raw := rawRootBytes(t, repo, pinned) + bundle := tr.auth.Sign(t, trustedIdentity, trustedIssuer, raw) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: pluginRoot, MediaType: "application/vnd.oci.image.index.v1+json", Size: int64(len(raw))} + attachBundleRaw(t, repo, pinned, subject, bundle) + result, err := repo.Verify(ctx, subject, policy) + require.NoError(t, err, "the existing plugin signing form still verifies") + assert.Equal(t, artifact.VerificationVerified, result.Status) + }) + + t.Run("cosign 3 style DSSE statement", func(t *testing.T) { + image := pushImage(t, reg, testPrefix+"/tools/dsse", "v1") + pinned := mustResolveRef(t, repo, artifact.Ref{Name: "tools/dsse", Version: "v1"}) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: "application/vnd.oci.image.manifest.v1+json"} + bundle := tr.auth.SignStatement(t, trustedIdentity, trustedIssuer, rawRootBytes(t, repo, pinned)) + attachBundleRaw(t, repo, pinned, subject, bundle) + result, err := repo.Verify(ctx, subject, policy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + }) + + t.Run("legacy cosign tag payloads are not evidence", func(t *testing.T) { + image := pushImage(t, reg, testPrefix+"/tools/legacy", "v1") + repoName := reg.server.Host + "/" + testPrefix + "/tools/legacy" + signatureTag := "sha256-" + image.manifest.Encoded() + ".sig" + reference, err := name.ParseReference(repoName + ":" + signatureTag) + require.NoError(t, err) + payload, err := random.Image(32, 1) + require.NoError(t, err) + require.NoError(t, remote.Write(reference, payload, remote.WithTransport(reg.recorder))) + mustResolveRef(t, repo, artifact.Ref{Name: "tools/legacy", Version: "v1"}) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: "application/vnd.oci.image.manifest.v1+json"} + _, err = repo.Verify(ctx, subject, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "unsupported required evidence fails closed") + }) +} + +// attachBundleRaw attaches bundle bytes produced outside the signer as evidence of the root. +func attachBundleRaw(t *testing.T, repo *Repository, pinned artifact.PinnedReference, subject artifact.SignatureSubject, bundle []byte) { + t.Helper() + st, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: bundleMediaTypePrefix + "v0.3+json", Data: bundle} + _, _, err = repo.storeAttachment(context.Background(), st, &attachRequest{root: loaded.desc, subject: subject, payload: payload}) + require.NoError(t, err) +} + +// A signed multi-platform root keeps its signature when a child is selected; an index rebuilt +// around that child is a different object and claims none of it. +func TestOriginalIndexSignatureSurvivesPlatformSelectionAndReconstructionClaimsNone(t *testing.T) { + tr := newTrust(t) + ctx := context.Background() + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg, WithVerifier(tr.verifier())) + policy := identityPolicy(trustedIdentity, trustedIssuer) + image := pushIndex(t, reg, testPrefix+"/tools/img", "v1", multiPlatform...) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: "tools/img", Version: "v1"}) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: "application/vnd.oci.image.index.v1+json"} + _, err := repo.Sign(ctx, subject, tr.keyless, artifact.SignOptions{}) + require.NoError(t, err) + + selected, child, err := repo.SelectPlatform(ctx, pinned, ocispec.Platform{OS: multiPlatform[1].OS, Architecture: multiPlatform[1].Architecture}) + require.NoError(t, err) + inspection, err := repo.Inspect(ctx, selected) + require.NoError(t, err) + require.NotNil(t, inspection.Representation.OriginalRoot) + signed := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: inspection.Representation.OriginalRoot.Digest, MediaType: inspection.Representation.OriginalRoot.MediaType} + result, err := repo.Verify(ctx, signed, policy) + require.NoError(t, err, "the selected child is verified through the original signed index") + assert.Equal(t, artifact.VerificationVerified, result.Status) + assert.Len(t, inspection.Evidence, 1) + + // Rebuild an index that holds only the selected child and publish it under another tag. + reconstructed := reconstructedIndex(t, reg, image, child.Digest.String()) + assert.NotEqual(t, image.manifest, reconstructed, "a reconstructed root has its own digest") + narrow := mustResolveRef(t, repo, artifact.Ref{Name: "tools/img", Version: "narrow"}) + assert.Equal(t, reconstructed, narrow.ResolvedDigest) + narrowSubject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: reconstructed, MediaType: "application/vnd.oci.image.index.v1+json"} + _, err = repo.Verify(ctx, narrowSubject, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "the rebuilt root does not inherit the original signature") +} + +// reconstructedIndex pushes tools/img:narrow holding only the child manifest and returns its digest. +func reconstructedIndex(t *testing.T, reg *registry, image foreignImage, childDigest string) digest.Digest { + t.Helper() + repoName := reg.server.Host + "/" + testPrefix + "/tools/img" + childReference, err := name.NewDigest(repoName + "@" + childDigest) + require.NoError(t, err) + child, err := remote.Image(childReference, remote.WithTransport(reg.recorder)) + require.NoError(t, err) + index := mutate.AppendManifests(empty.Index, mutate.IndexAddendum{Add: child}) + reference, err := name.ParseReference(repoName + ":narrow") + require.NoError(t, err) + require.NoError(t, remote.WriteIndex(reference, index, remote.WithTransport(reg.recorder))) + rebuilt, err := index.Digest() + require.NoError(t, err) + assert.NotEmpty(t, image.children) + return digest.Digest(rebuilt.String()) +} diff --git a/pkg/artifact/oci/testutil_test.go b/pkg/artifact/oci/testutil_test.go new file mode 100644 index 00000000000..a537e27b588 --- /dev/null +++ b/pkg/artifact/oci/testutil_test.go @@ -0,0 +1,32 @@ +package oci + +import ( + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +// corruptBlob overwrites a layout blob with other bytes of the same length. Only layouts are +// corrupted on disk; the helper skips registry repositories, whose blobs are not files. +func corruptBlob(t *testing.T, repo *Repository, dgst digest.Digest) { + t.Helper() + layout, ok := repo.be.(*layoutBackend) + if !ok { + t.Skip("blob corruption is exercised on layouts, where blobs are files") + } + path := filepath.Join(layout.root, "blobs", dgst.Algorithm().String(), dgst.Encoded()) + raw, err := os.ReadFile(path) + require.NoError(t, err) + raw[len(raw)-1] ^= 0xff + require.NoError(t, os.Chmod(path, 0o600)) + require.NoError(t, os.WriteFile(path, raw, 0o600)) +} + +// writeSource writes a tiny package source tree. +func writeSource(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("service source"), 0o644)) +} diff --git a/pkg/artifact/oci/transfer_test.go b/pkg/artifact/oci/transfer_test.go new file mode 100644 index 00000000000..0373661c3d1 --- /dev/null +++ b/pkg/artifact/oci/transfer_test.go @@ -0,0 +1,255 @@ +//nolint:gocritic // OCI descriptors and the pinned reference are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "context" + "errors" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" +) + +var multiPlatform = []v1.Platform{{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}, {OS: "darwin", Architecture: "arm64"}} + +// attachBundle stores a placeholder Sigstore bundle as a referrer of the pinned root, so a +// transfer has signature material to carry. +func attachBundle(t *testing.T, repo *Repository, pinned artifact.PinnedReference, bundle string) ocispec.Descriptor { + t.Helper() + st, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: loaded.desc.Digest, MediaType: loaded.desc.MediaType, Size: loaded.desc.Size} + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: bundleMediaTypePrefix + "v0.3+json", Data: []byte(bundle)} + evidence, _, err := repo.storeAttachment(context.Background(), st, &attachRequest{root: loaded.desc, subject: subject, payload: payload}) + require.NoError(t, err) + return evidence.Descriptor +} + +// A native multi-platform image moves registry -> layout -> second registry with distinct +// credentials and the source shut down in between. Every original digest and referrer survives. +func TestNativeImageSurvivesRegistryLayoutRegistryWithDistinctCredentials(t *testing.T) { + ctx := context.Background() + sourceRegistry := newRegistry(t, &ocitest.Credential{Username: "source-user", Password: "source-secret"}) + targetRegistry := newRegistry(t, &ocitest.Credential{Username: "target-user", Password: "target-secret"}) + source := newRegistryRepo(t, sourceRegistry) + carrier := newLayoutRepo(t) + target := newRegistryRepo(t, targetRegistry) + + image := pushIndex(t, sourceRegistry, testPrefix+"/tools/img", "v1", multiPlatform...) + ref := artifact.Ref{Name: "tools/img", Version: "v1"} + sourcePinned := mustResolveRef(t, source, ref) + assert.Equal(t, image.manifest, sourcePinned.ResolvedDigest) + attached := attachBundle(t, source, sourcePinned, `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json"}`) + rawRoot := rawRootBytes(t, source, sourcePinned) + + require.True(t, carrier.CanCopyFrom(source)) + hop, err := carrier.CopyFrom(ctx, source, sourcePinned, artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, image.manifest, hop.Pinned.ResolvedDigest, "the layout keeps the original index digest") + assert.Equal(t, artifact.PackagingOCIGraph, hop.Representation.Packaging) + + sourceRegistry.server.Close() + + carrierPinned := mustResolveRef(t, carrier, ref) + hop, err = target.CopyFrom(ctx, carrier, carrierPinned, artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, image.manifest, hop.Pinned.ResolvedDigest, "the second registry resolves to the same root digest") + + targetPinned := mustResolveRef(t, target, ref) + assert.Equal(t, rawRoot, rawRootBytes(t, target, targetPinned), "the original index bytes are untouched") + content, err := target.Read(ctx, targetPinned) + require.NoError(t, err) + defer content.Close() + nodes := walkGraph(t, content.Graph) + assert.GreaterOrEqual(t, nodes, 1+len(multiPlatform)*(1+1+2), "index, each platform manifest, config and two layers all arrived") + + inTarget, err := target.attachments(ctx, mustStore(t, target, ref.Name), ref.Name, targetPinnedDescriptor(t, target, targetPinned)) + require.NoError(t, err) + require.Len(t, inTarget.valid, 1, "the signature referrer travelled with the image") + assert.Equal(t, attached.Digest, inTarget.valid[0].layer.Digest) + assert.Empty(t, inTarget.diagnostics) +} + +func rawRootBytes(t *testing.T, repo *Repository, pinned artifact.PinnedReference) []byte { + t.Helper() + _, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + return loaded.raw +} + +func mustStore(t *testing.T, repo *Repository, name string) store { + t.Helper() + st, err := repo.be.open(context.Background(), name, false) + require.NoError(t, err) + return st +} + +func targetPinnedDescriptor(t *testing.T, repo *Repository, pinned artifact.PinnedReference) ocispec.Descriptor { + t.Helper() + _, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + return loaded.desc +} + +// The committed plugin release moves registry -> layout -> registry with its digests intact. +func TestPluginReleaseTransfersWithDigestsPreserved(t *testing.T) { + ctx := context.Background() + sourceRegistry := newRegistry(t, &ocitest.Credential{Username: "a", Password: "pa"}) + targetRegistry := newRegistry(t, &ocitest.Credential{Username: "b", Password: "pb"}) + source, carrier, target := newRegistryRepo(t, sourceRegistry), newLayoutRepo(t), newRegistryRepo(t, targetRegistry) + + ref := artifact.Ref{Name: "plugins/example", Version: "1.2.3"} + importPlugin(t, source, ref) + _, err := carrier.CopyFrom(ctx, source, mustResolveRef(t, source, ref), artifact.CopyFromOptions{}) + require.NoError(t, err) + sourceRegistry.server.Close() + result, err := target.CopyFrom(ctx, carrier, mustResolveRef(t, carrier, ref), artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, pluginRoot, result.Pinned.ResolvedDigest) + pinned := mustResolveRef(t, target, ref) + assert.Equal(t, pluginRoot, pinned.ResolvedDigest) + inspection, err := target.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, "example/fixture", inspection.Representation.OriginalRoot.Annotations[pluginAnnotationName]) +} + +// An Atmos package copied between OCI repositories is not repackaged: the envelope digest is +// identical too. Repacking through the service (tested elsewhere) keeps only Record identity. +func TestAtmosPackageNativeCopyKeepsTheEnvelopeDigest(t *testing.T) { + ctx := context.Background() + source, carrier := newRegistryRepo(t, newRegistry(t, nil)), newLayoutRepo(t) + published := publish(t, source, publishSpec{content: "native copy"}) + + result, err := carrier.CopyFrom(ctx, source, mustResolveRef(t, source, artifact.Ref{Name: appName, Version: appVersion}), artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, published.Pinned.ResolvedDigest, result.Pinned.ResolvedDigest) + assert.Equal(t, published.Pinned.RecordDigest, result.Pinned.RecordDigest) + assert.Equal(t, published.Pinned.ContentDigest, result.Pinned.ContentDigest) + + // By digest, a copy publishes under the Record's own name and version. + byDigest := artifact.PinnedReference{Ref: artifact.Ref{Name: appName, Digest: published.Pinned.RecordDigest}, ResolvedDigest: published.Pinned.ResolvedDigest} + other := newLayoutRepo(t) + copied, err := other.CopyFrom(ctx, source, byDigest, artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, appVersion, copied.Pinned.Ref.Version) + assert.False(t, other.CanCopyFrom(other)) + assert.False(t, other.CanCopyFrom(nil)) +} + +func TestCopyRefusesToDropReferrersItCannotList(t *testing.T) { + ctx := context.Background() + reg := newRegistry(t, nil) + healthy := newRegistryRepo(t, reg) + image := pushImage(t, reg, testPrefix+"/tools/img", "v1") + ref := artifact.Ref{Name: "tools/img", Version: "v1"} + pinned := mustResolveRef(t, healthy, ref) + attachBundle(t, healthy, pinned, `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json"}`) + + broken, err := NewRepository(ctx, "broken", registryConfig(reg.server.Host), WithTransport(noReferrers{reg.transport}), WithLockDir(t.TempDir())) + require.NoError(t, err) + destination := newLayoutRepo(t) + _, err = destination.CopyFrom(ctx, broken, mustResolveRef(t, broken, ref), artifact.CopyFromOptions{}) + require.Error(t, err, "a registry that cannot provide the signature material fails the copy") + assert.True(t, errors.Is(err, errUtils.ErrArtifactUploadFailed)) + _, err = destination.Resolve(ctx, ref) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "nothing was published, not even the unsigned image") + assert.NotEmpty(t, image.manifest) +} + +// noReferrers is a transport whose repositories fail every referrer lookup. +type noReferrers struct{ atmosoci.Transport } + +func (n noReferrers) Repository(ctx context.Context, repository atmosoci.Repository) (atmosoci.RepositoryStore, error) { + inner, err := n.Transport.Repository(ctx, repository) + if err != nil { + return nil, err + } + return brokenReferrers{inner}, nil +} + +type brokenReferrers struct{ atmosoci.RepositoryStore } + +func (brokenReferrers) Predecessors(context.Context, ocispec.Descriptor) ([]ocispec.Descriptor, error) { + return nil, errors.New("referrers are unavailable") +} + +func (brokenReferrers) Referrers(context.Context, ocispec.Descriptor, string) ([]ocispec.Descriptor, error) { + return nil, errors.New("referrers are unavailable") +} + +// A registry without the OCI 1.1 referrers API uses the tag-schema fallback, and signature +// material still travels with the image. +func TestReferrersTagSchemaFallbackStillCarriesSignatures(t *testing.T) { + ctx := context.Background() + server := ocitest.NewServerWithOptions(t, nil, ocitest.WithoutReferrersAPI()) + client := atmosoci.NewClient(nil, atmosoci.WithKeychain(credentialsKeychain{})) + reg := ®istry{server: server, recorder: &recorder{}, transport: client} + source := newRegistryRepo(t, reg) + pushImage(t, reg, testPrefix+"/tools/img", "v1") + ref := artifact.Ref{Name: "tools/img", Version: "v1"} + attached := attachBundle(t, source, mustResolveRef(t, source, ref), `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","fallback":true}`) + + carrier := newLayoutRepo(t) + _, err := carrier.CopyFrom(ctx, source, mustResolveRef(t, source, ref), artifact.CopyFromOptions{}) + require.NoError(t, err) + pinned := mustResolveRef(t, carrier, ref) + set, err := carrier.attachments(ctx, mustStore(t, carrier, ref.Name), ref.Name, targetPinnedDescriptor(t, carrier, pinned)) + require.NoError(t, err) + require.Len(t, set.valid, 1) + assert.Equal(t, attached.Digest, set.valid[0].layer.Digest) +} + +func TestSelectingAPlatformKeepsTheOriginalIndex(t *testing.T) { + ctx := context.Background() + reg := newRegistry(t, nil) + repo := newRegistryRepo(t, reg) + image := pushIndex(t, reg, testPrefix+"/tools/img", "v1", multiPlatform...) + pinned := mustResolveRef(t, repo, artifact.Ref{Name: "tools/img", Version: "v1"}) + + selected, child, err := repo.SelectPlatform(ctx, pinned, ocispec.Platform{OS: "linux", Architecture: "arm64"}) + require.NoError(t, err) + assert.Equal(t, image.manifest, selected.ResolvedDigest, "selection never reconstructs the root") + assert.Equal(t, image.children[1], child.Digest) + require.NotNil(t, selected.Platform) + + content, err := repo.Read(ctx, selected) + require.NoError(t, err) + defer content.Close() + assert.Equal(t, image.manifest, content.Graph.Root().Digest) + successors, err := content.Graph.Successors(ctx, content.Graph.Root()) + require.NoError(t, err) + require.Len(t, successors, 1, "only the selected child travels below the original index") + assert.Equal(t, image.children[1], successors[0].Digest) + assert.Equal(t, 1+4, walkGraph(t, content.Graph), "index, one manifest, its config and two layers") + + inspection, err := repo.Inspect(ctx, selected) + require.NoError(t, err) + assert.Equal(t, image.manifest, inspection.Representation.OriginalRoot.Digest) + assert.Len(t, inspection.Representation.Payload, 1) + + _, _, err = repo.SelectPlatform(ctx, pinned, ocispec.Platform{OS: "plan9", Architecture: "mips"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = newLayoutRepo(t).CopyFrom(ctx, repo, selected, artifact.CopyFromOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a native copy does not narrow an index") +} + +func TestCrossBackendCredentialsAreNotShared(t *testing.T) { + ctx := context.Background() + protected := newRegistry(t, &ocitest.Credential{Username: "u", Password: "p"}) + pushImage(t, protected, testPrefix+"/tools/img", "v1") + + anonymous := atmosoci.NewClient(nil, atmosoci.WithKeychain(credentialsKeychain{})) + repo, err := NewRepository(ctx, "wrong", registryConfig(protected.server.Host), WithTransport(anonymous), WithLockDir(t.TempDir())) + require.NoError(t, err) + _, err = repo.Resolve(ctx, artifact.Ref{Name: "tools/img", Version: "v1"}) + require.Error(t, err) + assert.NotErrorIs(t, err, errUtils.ErrArtifactNotFound, "an authentication failure is not reported as a missing artifact") + assert.Contains(t, err.Error(), "credential") +} diff --git a/pkg/artifact/sigstore/doc.go b/pkg/artifact/sigstore/doc.go new file mode 100644 index 00000000000..a0c58af0ed0 --- /dev/null +++ b/pkg/artifact/sigstore/doc.go @@ -0,0 +1,38 @@ +// Package sigstore provides the ready-to-use [artifact.Signer] and [artifact.Verifier] for Sigstore +// bundle evidence. It adapts the reusable primitives in pkg/signature to the artifact contracts, so +// composition roots can pass one verifier to every repository (for example +// local.Register(local.WithVerifier(v)) or an OCI repository) without pkg/artifact itself gaining a +// Sigstore dependency. +// +// # Subjects +// +// A signature subject is an immutable digest: the canonical portable Record of an Atmos file +// package, or the original manifest or index of a foreign OCI graph. Verification always derives +// the digest it checks from the subject the caller asked about, never from the evidence. +// +// # Statement form +// +// The Signer is digest-only, because the artifact.Signer contract does not carry subject bytes. It +// signs [SubjectStatement], a deterministic canonical JSON document naming the subject kind, +// digest, media type, and size, with the injected [signature.BlobSigner]. The resulting bundle is +// a message signature over the statement, so it works unchanged on every backend. +// +// # Raw-digest compatibility form +// +// The Verifier also accepts bundles whose signed artifact digest is the subject digest itself: +// a message signature over the subject bytes (as plugin signing and cosign sign-blob produce) or a +// DSSE in-toto statement naming the digest (as cosign OCI signatures produce). The raw form is +// tried first, then the statement form. +// +// # Key path separation +// +// Certificate (keyless) bundles are verified by [signature.Verifier] against trusted identity and +// issuer pairs. Bundles signed with a configured public key are verified by the separate +// [signature.KeyVerifier]. A public-key bundle never satisfies an identity requirement and a +// certificate bundle never satisfies a key requirement; the bundle's verification material and its +// declared evidence format must agree, or the evidence is ignored and reported. +// +// Trust roots come from the injected loader (by default the TUF-authenticated Sigstore public-good +// root); roots supplied by an artifact are never used. In offline mode the loader may use only +// cached trust material, and missing or expired trust fails verification without network access. +package sigstore diff --git a/pkg/artifact/sigstore/helpers_test.go b/pkg/artifact/sigstore/helpers_test.go new file mode 100644 index 00000000000..8dc38f19022 --- /dev/null +++ b/pkg/artifact/sigstore/helpers_test.go @@ -0,0 +1,124 @@ +package sigstore + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/signature/signaturetest" +) + +const ( + testIdentity = "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main" + testIssuer = "https://token.actions.githubusercontent.com" + keyHint = signaturetest.KeyHint +) + +// fixture is one signed-subject test setup. +type fixture struct { + t *testing.T + auth *signaturetest.Authority + data []byte + subject artifact.SignatureSubject +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + data := []byte(`{"name":"team/app","version":"1.0.0"}`) + return &fixture{ + t: t, + auth: signaturetest.New(t), + data: data, + subject: artifact.SignatureSubject{ + Kind: artifact.SubjectRecord, Digest: digest.FromBytes(data), + MediaType: artifact.MediaTypeConfig, Size: int64(len(data)), + }, + } +} + +func (f *fixture) loader(material root.TrustedMaterial) RootLoader { + return func(context.Context, bool) (root.TrustedMaterial, error) { return material, nil } +} + +func (f *fixture) verifier(opts ...VerifierOption) *Verifier { + return NewVerifier(append([]VerifierOption{WithRootLoader(f.loader(f.auth))}, opts...)...) +} + +func (f *fixture) blob(format artifact.EvidenceFormat, data []byte) artifact.EvidenceBlob { + return artifact.EvidenceBlob{ + Evidence: artifact.SignatureEvidence{ + Subject: f.subject, Format: format, + Descriptor: ocispec.Descriptor{MediaType: "application/vnd.dev.sigstore.bundle.v0.3+json", Digest: digest.FromBytes(data), Size: int64(len(data))}, + }, + Data: data, + } +} + +// keyless signs the subject bytes directly (the raw-digest compatibility form). +func (f *fixture) keyless(identity, issuer string) artifact.EvidenceBlob { + return f.blob(artifact.EvidenceSigstoreBundle, f.auth.Sign(f.t, identity, issuer, f.data)) +} + +// keylessStatement signs the subject statement through the Signer (the statement form). +func (f *fixture) keylessStatement(identity, issuer string) artifact.EvidenceBlob { + signer := NewSigner(&authoritySigner{t: f.t, auth: f.auth, identity: identity, issuer: issuer}) + payload, err := signer.Sign(context.Background(), f.subject) + require.NoError(f.t, err) + return f.blob(payload.Format, payload.Data) +} + +// keyed signs the subject bytes with a fresh key and returns the evidence and the key. +func (f *fixture) keyed() (artifact.EvidenceBlob, signaturetest.KeySignature) { + signed := f.auth.SignWithKey(f.t, f.data) + return f.blob(artifact.EvidencePublicKeyBundle, signed.Bundle), signed +} + +func keyPolicy(refs ...string) artifact.VerificationPolicy { + return artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: refs}}} +} + +func identityPolicy(identity, issuer string) artifact.VerificationPolicy { + return artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: identity, Issuer: issuer}}}}} +} + +// authoritySigner is a signature.BlobSigner backed by the isolated test authority. +type authoritySigner struct { + t *testing.T + auth *signaturetest.Authority + identity string + issuer string + // replace, when set, is signed instead of the requested bytes. + replace []byte +} + +func (a *authoritySigner) Sign(_ context.Context, data []byte) ([]byte, error) { + if a.replace != nil { + data = a.replace + } + return a.auth.Sign(a.t, a.identity, a.issuer, data), nil +} + +func codes(result *artifact.VerificationResult) []string { + out := make([]string, 0, len(result.Diagnostics)) + for _, diagnostic := range result.Diagnostics { + out = append(out, diagnostic.Code) + } + return out +} + +func pemFor(t *testing.T, key signaturetest.KeySignature) (encoded string, fingerprint string) { + t.Helper() + der, err := x509.MarshalPKIXPublicKey(key.PublicKey) + require.NoError(t, err) + sum := sha256.Sum256(der) + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), hex.EncodeToString(sum[:]) +} diff --git a/pkg/artifact/sigstore/keys.go b/pkg/artifact/sigstore/keys.go new file mode 100644 index 00000000000..112af75fecb --- /dev/null +++ b/pkg/artifact/sigstore/keys.go @@ -0,0 +1,92 @@ +package sigstore + +import ( + "crypto" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/signature" +) + +// maxKeyFileSize bounds one public-key file. +const maxKeyFileSize = 64 << 10 + +// resolvedKey is one trusted public key together with the identifiers a bundle may carry for it. +type resolvedKey struct { + fingerprint string + hints []string + key crypto.PublicKey +} + +// keySet is the trusted keys of one trust set. +type keySet struct { + keys []resolvedKey +} + +// trustedKeys expands the keys into the verifier's key list; one key is registered under each of +// its hints. +func (k *keySet) trustedKeys() []signature.TrustedKey { + var out []signature.TrustedKey + for index := range k.keys { + for _, hint := range k.keys[index].hints { + out = append(out, signature.TrustedKey{Hint: hint, PublicKey: k.keys[index].key}) + } + } + return out +} + +// fingerprintFor maps a bundle's key hint back to the fingerprint of the key that matched. +func (k *keySet) fingerprintFor(hint string) string { + for index := range k.keys { + for _, candidate := range k.keys[index].hints { + if candidate == hint { + return k.keys[index].fingerprint + } + } + } + return hint +} + +// newResolvedKey derives the fingerprint and the hint variants from a key's PKIX DER bytes. +func newResolvedKey(der []byte, key crypto.PublicKey, extraHints ...string) resolvedKey { + sum := sha256.Sum256(der) + fingerprint := hex.EncodeToString(sum[:]) + hints := []string{ + fingerprint, + base64.StdEncoding.EncodeToString(sum[:]), + base64.URLEncoding.EncodeToString(sum[:]), + base64.RawURLEncoding.EncodeToString(sum[:]), + } + hints = append(hints, extraHints...) + return resolvedKey{fingerprint: fingerprint, hints: hints, key: key} +} + +// programmaticKey builds a key registered through WithTrustedKey. +func programmaticKey(hint string, key crypto.PublicKey) (resolvedKey, error) { + der, err := x509.MarshalPKIXPublicKey(key) + if err != nil { + return resolvedKey{}, fmt.Errorf("%w: trusted key %q: %w", errUtils.ErrArtifactConfigInvalid, hint, err) + } + return newResolvedKey(der, key, hint), nil +} + +// parsePublicKeyFile decodes the first PEM "PUBLIC KEY" block of a key file. +func parsePublicKeyFile(path string, raw []byte) (resolvedKey, error) { + if len(raw) == 0 || len(raw) > maxKeyFileSize { + return resolvedKey{}, fmt.Errorf("%w: trusted key file %q must be 1 to %d bytes", errUtils.ErrArtifactConfigInvalid, path, maxKeyFileSize) + } + block, _ := pem.Decode(raw) + if block == nil || block.Type != "PUBLIC KEY" { + return resolvedKey{}, fmt.Errorf("%w: trusted key file %q is not a PEM PUBLIC KEY", errUtils.ErrArtifactConfigInvalid, path) + } + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return resolvedKey{}, fmt.Errorf("%w: trusted key file %q: %w", errUtils.ErrArtifactConfigInvalid, path, err) + } + return newResolvedKey(block.Bytes, key), nil +} diff --git a/pkg/artifact/sigstore/signer.go b/pkg/artifact/sigstore/signer.go new file mode 100644 index 00000000000..f90554c273e --- /dev/null +++ b/pkg/artifact/sigstore/signer.go @@ -0,0 +1,59 @@ +package sigstore + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/signature" +) + +// Signer implements [artifact.Signer] over an injected [signature.BlobSigner], typically the +// managed Cosign authoring path. It owns no cryptography and no registry credentials. +type Signer struct { + blob signature.BlobSigner +} + +var _ artifact.Signer = (*Signer)(nil) + +// NewSigner returns a Signer that signs subject statements with blob. +func NewSigner(blob signature.BlobSigner) *Signer { + defer perf.Track(nil, "sigstore.NewSigner")() + + return &Signer{blob: blob} +} + +// Formats implements [artifact.Signer]. The produced format follows the bundle the injected signer +// returns: a certificate yields a sigstore bundle and a configured key yields a public-key bundle. +func (s *Signer) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "sigstore.Signer.Formats")() + + return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle, artifact.EvidencePublicKeyBundle} +} + +// Sign implements [artifact.Signer]. It signs [SubjectStatement] for the subject and refuses +// output that is oversized, malformed, or bound to different bytes. +func (s *Signer) Sign(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + defer perf.Track(nil, "sigstore.Signer.Sign")() + + if s == nil || s.blob == nil { + return nil, artifact.NewUnsupportedCapability("", "sign", "signer", "no Sigstore blob signer is configured") + } + if err := subject.Validate(); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + bundle, err := signature.SignBlob(ctx, s.blob, SubjectStatement(subject)) + if err != nil { + return nil, fmt.Errorf("%w: sign %s %s: %w", errUtils.ErrArtifactVerificationFailed, subject.Kind, subject.Digest, err) + } + format, mediaType, err := EvidenceFormatOf(bundle) + if err != nil { + return nil, err + } + return &artifact.EvidencePayload{Format: format, MediaType: mediaType, Data: bundle}, nil +} diff --git a/pkg/artifact/sigstore/signer_test.go b/pkg/artifact/sigstore/signer_test.go new file mode 100644 index 00000000000..d6038634d84 --- /dev/null +++ b/pkg/artifact/sigstore/signer_test.go @@ -0,0 +1,189 @@ +package sigstore + +import ( + "context" + "errors" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/signature" +) + +func TestSigner_SignRoundTripsThroughVerifier(t *testing.T) { + f := newFixture(t) + signer := NewSigner(&authoritySigner{t: t, auth: f.auth, identity: testIdentity, issuer: testIssuer}) + for _, kind := range []artifact.SubjectKind{artifact.SubjectRecord, artifact.SubjectOCIRoot} { + t.Run(string(kind), func(t *testing.T) { + subject := f.subject + subject.Kind = kind + payload, err := signer.Sign(context.Background(), subject) + require.NoError(t, err) + assert.Equal(t, artifact.EvidenceSigstoreBundle, payload.Format) + assert.Equal(t, "application/vnd.dev.sigstore.bundle.v0.3+json", payload.MediaType) + + evidence := artifact.EvidenceBlob{ + Evidence: artifact.SignatureEvidence{Subject: subject, Format: payload.Format}, + Data: payload.Data, + } + result, err := f.verifier().Verify(context.Background(), subject, []artifact.EvidenceBlob{evidence}, identityPolicy(testIdentity, testIssuer)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + + // The statement binds kind, so the same bundle does not verify a different subject. + other := subject + other.MediaType = "application/octet-stream" + evidence.Evidence.Subject = other + _, err = f.verifier().Verify(context.Background(), other, []artifact.EvidenceBlob{evidence}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + } +} + +func TestSigner_KeyBundleIsAPublicKeyBundle(t *testing.T) { + f := newFixture(t) + statement := SubjectStatement(f.subject) + signed := f.auth.SignWithKey(t, statement) + ctrl := gomock.NewController(t) + blob := signature.NewMockBlobSigner(ctrl) + blob.EXPECT().Sign(gomock.Any(), statement).Return(signed.Bundle, nil) + + payload, err := NewSigner(blob).Sign(context.Background(), f.subject) + require.NoError(t, err) + assert.Equal(t, artifact.EvidencePublicKeyBundle, payload.Format) + + verifier := f.verifier(WithTrustedKey(keyHint, signed.PublicKey)) + result, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{{ + Evidence: artifact.SignatureEvidence{Subject: f.subject, Format: payload.Format}, + Data: payload.Data, + }}, keyPolicy(keyHint)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) +} + +func TestSigner_Failures(t *testing.T) { + f := newFixture(t) + tests := []struct { + name string + signer *Signer + subject artifact.SignatureSubject + ctx func() context.Context + wantErr error + }{ + { + name: "output bound to different bytes", + signer: NewSigner(&authoritySigner{t: t, auth: f.auth, identity: testIdentity, issuer: testIssuer, replace: []byte("other bytes")}), + subject: f.subject, + wantErr: errUtils.ErrArtifactVerificationFailed, + }, + { + name: "invalid subject", + signer: NewSigner(&authoritySigner{t: t, auth: f.auth, identity: testIdentity, issuer: testIssuer}), + subject: artifact.SignatureSubject{Kind: "bogus", Digest: digest.FromString("x")}, + wantErr: errUtils.ErrArtifactRefInvalid, + }, + { + name: "no blob signer", + signer: NewSigner(nil), + subject: f.subject, + wantErr: errUtils.ErrArtifactUnsupportedCapability, + }, + { + name: "canceled context", + signer: NewSigner(&authoritySigner{t: t, auth: f.auth, identity: testIdentity, issuer: testIssuer}), + subject: f.subject, + ctx: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }, + wantErr: context.Canceled, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + if tt.ctx != nil { + ctx = tt.ctx() + } + payload, err := tt.signer.Sign(ctx, tt.subject) + require.ErrorIs(t, err, tt.wantErr) + assert.Nil(t, payload) + }) + } + + t.Run("signer error is returned", func(t *testing.T) { + ctrl := gomock.NewController(t) + blob := signature.NewMockBlobSigner(ctrl) + boom := errors.New("cosign failed") + blob.EXPECT().Sign(gomock.Any(), gomock.Any()).Return(nil, boom) + _, err := NewSigner(blob).Sign(context.Background(), f.subject) + require.ErrorIs(t, err, boom) + }) +} + +func TestSigner_Formats(t *testing.T) { + assert.ElementsMatch(t, NewVerifier().Formats(), NewSigner(nil).Formats()) +} + +func TestSubjectStatement_IsDeterministicAndBindsEveryField(t *testing.T) { + subject := artifact.SignatureSubject{ + Kind: artifact.SubjectRecord, + Digest: digest.Digest("sha256:0000000000000000000000000000000000000000000000000000000000000000"), + MediaType: artifact.MediaTypeConfig, + Size: 42, + } + want := `{"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","kind":"record","media_type":"application/vnd.atmos.artifact.config.v1+json","size":42,"version":1}` + assert.Equal(t, want, string(SubjectStatement(subject))) + assert.Equal(t, SubjectStatement(subject), SubjectStatement(subject)) + + variants := map[string]artifact.SignatureSubject{} + for name, mutate := range map[string]func(*artifact.SignatureSubject){ + "kind": func(s *artifact.SignatureSubject) { s.Kind = artifact.SubjectOCIRoot }, + "digest": func(s *artifact.SignatureSubject) { s.Digest = digest.FromString("x") }, + "media": func(s *artifact.SignatureSubject) { s.MediaType = "x" }, + "size": func(s *artifact.SignatureSubject) { s.Size = 43 }, + } { + changed := subject + mutate(&changed) + variants[name] = changed + assert.NotEqual(t, SubjectStatement(subject), SubjectStatement(changed), name) + } +} + +func TestEvidenceFormatOf(t *testing.T) { + tests := []struct { + name string + bundle string + want artifact.EvidenceFormat + wantMedia string + wantErr bool + }{ + {name: "certificate", bundle: `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"certificate":{"rawBytes":"AA=="}}}`, want: artifact.EvidenceSigstoreBundle, wantMedia: "application/vnd.dev.sigstore.bundle.v0.3+json"}, + {name: "certificate chain defaults the media type", bundle: `{"verificationMaterial":{"x509CertificateChain":{"certificates":[]}}}`, want: artifact.EvidenceSigstoreBundle, wantMedia: defaultBundleMediaType}, + {name: "public key", bundle: `{"mediaType":"application/vnd.dev.sigstore.bundle+json;version=0.2","verificationMaterial":{"publicKey":{"hint":"k"}}}`, want: artifact.EvidencePublicKeyBundle, wantMedia: "application/vnd.dev.sigstore.bundle+json;version=0.2"}, + {name: "both", bundle: `{"verificationMaterial":{"certificate":{},"publicKey":{}}}`, wantErr: true}, + {name: "neither", bundle: `{"verificationMaterial":{}}`, wantErr: true}, + {name: "null material", bundle: `{"verificationMaterial":{"certificate":null}}`, wantErr: true}, + {name: "not json", bundle: `nope`, wantErr: true}, + {name: "empty", bundle: ``, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + format, media, err := EvidenceFormatOf([]byte(tt.bundle)) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrSignatureInvalid) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, format) + assert.Equal(t, tt.wantMedia, media) + }) + } + assert.Contains(t, defaultBundleMediaType, BundleMediaTypePrefix) +} diff --git a/pkg/artifact/sigstore/statement.go b/pkg/artifact/sigstore/statement.go new file mode 100644 index 00000000000..c6dc3995788 --- /dev/null +++ b/pkg/artifact/sigstore/statement.go @@ -0,0 +1,105 @@ +package sigstore + +import ( + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/signature" +) + +// BundleMediaTypePrefix is the prefix of every Sigstore bundle media type. +const BundleMediaTypePrefix = "application/vnd.dev.sigstore.bundle." + +// defaultBundleMediaType is used when a bundle does not declare its own media type. +const defaultBundleMediaType = BundleMediaTypePrefix + "v0.3+json" + +// statementVersion is the version of the subject statement schema. +const statementVersion = 1 + +// subjectStatement is the document the Signer signs. Field order is alphabetical by JSON key, so +// serialization is deterministic. +type subjectStatement struct { + Digest string `json:"digest"` + Kind string `json:"kind"` + MediaType string `json:"media_type"` + Size int64 `json:"size"` + Version int `json:"version"` +} + +// SubjectStatement returns the deterministic canonical JSON bytes that bind a subject's kind, +// digest, media type, and size: no whitespace and no trailing newline. +func SubjectStatement(subject artifact.SignatureSubject) []byte { + defer perf.Track(nil, "sigstore.SubjectStatement")() + + raw, err := json.Marshal(subjectStatement{ + Digest: subject.Digest.String(), + Kind: string(subject.Kind), + MediaType: subject.MediaType, + Size: subject.Size, + Version: statementVersion, + }) + if err != nil { + // A struct of strings and integers always marshals. + return nil + } + return raw +} + +// bundleShape is the part of a Sigstore bundle needed to classify it. +type bundleShape struct { + MediaType string `json:"mediaType"` + VerificationMaterial struct { + Certificate json.RawMessage `json:"certificate"` + X509CertificateChain json.RawMessage `json:"x509CertificateChain"` + PublicKey json.RawMessage `json:"publicKey"` + } `json:"verificationMaterial"` +} + +// EvidenceFormatOf classifies a Sigstore bundle by its verification material and returns the +// evidence format and the bundle's own media type. A certificate or certificate chain means a +// keyless bundle; a public key identifier means a public-key bundle. A bundle that carries neither, +// or both, is malformed. +func EvidenceFormatOf(bundle []byte) (artifact.EvidenceFormat, string, error) { + defer perf.Track(nil, "sigstore.EvidenceFormatOf")() + + if len(bundle) == 0 || int64(len(bundle)) > artifact.MaxEvidenceSize || len(bundle) > signature.MaxBundleSize { + return "", "", fmt.Errorf("%w: bundle must be 1 to %d bytes", errUtils.ErrSignatureInvalid, artifact.MaxEvidenceSize) + } + var shape bundleShape + if err := json.Unmarshal(bundle, &shape); err != nil { + return "", "", fmt.Errorf("%w: %w", errUtils.ErrSignatureInvalid, err) + } + format, err := formatOfMaterial(&shape) + if err != nil { + return "", "", err + } + mediaType := shape.MediaType + if mediaType == "" { + mediaType = defaultBundleMediaType + } + return format, mediaType, nil +} + +// formatOfMaterial maps the verification material to an evidence format. +func formatOfMaterial(shape *bundleShape) (artifact.EvidenceFormat, error) { + material := shape.VerificationMaterial + hasCertificate := isPresent(material.Certificate) || isPresent(material.X509CertificateChain) + hasKey := isPresent(material.PublicKey) + switch { + case hasCertificate && hasKey: + return "", fmt.Errorf("%w: bundle carries both a certificate and a public key", errUtils.ErrSignatureInvalid) + case hasCertificate: + return artifact.EvidenceSigstoreBundle, nil + case hasKey: + return artifact.EvidencePublicKeyBundle, nil + default: + return "", fmt.Errorf("%w: bundle carries neither a certificate nor a public key", errUtils.ErrSignatureInvalid) + } +} + +func isPresent(raw json.RawMessage) bool { + return len(raw) > 0 && string(raw) != "null" +} diff --git a/pkg/artifact/sigstore/verifier.go b/pkg/artifact/sigstore/verifier.go new file mode 100644 index 00000000000..e850225bc0d --- /dev/null +++ b/pkg/artifact/sigstore/verifier.go @@ -0,0 +1,402 @@ +package sigstore + +import ( + "context" + "crypto" + "errors" + "fmt" + "os" + + "github.com/opencontainers/go-digest" + "github.com/sigstore/sigstore-go/pkg/root" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/signature" +) + +// maxDiagnostics bounds the diagnostics one verification reports. +const maxDiagnostics = 16 + +// RootLoader supplies independently provisioned trust material. In offline mode it may use only +// cached trust and must fail otherwise. +type RootLoader func(ctx context.Context, offline bool) (root.TrustedMaterial, error) + +// Verifier implements [artifact.Verifier] for Sigstore bundle evidence. +type Verifier struct { + loadRoot RootLoader + readFile func(path string) ([]byte, error) + keys map[string]crypto.PublicKey +} + +var _ artifact.Verifier = (*Verifier)(nil) + +// VerifierOption configures a [Verifier]. +type VerifierOption func(*Verifier) + +// WithRootLoader injects the trust-root loader. The default is [signature.LoadPublicRoot], the +// TUF-authenticated Sigstore public-good root. +func WithRootLoader(loader func(ctx context.Context, offline bool) (root.TrustedMaterial, error)) VerifierOption { + defer perf.Track(nil, "sigstore.WithRootLoader")() + + return func(v *Verifier) { v.loadRoot = loader } +} + +// WithTrustedKey registers an operator-provisioned public key. A trust set refers to it by naming +// hint in its Keys list, exactly like a key file path. The bundle may identify the key by hint or +// by the SHA-256 fingerprint of its PKIX encoding. +func WithTrustedKey(hint string, key crypto.PublicKey) VerifierOption { + defer perf.Track(nil, "sigstore.WithTrustedKey")() + + return func(v *Verifier) { v.keys[hint] = key } +} + +// WithKeyFileReader injects how trusted key files named by a trust set are read. The default +// reads the operating system file. +func WithKeyFileReader(read func(path string) ([]byte, error)) VerifierOption { + defer perf.Track(nil, "sigstore.WithKeyFileReader")() + + return func(v *Verifier) { v.readFile = read } +} + +// NewVerifier returns a Verifier. +func NewVerifier(opts ...VerifierOption) *Verifier { + defer perf.Track(nil, "sigstore.NewVerifier")() + + v := &Verifier{loadRoot: signature.LoadPublicRoot, readFile: os.ReadFile, keys: map[string]crypto.PublicKey{}} + for _, opt := range opts { + if opt != nil { + opt(v) + } + } + if v.loadRoot == nil { + v.loadRoot = signature.LoadPublicRoot + } + if v.readFile == nil { + v.readFile = os.ReadFile + } + return v +} + +// Formats implements [artifact.Verifier]. +func (v *Verifier) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "sigstore.Verifier.Formats")() + + return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle, artifact.EvidencePublicKeyBundle} +} + +// session holds the state of one Verify call. +type session struct { + verifier *Verifier + subject artifact.SignatureSubject + policy artifact.VerificationPolicy + digests []string + loader RootLoader + result *artifact.VerificationResult + dropped int +} + +// candidate is evidence that passed the structural screen. +type candidate struct { + blob *artifact.EvidenceBlob + format artifact.EvidenceFormat + digest digest.Digest +} + +// Verify implements [artifact.Verifier]. Every trust set must be satisfied by at least one valid +// evidence item. Invalid, unrelated, or unsupported evidence is reported (bounded) and ignored; +// it can never override matching valid evidence, and a public-key bundle never satisfies an +// identity requirement. +func (v *Verifier) Verify(ctx context.Context, subject artifact.SignatureSubject, evidence []artifact.EvidenceBlob, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "sigstore.Verifier.Verify")() + + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, err := policy.Fingerprint(); err == nil { + result.PolicyFingerprint = fingerprint + } + if !policy.HasTrustConstraints() { + result.Status = artifact.VerificationUnchecked + return result, nil + } + s := &session{verifier: v, subject: subject, policy: policy, result: result} + s.digests = []string{subject.Digest.String(), digest.FromBytes(SubjectStatement(subject)).String()} + s.loader = memoizeLoader(v.loadRoot) + if err := ctx.Err(); err != nil { + cancelled := errors.Join(errUtils.ErrArtifactVerificationFailed, err) + return s.fail(), cancelled + } + candidates := s.screen(evidence) + return s.evaluate(ctx, candidates) +} + +// evaluate checks every trust set against the candidates. +func (s *session) evaluate(ctx context.Context, candidates []candidate) (*artifact.VerificationResult, error) { + var claims []artifact.SignerClaim + verified := map[int]bool{} + for index := range s.policy.TrustSets { + set := &s.policy.TrustSets[index] + if len(set.Identities) == 0 && len(set.Keys) == 0 { + continue + } + found, err := s.satisfy(ctx, set, candidates) + if err != nil { + wrapped := errors.Join(errUtils.ErrArtifactVerificationFailed, err) + return s.fail(), wrapped + } + claim, matched := found.claim, found.index + if matched < 0 { + s.add(artifact.SeverityError, "no-permitted-signature", fmt.Sprintf("no valid permitted signature satisfies trust set %d", index+1)) + for _, failure := range found.failures { + s.add(artifact.SeverityWarning, "verification-failed", failure) + } + wrapped := fmt.Errorf("%w: no valid permitted signature for %s", errUtils.ErrArtifactVerificationFailed, s.subject.Digest) + return s.fail(), wrapped + } + verified[matched] = true + if !containsClaim(claims, claim) { + claims = append(claims, claim) + } + } + for index := range candidates { + if verified[index] { + s.result.EvidenceDigests = append(s.result.EvidenceDigests, candidates[index].digest) + } + } + s.result.Claims = claims + s.result.Status = artifact.VerificationVerified + s.flush() + return s.result, nil +} + +// satisfaction is the outcome of checking one trust set: the matched candidate index (-1 when none +// matched), its claim, and the reasons other candidates failed. +type satisfaction struct { + claim artifact.SignerClaim + index int + failures []string +} + +// satisfy looks for one candidate that satisfies the set. It returns a non-nil error only for +// cancellation. +func (s *session) satisfy(ctx context.Context, set *artifact.TrustSet, candidates []candidate) (satisfaction, error) { + var keys *keySet + var keyErr error + if len(set.Keys) > 0 { + keys, keyErr = s.verifier.resolveKeys(set.Keys) + } + var failures []string + if keyErr != nil { + failures = append(failures, keyErr.Error()) + } + for index := range candidates { + claim, err := s.authenticate(ctx, set, keys, &candidates[index]) + if err == nil { + return satisfaction{claim: claim, index: index}, nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return satisfaction{index: -1}, ctxErr + } + if !errors.Is(err, errNotApplicable) { + failures = append(failures, err.Error()) + } + } + return satisfaction{index: -1, failures: failures}, nil +} + +// errNotApplicable marks evidence that cannot count toward a trust set at all, for example a +// public-key bundle offered to an identity-only set. +var errNotApplicable = errors.New("evidence does not apply to this trust set") + +// authenticate verifies one candidate against one trust set through the verifier that matches the +// evidence format. Identities are checked only for certificate bundles and keys only for +// public-key bundles. +func (s *session) authenticate(ctx context.Context, set *artifact.TrustSet, keys *keySet, c *candidate) (artifact.SignerClaim, error) { + switch c.format { + case artifact.EvidenceSigstoreBundle: + if len(set.Identities) == 0 { + return artifact.SignerClaim{}, errNotApplicable + } + return s.verifyKeyless(ctx, set.Identities, c) + case artifact.EvidencePublicKeyBundle: + if keys == nil || len(keys.keys) == 0 { + return artifact.SignerClaim{}, errNotApplicable + } + return s.verifyKey(ctx, keys, c) + default: + return artifact.SignerClaim{}, errNotApplicable + } +} + +func (s *session) verifyKeyless(ctx context.Context, trusted []artifact.TrustedIdentity, c *candidate) (artifact.SignerClaim, error) { + identities := make([]signature.Identity, 0, len(trusted)) + for _, identity := range trusted { + identities = append(identities, signature.Identity{Identity: identity.Identity, Issuer: identity.Issuer}) + } + verifier := signature.Verifier{LoadRoot: s.loader} + var failure error + for _, subjectDigest := range s.digests { + result, err := verifier.Verify(ctx, c.blob.Data, subjectDigest, identities, s.policy.Offline) + if err == nil { + return artifact.SignerClaim{Identity: result.Identity, Issuer: result.Issuer}, nil + } + if ctx.Err() != nil { + return artifact.SignerClaim{}, err + } + failure = err + } + return artifact.SignerClaim{}, fmt.Errorf("%s: %w", c.digest, failure) +} + +func (s *session) verifyKey(ctx context.Context, keys *keySet, c *candidate) (artifact.SignerClaim, error) { + verifier := signature.KeyVerifier{Keys: keys.trustedKeys(), LoadRoot: s.loader} + var failure error + for _, subjectDigest := range s.digests { + result, err := verifier.Verify(ctx, c.blob.Data, subjectDigest, s.policy.Offline) + if err == nil { + return artifact.SignerClaim{KeyID: keys.fingerprintFor(result.Hint)}, nil + } + if ctx.Err() != nil { + return artifact.SignerClaim{}, err + } + failure = err + } + return artifact.SignerClaim{}, fmt.Errorf("%s: %w", c.digest, failure) +} + +// resolveKeys loads the keys a trust set names: a registered programmatic key by name, otherwise a +// PEM public-key file. Any unresolvable key is an error, so a typo never silently narrows trust +// into acceptance of nothing while hiding why. Keys that do resolve are still returned. +func (v *Verifier) resolveKeys(refs []string) (*keySet, error) { + set := &keySet{} + var errs []error + for _, ref := range refs { + if key, ok := v.keys[ref]; ok { + resolved, err := programmaticKey(ref, key) + if err != nil { + errs = append(errs, err) + continue + } + set.keys = append(set.keys, resolved) + continue + } + raw, err := v.readFile(ref) + if err != nil { + errs = append(errs, fmt.Errorf("%w: read trusted key %q: %w", errUtils.ErrArtifactConfigInvalid, ref, err)) + continue + } + resolved, err := parsePublicKeyFile(ref, raw) + if err != nil { + errs = append(errs, err) + continue + } + set.keys = append(set.keys, resolved) + } + return set, errors.Join(errs...) +} + +// screen applies the structural checks and reports what it ignores. +func (s *session) screen(evidence []artifact.EvidenceBlob) []candidate { + var out []candidate + for index := range evidence { + blob := &evidence[index] + format, ok := s.screenOne(blob) + if !ok { + continue + } + out = append(out, candidate{blob: blob, format: format, digest: evidenceDigest(blob)}) + } + return out +} + +func evidenceDigest(blob *artifact.EvidenceBlob) digest.Digest { + if blob.Evidence.Descriptor.Digest != "" { + return blob.Evidence.Descriptor.Digest + } + return digest.FromBytes(blob.Data) +} + +func (s *session) screenOne(blob *artifact.EvidenceBlob) (artifact.EvidenceFormat, bool) { + held := blob.Evidence + switch { + case held.Subject.Kind != s.subject.Kind || held.Subject.Digest != s.subject.Digest: + s.add(artifact.SeverityWarning, "subject-mismatch", "evidence bound to a different subject was ignored") + case held.Format != artifact.EvidenceSigstoreBundle && held.Format != artifact.EvidencePublicKeyBundle: + s.add(artifact.SeverityWarning, "unsupported-evidence", fmt.Sprintf("evidence in unsupported format %q was ignored", held.Format)) + case held.Descriptor.Digest != "" && held.Descriptor.Digest != digest.FromBytes(blob.Data): + s.add(artifact.SeverityWarning, "malformed-evidence", "evidence bytes do not match their descriptor and were ignored") + default: + return s.classify(blob) + } + return "", false +} + +func (s *session) classify(blob *artifact.EvidenceBlob) (artifact.EvidenceFormat, bool) { + sniffed, _, err := EvidenceFormatOf(blob.Data) + if err != nil { + s.add(artifact.SeverityWarning, "malformed-evidence", "malformed evidence was ignored: "+err.Error()) + return "", false + } + if sniffed != blob.Evidence.Format { + s.add(artifact.SeverityWarning, "format-mismatch", fmt.Sprintf("evidence declared as %q carries %q verification material and was ignored", blob.Evidence.Format, sniffed)) + return "", false + } + return sniffed, true +} + +// add records a bounded diagnostic. +func (s *session) add(severity artifact.Severity, code, message string) { + if len(s.result.Diagnostics) >= maxDiagnostics { + s.dropped++ + return + } + s.result.Diagnostics = append(s.result.Diagnostics, artifact.Diagnostic{Severity: severity, Code: code, Message: message}) +} + +// flush appends the truncation summary when diagnostics were dropped. +func (s *session) flush() { + if s.dropped > 0 { + s.result.Diagnostics = append(s.result.Diagnostics, artifact.Diagnostic{ + Severity: artifact.SeverityWarning, Code: "diagnostics-truncated", + Message: fmt.Sprintf("%d further diagnostics were omitted", s.dropped), + }) + s.dropped = 0 + } +} + +// fail finalizes a failed result. +func (s *session) fail() *artifact.VerificationResult { + s.result.Status = artifact.VerificationFailed + s.result.Claims = nil + s.result.EvidenceDigests = nil + s.flush() + return s.result +} + +func containsClaim(claims []artifact.SignerClaim, claim artifact.SignerClaim) bool { + for _, have := range claims { + if have == claim { + return true + } + } + return false +} + +// memoizeLoader loads trust material at most once per offline mode during one verification. +func memoizeLoader(load RootLoader) RootLoader { + type loaded struct { + material root.TrustedMaterial + err error + } + cache := map[bool]*loaded{} + return func(ctx context.Context, offline bool) (root.TrustedMaterial, error) { + if hit, ok := cache[offline]; ok { + return hit.material, hit.err + } + material, err := load(ctx, offline) + if ctx.Err() == nil { + cache[offline] = &loaded{material: material, err: err} + } + return material, err + } +} diff --git a/pkg/artifact/sigstore/verifier_test.go b/pkg/artifact/sigstore/verifier_test.go new file mode 100644 index 00000000000..a9646d54d7b --- /dev/null +++ b/pkg/artifact/sigstore/verifier_test.go @@ -0,0 +1,466 @@ +package sigstore + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/sigstore/sigstore-go/pkg/root" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +var _ = artifact.SignatureSubject{Kind: artifact.SubjectRecord} + +func TestVerify_Matrix(t *testing.T) { + tests := []struct { + name string + policy func(f *fixture) artifact.VerificationPolicy + evidence func(f *fixture) []artifact.EvidenceBlob + opts func(f *fixture) []VerifierOption + wantStatus artifact.VerificationStatus + wantCodes []string + wantClaim artifact.SignerClaim + }{ + { + name: "keyless message signature over the subject digest", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)} + }, + wantStatus: artifact.VerificationVerified, + wantClaim: artifact.SignerClaim{Identity: testIdentity, Issuer: testIssuer}, + }, + { + name: "keyless statement form from the Signer", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keylessStatement(testIdentity, testIssuer)} + }, + wantStatus: artifact.VerificationVerified, + wantClaim: artifact.SignerClaim{Identity: testIdentity, Issuer: testIssuer}, + }, + { + name: "DSSE in-toto statement naming the subject digest", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.blob(artifact.EvidenceSigstoreBundle, f.auth.SignStatement(f.t, testIdentity, testIssuer, f.data))} + }, + wantStatus: artifact.VerificationVerified, + wantClaim: artifact.SignerClaim{Identity: testIdentity, Issuer: testIssuer}, + }, + { + name: "identity wildcard", + policy: func(*fixture) artifact.VerificationPolicy { + return identityPolicy("https://github.com/acme/*", testIssuer) + }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)} + }, + wantStatus: artifact.VerificationVerified, + wantClaim: artifact.SignerClaim{Identity: testIdentity, Issuer: testIssuer}, + }, + { + name: "no evidence", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(*fixture) []artifact.EvidenceBlob { return nil }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature"}, + }, + { + name: "wrong issuer", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keyless(testIdentity, "https://evil.example.com")} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature", "verification-failed"}, + }, + { + name: "policy issuer differs from the signed issuer", + policy: func(*fixture) artifact.VerificationPolicy { + return identityPolicy(testIdentity, "https://other.example.com") + }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature", "verification-failed"}, + }, + { + name: "wrong identity", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.keyless("https://github.com/evil/repo/.github/workflows/x.yml@refs/heads/main", testIssuer)} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature", "verification-failed"}, + }, + { + name: "signature over different bytes is wrong for the subject", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.blob(artifact.EvidenceSigstoreBundle, f.auth.Sign(f.t, testIdentity, testIssuer, []byte("another record")))} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature", "verification-failed"}, + }, + { + name: "tampered bundle", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + tampered := tamperSignature(f.t, f.auth.Sign(f.t, testIdentity, testIssuer, f.data)) + return []artifact.EvidenceBlob{f.blob(artifact.EvidenceSigstoreBundle, tampered)} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"no-permitted-signature", "verification-failed"}, + }, + { + name: "evidence bound to another subject is ignored", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + other := f.keyless(testIdentity, testIssuer) + other.Evidence.Subject.Digest = digest.FromString("another subject") + return []artifact.EvidenceBlob{other} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"subject-mismatch", "no-permitted-signature"}, + }, + { + name: "evidence of another subject kind is ignored", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + other := f.keyless(testIdentity, testIssuer) + other.Evidence.Subject.Kind = artifact.SubjectOCIRoot + return []artifact.EvidenceBlob{other} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"subject-mismatch", "no-permitted-signature"}, + }, + { + name: "declared format that disagrees with the verification material", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + return []artifact.EvidenceBlob{f.blob(artifact.EvidencePublicKeyBundle, f.auth.Sign(f.t, testIdentity, testIssuer, f.data))} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"format-mismatch", "no-permitted-signature"}, + }, + { + name: "descriptor digest that does not match the bytes", + policy: func(*fixture) artifact.VerificationPolicy { return identityPolicy(testIdentity, testIssuer) }, + evidence: func(f *fixture) []artifact.EvidenceBlob { + item := f.keyless(testIdentity, testIssuer) + item.Evidence.Descriptor.Digest = digest.FromString("something else") + return []artifact.EvidenceBlob{item} + }, + wantStatus: artifact.VerificationFailed, + wantCodes: []string{"malformed-evidence", "no-permitted-signature"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newFixture(t) + var opts []VerifierOption + if tt.opts != nil { + opts = tt.opts(f) + } + result, err := f.verifier(opts...).Verify(context.Background(), f.subject, tt.evidence(f), tt.policy(f)) + require.NotNil(t, result) + assert.Equal(t, tt.wantStatus, result.Status) + assert.NotEmpty(t, result.PolicyFingerprint) + assert.Equal(t, f.subject, result.Subject) + if tt.wantStatus == artifact.VerificationVerified { + require.NoError(t, err) + require.Len(t, result.Claims, 1) + assert.Equal(t, tt.wantClaim, result.Claims[0]) + assert.Len(t, result.EvidenceDigests, 1) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Empty(t, result.Claims) + for _, code := range tt.wantCodes { + assert.Contains(t, codes(result), code) + } + }) + } +} + +func TestVerify_PublicKeyPath(t *testing.T) { + t.Run("configured key verifies and never satisfies an identity policy", func(t *testing.T) { + f := newFixture(t) + item, signed := f.keyed() + verifier := f.verifier(WithTrustedKey(keyHint, signed.PublicKey)) + + result, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, keyPolicy(keyHint)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + require.Len(t, result.Claims, 1) + assert.Empty(t, result.Claims[0].Identity, "a key never produces an identity claim") + assert.NotEmpty(t, result.Claims[0].KeyID) + + result, err = verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + }) + + t.Run("a key-only identity cannot be forged by naming it in the identity", func(t *testing.T) { + f := newFixture(t) + item, signed := f.keyed() + verifier := f.verifier(WithTrustedKey(keyHint, signed.PublicKey)) + _, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, identityPolicy(keyHint, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + + t.Run("a keyless bundle never satisfies a key policy", func(t *testing.T) { + f := newFixture(t) + _, signed := f.keyed() + verifier := f.verifier(WithTrustedKey(keyHint, signed.PublicKey)) + result, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, keyPolicy(keyHint)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + }) + + t.Run("an untrusted key fails", func(t *testing.T) { + f := newFixture(t) + item, _ := f.keyed() + _, trusted := f.keyed() + verifier := f.verifier(WithTrustedKey(keyHint, trusted.PublicKey)) + result, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, keyPolicy(keyHint)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + }) + + t.Run("an unregistered key reference fails with the reason", func(t *testing.T) { + f := newFixture(t) + item, _ := f.keyed() + reader := func(string) ([]byte, error) { return nil, os.ErrNotExist } + result, err := f.verifier(WithKeyFileReader(reader)).Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, keyPolicy("missing.pub")) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Contains(t, fmt.Sprint(result.Diagnostics), "missing.pub") + }) + + t.Run("public key file loaded from disk by the default reader", func(t *testing.T) { + f := newFixture(t) + item, signed := f.keyed() + encoded, fingerprint := pemFor(t, signed) + path := filepath.Join(t.TempDir(), "trusted.pub") + require.NoError(t, os.WriteFile(path, []byte(encoded), 0o600)) + // Bundles from real signers identify the key by its fingerprint. + item.Data = bytes.Replace(item.Data, []byte(keyHint), []byte(fingerprint), 1) + item.Evidence.Descriptor.Digest = digest.FromBytes(item.Data) + + result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, keyPolicy(path)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + assert.Equal(t, fingerprint, result.Claims[0].KeyID) + }) + + t.Run("malformed key file is reported", func(t *testing.T) { + f := newFixture(t) + item, _ := f.keyed() + path := filepath.Join(t.TempDir(), "bad.pub") + require.NoError(t, os.WriteFile(path, []byte("not pem"), 0o600)) + result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{item}, keyPolicy(path)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Contains(t, fmt.Sprint(result.Diagnostics), "not a PEM PUBLIC KEY") + }) +} + +func TestVerify_TrustSetsAreAllRequired(t *testing.T) { + f := newFixture(t) + item, signed := f.keyed() + keyless := f.keyless(testIdentity, testIssuer) + verifier := f.verifier(WithTrustedKey(keyHint, signed.PublicKey)) + policy := artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: testIdentity, Issuer: testIssuer}}}, + {Keys: []string{keyHint}}, + }} + + result, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{keyless, item}, policy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + assert.Len(t, result.Claims, 2) + assert.Len(t, result.EvidenceDigests, 2) + + for name, held := range map[string][]artifact.EvidenceBlob{"only keyless": {keyless}, "only key": {item}} { + t.Run(name, func(t *testing.T) { + result, err := verifier.Verify(context.Background(), f.subject, held, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + }) + } + + t.Run("one set with identities and keys accepts either signer", func(t *testing.T) { + either := artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{ + Identities: []artifact.TrustedIdentity{{Identity: testIdentity, Issuer: testIssuer}}, + Keys: []string{keyHint}, + }}} + for _, held := range [][]artifact.EvidenceBlob{{keyless}, {item}} { + result, err := verifier.Verify(context.Background(), f.subject, held, either) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + } + }) +} + +func TestVerify_TrustMaterial(t *testing.T) { + tests := []struct { + name string + loader RootLoader + offline bool + message string + }{ + { + name: "offline with no cached trust", + loader: func(context.Context, bool) (root.TrustedMaterial, error) { return nil, errUtils.ErrSignatureOffline }, + offline: true, + message: errUtils.ErrSignatureOffline.Error(), + }, + { + name: "expired trust material", + loader: func(context.Context, bool) (root.TrustedMaterial, error) { + return nil, errors.New("trusted root metadata expired") + }, + message: "trusted root metadata expired", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newFixture(t) + policy := identityPolicy(testIdentity, testIssuer) + policy.Offline = tt.offline + result, err := NewVerifier(WithRootLoader(tt.loader)).Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, policy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + assert.Contains(t, fmt.Sprint(result.Diagnostics), tt.message) + }) + } + + t.Run("trust from a different authority does not verify", func(t *testing.T) { + f := newFixture(t) + other := newFixture(t) + verifier := NewVerifier(WithRootLoader(f.loader(other.auth))) + _, err := verifier.Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + + t.Run("the offline flag reaches the loader", func(t *testing.T) { + f := newFixture(t) + var seen []bool + loader := func(_ context.Context, offline bool) (root.TrustedMaterial, error) { + seen = append(seen, offline) + return f.auth, nil + } + policy := identityPolicy(testIdentity, testIssuer) + policy.Offline = true + _, err := NewVerifier(WithRootLoader(loader)).Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, policy) + require.NoError(t, err) + require.NotEmpty(t, seen) + for _, offline := range seen { + assert.True(t, offline) + } + }) +} + +func TestVerify_UnrelatedEvidenceNeverOverridesValidEvidence(t *testing.T) { + f := newFixture(t) + valid := f.keyless(testIdentity, testIssuer) + malformed := f.blob(artifact.EvidenceSigstoreBundle, []byte("{not json")) + unsupported := f.blob("cosign-tag-payload", []byte("legacy")) + foreign := f.keyless("https://github.com/evil/repo/.github/workflows/x.yml@refs/heads/main", testIssuer) + evidence := []artifact.EvidenceBlob{malformed, unsupported, foreign, valid} + + result, err := f.verifier().Verify(context.Background(), f.subject, evidence, identityPolicy(testIdentity, testIssuer)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + assert.Equal(t, []digestValue{valid.Evidence.Descriptor.Digest}, toValues(result.EvidenceDigests)) + assert.Contains(t, codes(result), "malformed-evidence") + assert.Contains(t, codes(result), "unsupported-evidence") + + t.Run("unsupported-only evidence fails closed", func(t *testing.T) { + result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{unsupported}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) + }) + + t.Run("diagnostics are bounded", func(t *testing.T) { + many := make([]artifact.EvidenceBlob, 0, 60) + for index := range 59 { + many = append(many, f.blob(artifact.EvidenceSigstoreBundle, fmt.Appendf(nil, "{bad %d", index))) + } + many = append(many, valid) + result, err := f.verifier().Verify(context.Background(), f.subject, many, identityPolicy(testIdentity, testIssuer)) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, result.Status) + assert.Len(t, result.Diagnostics, maxDiagnostics+1) + assert.Equal(t, "diagnostics-truncated", result.Diagnostics[len(result.Diagnostics)-1].Code) + }) + + t.Run("oversize evidence is malformed", func(t *testing.T) { + huge := f.blob(artifact.EvidenceSigstoreBundle, bytes.Repeat([]byte("x"), int(artifact.MaxEvidenceSize)+1)) + result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{huge, valid}, identityPolicy(testIdentity, testIssuer)) + require.NoError(t, err) + assert.Contains(t, codes(result), "malformed-evidence") + }) +} + +type digestValue = digest.Digest + +func toValues(in []digest.Digest) []digestValue { return in } + +func TestVerify_NoTrustConstraintsIsUnchecked(t *testing.T) { + f := newFixture(t) + for _, policy := range []artifact.VerificationPolicy{{}, {RequireSignature: true}, {TrustSets: []artifact.TrustSet{{}}}} { + result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, policy) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationUnchecked, result.Status, "nothing is claimed without trust configuration") + assert.Empty(t, result.Claims) + } +} + +func TestVerify_CanceledContext(t *testing.T) { + f := newFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result, err := f.verifier().Verify(ctx, f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) +} + +func TestVerifier_Formats(t *testing.T) { + assert.ElementsMatch(t, []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle, artifact.EvidencePublicKeyBundle}, NewVerifier().Formats()) +} + +func TestNewVerifier_DefaultsAndNilOptions(t *testing.T) { + verifier := NewVerifier(nil, WithRootLoader(nil), WithKeyFileReader(nil)) + require.NotNil(t, verifier.loadRoot) + require.NotNil(t, verifier.readFile) +} + +// tamperSignature flips one byte of the bundle's message signature and re-encodes the bundle. +func tamperSignature(t *testing.T, bundle []byte) []byte { + t.Helper() + var parsed map[string]any + require.NoError(t, json.Unmarshal(bundle, &parsed)) + message, ok := parsed["messageSignature"].(map[string]any) + require.True(t, ok) + raw, err := base64.StdEncoding.DecodeString(message["signature"].(string)) + require.NoError(t, err) + raw[len(raw)-1] ^= 0xff + message["signature"] = base64.StdEncoding.EncodeToString(raw) + out, err := json.Marshal(parsed) + require.NoError(t, err) + return out +} diff --git a/pkg/oci/ocitest/registry.go b/pkg/oci/ocitest/registry.go index 48854f61491..3f5fca763b0 100644 --- a/pkg/oci/ocitest/registry.go +++ b/pkg/oci/ocitest/registry.go @@ -6,6 +6,7 @@ import ( "archive/tar" "archive/zip" "bytes" + "encoding/json" "io" "log" "net/http" @@ -164,6 +165,32 @@ type Server struct { server *httptest.Server } +// ServerOption customizes a test registry. +type ServerOption func(*serverConfig) + +type serverConfig struct { + tagsFollowManifestDelete bool + referrers bool +} + +// WithTagsFollowManifestDelete makes deleting a manifest by digest also delete every tag that +// resolves to it, like a production registry, where a tag whose manifest is gone no longer +// resolves. The bare in-process registry keeps such tags, which would hide a delete that is not +// tag-safe. +// +//nolint:lintroller // Functional option only. +func WithTagsFollowManifestDelete() ServerOption { + return func(config *serverConfig) { config.tagsFollowManifestDelete = true } +} + +// WithoutReferrersAPI turns off the OCI 1.1 referrers API, so clients must use the referrers +// tag-schema fallback. +// +//nolint:lintroller // Functional option only. +func WithoutReferrersAPI() ServerOption { + return func(config *serverConfig) { config.referrers = false } +} + // NewServer starts an empty in-process OCI registry with the native OCI 1.1 referrers API enabled. // When credential is non-nil, every request must carry exactly that Basic identity; any other // request receives a 401 challenge, so a client that downgraded to anonymous or reused @@ -173,7 +200,7 @@ func NewServer(t *testing.T, credential *Credential) *Server { t.Helper() - return newServer(t, credential, true) + return NewServerWithOptions(t, credential) } // NewServerWithoutReferrers is NewServer for a registry without the OCI 1.1 referrers API, so attachment @@ -183,13 +210,23 @@ func NewServerWithoutReferrers(t *testing.T, credential *Credential) *Server { t.Helper() - return newServer(t, credential, false) + return NewServerWithOptions(t, credential, WithoutReferrersAPI()) } -func newServer(t *testing.T, credential *Credential, referrers bool) *Server { +// NewServerWithOptions is NewServer with registry behavior options. +func NewServerWithOptions(t *testing.T, credential *Credential, options ...ServerOption) *Server { + defer perf.Track(nil, "ocitest.NewServerWithOptions")() + t.Helper() - handler := registry.New(registry.Logger(log.New(io.Discard, "", 0)), registry.WithReferrersSupport(referrers)) + config := serverConfig{referrers: true} + for _, option := range options { + option(&config) + } + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0)), registry.WithReferrersSupport(config.referrers)) + if config.tagsFollowManifestDelete { + handler = tagsFollowDelete{next: handler} + } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if credential != nil { user, password, ok := r.BasicAuth() @@ -205,6 +242,42 @@ func newServer(t *testing.T, credential *Credential, referrers bool) *Server { return &Server{Host: strings.TrimPrefix(srv.URL, "http://"), server: srv} } +// tagsFollowDelete deletes the tags of a manifest together with the manifest itself. +type tagsFollowDelete struct{ next http.Handler } + +//nolint:lintroller // Test-registry request wrapper. +func (d tagsFollowDelete) ServeHTTP(w http.ResponseWriter, r *http.Request) { + repository, reference, isManifest := splitManifestPath(r.URL.Path) + if r.Method != http.MethodDelete || !isManifest || !strings.HasPrefix(reference, "sha256:") { + d.next.ServeHTTP(w, r) + return + } + tags := httptest.NewRecorder() + d.next.ServeHTTP(tags, httptest.NewRequest(http.MethodGet, "/v2/"+repository+"/tags/list", nil)) + var listing struct { + Tags []string `json:"tags"` + } + _ = json.Unmarshal(tags.Body.Bytes(), &listing) + for _, tag := range listing.Tags { + head := httptest.NewRecorder() + d.next.ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/v2/"+repository+"/manifests/"+tag, nil)) + if head.Header().Get("Docker-Content-Digest") == reference { + d.next.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodDelete, "/v2/"+repository+"/manifests/"+tag, nil)) + } + } + d.next.ServeHTTP(w, r) +} + +// splitManifestPath splits /v2//manifests/. +func splitManifestPath(path string) (repository, reference string, ok bool) { + trimmed := strings.TrimPrefix(path, "/v2/") + index := strings.LastIndex(trimmed, "/manifests/") + if trimmed == path || index < 0 { + return "", "", false + } + return trimmed[:index], trimmed[index+len("/manifests/"):], true +} + // Close shuts the registry down now; calling it again is harmless. func (s *Server) Close() { defer perf.Track(nil, "ocitest.Server.Close")() diff --git a/pkg/oci/status.go b/pkg/oci/status.go index 4cfec1bc969..d1ac10b97b7 100644 --- a/pkg/oci/status.go +++ b/pkg/oci/status.go @@ -9,6 +9,14 @@ import ( "oras.land/oras-go/v2/registry/remote/errcode" ) +// IsAlreadyExists reports whether a push found the identical content already stored, which +// content addressing makes a success for the writer. +// +//nolint:lintroller // Trivial accessor, pure lookup, factory option, or delegation; exempt under repository tracking policy. +func IsAlreadyExists(err error) bool { + return errors.Is(err, errdef.ErrAlreadyExists) +} + // IsNotFound distinguishes a missing manifest from auth and transport failures. // It recognizes errors from both the legacy and the ORAS-backed transport. // diff --git a/pkg/oci/transport_catalog.go b/pkg/oci/transport_catalog.go new file mode 100644 index 00000000000..35d9081a7e5 --- /dev/null +++ b/pkg/oci/transport_catalog.go @@ -0,0 +1,58 @@ +package oci + +import ( + "context" + "net/http" + + "github.com/google/go-containerregistry/pkg/name" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// RegistryCatalog lists the repositories a registry exposes through its repository catalog +// (GET /v2/_catalog). Many registries do not implement it; callers must treat a failure as +// "enumeration is unsupported" and never as an empty registry. +type RegistryCatalog interface { + // Repositories lists every repository name the registry reports, in registry order. + Repositories(ctx context.Context, registry string) ([]string, error) +} + +// Repositories lists the repositories of a registry with credentials resolved up front, exactly +// like Repository. A rejected or unsupported catalog request is returned as an error. +func (c *Client) Repositories(ctx context.Context, registry string) ([]string, error) { + defer perf.Track(c.config, "oci.Client.Repositories")() + + resource, err := name.NewRegistry(registry) + if err != nil { + return nil, wrapRegistry(err) + } + authenticator, err := c.authentication(ctx, resource) + if err != nil { + return nil, err + } + host := orasHost(resource.RegistryStr()) + remoteRegistry, err := remote.NewRegistry(host) + if err != nil { + return nil, wrapRegistry(err) + } + remoteRegistry.PlainHTTP = resource.Scheme() == "http" + remoteRegistry.Client = &auth.Client{ + Client: &http.Client{Transport: newRegistryTransport(c.transport)}, + Header: http.Header{"User-Agent": {userAgent}}, + Cache: auth.NewCache(), + Credential: hostScopedCredential(host, authenticator), + } + var repositories []string + err = remoteRegistry.Repositories(ctx, "", func(page []string) error { + repositories = append(repositories, page...) + return nil + }) + if err != nil { + return nil, wrapRegistry(err) + } + return repositories, nil +} + +var _ RegistryCatalog = (*Client)(nil) diff --git a/pkg/oci/transport_catalog_test.go b/pkg/oci/transport_catalog_test.go new file mode 100644 index 00000000000..4fdc2a23bdd --- /dev/null +++ b/pkg/oci/transport_catalog_test.go @@ -0,0 +1,147 @@ +package oci_test + +import ( + "bytes" + "context" + "errors" + "strconv" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/errdef" + + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" +) + +// emptyManifest pushes a minimal artifact manifest under reference and returns its descriptor. +func pushManifest(t *testing.T, store oci.RepositoryStore, reference, title string) ocispec.Descriptor { + t.Helper() + config := oci.Descriptor(ocispec.MediaTypeEmptyJSON, []byte("{}")) + exists, err := store.Exists(t.Context(), config) + require.NoError(t, err) + if !exists { + require.NoError(t, store.Push(t.Context(), config, bytes.NewReader([]byte("{}")))) + } + manifest := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/vnd.example","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"` + config.Digest.String() + `","size":2},"layers":[],"annotations":{"title":"` + title + `"}}`) + descriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, manifest) + require.NoError(t, store.PushReference(t.Context(), descriptor, bytes.NewReader(manifest), reference)) + return descriptor +} + +func TestClientRepositoriesListsTheRegistryCatalogWithCredentials(t *testing.T) { + server, client := authenticatedRegistry(t, "secret") + for _, path := range []string{"acme/one", "acme/two/deep", "other/three"} { + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: path}) + require.NoError(t, err) + pushManifest(t, store, "v1", path) + } + var catalog oci.RegistryCatalog = client + repositories, err := catalog.Repositories(t.Context(), server.Host) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"acme/one", "acme/two/deep", "other/three"}, repositories) + + anonymous := oci.NewClient(nil, oci.WithKeychain(authn.NewMultiKeychain())) + _, err = anonymous.Repositories(t.Context(), server.Host) + require.Error(t, err, "a missing credential is an error, never an empty catalog") + + _, err = client.Repositories(t.Context(), "not a registry!") + require.Error(t, err) + canceled, cancel := context.WithCancel(t.Context()) + cancel() + _, err = client.Repositories(canceled, server.Host) + require.Error(t, err) +} + +func TestLayoutUntagRemovesOnlyTheReference(t *testing.T) { + directory := t.TempDir() + layout, err := oci.OpenLayout(t.Context(), directory) + require.NoError(t, err) + body := []byte("blob") + blob := oci.Descriptor("application/octet-stream", body) + require.NoError(t, layout.Push(t.Context(), blob, bytes.NewReader(body))) + config := oci.Descriptor(ocispec.MediaTypeEmptyJSON, []byte("{}")) + require.NoError(t, layout.Push(t.Context(), config, bytes.NewReader([]byte("{}")))) + manifest := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"` + config.Digest.String() + `","size":2},"layers":[{"mediaType":"application/octet-stream","digest":"` + blob.Digest.String() + `","size":4}]}`) + root := oci.Descriptor(ocispec.MediaTypeImageManifest, manifest) + require.NoError(t, layout.Push(t.Context(), root, bytes.NewReader(manifest))) + require.NoError(t, layout.Tag(t.Context(), root, "a:1")) + require.NoError(t, layout.Tag(t.Context(), root, "a:2")) + assert.Equal(t, []string{"a:1", "a:2"}, layout.References()) + + require.NoError(t, layout.Untag(t.Context(), "a:1")) + assert.Equal(t, []string{"a:2"}, layout.References()) + _, err = layout.Resolve(t.Context(), "a:1") + require.ErrorIs(t, err, errdef.ErrNotFound) + resolved, err := layout.Resolve(t.Context(), "a:2") + require.NoError(t, err) + assert.Equal(t, root.Digest, resolved.Digest) + exists, err := layout.Exists(t.Context(), blob) + require.NoError(t, err) + assert.True(t, exists, "blobs are never deleted: they may be shared") + + reopened, err := oci.OpenLayout(t.Context(), directory) + require.NoError(t, err) + assert.Equal(t, []string{"a:2"}, reopened.References(), "the removal was persisted") + require.Error(t, layout.Untag(t.Context(), "a:1"), "an absent reference cannot be untagged") + require.Error(t, layout.Untag(t.Context(), root.Digest.String()), "a digest is not a tag") + require.Error(t, layout.Untag(t.Context(), "")) +} + +func TestIsAlreadyExists(t *testing.T) { + assert.True(t, oci.IsAlreadyExists(errdef.ErrAlreadyExists)) + assert.True(t, oci.IsAlreadyExists(errors.Join(errors.New("push"), errdef.ErrAlreadyExists))) + assert.False(t, oci.IsAlreadyExists(errdef.ErrNotFound)) + assert.False(t, oci.IsAlreadyExists(nil)) +} + +func TestTestRegistryOptions(t *testing.T) { + t.Run("tags follow a manifest delete like a production registry", func(t *testing.T) { + server := ocitest.NewServerWithOptions(t, nil, ocitest.WithTagsFollowManifestDelete()) + client := oci.NewClient(nil, oci.WithKeychain(authn.NewMultiKeychain())) + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: "acme/app"}) + require.NoError(t, err) + first := pushManifest(t, store, "v1", "first") + require.NoError(t, store.Tag(t.Context(), first, "stable")) + kept := pushManifest(t, store, "v2", "second") + + require.NoError(t, store.Delete(t.Context(), first)) + for _, tag := range []string{"v1", "stable"} { + _, err = store.Resolve(t.Context(), tag) + require.ErrorIs(t, err, errdef.ErrNotFound, tag) + } + resolved, err := store.Resolve(t.Context(), "v2") + require.NoError(t, err) + assert.Equal(t, kept.Digest, resolved.Digest) + }) + t.Run("the bare registry keeps tags of a deleted manifest", func(t *testing.T) { + server := ocitest.NewServer(t, nil) + client := oci.NewClient(nil, oci.WithKeychain(authn.NewMultiKeychain())) + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: "acme/app"}) + require.NoError(t, err) + first := pushManifest(t, store, "v1", "first") + require.NoError(t, store.Delete(t.Context(), first)) + _, err = store.Resolve(t.Context(), "v1") + require.NoError(t, err) + }) + t.Run("referrers fall back to the tag schema", func(t *testing.T) { + server := ocitest.NewServerWithOptions(t, nil, ocitest.WithoutReferrersAPI()) + client := oci.NewClient(nil, oci.WithKeychain(authn.NewMultiKeychain())) + store, err := client.Repository(t.Context(), oci.Repository{Registry: server.Host, Path: "acme/app"}) + require.NoError(t, err) + subject := pushManifest(t, store, "v1", "subject") + attachment := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/vnd.example.sig","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"` + digest.FromString("{}").String() + `","size":2},"layers":[],"subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"` + subject.Digest.String() + `","size":` + itoa(subject.Size) + `}}`) + descriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, attachment) + require.NoError(t, store.Push(t.Context(), descriptor, bytes.NewReader(attachment))) + found, err := store.Referrers(t.Context(), subject, "") + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equal(t, descriptor.Digest, found[0].Digest) + }) +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index a009fd1fd73..4820d605e2f 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -322,6 +322,26 @@ func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []oci return append(entries, named) } +// Untag removes a named root from index.json without touching blobs, which may be shared by +// other references. Digest references are implicit and cannot be untagged. +func (l *Layout) Untag(ctx context.Context, reference string) error { + defer perf.Track(nil, "oci.Layout.Untag")() + + if err := l.store.Untag(ctx, reference); err != nil { + return layoutError(err) + } + l.mu.Lock() + defer l.mu.Unlock() + kept := l.index[:0:0] + for i := range l.index { + if l.index[i].Annotations[ocispec.AnnotationRefName] != reference { + kept = append(kept, l.index[i]) + } + } + l.index = kept + return l.writeIndex() +} + // References returns the named roots in index.json in insertion order. func (l *Layout) References() []string { defer perf.Track(nil, "oci.Layout.References")() From ef0a81ceb17f24511b089c66f0768511d5cf3cd8 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 15:41:41 -0600 Subject: [PATCH 04/53] fix: avoid overflow in S3 artifact pagination --- pkg/artifact/s3/list_test.go | 6 ++++++ pkg/artifact/s3/resolve.go | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/s3/list_test.go b/pkg/artifact/s3/list_test.go index a332717e06a..576fd9c1b9a 100644 --- a/pkg/artifact/s3/list_test.go +++ b/pkg/artifact/s3/list_test.go @@ -3,6 +3,7 @@ package s3 import ( "context" "fmt" + "math" "sort" "strings" "testing" @@ -66,6 +67,11 @@ func TestList_PaginatesInLexicalOrderAndFiltersByLabel(t *testing.T) { assert.Equal(t, "a/app:10", page.Entries[0].Name+":"+page.Entries[0].Version) assert.NotEmpty(t, page.NextCursor, "the label filter pages too") + large, err := repo.List(context.Background(), artifact.ListQuery{PageSize: math.MaxInt}) + require.NoError(t, err) + assert.Len(t, large.Entries, 5) + assert.Empty(t, large.NextCursor) + named, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) require.NoError(t, err) assert.Len(t, named.Entries, 3) diff --git a/pkg/artifact/s3/resolve.go b/pkg/artifact/s3/resolve.go index 3d6b4e08bd3..ed1d4cb67a9 100644 --- a/pkg/artifact/s3/resolve.go +++ b/pkg/artifact/s3/resolve.go @@ -122,7 +122,11 @@ func (r *Repository) page(ctx context.Context, candidates []listCandidate, query } page := &artifact.ListPage{} for len(candidates) > 0 { - batch := candidates[:min(len(candidates), size+1)] + batchSize := len(candidates) + if size < batchSize { + batchSize = size + 1 + } + batch := candidates[:batchSize] candidates = candidates[len(batch):] entries, matched, err := r.loadEntries(ctx, batch, query.Labels) if err != nil { From 736af73a4dee0bfc43204f8d30fc1296ad515caf Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:05:11 -0600 Subject: [PATCH 05/53] fix: harden artifact backends and evidence handling --- pkg/artifact/oci/assemble.go | 5 ++ pkg/artifact/oci/assemble_test.go | 8 ++ pkg/artifact/oci/attach.go | 12 ++- pkg/artifact/oci/backend.go | 5 +- pkg/artifact/oci/copy.go | 40 ++++++++- pkg/artifact/oci/delete.go | 3 + pkg/artifact/oci/delete_test.go | 8 ++ pkg/artifact/oci/edge_test.go | 27 +++++++ pkg/artifact/oci/fetch.go | 4 + pkg/artifact/oci/list.go | 12 ++- pkg/artifact/oci/list_test.go | 13 ++- pkg/artifact/oci/lock.go | 14 +++- pkg/artifact/oci/lock_test.go | 16 ++++ pkg/artifact/oci/publish.go | 6 +- pkg/artifact/oci/repository.go | 48 +++++++++-- pkg/artifact/oci/resolve.go | 2 +- pkg/artifact/oci/roundtrip_test.go | 5 ++ pkg/artifact/oci/sign.go | 9 ++- pkg/artifact/oci/sign_test.go | 17 +++- pkg/artifact/oci/transfer_test.go | 108 +++++++++++++++++++++++++ pkg/artifact/s3/build.go | 32 +++++--- pkg/artifact/s3/delete.go | 18 ++--- pkg/artifact/s3/delete_test.go | 16 ++++ pkg/artifact/s3/evidence.go | 102 +++++++++++++++++++---- pkg/artifact/s3/identity.go | 10 ++- pkg/artifact/s3/identity_test.go | 9 ++- pkg/artifact/s3/integrity_test.go | 8 ++ pkg/artifact/s3/list_test.go | 12 +++ pkg/artifact/s3/lock.go | 38 ++++++++- pkg/artifact/s3/objects.go | 7 +- pkg/artifact/s3/publish.go | 13 +++ pkg/artifact/s3/read.go | 3 + pkg/artifact/s3/resolve.go | 19 +++-- pkg/artifact/s3/signature_test.go | 37 +++++++++ pkg/artifact/service/commit.go | 19 +++-- pkg/artifact/service/copy.go | 18 ++--- pkg/artifact/service/edge_test.go | 2 +- pkg/artifact/sigstore/keys.go | 12 --- pkg/artifact/sigstore/signer.go | 3 + pkg/artifact/sigstore/signer_test.go | 12 +++ pkg/artifact/sigstore/statement.go | 4 + pkg/artifact/sigstore/verifier.go | 30 ++++--- pkg/artifact/sigstore/verifier_test.go | 23 +++++- pkg/oci/status.go | 12 +++ pkg/oci/transport_layout.go | 4 +- 45 files changed, 706 insertions(+), 119 deletions(-) diff --git a/pkg/artifact/oci/assemble.go b/pkg/artifact/oci/assemble.go index 42ba552fbac..7b3326769de 100644 --- a/pkg/artifact/oci/assemble.go +++ b/pkg/artifact/oci/assemble.go @@ -117,7 +117,12 @@ func stagedLayers(prepared *artifact.PreparedArtifact) (stagedBlob, []stagedBlob if err != nil { return stagedBlob{}, nil, err } + if _, exists := documents[blob.DocumentRole]; exists { + return stagedBlob{}, nil, fmt.Errorf("%w: duplicate %s document", errUtils.ErrArtifactPackagingInvalid, blob.DocumentRole) + } documents[blob.DocumentRole] = doc + default: + return stagedBlob{}, nil, fmt.Errorf("%w: unsupported blob role %q", errUtils.ErrArtifactPackagingInvalid, blob.Role) } } if payload == nil { diff --git a/pkg/artifact/oci/assemble_test.go b/pkg/artifact/oci/assemble_test.go index 34e8d11bded..a30002075be 100644 --- a/pkg/artifact/oci/assemble_test.go +++ b/pkg/artifact/oci/assemble_test.go @@ -122,6 +122,14 @@ func TestAssemble_Rejections(t *testing.T) { p.Blobs = append(p.Blobs, p.Blobs[0]) return p }}, + {"duplicate document", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs = append(p.Blobs, p.Blobs[1]) + return p + }}, + {"unknown blob role", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + p.Blobs[0].Role = "unknown" + return p + }}, {"document the Record does not describe", func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { p.Blobs[1].DocumentRole = artifact.DocumentChangelog p.Record.Documents = p.Record.Documents[:1] diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index 318796ae47d..0bb2c378d0a 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -36,7 +36,7 @@ type attachmentSet struct { } // attachments lists the Sigstore bundle referrers of a root, in digest order, inspecting at most -// [maxAttachments] of them. Referrers of other types (SBOMs, attestations) and manifests that +// [maxAttachments] valid signatures and a bounded number of candidates. Referrers of other types and manifests that // merely contain the root as a child are not evidence and are skipped. A failure to list // referrers is an error: required signature material must never be silently omitted. func (r *Repository) attachments(ctx context.Context, st store, name string, rootDesc ocispec.Descriptor) (*attachmentSet, error) { @@ -54,8 +54,8 @@ func (r *Repository) attachments(ctx context.Context, st store, name string, roo if !isManifestMediaType(predecessors[index].MediaType) { continue } - if inspected >= maxAttachments { - set.warn("evidence-truncated", fmt.Sprintf("more than %d referrers of %s; the rest were not inspected", maxAttachments, rootDesc.Digest)) + if inspected >= maxAttachments*4 { + set.warn("evidence-truncated", fmt.Sprintf("more than %d candidate referrers of %s; the rest were not inspected", maxAttachments*4, rootDesc.Digest)) break } inspected++ @@ -66,6 +66,10 @@ func (r *Repository) attachments(ctx context.Context, st store, name string, roo case problem != "": set.warn("malformed-evidence", problem) case item != nil: + if len(set.valid) >= maxAttachments { + set.warn("evidence-truncated", fmt.Sprintf("more than %d signature referrers of %s", maxAttachments, rootDesc.Digest)) + return set, nil + } set.valid = append(set.valid, *item) } } @@ -84,7 +88,7 @@ func (r *Repository) readAttachment(ctx context.Context, st store, name string, if ctx.Err() != nil { return nil, "", ctx.Err() } - return nil, fmt.Sprintf("referrer %s could not be read: %v", desc.Digest, err), nil + return nil, "", fmt.Errorf("%w: read referrer %s: %w", errUtils.ErrArtifactDownloadFailed, desc.Digest, err) } var manifest ocispec.Manifest if err := json.Unmarshal(raw, &manifest); err != nil { diff --git a/pkg/artifact/oci/backend.go b/pkg/artifact/oci/backend.go index fbbef01f2f8..a273cb505c5 100644 --- a/pkg/artifact/oci/backend.go +++ b/pkg/artifact/oci/backend.go @@ -156,7 +156,10 @@ func (b *registryBackend) names(ctx context.Context) ([]string, error) { if ctx.Err() != nil { return nil, ctx.Err() } - return nil, b.unsupportedEnumeration(fmt.Sprintf("the registry did not provide a repository catalog (%v)", err)) + if atmosoci.IsCatalogUnavailable(err) { + return nil, b.unsupportedEnumeration(fmt.Sprintf("the registry did not provide a repository catalog (%v)", err)) + } + return nil, fmt.Errorf("%w: list registry catalog: %w", errUtils.ErrArtifactListFailed, err) } scope := "" if b.prefix != "" { diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index c1179dd24fa..98684e75828 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" @@ -90,7 +91,7 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto if err != nil { return nil, err } - if outcome != artifact.OutcomeUnchanged { + { _, err = atmosoci.ExtendedCopy(ctx, atmosoci.GraphSource{Store: src, Reference: loaded.desc.Digest.String()}, atmosoci.Destination{Store: dst, Reference: reference}, @@ -173,6 +174,9 @@ func (r *Repository) ExportLayout(ctx context.Context, pinned artifact.PinnedRef if err != nil { return ocispec.Descriptor{}, fmt.Errorf("%w: open layout %q: %w", errUtils.ErrArtifactUploadFailed, directory, err) } + if err := validateExportRoots(ctx, destination, directory, loaded.desc.Digest); err != nil { + return ocispec.Descriptor{}, err + } reference := pinned.Ref.String() if _, err := atmosoci.ExtendedCopy(ctx, atmosoci.GraphSource{Store: st, Reference: loaded.desc.Digest.String()}, @@ -183,6 +187,35 @@ func (r *Repository) ExportLayout(ctx context.Context, pinned artifact.PinnedRef return loaded.desc, nil } +// validateExportRoots permits the requested root and unnamed referrers, but rejects other roots. +func validateExportRoots(ctx context.Context, destination store, directory string, target digest.Digest) error { + raw, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + if err != nil { + return fmt.Errorf("%w: read export index: %w", errUtils.ErrArtifactUploadFailed, err) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return fmt.Errorf("%w: parse export index: %w", errUtils.ErrArtifactUploadFailed, err) + } + for _, entry := range index.Manifests { + if entry.Digest == target { + continue + } + if entry.Annotations[ocispec.AnnotationRefName] == "" { + raw, err := fetchBounded(ctx, destination, entry, maxManifestSize) + if err != nil { + return err + } + var manifest ocispec.Manifest + if json.Unmarshal(raw, &manifest) == nil && manifest.Subject != nil { + continue + } + } + return fmt.Errorf("%w: export layout already contains another root %s", errUtils.ErrArtifactConflict, entry.Digest) + } + return nil +} + // layoutRoot reads a layout's index.json and picks the root to import. func layoutRoot(directory, selector string) (ocispec.Descriptor, error) { raw, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) @@ -195,10 +228,13 @@ func layoutRoot(directory, selector string) (ocispec.Descriptor, error) { } var matches []ocispec.Descriptor for _, entry := range index.Manifests { - if selector == "" || entry.Digest.String() == selector || entry.Annotations[ocispec.AnnotationRefName] == selector { + if (selector == "" && entry.Annotations[ocispec.AnnotationRefName] != "") || (selector != "" && (entry.Digest.String() == selector || entry.Annotations[ocispec.AnnotationRefName] == selector)) { matches = append(matches, entry) } } + if selector == "" && len(matches) == 0 && len(index.Manifests) == 1 { + matches = index.Manifests + } if len(matches) != 1 { return ocispec.Descriptor{}, fmt.Errorf("%w: %d roots in %q match %q; select exactly one by reference name or digest", errUtils.ErrArtifactRefInvalid, len(matches), directory, selector) } diff --git a/pkg/artifact/oci/delete.go b/pkg/artifact/oci/delete.go index 4af28ee5628..f4274ec5c5f 100644 --- a/pkg/artifact/oci/delete.go +++ b/pkg/artifact/oci/delete.go @@ -25,6 +25,9 @@ import ( func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { defer perf.Track(nil, "oci.Repository.Delete")() + if options.Scope == artifact.DeleteReference && r.be.deleteScope() != artifact.DeleteReference { + return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/tag", "the registry can only delete a manifest") + } name := pinned.Ref.Name if err := artifact.ValidateName(name); err != nil { return nil, err diff --git a/pkg/artifact/oci/delete_test.go b/pkg/artifact/oci/delete_test.go index 5588711a351..ced1aa14b23 100644 --- a/pkg/artifact/oci/delete_test.go +++ b/pkg/artifact/oci/delete_test.go @@ -126,3 +126,11 @@ func TestDeleteRefusesAMovedTag(t *testing.T) { _, err = repo.Delete(context.Background(), old, artifact.DeleteOptions{Scope: artifact.DeleteReference}) require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the tag no longer names the pinned manifest") } + +func TestRegistryRejectsReferenceScope(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + first := publish(t, repo, publishSpec{content: "keep"}) + _, err := repo.Delete(context.Background(), first.Pinned, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.True(t, resolves(repo, appName, appVersion)) +} diff --git a/pkg/artifact/oci/edge_test.go b/pkg/artifact/oci/edge_test.go index 7c524afcb2f..439010ca9e8 100644 --- a/pkg/artifact/oci/edge_test.go +++ b/pkg/artifact/oci/edge_test.go @@ -4,10 +4,12 @@ package oci import ( "context" "errors" + "fmt" "io" "testing" "time" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -154,3 +156,28 @@ func TestReaderClosedBeforeReadingFetchesNothing(t *testing.T) { require.NoError(t, content.Close()) assert.Zero(t, reg.recorder.count("GET", payloadDigest(t, result).Encoded())) } + +func TestFinishVerificationRejectsUnchecked(t *testing.T) { + base := &artifact.VerificationResult{Status: artifact.VerificationFailed} + result, err := finishVerification(base, &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationUnchecked, result.Status) +} + +func TestLazyReaderRejectsUnavailableDigestAlgorithm(t *testing.T) { + reader := newLazyReader(context.Background(), nil, ocispec.Descriptor{Digest: "sha999:abcd"}) + _, err := reader.Read(make([]byte, 1)) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestTrackedRootsAreBounded(t *testing.T) { + repo := newLayoutRepo(t) + for i := 0; i < manifestCacheEntries+1; i++ { + desc := ocispec.Descriptor{Digest: digest.FromString(fmt.Sprint(i))} + repo.track("example", &root{desc: desc, class: classPlugin}) + } + assert.Len(t, repo.pinned, manifestCacheEntries) + assert.LessOrEqual(t, len(repo.roots), manifestCacheEntries) + _, found := repo.descriptorFor("example", digest.FromString("0")) + assert.False(t, found) +} diff --git a/pkg/artifact/oci/fetch.go b/pkg/artifact/oci/fetch.go index f94f667b427..41f4f48650c 100644 --- a/pkg/artifact/oci/fetch.go +++ b/pkg/artifact/oci/fetch.go @@ -94,6 +94,10 @@ func (l *lazyReader) Read(p []byte) (int, error) { } func (l *lazyReader) open() { + if !l.desc.Digest.Algorithm().Available() { + l.err = fmt.Errorf("%w: unavailable digest algorithm for %s", errUtils.ErrArtifactIntegrityFailed, l.desc.Digest) + return + } l.hasher = l.desc.Digest.Algorithm().Hash() source, err := l.st.Fetch(l.ctx, l.desc) if err != nil { diff --git a/pkg/artifact/oci/list.go b/pkg/artifact/oci/list.go index 330cd55b867..0011bced11f 100644 --- a/pkg/artifact/oci/list.go +++ b/pkg/artifact/oci/list.go @@ -38,11 +38,11 @@ func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artif if cursor.skipsName(name) { continue } - items, err := r.entriesOf(ctx, name, query.Labels) + items, err := r.entriesOf(ctx, name, query.Labels, cursor) if err != nil { return nil, err } - entries = append(entries, cursor.remaining(name, items)...) + entries = append(entries, items...) if len(entries) > size { last := entries[size-1] return &artifact.ListPage{Entries: entries[:size], NextCursor: last.Name + cursorSeparator + last.Version}, nil @@ -66,6 +66,10 @@ func parseCursor(raw string) listCursor { // skipsName reports whether every entry of name precedes the cursor. func (c listCursor) skipsName(name string) bool { return c.set && name < c.name } +func (c listCursor) skipsVersion(name, version string) bool { + return c.set && name == c.name && version <= c.version +} + // remaining returns the entries of name that follow the cursor. func (c listCursor) remaining(name string, items []artifact.ListEntry) []artifact.ListEntry { if !c.set || name != c.name { @@ -93,7 +97,7 @@ func (r *Repository) namesFor(ctx context.Context, name string) ([]string, error // entriesOf lists the Atmos package versions of one name, sorted by version, that match labels. // Each tag costs a manifest read and a Record read, bounded by the configured concurrency. -func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifact.Label) ([]artifact.ListEntry, error) { +func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifact.Label, cursor listCursor) ([]artifact.ListEntry, error) { versions, err := r.be.versions(ctx, name) if err != nil { return nil, err @@ -103,7 +107,7 @@ func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifa group, groupCtx := errgroup.WithContext(ctx) group.SetLimit(r.settings.concurrency) for index, version := range versions { - if artifact.ValidateVersion(version) != nil { + if cursor.skipsVersion(name, version) || artifact.ValidateVersion(version) != nil { continue } group.Go(func() error { diff --git a/pkg/artifact/oci/list_test.go b/pkg/artifact/oci/list_test.go index c6f88884686..e55029e94a2 100644 --- a/pkg/artifact/oci/list_test.go +++ b/pkg/artifact/oci/list_test.go @@ -2,11 +2,12 @@ package oci import ( "context" - "errors" + "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/registry/remote/errcode" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -93,7 +94,7 @@ func TestNamedListingWorksWithoutACatalogAndCollectionListingFailsLoudly(t *test tests := map[string]atmosoci.Transport{ "transport without a catalog": withoutCatalog{reg.transport}, "registry rejects the catalog": failingCatalog{ - Transport: reg.transport, err: errors.New("GET _catalog: response status code 404"), + Transport: reg.transport, err: &errcode.ErrorResponse{StatusCode: http.StatusNotFound}, }, } for name, transport := range tests { @@ -129,3 +130,11 @@ func TestListingNeverPresentsForeignImagesOrPluginsAsArtifacts(t *testing.T) { assert.Equal(t, []string{"shared/thing:1.0.0"}, listAll(t, repo, artifact.ListQuery{Name: "shared/thing"})) assert.Equal(t, []string{"shared/thing:1.0.0"}, listAll(t, repo, artifact.ListQuery{})) } + +func TestCatalogFailuresAreNotUnsupported(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusInternalServerError} { + b := newRegistryBackend("example.test", "", failingCatalog{err: &errcode.ErrorResponse{StatusCode: status}}, false) + _, err := b.names(context.Background()) + require.ErrorIs(t, err, errUtils.ErrArtifactListFailed) + } +} diff --git a/pkg/artifact/oci/lock.go b/pkg/artifact/oci/lock.go index 4ed4b2c9613..02696a98adb 100644 --- a/pkg/artifact/oci/lock.go +++ b/pkg/artifact/oci/lock.go @@ -78,9 +78,21 @@ func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (ar return r.acquireHost(ctx, request.Resource, owner) } +type lockChainKey struct{} + // delegateLock takes the lock from the configured delegate repository. The resource is scoped by // this repository's location, so unrelated registries sharing one coordinator never contend. func (r *Repository) delegateLock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + chain, _ := ctx.Value(lockChainKey{}).(map[*Repository]bool) + if chain[r] { + return nil, fmt.Errorf("%w: cyclic lock delegation at repository %q", errUtils.ErrArtifactConfigInvalid, r.alias) + } + next := make(map[*Repository]bool, len(chain)+1) + for repo := range chain { + next[repo] = true + } + next[r] = true + ctx = context.WithValue(ctx, lockChainKey{}, next) if r.lookup == nil { return nil, fmt.Errorf("%w: repository %q delegates locking to %q but no repository lookup was supplied", errUtils.ErrArtifactConfigInvalid, r.alias, r.delegate) } @@ -100,7 +112,7 @@ func (r *Repository) delegateLock(ctx context.Context, request artifact.LockRequ return nil, artifact.NewUnsupportedCapability(r.delegate, opLock, "lock/"+string(required), fmt.Sprintf("the lock delegate of repository %q cannot provide %s-scope coordination", r.alias, required)) } - request.Resource = r.lockNamespace() + "/" + request.Resource + request.Resource = fmt.Sprintf("%d:%s/%s", len(r.lockNamespace()), r.lockNamespace(), request.Resource) handle, err := delegate.Lock(ctx, request) if err != nil { return nil, fmt.Errorf("lock delegate %q of repository %q: %w", r.delegate, r.alias, err) diff --git a/pkg/artifact/oci/lock_test.go b/pkg/artifact/oci/lock_test.go index b0de392da7c..8dddf2059e9 100644 --- a/pkg/artifact/oci/lock_test.go +++ b/pkg/artifact/oci/lock_test.go @@ -171,3 +171,19 @@ func TestServicePublishesUnderTheDelegatedLockAndReportsItsScope(t *testing.T) { assert.Equal(t, artifact.LockScopeRepository, outcome.Operation.LockScope) assert.Empty(t, env.coordinator.HeldLocks(), "the delegated lock was released") } + +func TestDelegationDetectsIndirectCycle(t *testing.T) { + a := newLayoutRepo(t) + b := newLayoutRepo(t) + a.alias, b.alias = "a", "b" + a.delegate, b.delegate = "b", "a" + lookup := func(name string) (artifact.Repository, error) { + if name == "a" { + return a, nil + } + return b, nil + } + a.lookup, b.lookup = lookup, lookup + _, err := a.Lock(context.Background(), artifact.LockRequest{Resource: "app:1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} diff --git a/pkg/artifact/oci/publish.go b/pkg/artifact/oci/publish.go index ef0c889bdd8..34323a2871d 100644 --- a/pkg/artifact/oci/publish.go +++ b/pkg/artifact/oci/publish.go @@ -96,6 +96,9 @@ func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, op } return artifact.OutcomeCreated, nil } + if options.ExpectedPrior != nil && options.ExpectedPrior.ResolvedDigest != state.desc.Digest { + return "", fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest, options.ExpectedPrior.ResolvedDigest) + } if state.desc.Digest == incoming.digest { return artifact.OutcomeUnchanged, nil } @@ -108,9 +111,6 @@ func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, op if options.ExpectedPrior == nil { return "", fmt.Errorf("%w: %s already exists (%s) but the caller expected it not to", errUtils.ErrArtifactConflict, ref, state.desc.Digest) } - if options.ExpectedPrior.ResolvedDigest != state.desc.Digest { - return "", fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest, options.ExpectedPrior.ResolvedDigest) - } if !options.Force { return "", fmt.Errorf("%w: %s already exists with different content; use force to replace it", errUtils.ErrArtifactConflict, ref) } diff --git a/pkg/artifact/oci/repository.go b/pkg/artifact/oci/repository.go index 72c0abd25d3..773e79ef067 100644 --- a/pkg/artifact/oci/repository.go +++ b/pkg/artifact/oci/repository.go @@ -28,11 +28,12 @@ type Repository struct { delegate string lookup artifact.RepositoryLookup - mu sync.Mutex - cache map[string][]byte - order []string - roots map[digest.Digest][]located - pinned map[string]ocispec.Descriptor + mu sync.Mutex + cache map[string][]byte + order []string + roots map[digest.Digest][]located + pinned map[string]ocispec.Descriptor + tracked []trackedRoot } var ( @@ -50,6 +51,12 @@ type located struct { desc ocispec.Descriptor } +type trackedRoot struct { + name string + desc ocispec.Descriptor + keys []digest.Digest +} + func newRepositoryCore(alias string, be backend, applied settings, delegate string, lookup artifact.RepositoryLookup) *Repository { return &Repository{ alias: alias, be: be, settings: applied, delegate: delegate, lookup: lookup, @@ -127,6 +134,29 @@ func (r *Repository) track(name string, loaded *root) { if loaded.class == classAtmos { keys = append(keys, loaded.manifest.Config.Digest) } + keyName := cacheKey(name, loaded.desc) + if _, exists := r.pinned[keyName]; exists { + return + } + if len(r.tracked) >= manifestCacheEntries { + old := r.tracked[0] + r.tracked = r.tracked[1:] + delete(r.pinned, cacheKey(old.name, old.desc)) + for _, key := range old.keys { + items := r.roots[key][:0] + for _, item := range r.roots[key] { + if item.name != old.name || item.desc.Digest != old.desc.Digest { + items = append(items, item) + } + } + if len(items) == 0 { + delete(r.roots, key) + } else { + r.roots[key] = items + } + } + } + r.tracked = append(r.tracked, trackedRoot{name: name, desc: loaded.desc, keys: keys}) for _, key := range keys { known := r.roots[key] duplicate := false @@ -137,7 +167,7 @@ func (r *Repository) track(name string, loaded *root) { r.roots[key] = append(known, located{name: name, desc: loaded.desc}) } } - r.pinned[cacheKey(name, loaded.desc)] = loaded.desc + r.pinned[keyName] = loaded.desc } // descriptorFor returns the descriptor remembered for a resolved digest of a name. @@ -158,7 +188,11 @@ func (r *Repository) locations(dgst digest.Digest) []located { // loadCachedRoot loads a root through the verified cache. func (r *Repository) loadCachedRoot(ctx context.Context, st store, name string, desc ocispec.Descriptor) (*root, error) { if raw, ok := r.cached(name, desc); ok { - return classify(desc, raw) + loaded, err := classify(desc, raw) + if err == nil { + r.track(name, loaded) + } + return loaded, err } loaded, err := loadRoot(ctx, st, desc) if err != nil { diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go index 4d8be652b54..4149282ddcc 100644 --- a/pkg/artifact/oci/resolve.go +++ b/pkg/artifact/oci/resolve.go @@ -151,7 +151,7 @@ func (r *Repository) scanForDigest(ctx context.Context, st store, ref artifact.R } sort.Strings(versions) if len(versions) > maxTagsScanned { - versions = versions[:maxTagsScanned] + return ocispec.Descriptor{}, fmt.Errorf("%w: %s has %d tags; digest lookup inspects at most %d", errUtils.ErrArtifactDownloadFailed, ref.Name, len(versions), maxTagsScanned) } for _, version := range versions { desc, found, err := r.digestAt(ctx, st, ref, version) diff --git a/pkg/artifact/oci/roundtrip_test.go b/pkg/artifact/oci/roundtrip_test.go index 5a39cb42082..2d0b5390e80 100644 --- a/pkg/artifact/oci/roundtrip_test.go +++ b/pkg/artifact/oci/roundtrip_test.go @@ -63,6 +63,11 @@ func TestPublishIsIdempotentAndGuardsTheReference(t *testing.T) { assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome) assert.Equal(t, first.Pinned.ResolvedDigest, again.Pinned.ResolvedDigest) + staleSame := first.Pinned + staleSame.ResolvedDigest = digest.FromString("stale") + _, err = repo.Publish(ctx, prepare(t, publishSpec{content: "base"}), artifact.PublishOptions{ExpectedPrior: &staleSame}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + changed := prepare(t, publishSpec{content: "changed"}) _, err = repo.Publish(ctx, changed, artifact.PublishOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "an existing reference expected to be absent") diff --git a/pkg/artifact/oci/sign.go b/pkg/artifact/oci/sign.go index d28b0c0c5bf..f99fe5f03ed 100644 --- a/pkg/artifact/oci/sign.go +++ b/pkg/artifact/oci/sign.go @@ -99,7 +99,7 @@ func bindSubject(subject artifact.SignatureSubject, loaded *root) error { return fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, loaded.manifest.Config.Size) } case artifact.SubjectOCIRoot: - if loaded.desc.Digest != subject.Digest { + if loaded.desc.Digest != subject.Digest || loaded.desc.MediaType != subject.MediaType || (subject.Size != 0 && loaded.desc.Size != subject.Size) { return fmt.Errorf("%w: %s is not the manifest or index digest of the located root", errUtils.ErrArtifactRefInvalid, subject.Digest) } default: @@ -262,7 +262,7 @@ func finishVerification(base, verified *artifact.VerificationResult, verifyErr e return result, verifyErr case verifyErr != nil: return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) - case result.Status == artifact.VerificationFailed: + case result.Status != artifact.VerificationVerified: return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) } return result, nil @@ -292,6 +292,11 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe if err != nil { return nil, err } + for _, diagnostic := range set.diagnostics { + if diagnostic.Code == "evidence-truncated" { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDownloadFailed, diagnostic.Message) + } + } var blobs []artifact.EvidenceBlob for item := range set.valid { att := &set.valid[item] diff --git a/pkg/artifact/oci/sign_test.go b/pkg/artifact/oci/sign_test.go index 284cf4d7ead..bda77227c43 100644 --- a/pkg/artifact/oci/sign_test.go +++ b/pkg/artifact/oci/sign_test.go @@ -378,7 +378,8 @@ func TestNativeImagePluginAndLegacyCosignSignatures(t *testing.T) { t.Run("image signed through the signer", func(t *testing.T) { image := pushImage(t, reg, testPrefix+"/tools/img", "v1") pinned := mustResolveRef(t, repo, artifact.Ref{Name: "tools/img", Version: "v1"}) - subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: "application/vnd.oci.image.manifest.v1+json"} + desc := targetPinnedDescriptor(t, repo, pinned) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: image.manifest, MediaType: desc.MediaType} _, err := repo.Verify(ctx, subject, policy) require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) _, err = repo.Sign(ctx, subject, tr.keyless, artifact.SignOptions{}) @@ -493,3 +494,17 @@ func reconstructedIndex(t *testing.T, reg *registry, image foreignImage, childDi assert.NotEmpty(t, image.children) return digest.Digest(rebuilt.String()) } + +func TestBindSubjectRequiresRootDescriptorMetadata(t *testing.T) { + desc := ocispec.Descriptor{Digest: digest.FromString("root"), MediaType: "application/example", Size: 42} + loaded := &root{desc: desc} + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: desc.Digest, MediaType: desc.MediaType, Size: desc.Size} + require.NoError(t, bindSubject(subject, loaded)) + subject.Size = 0 + require.NoError(t, bindSubject(subject, loaded)) + subject.Size = 41 + require.ErrorIs(t, bindSubject(subject, loaded), errUtils.ErrArtifactRefInvalid) + subject.Size = 42 + subject.MediaType = "application/other" + require.ErrorIs(t, bindSubject(subject, loaded), errUtils.ErrArtifactRefInvalid) +} diff --git a/pkg/artifact/oci/transfer_test.go b/pkg/artifact/oci/transfer_test.go index 0373661c3d1..9c7c9062a07 100644 --- a/pkg/artifact/oci/transfer_test.go +++ b/pkg/artifact/oci/transfer_test.go @@ -4,9 +4,11 @@ package oci import ( "context" "errors" + "io" "testing" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -253,3 +255,109 @@ func TestCrossBackendCredentialsAreNotShared(t *testing.T) { assert.NotErrorIs(t, err, errUtils.ErrArtifactNotFound, "an authentication failure is not reported as a missing artifact") assert.Contains(t, err.Error(), "credential") } + +// failingFetchStore exposes a failed manifest fetch while preserving the underlying graph. +type failingFetchStore struct{ store } + +func (f failingFetchStore) Fetch(context.Context, ocispec.Descriptor) (io.ReadCloser, error) { + return nil, errors.New("fetch failed") +} + +func TestAttachmentFetchFailureIsAnError(t *testing.T) { + repo := newLayoutRepo(t) + pinned := publish(t, repo, publishSpec{content: "root"}).Pinned + st, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + desc := ocispec.Descriptor{Digest: digest.FromString("other"), Size: loaded.desc.Size, MediaType: loaded.desc.MediaType} + _, _, err = repo.readAttachment(context.Background(), failingFetchStore{st}, pinned.Ref.Name, loaded.desc, desc) + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) +} + +type manyVersionsBackend struct{ backend } + +func (m manyVersionsBackend) versions(context.Context, string) ([]string, error) { + return make([]string, maxTagsScanned+1), nil +} + +func TestDigestScanFailsWhenTagBoundExceeded(t *testing.T) { + repo := newLayoutRepo(t) + repo.be = manyVersionsBackend{repo.be} + _, err := repo.scanForDigest(context.Background(), nil, artifact.Ref{Name: appName}) + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) +} + +type overflowingStore struct { + store + desc ocispec.Descriptor +} + +func (s overflowingStore) Predecessors(context.Context, ocispec.Descriptor) ([]ocispec.Descriptor, error) { + return makeRepeatedDescriptors(s.desc, maxAttachments+1), nil +} + +func makeRepeatedDescriptors(desc ocispec.Descriptor, count int) []ocispec.Descriptor { + items := make([]ocispec.Descriptor, count) + for i := range items { + items[i] = desc + } + return items +} + +type overflowingBackend struct { + backend + desc ocispec.Descriptor +} + +func (b overflowingBackend) open(ctx context.Context, name string, write bool) (store, error) { + st, err := b.backend.open(ctx, name, write) + if err != nil { + return nil, err + } + return overflowingStore{store: st, desc: b.desc}, nil +} + +func TestReadEvidenceRejectsTruncatedReferrers(t *testing.T) { + repo := newLayoutRepo(t) + pinned := publish(t, repo, publishSpec{content: "root"}).Pinned + attachBundle(t, repo, pinned, `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json"}`) + st, loaded, err := repo.rootOf(context.Background(), &pinned) + require.NoError(t, err) + refs, err := st.Predecessors(context.Background(), loaded.desc) + require.NoError(t, err) + require.NotEmpty(t, refs) + repo.be = overflowingBackend{backend: repo.be, desc: refs[0]} + _, err = repo.ReadEvidence(context.Background(), &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) +} + +func TestCopyUnchangedRootStillCopiesNewReferrer(t *testing.T) { + ctx := context.Background() + source := newLayoutRepo(t) + destination := newLayoutRepo(t) + pinned := publish(t, source, publishSpec{content: "root"}).Pinned + first, err := destination.CopyFrom(ctx, source, pinned, artifact.CopyFromOptions{}) + require.NoError(t, err) + attached := attachBundle(t, source, pinned, `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json"}`) + second, err := destination.CopyFrom(ctx, source, pinned, artifact.CopyFromOptions{ExpectedPrior: &first.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, second.Outcome) + set, err := destination.attachments(ctx, mustStore(t, destination, pinned.Ref.Name), pinned.Ref.Name, targetPinnedDescriptor(t, destination, second.Pinned)) + require.NoError(t, err) + require.Len(t, set.valid, 1) + assert.Equal(t, attached.Digest, set.valid[0].layer.Digest) +} + +func TestExportRejectsAnotherRootAndImportPrefersNamedRoot(t *testing.T) { + ctx := context.Background() + repo := newLayoutRepo(t) + first := publish(t, repo, publishSpec{content: "one"}).Pinned + second := publish(t, repo, publishSpec{version: "2.0.0", content: "two"}).Pinned + dir := t.TempDir() + _, err := repo.ExportLayout(ctx, first, dir) + require.NoError(t, err) + _, err = repo.ExportLayout(ctx, second, dir) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + root, err := layoutRoot(dir, "") + require.NoError(t, err) + assert.Equal(t, first.ResolvedDigest, root.Digest) +} diff --git a/pkg/artifact/s3/build.go b/pkg/artifact/s3/build.go index ab24a76dce0..a47ead42452 100644 --- a/pkg/artifact/s3/build.go +++ b/pkg/artifact/s3/build.go @@ -78,34 +78,44 @@ func (r *Repository) retire(ctx context.Context, job *publishJob, plan *publishP return } if plan.abandoned != nil { - r.removePayload(cleanup, plan.abandoned, keep) + if err := r.removePayload(cleanup, plan.abandoned, keep); err != nil { + return + } } if plan.prior != nil { - r.retirePrior(cleanup, plan.prior, job, keep) + if err := r.retirePrior(cleanup, plan.prior, job, keep); err != nil { + return + } } _ = r.deleteObject(cleanup, layout.OperationPath(job.opID)) } // retirePrior removes the payload, Record, and evidence of the replaced publication. -func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job *publishJob, keep []string) { +func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job *publishJob, keep []string) error { if prior.RecordDigest == "" || prior.RecordDigest == job.recordDigest { - return + return nil } if prior.Location != "" { - if record, _, err := r.loadRecord(ctx, prior.RecordDigest); err == nil { - r.removePayload(ctx, &ownedPayload{packaging: prior.Packaging, location: prior.Location, set: setOfRecord(record)}, keep) + record, _, err := r.loadRecord(ctx, prior.RecordDigest) + if err != nil { + return err } + if err := r.removePayload(ctx, &ownedPayload{packaging: prior.Packaging, location: prior.Location, set: setOfRecord(record)}, keep); err != nil { + return err + } + } + if err := r.deleteEvidence(ctx, prior.RecordDigest); err != nil { + return err } - _ = r.deleteEvidence(ctx, prior.RecordDigest) - _ = r.deleteObject(ctx, layout.RecordPath(prior.RecordDigest)) + return r.deleteObject(ctx, layout.RecordPath(prior.RecordDigest)) } // removePayload deletes the objects of a payload that the new publication does not reuse. -func (r *Repository) removePayload(ctx context.Context, owned *ownedPayload, keep []string) { +func (r *Repository) removePayload(ctx context.Context, owned *ownedPayload, keep []string) error { keys, err := owned.set.keys(owned.packaging, owned.location) if err != nil { - return + return err } keys = slices.DeleteFunc(keys, func(key string) bool { return slices.Contains(keep, key) }) - _ = r.deleteAll(ctx, keys) + return r.deleteAll(ctx, keys) } diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go index 7737b8cb7b4..08f2dbe2f6a 100644 --- a/pkg/artifact/s3/delete.go +++ b/pkg/artifact/s3/delete.go @@ -33,17 +33,15 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference if err != nil { return nil, err } - if options.Lock == nil { - lock, lockErr := r.acquire(ctx, record.Name+":"+record.Version, r.settings.owner, "") - if lockErr != nil { - return nil, lockErr - } - defer func() { - if releaseErr := lock.Release(ctx); err == nil { - err = releaseErr - } - }() + release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version) + if err != nil { + return nil, err } + defer func() { + if releaseErr := release(); err == nil { + err = releaseErr + } + }() return r.deleteLocked(ctx, &pinned, &options) } diff --git a/pkg/artifact/s3/delete_test.go b/pkg/artifact/s3/delete_test.go index db5940160ae..c08bc960ff5 100644 --- a/pkg/artifact/s3/delete_test.go +++ b/pkg/artifact/s3/delete_test.go @@ -119,3 +119,19 @@ func TestDelete_ThenRepublish(t *testing.T) { assert.Equal(t, artifact.OutcomeCreated, again.Outcome) assertIntact(t, repo, defaultRef()) } + +func TestDeleteRejectsUnrelatedAndReleasedLocks(t *testing.T) { + repo := newRepo(t, noneSpec()) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + ctx := context.Background() + wrong, err := repo.Lock(ctx, artifact.LockRequest{Resource: "other:1"}) + require.NoError(t, err) + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Lock: wrong}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NoError(t, wrong.Release(ctx)) + lock, err := repo.Lock(ctx, artifact.LockRequest{Resource: pinned.Ref.Name + ":" + pinned.Ref.Version}) + require.NoError(t, err) + require.NoError(t, lock.Release(ctx)) + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Lock: lock}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) +} diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index a10e13811af..69529df4a7a 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -37,16 +37,24 @@ func (r *Repository) loadEvidence(ctx context.Context, subject digest.Digest) ([ loaded []loadedEvidence diags []artifact.Diagnostic ) + attempted := 0 for index := range objects { stem, ok := strings.CutSuffix(strings.TrimPrefix(objects[index].Key, dir), ".json") if !ok || strings.Contains(stem, "/") { continue } - if len(loaded) >= maxEvidenceFiles { + if digest.NewDigestFromEncoded(digest.SHA256, stem).Validate() != nil { + continue + } + if attempted >= maxEvidenceFiles { diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: fmt.Sprintf("more than %d evidence files; the rest were not read", maxEvidenceFiles)}) break } - item, problem := r.readEvidenceObject(ctx, objects[index].Key, stem) + attempted++ + item, problem, readErr := r.readEvidenceObject(ctx, objects[index].Key, stem) + if readErr != nil { + return nil, nil, readErr + } switch { case errors.Is(problem, context.Canceled), errors.Is(problem, context.DeadlineExceeded): return nil, nil, problem @@ -59,19 +67,20 @@ func (r *Repository) loadEvidence(ctx context.Context, subject digest.Digest) ([ return loaded, diags, nil } -func (r *Repository) readEvidenceObject(ctx context.Context, key, hexDigest string) (*layout.Evidence, error) { +func (r *Repository) readEvidenceObject(ctx context.Context, key, hexDigest string) (*layout.Evidence, error, error) { want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) if err := want.Validate(); err != nil { - return nil, err + return nil, err, nil } raw, _, err := r.readObject(ctx, key, layout.MaxEvidenceFileBytes) if err != nil { - return nil, err + return nil, nil, err } if raw == nil { - return nil, fmt.Errorf("%w: evidence object disappeared", errUtils.ErrArtifactNotFound) + return nil, nil, fmt.Errorf("%w: evidence object disappeared", errUtils.ErrArtifactNotFound) } - return layout.DecodeEvidence(raw, want) + item, err := layout.DecodeEvidence(raw, want) + return item, err, nil } // evidenceRefs converts loaded evidence into descriptors. @@ -111,7 +120,7 @@ func (r *Repository) checkSubject(operation string, subject artifact.SignatureSu // Sign implements [artifact.Repository]: it asks the signer for evidence over the Record subject // and stores it as an object named by the evidence digest below the subject's directory. Evidence // lives beside the Record, so signing never changes the Record, the reference, or the payload. -func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, options artifact.SignOptions) (result []artifact.SignatureEvidence, err error) { defer perf.Track(nil, "s3.Repository.Sign")() if err := r.checkSubject("sign", subject); err != nil { @@ -120,6 +129,23 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject if signer == nil { return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) } + record, _, err := r.loadRecord(ctx, subject.Digest) + if err != nil { + return nil, err + } + release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := release(); err == nil { + err = releaseErr + } + }() + return r.signLocked(ctx, subject, signer) +} + +func (r *Repository) signLocked(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer) ([]artifact.SignatureEvidence, error) { raw, err := r.requireCurrentRecord(ctx, subject.Digest) if err != nil { return nil, err @@ -156,6 +182,24 @@ func (r *Repository) storeEvidence(ctx context.Context, subject artifact.Signatu return artifact.SignatureEvidence{}, err } key := layout.SignaturePath(subject.Kind, subject.Digest, evidence.Digest()) + existing, _, err := r.readObject(ctx, key, layout.MaxEvidenceFileBytes) + if err != nil { + return artifact.SignatureEvidence{}, err + } + if existing == nil { + loaded, diagnostics, err := r.loadEvidence(ctx, subject.Digest) + if err != nil { + return artifact.SignatureEvidence{}, err + } + if len(loaded) >= maxEvidenceFiles { + return artifact.SignatureEvidence{}, fmt.Errorf("%w: evidence limit reached", errUtils.ErrArtifactTooLarge) + } + for _, diagnostic := range diagnostics { + if diagnostic.Code == "evidence-truncated" { + return artifact.SignatureEvidence{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactTooLarge, diagnostic.Message) + } + } + } if err := r.putImmutable(ctx, key, raw, contentTypeJSON, layout.MaxEvidenceFileBytes); err != nil { return artifact.SignatureEvidence{}, wrapWrite("store evidence", err) } @@ -188,9 +232,13 @@ func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubje if verifier == nil { return nil, artifact.NewUnsupportedCapability(r.alias, "verify", "evidence/verify", "no signature verifier is configured for this repository") } - if _, err := r.requireCurrentRecord(ctx, subject.Digest); err != nil { + raw, err := r.requireCurrentRecord(ctx, subject.Digest) + if err != nil { return nil, err } + if subject.Size != 0 && subject.Size != int64(len(raw)) { + return nil, fmt.Errorf("%w: subject size differs from the Record", errUtils.ErrArtifactRefInvalid) + } loaded, diags, err := r.loadEvidence(ctx, subject.Digest) if err != nil { return nil, err @@ -240,7 +288,7 @@ func finishVerification(base, verified *artifact.VerificationResult, verifyErr e return result, verifyErr case verifyErr != nil: return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) - case result.Status == artifact.VerificationFailed: + case result.Status != artifact.VerificationVerified: return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) } return result, nil @@ -258,10 +306,15 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe if _, err := r.requireCurrentRecord(ctx, pinned.RecordDigest); err != nil { return nil, err } - loaded, _, err := r.loadEvidence(ctx, pinned.RecordDigest) + loaded, diags, err := r.loadEvidence(ctx, pinned.RecordDigest) if err != nil { return nil, err } + for _, diag := range diags { + if diag.Code == "evidence-truncated" { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDownloadFailed, diag.Message) + } + } blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) for index := range loaded { blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) @@ -272,15 +325,33 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe // WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this // repository already holds. The evidence must bind to the pinned Record and carry matching bytes; // importing never changes the Record or the reference. -func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, lock artifact.LockHandle) (stored []artifact.SignatureEvidence, err error) { defer perf.Track(nil, "s3.Repository.WriteEvidence")() if err := ctx.Err(); err != nil { return nil, err } - if _, err := r.requireCurrentRecord(ctx, pinned.RecordDigest); err != nil { + record, _, err := r.loadRecord(ctx, pinned.RecordDigest) + if err != nil { return nil, err } + release, err := r.holdClaim(ctx, lock, record.Name+":"+record.Version) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := release(); err == nil { + err = releaseErr + } + }() + raw, err := r.requireCurrentRecord(ctx, pinned.RecordDigest) + if err != nil { + return nil, err + } + return r.writeEvidenceLocked(ctx, pinned, evidence, raw) +} + +func (r *Repository) writeEvidenceLocked(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, raw []byte) ([]artifact.SignatureEvidence, error) { stored := make([]artifact.SignatureEvidence, 0, len(evidence)) for index := range evidence { blobEvidence := &evidence[index] @@ -291,7 +362,10 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR if subject.Digest != pinned.RecordDigest { return stored, fmt.Errorf("%w: evidence is bound to %s, not to the pinned Record %s", errUtils.ErrArtifactRefInvalid, subject.Digest, pinned.RecordDigest) } - if blobEvidence.Evidence.Descriptor.Digest != digest.FromBytes(blobEvidence.Data) { + if subject.Size != 0 && subject.Size != int64(len(raw)) { + return stored, fmt.Errorf("%w: subject size differs from the Record", errUtils.ErrArtifactRefInvalid) + } + if blobEvidence.Evidence.Descriptor.Size != int64(len(blobEvidence.Data)) || blobEvidence.Evidence.Descriptor.Digest != digest.FromBytes(blobEvidence.Data) { return stored, fmt.Errorf("%w: evidence bytes do not match their descriptor %s", errUtils.ErrArtifactIntegrityFailed, blobEvidence.Evidence.Descriptor.Digest) } item, err := r.storeEvidence(ctx, subject, &artifact.EvidencePayload{Format: blobEvidence.Evidence.Format, MediaType: blobEvidence.Evidence.Descriptor.MediaType, Data: blobEvidence.Data}) diff --git a/pkg/artifact/s3/identity.go b/pkg/artifact/s3/identity.go index 7d94e37007a..581bb374490 100644 --- a/pkg/artifact/s3/identity.go +++ b/pkg/artifact/s3/identity.go @@ -27,11 +27,17 @@ func (a identityAdapter) ResolveAWSAuthContext(ctx context.Context, identity str } switch typed := value.(type) { case *store.AWSAuthConfig: - if typed != nil { + if typed != nil && hasAWSIdentitySource(typed) { return typed, nil } case store.AWSAuthConfig: - return &typed, nil + if hasAWSIdentitySource(&typed) { + return &typed, nil + } } return nil, fmt.Errorf("%w: identity %q did not resolve to AWS credentials (got %T)", errUtils.ErrBlobIdentityUnresolved, identity, value) } + +func hasAWSIdentitySource(config *store.AWSAuthConfig) bool { + return config.CredentialsFile != "" || config.ConfigFile != "" || config.Profile != "" +} diff --git a/pkg/artifact/s3/identity_test.go b/pkg/artifact/s3/identity_test.go index 2e31893e0c4..37630521388 100644 --- a/pkg/artifact/s3/identity_test.go +++ b/pkg/artifact/s3/identity_test.go @@ -3,6 +3,7 @@ package s3 import ( "context" "errors" + "os" "path/filepath" "sync/atomic" "testing" @@ -25,6 +26,7 @@ func isolateAWS(t *testing.T) { t.Setenv("AWS_ACCESS_KEY_ID", "ambient-test-key") t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-test-secret") t.Setenv("AWS_REGION", "us-east-1") + require.NoError(t, os.WriteFile(filepath.Join(dir, "credentials"), []byte("[default]\naws_access_key_id = test-key\naws_secret_access_key = test-secret\n"), 0o600)) t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) t.Setenv("AWS_EC2_METADATA_DISABLED", "true") @@ -65,7 +67,7 @@ func identityRepo(t *testing.T, fake *s3test.Server, resolver artifact.IdentityR func TestIdentity_IsResolvedLazily(t *testing.T) { isolateAWS(t) fake := s3test.NewServer(t) - stub := &resolverStub{values: []any{&store.AWSAuthConfig{Region: "us-east-1"}}, errs: []error{nil}} + stub := &resolverStub{values: []any{&store.AWSAuthConfig{Region: "us-east-1", Profile: "default"}}, errs: []error{nil}} repo := identityRepo(t, fake, stub.resolve, "deploy") _ = repo.Capabilities() _ = repo.Location() @@ -98,6 +100,9 @@ func TestIdentity_FailsClosed(t *testing.T) { {name: "resolver returns another cloud's credentials", resolver: func() artifact.IdentityResolver { return (&resolverStub{values: []any{"azure-token"}, errs: []error{nil}}).resolve }}, + {name: "resolver returns empty AWS config", resolver: func() artifact.IdentityResolver { + return (&resolverStub{values: []any{store.AWSAuthConfig{Region: "us-east-1"}}, errs: []error{nil}}).resolve + }}, {name: "resolver returns a nil AWS config", resolver: func() artifact.IdentityResolver { var none *store.AWSAuthConfig return (&resolverStub{values: []any{none}, errs: []error{nil}}).resolve @@ -131,7 +136,7 @@ func TestIdentity_TransientFailureIsRetried(t *testing.T) { isolateAWS(t) fake := s3test.NewServer(t) stub := &resolverStub{ - values: []any{nil, &store.AWSAuthConfig{Region: "us-east-1"}}, + values: []any{nil, &store.AWSAuthConfig{Region: "us-east-1", Profile: "default"}}, errs: []error{errors.New("temporarily unavailable"), nil}, } repo := identityRepo(t, fake, stub.resolve, "deploy") diff --git a/pkg/artifact/s3/integrity_test.go b/pkg/artifact/s3/integrity_test.go index d58904036c0..4a8a920aed3 100644 --- a/pkg/artifact/s3/integrity_test.go +++ b/pkg/artifact/s3/integrity_test.go @@ -6,6 +6,7 @@ import ( "io" "testing" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -276,3 +277,10 @@ func TestRead_PayloadThatGrowsIsRejected(t *testing.T) { defer content.Close() require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) } + +func TestStreamsRejectUnsupportedPayloadDigest(t *testing.T) { + repo := newRepo(t, noneSpec()) + v := &view{rep: &layout.Representation{Packaging: artifact.PackagingTgz, Payload: []ocispec.Descriptor{{Digest: "sha999:abcd"}}}} + _, err := repo.streams(context.Background(), v, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} diff --git a/pkg/artifact/s3/list_test.go b/pkg/artifact/s3/list_test.go index 576fd9c1b9a..ccd20c5fc07 100644 --- a/pkg/artifact/s3/list_test.go +++ b/pkg/artifact/s3/list_test.go @@ -154,3 +154,15 @@ func seedReferences(t *testing.T, repo *Repository, template *artifact.Record, n return repo.putBytes(ctx, layout.ReferencePath(completed.Key()), encoded) })) } + +func TestListEntrySkipsRetiredReference(t *testing.T) { + repo := newRepo(t, noneSpec()) + first := publish(t, repo, &prepareSpec{files: map[string]string{"file": "old"}}, artifact.PublishOptions{}) + state, err := repo.loadReference(context.Background(), layout.ReferenceKey(first.Pinned.Ref.Name, first.Pinned.Ref.Version)) + require.NoError(t, err) + candidate := listCandidate{name: state.ref.Name, version: state.ref.Version, digest: state.ref.RecordDigest, generation: state.ref.Generation} + publish(t, repo, &prepareSpec{files: map[string]string{"file": "new"}, opID: "new"}, artifact.PublishOptions{ExpectedPrior: &first.Pinned, Force: true}) + _, found, err := repo.listEntry(context.Background(), &candidate, nil) + require.NoError(t, err) + assert.False(t, found) +} diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 6103231a074..3ffdf5ff9ef 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -75,8 +75,8 @@ func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (ar // acquire takes the claim for resource, waiting for a competing owner. func (r *Repository) acquire(ctx context.Context, resource, owner, operationID string) (*claimHandle, error) { - if !r.store.Capabilities().ConditionalPut { - return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lock/repository", "the S3 endpoint cannot enforce conditional writes, which owner claims require") + if !r.store.Capabilities().ConditionalPut || !r.store.Capabilities().ConditionalDelete { + return nil, artifact.NewUnsupportedCapability(r.alias, opLock, "lock/repository", "the S3 endpoint cannot enforce conditional writes and deletes, which owner claims require") } doc := claimDoc{ SchemaVersion: layout.SchemaVersion, Resource: resource, Owner: owner, OperationID: operationID, @@ -188,12 +188,12 @@ func (r *Repository) heldError(ctx context.Context, key, resource string, cause readCtx, cancel := detached(ctx) defer cancel() holder, _, _ := r.readClaim(readCtx, key) - message := fmt.Sprintf("lock %q is held through claim object %s", resource, r.uri(key)) + message := fmt.Sprintf("lock %q claim object %s is no longer present", resource, r.uri(key)) if holder != nil { message = fmt.Sprintf("lock %q is held by owner %q (operation %q, since %s) through claim object %s", resource, holder.Owner, holder.OperationID, holder.AcquiredAt, r.uri(key)) + message += "; claims never expire and are never taken over automatically. If that writer has stopped, confirm it is no longer running and delete the claim object to recover" } - message += "; claims never expire and are never taken over automatically. If that writer has stopped, confirm it is no longer running and delete the claim object to recover" if cause != nil { return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactConflict, message, cause) } @@ -282,3 +282,33 @@ func (l *claimHandle) goneAfter(ctx context.Context, err error) bool { current, _, readErr := l.repo.readClaim(ctx, l.key) return readErr == nil && current == nil } + +// holdClaim validates an existing claim or acquires one, and keeps it held through a mutation. +func (r *Repository) holdClaim(ctx context.Context, supplied artifact.LockHandle, resource string) (func() error, error) { + if supplied == nil { + handle, err := r.acquire(ctx, resource, r.settings.owner, "") + if err != nil { + return nil, err + } + return func() error { return handle.Release(ctx) }, nil + } + handle, ok := supplied.(*claimHandle) + if !ok || handle.repo != r || handle.resource != resource { + return nil, fmt.Errorf("%w: supplied lock does not guard %s", errUtils.ErrArtifactConflict, resource) + } + handle.mu.Lock() + if handle.released { + handle.mu.Unlock() + return nil, fmt.Errorf("%w: supplied lock is released", errUtils.ErrArtifactConflict) + } + claim, _, err := r.readClaim(ctx, handle.key) + if err != nil { + handle.mu.Unlock() + return nil, err + } + if claim == nil || claim.Token != handle.token { + handle.mu.Unlock() + return nil, fmt.Errorf("%w: supplied lock is no longer held", errUtils.ErrArtifactConflict) + } + return func() error { handle.mu.Unlock(); return nil }, nil +} diff --git a/pkg/artifact/s3/objects.go b/pkg/artifact/s3/objects.go index ce87da5925e..ca4a1833714 100644 --- a/pkg/artifact/s3/objects.go +++ b/pkg/artifact/s3/objects.go @@ -160,7 +160,12 @@ func (r *Repository) forEach(ctx context.Context, count int, fn func(ctx context group, groupCtx := errgroup.WithContext(ctx) group.SetLimit(max(r.settings.concurrency, 1)) for index := range count { - group.Go(func() error { return fn(groupCtx, index) }) + group.Go(func() error { + if err := groupCtx.Err(); err != nil { + return err + } + return fn(groupCtx, index) + }) } return group.Wait() } diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index cfbea73e25c..e90baa2937e 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -154,6 +154,19 @@ func (r *Repository) unchanged(ctx context.Context, job *publishJob, plan *publi if rep == nil || rep.RecordDigest != current.RecordDigest { return nil, fmt.Errorf("%w: %s:%s is completed but has no matching representation", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) } + journal, err := r.loadOperation(ctx, current.OperationID) + if err != nil { + return nil, err + } + if journal != nil && journal.Prior != nil { + keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) + if err != nil { + return nil, err + } + if err := r.retirePrior(ctx, journal.Prior, job, keep); err != nil { + return nil, err + } + } r.tidyOperation(ctx, current.OperationID) return r.result(record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome(), plan.scope), nil } diff --git a/pkg/artifact/s3/read.go b/pkg/artifact/s3/read.go index a8617c275ae..96e201ca06e 100644 --- a/pkg/artifact/s3/read.go +++ b/pkg/artifact/s3/read.go @@ -93,6 +93,9 @@ func (r *Repository) streams(ctx context.Context, v *view, guard func() error) ( return nil, fmt.Errorf("%w: an archive representation must have exactly one payload, found %d", errUtils.ErrArtifactIntegrityFailed, len(v.rep.Payload)) } descriptor := v.rep.Payload[0] + if err := artifact.ValidateDigest(descriptor.Digest); err != nil { + return nil, fmt.Errorf("%w: invalid payload digest: %w", errUtils.ErrArtifactIntegrityFailed, err) + } streams = append(streams, artifact.ContentStream{ Descriptor: descriptor, Role: artifact.BlobPayload, Reader: r.objectReader(ctx, v.rep.Location, descriptor.Digest, descriptor.Size, guard), diff --git a/pkg/artifact/s3/resolve.go b/pkg/artifact/s3/resolve.go index ed1d4cb67a9..d401b9e2180 100644 --- a/pkg/artifact/s3/resolve.go +++ b/pkg/artifact/s3/resolve.go @@ -11,6 +11,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" "github.com/cloudposse/atmos/pkg/perf" ) @@ -76,9 +77,10 @@ func (r *Repository) Inspect(ctx context.Context, pinned artifact.PinnedReferenc // listCandidate is one completed reference awaiting its Record. type listCandidate struct { - name string - version string - digest digest.Digest + name string + version string + digest digest.Digest + generation int64 } func (c *listCandidate) less(other *listCandidate) bool { @@ -102,7 +104,7 @@ func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artif candidates := make([]listCandidate, 0, len(refs)) for _, state := range refs { if state.ref.Available() && (query.Name == "" || state.ref.Name == query.Name) { - candidates = append(candidates, listCandidate{name: state.ref.Name, version: state.ref.Version, digest: state.ref.RecordDigest}) + candidates = append(candidates, listCandidate{name: state.ref.Name, version: state.ref.Version, digest: state.ref.RecordDigest, generation: state.ref.Generation}) } } sort.Slice(candidates, func(i, j int) bool { return candidates[i].less(&candidates[j]) }) @@ -170,10 +172,17 @@ func cursorAfter(candidate *listCandidate, cursor string) bool { } func (r *Repository) listEntry(ctx context.Context, candidate *listCandidate, labels []artifact.Label) (artifact.ListEntry, bool, error) { - record, _, err := r.loadRecord(ctx, candidate.digest) + record, _, recordErr := r.loadRecord(ctx, candidate.digest) + current, err := r.loadReference(ctx, layout.ReferenceKey(candidate.name, candidate.version)) if err != nil { return artifact.ListEntry{}, false, err } + if current == nil || !current.ref.Available() || current.ref.Generation != candidate.generation || current.ref.RecordDigest != candidate.digest { + return artifact.ListEntry{}, false, nil + } + if recordErr != nil { + return artifact.ListEntry{}, false, recordErr + } if record.Name != candidate.name || record.Version != candidate.version { return artifact.ListEntry{}, false, fmt.Errorf("%w: record %s describes %s:%s but its reference is %s:%s", errUtils.ErrArtifactIntegrityFailed, candidate.digest, record.Name, record.Version, candidate.name, candidate.version) diff --git a/pkg/artifact/s3/signature_test.go b/pkg/artifact/s3/signature_test.go index 48bd7905448..d69fc4927f1 100644 --- a/pkg/artifact/s3/signature_test.go +++ b/pkg/artifact/s3/signature_test.go @@ -2,6 +2,7 @@ package s3 import ( "context" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -147,6 +148,9 @@ func TestSignature_SubjectBinding(t *testing.T) { _, err := repo.Sign(context.Background(), wrongSize, newSigner(), artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + _, err = repo.Verify(context.Background(), wrongSize, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + badKind := subject badKind.Kind = artifact.SubjectOCIRoot _, err = repo.Sign(context.Background(), badKind, newSigner(), artifact.SignOptions{}) @@ -191,6 +195,15 @@ func TestSignature_ExportAndImportEvidenceAcrossRepositories(t *testing.T) { require.Len(t, imported, 1) assert.Equal(t, blobs[0].Evidence.Descriptor.Digest, imported[0].Descriptor.Digest, "re-import is idempotent") + wrongSize := blobs[0] + wrongSize.Evidence.Subject.Size++ + _, err = source.WriteEvidence(context.Background(), &sourcePinned, []artifact.EvidenceBlob{wrongSize}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + wrongSize = blobs[0] + wrongSize.Evidence.Descriptor.Size++ + _, err = source.WriteEvidence(context.Background(), &sourcePinned, []artifact.EvidenceBlob{wrongSize}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + corrupted := blobs[0] corrupted.Data = append([]byte(nil), corrupted.Data...) corrupted.Data[0] ^= 0xff @@ -222,3 +235,27 @@ func TestSignature_EvidenceFollowsTheRecordLifecycle(t *testing.T) { require.NoError(t, err) assert.Empty(t, managedKeys(spec.fake, "", currentDir), "delete removes the evidence the artifact exclusively owns") } + +func TestEvidenceExportFailsWhenAttemptsAreTruncated(t *testing.T) { + spec := signedSpec(t, noneSpec()) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + dir := layout.SignatureDir(artifact.SubjectRecord, pinned.RecordDigest) + for i := 0; i < maxEvidenceFiles+1; i++ { + spec.fake.Seed(fmt.Sprintf("%s/%064x.json", dir, i), []byte("bad"), nil) + } + loaded, diags, err := repo.loadEvidence(context.Background(), pinned.RecordDigest) + require.NoError(t, err) + assert.Empty(t, loaded) + require.Len(t, diags, maxEvidenceFiles+1) + assert.Equal(t, "evidence-truncated", diags[len(diags)-1].Code) + _, err = repo.ReadEvidence(context.Background(), &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) +} + +func TestFinishVerificationRejectsUnchecked(t *testing.T) { + base := &artifact.VerificationResult{Status: artifact.VerificationFailed} + result, err := finishVerification(base, &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationUnchecked, result.Status) +} diff --git a/pkg/artifact/service/commit.go b/pkg/artifact/service/commit.go index 5c2c660b4ae..69336062fd9 100644 --- a/pkg/artifact/service/commit.go +++ b/pkg/artifact/service/commit.go @@ -78,12 +78,13 @@ type publishState struct { // publication bundles everything one write needs: the operation, the target repository, the // incoming artifact, the force flag, the write closure, and the state commit fills in. type publication struct { - op *operation - tgt *target - in *incoming - force bool - write writer - state *publishState + op *operation + tgt *target + in *incoming + force bool + write writer + state *publishState + afterWrite func(context.Context, artifact.LockHandle) error } // commit runs the shared publication sequence under the writer lock: read the current reference, @@ -117,6 +118,9 @@ func (s *Service) commit(ctx context.Context, pub *publication) (err error) { } if decision == verdictUnchanged { s.reportUnchanged(pub, cur) + if pub.afterWrite != nil { + return pub.afterWrite(ctx, locks.handleFor(pub.tgt.alias)) + } return nil } result, err := s.execute(ctx, pub, cur, decision, locks.handleFor(pub.tgt.alias)) @@ -124,6 +128,9 @@ func (s *Service) commit(ctx context.Context, pub *publication) (err error) { return err } s.reportWritten(pub, result, locks, decision, cur) + if pub.afterWrite != nil { + return pub.afterWrite(ctx, locks.handleFor(pub.tgt.alias)) + } return nil } diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index ac25d29cb44..db44ff588b5 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -71,6 +71,7 @@ func (s *Service) copyArtifact(ctx context.Context, op *operation, request *arti defer cw.cleanup() pub := &publication{op: op, tgt: plan.dst, in: cw.in, force: request.Force, write: cw.write, state: &publishState{}} + pub.afterWrite = func(ctx context.Context, lock artifact.LockHandle) error { return s.carryEvidence(ctx, op, plan, lock) } if err := s.commit(ctx, pub); err != nil { return err } @@ -78,9 +79,6 @@ func (s *Service) copyArtifact(ctx context.Context, op *operation, request *arti op.result.Outcome = artifact.OutcomeCreated } op.result.Repository = request.To - // Evidence is carried even when the destination already holds the Record, because signatures - // added to the source after the first copy would otherwise never reach the destination. - s.carryEvidence(ctx, op, plan) return nil } @@ -291,38 +289,38 @@ func documentRoleFor(record *artifact.Record, digestValue string) (artifact.Docu // Evidence is never invented: without both optional interfaces, a diagnostic explains that the // destination holds no signatures. Evidence is written only to the exact record the copy produced // or found, never to a record a concurrent writer repointed the destination reference to. -func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPlan) { +func (s *Service) carryEvidence(ctx context.Context, op *operation, plan *copyPlan, lock artifact.LockHandle) error { reader, canRead := plan.src.repo.(artifact.EvidenceReader) writer, canWrite := plan.dst.repo.(artifact.EvidenceWriter) if !canRead || !canWrite { if verificationRequired(plan.dst.info.Policy) { op.warn("evidence-not-carried", "the destination policy requires verification but the repositories cannot transfer signature evidence; consumers of the destination copy must verify against the source evidence") } - return + return nil } blobs, err := reader.ReadEvidence(ctx, &plan.pinned) if err != nil { op.warn("evidence-read-failed", err.Error()) - return + return fmt.Errorf("read copy evidence: %w", err) } if len(blobs) == 0 { - return + return nil } dstPinned, err := resolveWritten(ctx, op, plan) if err == nil { blobs = missingEvidence(ctx, plan.dst.repo, &dstPinned, blobs) if len(blobs) > 0 { - _, err = writer.WriteEvidence(ctx, &dstPinned, blobs, nil) + _, err = writer.WriteEvidence(ctx, &dstPinned, blobs, lock) } } if err != nil { op.warn("evidence-write-failed", err.Error()) } + return nil } // resolveWritten resolves the destination reference and confirms it still points at the digest the -// copy wrote or found. A mismatch means another writer repointed the reference after the -// destination lock was released. +// copy wrote or found. A mismatch means the reference no longer has the expected digest. func resolveWritten(ctx context.Context, op *operation, plan *copyPlan) (artifact.PinnedReference, error) { dstRef, err := nativeRef(plan) if err != nil { diff --git a/pkg/artifact/service/edge_test.go b/pkg/artifact/service/edge_test.go index 91bf1d996cb..3c355bdee0d 100644 --- a/pkg/artifact/service/edge_test.go +++ b/pkg/artifact/service/edge_test.go @@ -399,7 +399,7 @@ func TestCopy_EvidenceCarryFailuresAreReported(t *testing.T) { }{e.repos["src"], reader} result, err := e.svc.Copy(context.Background(), artifact.CopyRequest{Ref: exampleRef, From: "src", To: "dst"}) - require.NoError(t, err) + require.Error(t, err) var warned bool for _, diagnostic := range result.Diagnostics { warned = warned || diagnostic.Code == "evidence-read-failed" diff --git a/pkg/artifact/sigstore/keys.go b/pkg/artifact/sigstore/keys.go index 112af75fecb..18e94e52246 100644 --- a/pkg/artifact/sigstore/keys.go +++ b/pkg/artifact/sigstore/keys.go @@ -40,18 +40,6 @@ func (k *keySet) trustedKeys() []signature.TrustedKey { return out } -// fingerprintFor maps a bundle's key hint back to the fingerprint of the key that matched. -func (k *keySet) fingerprintFor(hint string) string { - for index := range k.keys { - for _, candidate := range k.keys[index].hints { - if candidate == hint { - return k.keys[index].fingerprint - } - } - } - return hint -} - // newResolvedKey derives the fingerprint and the hint variants from a key's PKIX DER bytes. func newResolvedKey(der []byte, key crypto.PublicKey, extraHints ...string) resolvedKey { sum := sha256.Sum256(der) diff --git a/pkg/artifact/sigstore/signer.go b/pkg/artifact/sigstore/signer.go index f90554c273e..cdf4c1dc88f 100644 --- a/pkg/artifact/sigstore/signer.go +++ b/pkg/artifact/sigstore/signer.go @@ -51,6 +51,9 @@ func (s *Signer) Sign(ctx context.Context, subject artifact.SignatureSubject) (* if err != nil { return nil, fmt.Errorf("%w: sign %s %s: %w", errUtils.ErrArtifactVerificationFailed, subject.Kind, subject.Digest, err) } + if err := ctx.Err(); err != nil { + return nil, err + } format, mediaType, err := EvidenceFormatOf(bundle) if err != nil { return nil, err diff --git a/pkg/artifact/sigstore/signer_test.go b/pkg/artifact/sigstore/signer_test.go index d6038634d84..122c8e4d8a5 100644 --- a/pkg/artifact/sigstore/signer_test.go +++ b/pkg/artifact/sigstore/signer_test.go @@ -167,6 +167,7 @@ func TestEvidenceFormatOf(t *testing.T) { {name: "certificate", bundle: `{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"certificate":{"rawBytes":"AA=="}}}`, want: artifact.EvidenceSigstoreBundle, wantMedia: "application/vnd.dev.sigstore.bundle.v0.3+json"}, {name: "certificate chain defaults the media type", bundle: `{"verificationMaterial":{"x509CertificateChain":{"certificates":[]}}}`, want: artifact.EvidenceSigstoreBundle, wantMedia: defaultBundleMediaType}, {name: "public key", bundle: `{"mediaType":"application/vnd.dev.sigstore.bundle+json;version=0.2","verificationMaterial":{"publicKey":{"hint":"k"}}}`, want: artifact.EvidencePublicKeyBundle, wantMedia: "application/vnd.dev.sigstore.bundle+json;version=0.2"}, + {name: "foreign media type", bundle: `{"mediaType":"application/json","verificationMaterial":{"publicKey":{"hint":"k"}}}`, wantErr: true}, {name: "both", bundle: `{"verificationMaterial":{"certificate":{},"publicKey":{}}}`, wantErr: true}, {name: "neither", bundle: `{"verificationMaterial":{}}`, wantErr: true}, {name: "null material", bundle: `{"verificationMaterial":{"certificate":null}}`, wantErr: true}, @@ -187,3 +188,14 @@ func TestEvidenceFormatOf(t *testing.T) { } assert.Contains(t, defaultBundleMediaType, BundleMediaTypePrefix) } + +func TestSignerRejectsCancellationAfterBlobSigning(t *testing.T) { + f := newFixture(t) + signed := f.auth.SignWithKey(t, SubjectStatement(f.subject)) + ctx, cancel := context.WithCancel(context.Background()) + ctrl := gomock.NewController(t) + blob := signature.NewMockBlobSigner(ctrl) + blob.EXPECT().Sign(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, []byte) ([]byte, error) { cancel(); return signed.Bundle, nil }) + _, err := NewSigner(blob).Sign(ctx, f.subject) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/pkg/artifact/sigstore/statement.go b/pkg/artifact/sigstore/statement.go index c6dc3995788..2e7c7d6335f 100644 --- a/pkg/artifact/sigstore/statement.go +++ b/pkg/artifact/sigstore/statement.go @@ -3,6 +3,7 @@ package sigstore import ( "encoding/json" "fmt" + "strings" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -80,6 +81,9 @@ func EvidenceFormatOf(bundle []byte) (artifact.EvidenceFormat, string, error) { if mediaType == "" { mediaType = defaultBundleMediaType } + if !strings.HasPrefix(mediaType, BundleMediaTypePrefix) && !strings.HasPrefix(mediaType, "application/vnd.dev.sigstore.bundle+") { + return "", "", fmt.Errorf("%w: unsupported bundle media type %q", errUtils.ErrSignatureInvalid, mediaType) + } return format, mediaType, nil } diff --git a/pkg/artifact/sigstore/verifier.go b/pkg/artifact/sigstore/verifier.go index e850225bc0d..64083811b45 100644 --- a/pkg/artifact/sigstore/verifier.go +++ b/pkg/artifact/sigstore/verifier.go @@ -111,11 +111,17 @@ type candidate struct { func (v *Verifier) Verify(ctx context.Context, subject artifact.SignatureSubject, evidence []artifact.EvidenceBlob, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { defer perf.Track(nil, "sigstore.Verifier.Verify")() + if err := subject.Validate(); err != nil { + return nil, err + } result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} if fingerprint, err := policy.Fingerprint(); err == nil { result.PolicyFingerprint = fingerprint } if !policy.HasTrustConstraints() { + if policy.RequireSignature { + return result, fmt.Errorf("%w: no trust constraints configured", errUtils.ErrArtifactVerificationFailed) + } result.Status = artifact.VerificationUnchecked return result, nil } @@ -188,6 +194,7 @@ func (s *session) satisfy(ctx context.Context, set *artifact.TrustSet, candidate var failures []string if keyErr != nil { failures = append(failures, keyErr.Error()) + return satisfaction{index: -1, failures: failures}, nil } for index := range candidates { claim, err := s.authenticate(ctx, set, keys, &candidates[index]) @@ -249,17 +256,20 @@ func (s *session) verifyKeyless(ctx context.Context, trusted []artifact.TrustedI } func (s *session) verifyKey(ctx context.Context, keys *keySet, c *candidate) (artifact.SignerClaim, error) { - verifier := signature.KeyVerifier{Keys: keys.trustedKeys(), LoadRoot: s.loader} var failure error - for _, subjectDigest := range s.digests { - result, err := verifier.Verify(ctx, c.blob.Data, subjectDigest, s.policy.Offline) - if err == nil { - return artifact.SignerClaim{KeyID: keys.fingerprintFor(result.Hint)}, nil - } - if ctx.Err() != nil { - return artifact.SignerClaim{}, err + for _, key := range keys.keys { + one := &keySet{keys: []resolvedKey{key}} + verifier := signature.KeyVerifier{Keys: one.trustedKeys(), LoadRoot: s.loader} + for _, subjectDigest := range s.digests { + _, err := verifier.Verify(ctx, c.blob.Data, subjectDigest, s.policy.Offline) + if err == nil { + return artifact.SignerClaim{KeyID: key.fingerprint}, nil + } + if ctx.Err() != nil { + return artifact.SignerClaim{}, err + } + failure = err } - failure = err } return artifact.SignerClaim{}, fmt.Errorf("%s: %w", c.digest, failure) } @@ -319,7 +329,7 @@ func evidenceDigest(blob *artifact.EvidenceBlob) digest.Digest { func (s *session) screenOne(blob *artifact.EvidenceBlob) (artifact.EvidenceFormat, bool) { held := blob.Evidence switch { - case held.Subject.Kind != s.subject.Kind || held.Subject.Digest != s.subject.Digest: + case held.Subject != s.subject: s.add(artifact.SeverityWarning, "subject-mismatch", "evidence bound to a different subject was ignored") case held.Format != artifact.EvidenceSigstoreBundle && held.Format != artifact.EvidencePublicKeyBundle: s.add(artifact.SeverityWarning, "unsupported-evidence", fmt.Sprintf("evidence in unsupported format %q was ignored", held.Format)) diff --git a/pkg/artifact/sigstore/verifier_test.go b/pkg/artifact/sigstore/verifier_test.go index a9646d54d7b..73610e199c6 100644 --- a/pkg/artifact/sigstore/verifier_test.go +++ b/pkg/artifact/sigstore/verifier_test.go @@ -421,12 +421,15 @@ func toValues(in []digest.Digest) []digestValue { return in } func TestVerify_NoTrustConstraintsIsUnchecked(t *testing.T) { f := newFixture(t) - for _, policy := range []artifact.VerificationPolicy{{}, {RequireSignature: true}, {TrustSets: []artifact.TrustSet{{}}}} { + for _, policy := range []artifact.VerificationPolicy{{}, {TrustSets: []artifact.TrustSet{{}}}} { result, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{f.keyless(testIdentity, testIssuer)}, policy) require.NoError(t, err) assert.Equal(t, artifact.VerificationUnchecked, result.Status, "nothing is claimed without trust configuration") assert.Empty(t, result.Claims) } + result, err := f.verifier().Verify(context.Background(), f.subject, nil, artifact.VerificationPolicy{RequireSignature: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + assert.Equal(t, artifact.VerificationFailed, result.Status) } func TestVerify_CanceledContext(t *testing.T) { @@ -464,3 +467,21 @@ func tamperSignature(t *testing.T, bundle []byte) []byte { require.NoError(t, err) return out } + +func TestVerifyRejectsInvalidSubject(t *testing.T) { + f := newFixture(t) + subject := f.subject + subject.Digest = "invalid" + _, err := f.verifier().Verify(context.Background(), subject, nil, artifact.VerificationPolicy{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) +} + +func TestVerifyRejectsEvidenceWithDifferentSubjectMetadata(t *testing.T) { + f := newFixture(t) + for _, mutate := range []func(*artifact.SignatureSubject){func(s *artifact.SignatureSubject) { s.Size++ }, func(s *artifact.SignatureSubject) { s.MediaType = "different" }} { + blob := f.keyless(testIdentity, testIssuer) + mutate(&blob.Evidence.Subject) + _, err := f.verifier().Verify(context.Background(), f.subject, []artifact.EvidenceBlob{blob}, identityPolicy(testIdentity, testIssuer)) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + } +} diff --git a/pkg/oci/status.go b/pkg/oci/status.go index d1ac10b97b7..5ebef0a29ed 100644 --- a/pkg/oci/status.go +++ b/pkg/oci/status.go @@ -7,6 +7,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote/transport" "oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/registry/remote/errcode" + + "github.com/cloudposse/atmos/pkg/perf" ) // IsAlreadyExists reports whether a push found the identical content already stored, which @@ -32,3 +34,13 @@ func IsNotFound(err error) bool { } return errors.Is(err, errdef.ErrNotFound) } + +// IsCatalogUnavailable identifies responses indicating that a registry does not expose a catalog. +func IsCatalogUnavailable(err error) bool { + defer perf.Track(nil, "oci.IsCatalogUnavailable")() + if IsNotFound(err) { + return true + } + code := statusOf(err) + return code == http.StatusMethodNotAllowed || code == http.StatusNotImplemented +} diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 4820d605e2f..64488c4ebc9 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -327,11 +327,11 @@ func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []oci func (l *Layout) Untag(ctx context.Context, reference string) error { defer perf.Track(nil, "oci.Layout.Untag")() + l.mu.Lock() + defer l.mu.Unlock() if err := l.store.Untag(ctx, reference); err != nil { return layoutError(err) } - l.mu.Lock() - defer l.mu.Unlock() kept := l.index[:0:0] for i := range l.index { if l.index[i].Annotations[ocispec.AnnotationRefName] != reference { From 718d12b66cafc660e958bb9cba53d5778d958c3e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:07:47 -0600 Subject: [PATCH 06/53] fix: retain oversized evidence diagnostics --- pkg/artifact/s3/evidence.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index 69529df4a7a..f3e552db8b4 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -73,6 +73,9 @@ func (r *Repository) readEvidenceObject(ctx context.Context, key, hexDigest stri return nil, err, nil } raw, _, err := r.readObject(ctx, key, layout.MaxEvidenceFileBytes) + if errors.Is(err, errUtils.ErrArtifactTooLarge) { + return nil, err, nil + } if err != nil { return nil, nil, err } From 4cab69c686eefafac114267652f36825915424ae Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:09:01 -0600 Subject: [PATCH 07/53] test: cover S3 replacement cleanup recovery --- pkg/artifact/s3/edge_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index d7346a90b65..b1a47693164 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -291,3 +291,23 @@ func TestOptions_OwnerAndPartSize(t *testing.T) { assert.GreaterOrEqual(t, fake.CompletedUploads(), 1, "the configured part size forced multipart uploads") assert.NotEmpty(t, defaultOwner()) } + +func TestReplacementRetainsRecoveryMetadataUntilCleanupSucceeds(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{files: map[string]string{"old.txt": "old"}}, artifact.PublishOptions{}) + spec.fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, KeyPrefix: "team/app/1.0.0/old.txt", Status: 403, Code: "AccessDenied", Times: -1}) + prep := &prepareSpec{files: map[string]string{"new.txt": "new"}, opID: "replacement"} + second := publish(t, repo, prep, artifact.PublishOptions{Force: true, ExpectedPrior: &first.Pinned}) + _, ok := spec.fake.Object(layout.RecordPath(first.Pinned.RecordDigest)) + require.True(t, ok, "the old Record is needed to retry cleanup") + _, ok = spec.fake.Object(layout.OperationPath("replacement")) + require.True(t, ok, "the journal identifies the old location") + spec.fake.ClearFaults() + _, err := repo.Publish(context.Background(), prepare(t, repo, prep), artifact.PublishOptions{ExpectedPrior: &second.Pinned}) + require.NoError(t, err) + _, ok = spec.fake.Object(layout.RecordPath(first.Pinned.RecordDigest)) + assert.False(t, ok) + _, ok = spec.fake.Object(layout.OperationPath("replacement")) + assert.False(t, ok) +} From 45b1957f4f63074aa4e155adbf87987a97f6e7c3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:16:29 -0600 Subject: [PATCH 08/53] fix: recover interrupted S3 deletions and refresh identity --- pkg/artifact/blob/s3/client_test.go | 28 ++++++++ pkg/artifact/blob/s3/store.go | 10 +-- pkg/artifact/s3/delete.go | 103 ++++++++++++++++++++++++---- pkg/artifact/s3/delete_test.go | 21 ++++-- pkg/artifact/s3/edge_test.go | 13 +++- pkg/artifact/s3/identity_test.go | 2 +- pkg/artifact/s3/publish.go | 6 ++ 7 files changed, 161 insertions(+), 22 deletions(-) diff --git a/pkg/artifact/blob/s3/client_test.go b/pkg/artifact/blob/s3/client_test.go index 8074d3a8586..fc690bd913d 100644 --- a/pkg/artifact/blob/s3/client_test.go +++ b/pkg/artifact/blob/s3/client_test.go @@ -8,6 +8,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -204,3 +206,29 @@ func TestHTTPClientInjectionReachesTheSDK(t *testing.T) { type httpClientFunc func(*http.Request) (*http.Response, error) func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { return f(r) } + +func TestNamedIdentityReloadsRotatedCredentials(t *testing.T) { + headers := make(chan string, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + headers <- r.Header.Get("Authorization") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + creds := filepath.Join(t.TempDir(), "credentials") + write := func(key string) { + require.NoError(t, os.WriteFile(creds, []byte("[default]\naws_access_key_id = "+key+"\naws_secret_access_key = secret\n"), 0o600)) + } + write("first-key") + resolver := &fakeResolver{result: &store.AWSAuthConfig{CredentialsFile: creds, Profile: "default", Region: "us-east-1"}} + s, err := New("artifacts", WithIdentity("deploy", resolver), WithEndpoint(server.URL)) + require.NoError(t, err) + _, err = s.Stat(context.Background(), "file", blob.StatOptions{}) + require.NoError(t, err) + assert.Contains(t, <-headers, "Credential=first-key/") + write("second-key") + _, err = s.Stat(context.Background(), "file", blob.StatOptions{}) + require.NoError(t, err) + assert.Contains(t, <-headers, "Credential=second-key/") + assert.Equal(t, int32(2), resolver.calls.Load()) +} diff --git a/pkg/artifact/blob/s3/store.go b/pkg/artifact/blob/s3/store.go index 994392b71b4..bc5d8089d32 100644 --- a/pkg/artifact/blob/s3/store.go +++ b/pkg/artifact/blob/s3/store.go @@ -82,8 +82,8 @@ func (s *Store) Capabilities() blob.Capabilities { } } -// client returns the S3 client, building it on first use. A failed build is not cached, so a -// transient identity failure can be retried by the next operation. +// client builds named-identity clients per operation so refreshed identity files and endpoint +// configuration are observed. Injected clients and the default credential chain stay cached. func (s *Store) client(ctx context.Context) (API, error) { s.mu.Lock() defer s.mu.Unlock() @@ -94,8 +94,10 @@ func (s *Store) client(ctx context.Context) (API, error) { if err != nil { return nil, blob.NewError(blob.KindIdentity, "client", "", err) } - s.api = client - return s.api, nil + if s.cfg.client.Identity == "" { + s.api = client + } + return client, nil } // validateKey applies the configured key rules. diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go index 08f2dbe2f6a..5523358e8b7 100644 --- a/pkg/artifact/s3/delete.go +++ b/pkg/artifact/s3/delete.go @@ -3,6 +3,11 @@ package s3 import ( "context" "fmt" + "strings" + + "github.com/google/uuid" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -16,10 +21,9 @@ import ( // exclusively owns. Objects the repository does not manage (a user's extra object below a payload // prefix) stay in place, and nothing outside the repository prefix is touched. // -// The reference is removed first, with a conditional delete on the observed ETag, so the artifact -// disappears atomically; an interruption afterwards leaves garbage that a later publication of the -// same reference clears, never a reference that points at missing bytes. A pending publication is -// never deleted: its owner must repair it with the same intent. +// The reference is first made unavailable using the existing pending state; its journal retains +// payload ownership until cleanup succeeds. A later publication can finish interrupted deletion. +// A pending publication is never deleted: its owner must repair it with the same intent. // //nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (result *artifact.DeleteResult, err error) { @@ -45,7 +49,7 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference return r.deleteLocked(ctx, &pinned, &options) } -func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedReference, options *artifact.DeleteOptions) (*artifact.DeleteResult, error) { +func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedReference, options *artifact.DeleteOptions) (result *artifact.DeleteResult, err error) { v, err := r.openView(ctx, pinned) if err != nil { return nil, err @@ -54,26 +58,101 @@ func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedRe if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != stored.RecordDigest { return nil, fmt.Errorf("%w: %s:%s is %s but the delete expected %s", errUtils.ErrArtifactConflict, stored.Name, stored.Version, stored.RecordDigest, options.ExpectedCurrent.ResolvedDigest) } - keys, err := setOfRecord(v.record).keys(v.rep.Packaging, v.rep.Location) + claims, err := r.acquire(ctx, claimsResource, r.settings.owner, "") + if err != nil { + return nil, err + } + defer func() { + if releaseErr := claims.Release(ctx); err == nil { + err = releaseErr + } + }() + id := deletionPrefix + uuid.NewString() + intent := deletionIntent(id) + op := &layout.Operation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: artifact.RepositoryKindS3, + OperationID: id, Name: stored.Name, Version: stored.Version, IntentDigest: intent, + RecordDigest: stored.RecordDigest, CapturedAt: v.record.CreatedAt, Packaging: v.rep.Packaging, + Location: v.rep.Location, Payload: append([]ocispec.Descriptor{}, v.rep.Payload...), + Documents: append([]ocispec.Descriptor{}, v.rep.Documents...), Owner: r.settings.owner, + } + if err := r.writeOperation(ctx, op); err != nil { + return nil, deleteFailed(err) + } + pending := layout.NewPendingReference(stored.Name, stored.Version, id, intent, stored) + etag, err := r.writeReference(ctx, &pending, blob.IfMatch(v.ref.etag)) if err != nil { return nil, deleteFailed(err) } - if err := r.removeReference(ctx, v.ref); err != nil { + pendingState := &refState{ref: &pending, etag: etag} + keys, err := r.finishDeletion(ctx, pendingState, op) + if err != nil { return nil, err } + r.tidyOperation(ctx, stored.OperationID) + return &artifact.DeleteResult{Deleted: r.deletedLocators(v, keys), Digest: stored.RecordDigest}, nil +} + +// Deletions use the existing pending-reference and operation format so old writers refuse +// competing publications and reserve the old location until all payload cleanup is complete. +const deletionPrefix = "delete-" + +func deletionIntent(id string) digest.Digest { return digest.FromString("atmos/s3-delete/" + id) } + +func isDeletion(ref *layout.Reference) bool { + return ref != nil && !ref.Available() && strings.HasPrefix(ref.OperationID, deletionPrefix) && + ref.IntentDigest == deletionIntent(ref.OperationID) && ref.PriorRecordDigest != "" +} + +func validDeletionJournal(state *refState, op *layout.Operation) bool { + return isDeletion(state.ref) && op != nil && op.OperationID == state.ref.OperationID && + op.IntentDigest == state.ref.IntentDigest && op.RecordDigest == state.ref.PriorRecordDigest && + op.Name == state.ref.Name && op.Version == state.ref.Version +} + +func (r *Repository) finishDeletion(ctx context.Context, state *refState, op *layout.Operation) ([]string, error) { + if !validDeletionJournal(state, op) { + return nil, fmt.Errorf("%w: invalid deletion journal", errUtils.ErrArtifactIntegrityFailed) + } + keys, err := setOfDescriptors(op.Payload, op.Documents).keys(op.Packaging, op.Location) + if err != nil { + return nil, deleteFailed(err) + } if err := r.deleteAll(ctx, keys); err != nil { return nil, deleteFailed(err) } - if err := r.deleteEvidence(ctx, stored.RecordDigest); err != nil { + if err := r.deleteEvidence(ctx, op.RecordDigest); err != nil { return nil, deleteFailed(err) } - for _, owned := range []string{layout.RecordPath(stored.RecordDigest), layout.RepresentationPath(stored.Key())} { - if err := r.deleteObject(ctx, owned); err != nil { + for _, key := range []string{layout.RecordPath(op.RecordDigest), layout.RepresentationPath(state.ref.Key())} { + if err := r.deleteObject(ctx, key); err != nil { return nil, deleteFailed(err) } } - r.tidyOperation(ctx, stored.OperationID) - return &artifact.DeleteResult{Deleted: r.deletedLocators(v, keys), Digest: stored.RecordDigest}, nil + if err := r.removeReference(ctx, state); err != nil { + return nil, err + } + r.tidyOperation(ctx, op.OperationID) + return keys, nil +} + +// recoverDeletion is called with the reference lock held, before admitting a new publication. +func (r *Repository) recoverDeletion(ctx context.Context, state *refState) (err error) { + claims, err := r.acquire(ctx, claimsResource, r.settings.owner, "") + if err != nil { + return err + } + defer func() { + if releaseErr := claims.Release(ctx); err == nil { + err = releaseErr + } + }() + op, err := r.loadOperation(ctx, state.ref.OperationID) + if err != nil { + return err + } + _, err = r.finishDeletion(ctx, state, op) + return err } // removeReference deletes the reference only while its ETag is unchanged. A lost response is settled diff --git a/pkg/artifact/s3/delete_test.go b/pkg/artifact/s3/delete_test.go index c08bc960ff5..d0562df8413 100644 --- a/pkg/artifact/s3/delete_test.go +++ b/pkg/artifact/s3/delete_test.go @@ -53,10 +53,23 @@ func TestDelete_IsScopedToTheSelectedArtifact(t *testing.T) { assert.True(t, ok, "an unmanaged object inside the payload prefix stays") } - deletes := spec.fake.RequestsFor(s3test.OpDeleteObject) - require.NotEmpty(t, deletes) - assert.Equal(t, layout.ReferencePath(layout.ReferenceKey(testName, "2.0.0")), deletes[0].Key, "the reference is removed first") - assert.NotEmpty(t, deletes[0].Header.Get("If-Match"), "and conditionally on the observed ETag") + refKey := layout.ReferencePath(layout.ReferenceKey(testName, "2.0.0")) + var pendingSeq, firstPayloadDelete, referenceDelete int + for _, request := range spec.fake.Requests() { + if request.Op == s3test.OpPutObject && request.Key == refKey && request.Header.Get("If-Match") != "" { + pendingSeq = request.Seq + } + if request.Op == s3test.OpDeleteObject && firstPayloadDelete == 0 { + firstPayloadDelete = request.Seq + } + if request.Op == s3test.OpDeleteObject && request.Key == refKey { + referenceDelete = request.Seq + assert.NotEmpty(t, request.Header.Get("If-Match")) + } + } + require.Positive(t, pendingSeq, "reference is made unavailable conditionally") + assert.Less(t, pendingSeq, firstPayloadDelete) + assert.Greater(t, referenceDelete, firstPayloadDelete, "ownership remains until cleanup") }) } } diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index b1a47693164..e20f181bf8b 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -206,12 +206,23 @@ func TestDelete_PartialFailureLeavesGarbageNotADanglingReference(t *testing.T) { _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactDeleteFailed) _, err = repo.Resolve(context.Background(), defaultRef()) - require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the reference was removed first") + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "pending deletion makes the reference unavailable while retaining ownership") + state, err := repo.loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + oldClaim, err := repo.claimOf(context.Background(), state.ref) + require.NoError(t, err) + require.NotNil(t, oldClaim) + assert.Contains(t, oldClaim.Paths, "team/app/1.0.0", "older writers still observe the old location as claimed") + _, err = layout.Admit(&layout.AdmitInput{Current: state.ref, IncomingIntent: digest.FromString("old writer")}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "older writers cannot steal the pending deletion") spec.fake.ClearFaults() + repo.keyPattern = "new/{{ .Name }}/{{ .Version }}" again := publish(t, repo, &prepareSpec{files: map[string]string{"a.txt": "a2", "b.txt": "b2"}, opID: "op-again"}, artifact.PublishOptions{}) assert.Equal(t, artifact.OutcomeCreated, again.Outcome, "leftover payload objects are still owned by the reference's representation") assertIntact(t, repo, defaultRef()) + _, exists := spec.fake.Object("team/app/1.0.0/b.txt") + assert.False(t, exists, "recovery deletes leftovers at the old location") } // A lost response to the reference delete is settled by reading the reference. diff --git a/pkg/artifact/s3/identity_test.go b/pkg/artifact/s3/identity_test.go index 37630521388..3e469758d23 100644 --- a/pkg/artifact/s3/identity_test.go +++ b/pkg/artifact/s3/identity_test.go @@ -79,7 +79,7 @@ func TestIdentity_IsResolvedLazily(t *testing.T) { require.NoError(t, err) _, err = repo.List(context.Background(), artifact.ListQuery{}) require.NoError(t, err) - assert.Equal(t, int32(1), stub.calls.Load(), "the resolved client is reused") + assert.GreaterOrEqual(t, stub.calls.Load(), int32(2), "identity credentials are re-resolved for later operations") assert.NotEmpty(t, fake.Requests()) } diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index e90baa2937e..57b5320f4ff 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -96,6 +96,12 @@ func (r *Repository) admit(ctx context.Context, job *publishJob, options *artifa if err != nil { return nil, err } + if current != nil && isDeletion(current.ref) { + if err := r.recoverDeletion(ctx, current); err != nil { + return nil, err + } + current = nil + } input := layout.AdmitInput{ Current: refOf(current), IncomingRecord: job.recordDigest, IncomingIntent: job.prepared.IntentDigest, ExpectedPrior: options.ExpectedPrior, Force: options.Force, From 5079f7baade80a16b0bacffd9581cd235304c516 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:18:00 -0600 Subject: [PATCH 09/53] fix: report unresolved S3 claim creation --- pkg/artifact/s3/edge_test.go | 9 +++++---- pkg/artifact/s3/lock.go | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index e20f181bf8b..68e3b09d936 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -237,14 +237,15 @@ func TestDelete_LostResponseIsReconciled(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) } -// A create that never reached the endpoint is retried transparently. -func TestLock_CreateThatNeverArrivedIsRetried(t *testing.T) { +// An absent claim does not prove an ambiguous create cannot arrive later. +func TestLock_MissingAfterAmbiguousCreateRemainsAmbiguous(t *testing.T) { spec := noneSpec().fresh(t) repo := newRepo(t, spec) spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.DropBeforeCommit}) handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) - require.NoError(t, err) - require.NoError(t, handle.Release(context.Background())) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + assert.Nil(t, handle) + assert.Len(t, spec.fake.RequestsFor(s3test.OpPutObject), 1, "do not retry an unsettled create") } // More evidence than the read bound is reported, not silently dropped. diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 3ffdf5ff9ef..fe281718eb5 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -150,8 +150,10 @@ func (r *Repository) settleClaim(ctx context.Context, key string, doc *claimDoc) return nil, fmt.Errorf("%w: settle claim %s: %w", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key), err) case current != nil && current.Token == doc.Token: return r.adopt(ctx, key, doc) + case current == nil: + return nil, fmt.Errorf("%w: claim %s was not observed, but its create may still commit; inspect the claim before retrying", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key)) } - // Another owner's claim, or no claim at all: the create did not take effect. + // Another owner holds the claim. return nil, nil } From ba1a9d5ba33bdfbe1f81ff5b5d1604b07fa83347 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:29:45 -0600 Subject: [PATCH 10/53] fix: fence uncertain S3 lock claims --- pkg/artifact/s3/edge_test.go | 15 +++++++++++- pkg/artifact/s3/location_test.go | 13 ++++++++++ pkg/artifact/s3/lock.go | 42 +++++++++++++++++++++++++++++--- pkg/artifact/s3/lock_test.go | 31 +++++++++++++++++++++++ 4 files changed, 96 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index 68e3b09d936..c1ce26b65c2 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -3,6 +3,7 @@ package s3 import ( "bytes" "context" + "encoding/json" "strings" "testing" "time" @@ -14,6 +15,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob" "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" "github.com/cloudposse/atmos/pkg/artifact/layout" ) @@ -245,7 +247,18 @@ func TestLock_MissingAfterAmbiguousCreateRemainsAmbiguous(t *testing.T) { handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) assert.Nil(t, handle) - assert.Len(t, spec.fake.RequestsFor(s3test.OpPutObject), 1, "do not retry an unsettled create") + puts := spec.fake.RequestsFor(s3test.OpPutObject) + require.Len(t, puts, 2, "the second create installs a distinct fence, not a retry") + key := layout.LockPath(lockResource) + claim, _, err := repo.readClaim(context.Background(), key) + require.NoError(t, err) + require.NotNil(t, claim) + assert.Contains(t, claim.Owner, "pending reconciliation") + assert.ErrorContains(t, repo.heldError(context.Background(), key, lockResource, nil), "Do not delete it until the original remote request is known to have finished") + delayed, err := json.Marshal(&claimDoc{SchemaVersion: layout.SchemaVersion, Resource: lockResource, Owner: "owner-a", Token: "late-request"}) + require.NoError(t, err) + _, err = repo.putConditional(context.Background(), key, delayed, contentTypeJSON, blob.IfAbsent()) + require.True(t, blob.IsPreconditionFailed(err), "the retained fence rejects a late original create") } // More evidence than the read bound is reported, not silently dropped. diff --git a/pkg/artifact/s3/location_test.go b/pkg/artifact/s3/location_test.go index 1d4f3bc65b3..17aec514b50 100644 --- a/pkg/artifact/s3/location_test.go +++ b/pkg/artifact/s3/location_test.go @@ -192,3 +192,16 @@ func TestDistinctLocationsCoexist(t *testing.T) { assertIntact(t, archives, ref("two/app", testVersion)) assert.Equal(t, []string{"one/app:1.0.0", "two/app:1.0.0"}, listNames(t, files), "both repositories share one namespace") } + +// A legacy writer can reserve an empty path without writing an index or a payload object. +// An optimized reader must still inspect the pending reference before allowing a collision. +func TestLegacyPendingClaimBlocksCollisionBeforeAnyPayloadExists(t *testing.T) { + fake := s3test.NewServer(t) + legacy := newRepo(t, noneSpec().withPattern("shared/empty").with(func(s *rspec) { s.fake = fake }).withOpts(withFaults(crashAt(faultBeforePayload)))) + current := newRepo(t, noneSpec().withPattern("shared/empty/child").with(func(s *rspec) { s.fake = fake })) + _, err := legacy.Publish(context.Background(), prepare(t, legacy, &prepareSpec{name: "legacy/app"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + assert.Empty(t, managedKeys(fake, "", "shared/empty"), "the pending claim precedes all payload writes") + _, err = current.Publish(context.Background(), prepare(t, current, &prepareSpec{name: "new/app"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the legacy reference must still be examined") +} diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index fe281718eb5..79e6fe86aa7 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "sync" "time" @@ -20,6 +21,8 @@ import ( // opLock names the lock operation in capability errors. const opLock = "lock" +const reconciliationOwnerPrefix = "pending reconciliation (do not delete until original remote request is known to have finished): " + // maxClaimBytes bounds a claim object read. const maxClaimBytes int64 = 64 << 10 @@ -151,10 +154,37 @@ func (r *Repository) settleClaim(ctx context.Context, key string, doc *claimDoc) case current != nil && current.Token == doc.Token: return r.adopt(ctx, key, doc) case current == nil: - return nil, fmt.Errorf("%w: claim %s was not observed, but its create may still commit; inspect the claim before retrying", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key)) + handle, fenceErr := r.fenceUnknownClaim(ctx, readCtx, key, doc) + if handle != nil { + return handle, nil + } + return nil, errors.Join(fenceErr, ctx.Err()) + } + return nil, errors.Join(ctx.Err(), errUtils.ErrArtifactConflict, fmt.Errorf("%w: claim %s is held by another owner, but the prior create may still commit after it is released", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key))) +} + +// fenceUnknownClaim leaves a visible claim at an empty key while the first PUT may still arrive. +// The fence is intentionally not released: a finite wait or an empty read cannot prove that the +// first request has stopped executing. Legacy writers understand the existing claim format. +func (r *Repository) fenceUnknownClaim(originalCtx, ctx context.Context, key string, original *claimDoc) (*claimHandle, error) { + fence := *original + fence.Token = uuid.NewString() + fence.Owner = reconciliationOwnerPrefix + original.Owner + raw, err := json.Marshal(&fence) + if err != nil { + return nil, fmt.Errorf("%w: encode claim fence: %w", errUtils.ErrArtifactPublicationAmbiguous, err) + } + _, err = r.putConditional(ctx, key, raw, contentTypeJSON, blob.IfAbsent()) + if err == nil { + return nil, fmt.Errorf("%w: claim %s is fenced pending reconciliation of a possibly late create; do not delete the fence until the original request is known to have finished", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key)) } - // Another owner holds the claim. - return nil, nil + if blob.IsPreconditionFailed(err) { + current, _, readErr := r.readClaim(ctx, key) + if readErr == nil && current != nil && current.Token == original.Token { + return r.adopt(originalCtx, key, original) + } + } + return nil, fmt.Errorf("%w: could not conclusively fence claim %s; its original create or fence may still commit: %w", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key), err) } // adopt turns a created claim into a handle. A context that ended while the claim was being created @@ -194,7 +224,11 @@ func (r *Repository) heldError(ctx context.Context, key, resource string, cause if holder != nil { message = fmt.Sprintf("lock %q is held by owner %q (operation %q, since %s) through claim object %s", resource, holder.Owner, holder.OperationID, holder.AcquiredAt, r.uri(key)) - message += "; claims never expire and are never taken over automatically. If that writer has stopped, confirm it is no longer running and delete the claim object to recover" + if strings.HasPrefix(holder.Owner, reconciliationOwnerPrefix) { + message += "; this claim fences an ambiguous create. Do not delete it until the original remote request is known to have finished" + } else { + message += "; claims never expire and are never taken over automatically. If that writer has stopped, confirm it is no longer running and delete the claim object to recover" + } } if cause != nil { return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactConflict, message, cause) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 46d2f627f55..3ea837b8392 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -319,3 +319,34 @@ func TestPublish_UsesTheCallersLock(t *testing.T) { assert.Len(t, lockObjects(spec.fake), 1, "the caller's lock is still held") require.NoError(t, handle.Release(context.Background())) } + +func TestSettleClaimDoesNotRetryBehindForeignOwner(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + ctx := context.Background() + other, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "other"}) + require.NoError(t, err) + defer other.Release(ctx) + before := len(spec.fake.RequestsFor(s3test.OpPutObject)) + handle, err := repo.settleClaim(ctx, layout.LockPath(lockResource), &claimDoc{Resource: lockResource, Owner: "original", Token: "original-token"}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + assert.Nil(t, handle) + assert.Len(t, spec.fake.RequestsFor(s3test.OpPutObject), before, "a foreign claim does not prove the original request has finished") +} + +func TestFenceAdoptsOriginalClaimThatWonTheRace(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + ctx := context.Background() + key := layout.LockPath(lockResource) + doc := &claimDoc{SchemaVersion: layout.SchemaVersion, Resource: lockResource, Owner: "original", Token: "original-token"} + raw, err := json.Marshal(doc) + require.NoError(t, err) + _, err = repo.putConditional(ctx, key, raw, contentTypeJSON, blob.IfAbsent()) + require.NoError(t, err) + handle, err := repo.fenceUnknownClaim(ctx, ctx, key, doc) + require.NoError(t, err) + require.NotNil(t, handle) + assert.Equal(t, doc.Token, handle.Token()) + require.NoError(t, handle.Release(ctx)) +} From 3814ae67c5fccee14302513bb9556da2022acf5f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 07:00:13 -0600 Subject: [PATCH 11/53] fix(artifacts): coordinate delegated S3 mutations --- pkg/artifact/s3/delegated_mutation_test.go | 115 +++++++++++++++++++++ pkg/artifact/s3/delete.go | 2 +- pkg/artifact/s3/evidence.go | 4 +- pkg/artifact/s3/lock.go | 10 +- pkg/artifact/s3/publish.go | 19 ++-- pkg/artifact/s3/service_test.go | 59 +++++++++++ 6 files changed, 191 insertions(+), 18 deletions(-) create mode 100644 pkg/artifact/s3/delegated_mutation_test.go diff --git a/pkg/artifact/s3/delegated_mutation_test.go b/pkg/artifact/s3/delegated_mutation_test.go new file mode 100644 index 00000000000..2a25afe7d9b --- /dev/null +++ b/pkg/artifact/s3/delegated_mutation_test.go @@ -0,0 +1,115 @@ +package s3 + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) { + for _, operation := range []string{"sign", "evidence", "delete"} { + t.Run(operation, func(t *testing.T) { + ctx := context.Background() + spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(time.Second)) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + signer := newSigner() + _, err := repo.Sign(ctx, subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + evidence, err := repo.ReadEvidence(ctx, &pinned) + require.NoError(t, err) + require.Len(t, evidence, 1) + coordinator := newRepo(t, spec.with(func(s *rspec) { s.prefix = "coordinator" })) + delegated, err := coordinator.Lock(ctx, artifact.LockRequest{Resource: "destination/" + lockResource, Owner: "delegate"}) + require.NoError(t, err) + t.Cleanup(func() { _ = delegated.Release(ctx) }) + mutate := func(ctx context.Context, lock artifact.LockHandle) error { + switch operation { + case "sign": + _, err := repo.Sign(ctx, subject, signer, artifact.SignOptions{Lock: lock}) + return err + case "evidence": + _, err := repo.WriteEvidence(ctx, &pinned, evidence, lock) + return err + default: + _, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{Lock: lock}) + return err + } + } + held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "direct"}) + require.NoError(t, err) + before := objectSnapshot(t, spec.fake) + wait, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + require.ErrorIs(t, mutate(wait, delegated), context.DeadlineExceeded) + assert.Equal(t, before, objectSnapshot(t, spec.fake), "failed acquisition must not mutate target data or evidence") + require.NoError(t, held.Release(ctx)) + require.NoError(t, mutate(ctx, delegated)) + if operation == "delete" { + _, err := repo.Resolve(ctx, defaultRef()) + require.Error(t, err) + } else { + after, err := repo.ReadEvidence(ctx, &pinned) + require.NoError(t, err) + require.NotEmpty(t, after) + } + }) + } +} + +func TestMutationsRejectInvalidSameRepositoryClaims(t *testing.T) { + for _, operation := range []string{"sign", "evidence", "delete"} { + for _, invalid := range []string{"wrong-resource", "released", "stale"} { + t.Run(operation+"/"+invalid, func(t *testing.T) { + ctx := context.Background() + spec := signedSpec(t, noneSpec()) + repo := newRepo(t, spec) + pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned + subject := recordSubject(t, repo, &pinned) + signer := newSigner() + _, err := repo.Sign(ctx, subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + evidence, err := repo.ReadEvidence(ctx, &pinned) + require.NoError(t, err) + resource := lockResource + if invalid == "wrong-resource" { + resource = "other:1" + } + lock, err := repo.Lock(ctx, artifact.LockRequest{Resource: resource, Owner: "owner"}) + require.NoError(t, err) + if invalid == "released" { + require.NoError(t, lock.Release(ctx)) + } + if invalid == "stale" { + claim := lock.(*claimHandle) + require.NoError(t, repo.store.Delete(ctx, claim.key, blob.DeleteOptions{})) + replacement, err := repo.Lock(ctx, artifact.LockRequest{Resource: resource, Owner: "replacement"}) + require.NoError(t, err) + t.Cleanup(func() { _ = replacement.Release(ctx) }) + } + before := objectSnapshot(t, spec.fake) + switch operation { + case "sign": + _, err = repo.Sign(ctx, subject, signer, artifact.SignOptions{Lock: lock}) + case "evidence": + _, err = repo.WriteEvidence(ctx, &pinned, evidence, lock) + default: + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Lock: lock}) + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, before, objectSnapshot(t, spec.fake)) + if invalid == "wrong-resource" { + require.NoError(t, lock.Release(ctx)) + } + }) + } + } +} diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go index 5523358e8b7..971349b766a 100644 --- a/pkg/artifact/s3/delete.go +++ b/pkg/artifact/s3/delete.go @@ -37,7 +37,7 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference if err != nil { return nil, err } - release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version) + release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version, "") if err != nil { return nil, err } diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index f3e552db8b4..33d331cb185 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -136,7 +136,7 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject if err != nil { return nil, err } - release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version) + release, err := r.holdClaim(ctx, options.Lock, record.Name+":"+record.Version, "") if err != nil { return nil, err } @@ -338,7 +338,7 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR if err != nil { return nil, err } - release, err := r.holdClaim(ctx, lock, record.Name+":"+record.Version) + release, err := r.holdClaim(ctx, lock, record.Name+":"+record.Version, "") if err != nil { return nil, err } diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 79e6fe86aa7..027b4b7add5 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -320,16 +320,16 @@ func (l *claimHandle) goneAfter(ctx context.Context, err error) bool { } // holdClaim validates an existing claim or acquires one, and keeps it held through a mutation. -func (r *Repository) holdClaim(ctx context.Context, supplied artifact.LockHandle, resource string) (func() error, error) { - if supplied == nil { - handle, err := r.acquire(ctx, resource, r.settings.owner, "") +func (r *Repository) holdClaim(ctx context.Context, supplied artifact.LockHandle, resource, operationID string) (func() error, error) { + handle, local := supplied.(*claimHandle) + if !local || handle == nil || handle.repo != r { + handle, err := r.acquire(ctx, resource, r.settings.owner, operationID) if err != nil { return nil, err } return func() error { return handle.Release(ctx) }, nil } - handle, ok := supplied.(*claimHandle) - if !ok || handle.repo != r || handle.resource != resource { + if handle.resource != resource { return nil, fmt.Errorf("%w: supplied lock does not guard %s", errUtils.ErrArtifactConflict, resource) } handle.mu.Lock() diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index 57b5320f4ff..9ed9e028c30 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -56,17 +56,16 @@ func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArt scope := artifact.LockScopeRepository if options.Lock != nil { scope = options.Lock.Scope() - } else { - lock, lockErr := r.acquire(ctx, prepared.Record.Name+":"+prepared.Record.Version, r.settings.owner, job.opID) - if lockErr != nil { - return nil, lockErr - } - defer func() { - if releaseErr := lock.Release(ctx); err == nil { - err = releaseErr - } - }() } + release, err := r.holdClaim(ctx, options.Lock, prepared.Record.Name+":"+prepared.Record.Version, job.opID) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := release(); err == nil { + err = releaseErr + } + }() return r.publishLocked(ctx, job, &options, scope) } diff --git a/pkg/artifact/s3/service_test.go b/pkg/artifact/s3/service_test.go index 0f6d731af63..8930889e569 100644 --- a/pkg/artifact/s3/service_test.go +++ b/pkg/artifact/s3/service_test.go @@ -156,3 +156,62 @@ func TestService_ReconcilesALostResponseAfterCommit(t *testing.T) { require.NoError(t, err) assert.Equal(t, artifact.OutcomeRead, pulled.Outcome) } + +// The shared service must hold both the subject lock and the canonical S3 reference lock. +func TestService_SignAndDeleteS3ByDigest(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithVerifier(&artifacttest.FakeVerifier{}))) + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "bucket": {Kind: artifact.RepositoryKindS3, Packaging: "none", Options: map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1"}}, + }} + svc, err := service.New(service.FromConfig(cfg, registry), nil, service.WithSigner(newSigner()), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + source := t.TempDir() + fixtures.WriteFilePackage(t, source) + published, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "bucket", Source: source, Name: testName, Version: testVersion}) + require.NoError(t, err) + ref := artifact.Ref{Name: testName, Digest: published.RecordDigest} + _, err = svc.Sign(context.Background(), artifact.SignRequest{Repository: "bucket", Ref: ref}) + require.NoError(t, err) + deleted, err := svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "bucket", Ref: ref}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDeleted, deleted.Outcome) +} + +// A delegated service lock must coexist with the S3 destination's own reference claim. +func TestService_DelegatedS3EvidenceAndDelete(t *testing.T) { + isolateAWS(t) + fake := s3test.NewServer(t) + registry := artifact.NewRepositoryRegistry() + require.NoError(t, local.RegisterWith(registry)) + require.NoError(t, RegisterWith(registry, WithVerifier(&artifacttest.FakeVerifier{}))) + options := func(prefix string) map[string]any { + return map[string]any{"bucket": fake.Bucket(), "endpoint": fake.Endpoint(), "region": "us-east-1", "prefix": prefix} + } + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "coordinator": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(t.TempDir(), "coordinator")}}, + "source": {Kind: artifact.RepositoryKindS3, Packaging: "none", Options: options("source")}, + "target": {Kind: artifact.RepositoryKindS3, Packaging: "none", Options: options("target"), Locking: &schema.ArtifactLockingConfig{Repository: "coordinator"}}, + }} + svc, err := service.New(service.FromConfig(cfg, registry), nil, service.WithSigner(newSigner()), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + ctx := context.Background() + src := t.TempDir() + fixtures.WriteFilePackage(t, src) + published, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "source", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + digestRef := artifact.Ref{Name: testName, Digest: published.RecordDigest} + _, err = svc.Sign(ctx, artifact.SignRequest{Repository: "source", Ref: digestRef}) + require.NoError(t, err) + _, err = svc.Copy(ctx, artifact.CopyRequest{From: "source", To: "target", Ref: defaultRef()}) + require.NoError(t, err) + inspection, err := svc.Inspect(ctx, artifact.InspectRequest{Repository: "target", Ref: defaultRef()}) + require.NoError(t, err) + require.NotEmpty(t, inspection.Inspection.Evidence, "copy carries evidence through the delegated lock") + _, err = svc.Sign(ctx, artifact.SignRequest{Repository: "target", Ref: digestRef}) + require.NoError(t, err) + _, err = svc.Delete(ctx, artifact.DeleteRequest{Repository: "target", Ref: defaultRef()}) + require.NoError(t, err) +} From ca5fa818133b6a8ec92b278fead3af8bfc936eac Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 16:16:46 -0600 Subject: [PATCH 12/53] fix(artifact): identify uncertain S3 lock attempts for recovery When the outcome of a lock claim create is uncertain, acquisition already stops without a lock, and nothing it protects is mutated. Its errors named only the claim object, so an operator could not tell whether a claim or fence left at that key came from the failed attempt. The errors now include the attempt's resource, owner, operation, and lock ID, plus the fence's lock ID when one is installed. The package documentation now describes the deliberate recovery: confirm the writer that reported the error has exited and that the claim object still names that attempt, then delete that one object. Claims and fences still never expire, another owner's claim is never deleted, and an empty read or elapsed timeout is still not treated as proof that the uncertain create cannot commit. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/s3/doc.go | 10 ++++++++++ pkg/artifact/s3/edge_test.go | 7 +++++-- pkg/artifact/s3/lock.go | 16 ++++++++++++---- pkg/artifact/s3/lock_test.go | 2 ++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/pkg/artifact/s3/doc.go b/pkg/artifact/s3/doc.go index 522aaa264df..aa35056e007 100644 --- a/pkg/artifact/s3/doc.go +++ b/pkg/artifact/s3/doc.go @@ -51,6 +51,16 @@ // (for example with the AWS console or "aws s3 rm"); nothing else needs to change. The brief // repository-wide claim taken while a location is reserved is recovered the same way. // +// An acquisition whose create outcome is uncertain stops without a lock, and nothing protected by +// that lock is mutated. The error names the claim object and the attempt's owner, operation, and +// lock ID. An empty read or an elapsed timeout never proves the uncertain create cannot still +// commit, so the attempt may leave behind either its own claim (with that lock ID) or a +// reconciliation fence whose owner starts with "pending reconciliation". Neither expires or is +// removed automatically, and another owner's claim is never deleted. To recover, the operator +// confirms the writer that reported the error has exited, checks that the claim object still names +// that attempt, and deletes that one object deliberately. Later attempts report the claim as held +// until then. +// // # Signatures // // Evidence is stored as digest-named objects below the signature directory of the canonical Record diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index c1ce26b65c2..f26df198d32 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -244,8 +244,8 @@ func TestLock_MissingAfterAmbiguousCreateRemainsAmbiguous(t *testing.T) { spec := noneSpec().fresh(t) repo := newRepo(t, spec) spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.DropBeforeCommit}) - handle, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) - require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + handle, lockErr := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.ErrorIs(t, lockErr, errUtils.ErrArtifactPublicationAmbiguous) assert.Nil(t, handle) puts := spec.fake.RequestsFor(s3test.OpPutObject) require.Len(t, puts, 2, "the second create installs a distinct fence, not a retry") @@ -254,6 +254,9 @@ func TestLock_MissingAfterAmbiguousCreateRemainsAmbiguous(t *testing.T) { require.NoError(t, err) require.NotNil(t, claim) assert.Contains(t, claim.Owner, "pending reconciliation") + assert.ErrorContains(t, lockErr, `owner "owner-a"`, "the error identifies the uncertain attempt") + assert.ErrorContains(t, lockErr, "lock ID "+claim.Token, "the error names the fence the operator would recover") + assert.ErrorContains(t, lockErr, repo.uri(key)) assert.ErrorContains(t, repo.heldError(context.Background(), key, lockResource, nil), "Do not delete it until the original remote request is known to have finished") delayed, err := json.Marshal(&claimDoc{SchemaVersion: layout.SchemaVersion, Resource: lockResource, Owner: "owner-a", Token: "late-request"}) require.NoError(t, err) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 027b4b7add5..d3fc21a7c1c 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -150,7 +150,7 @@ func (r *Repository) settleClaim(ctx context.Context, key string, doc *claimDoc) current, _, err := r.readClaim(readCtx, key) switch { case err != nil: - return nil, fmt.Errorf("%w: settle claim %s: %w", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key), err) + return nil, fmt.Errorf("%w: settle %s: %w", errUtils.ErrArtifactPublicationAmbiguous, r.attempt(key, doc), err) case current != nil && current.Token == doc.Token: return r.adopt(ctx, key, doc) case current == nil: @@ -160,7 +160,7 @@ func (r *Repository) settleClaim(ctx context.Context, key string, doc *claimDoc) } return nil, errors.Join(fenceErr, ctx.Err()) } - return nil, errors.Join(ctx.Err(), errUtils.ErrArtifactConflict, fmt.Errorf("%w: claim %s is held by another owner, but the prior create may still commit after it is released", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key))) + return nil, errors.Join(ctx.Err(), errUtils.ErrArtifactConflict, fmt.Errorf("%w: %s found the claim held by another owner, but the prior create may still commit after it is released", errUtils.ErrArtifactPublicationAmbiguous, r.attempt(key, doc))) } // fenceUnknownClaim leaves a visible claim at an empty key while the first PUT may still arrive. @@ -176,7 +176,7 @@ func (r *Repository) fenceUnknownClaim(originalCtx, ctx context.Context, key str } _, err = r.putConditional(ctx, key, raw, contentTypeJSON, blob.IfAbsent()) if err == nil { - return nil, fmt.Errorf("%w: claim %s is fenced pending reconciliation of a possibly late create; do not delete the fence until the original request is known to have finished", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key)) + return nil, fmt.Errorf("%w: %s is fenced (fence lock ID %s) pending reconciliation of a possibly late create; do not delete the fence until the original request is known to have finished", errUtils.ErrArtifactPublicationAmbiguous, r.attempt(key, original), fence.Token) } if blob.IsPreconditionFailed(err) { current, _, readErr := r.readClaim(ctx, key) @@ -184,7 +184,15 @@ func (r *Repository) fenceUnknownClaim(originalCtx, ctx context.Context, key str return r.adopt(originalCtx, key, original) } } - return nil, fmt.Errorf("%w: could not conclusively fence claim %s; its original create or fence may still commit: %w", errUtils.ErrArtifactPublicationAmbiguous, r.uri(key), err) + return nil, fmt.Errorf("%w: could not conclusively fence %s; its original create or fence may still commit: %w", errUtils.ErrArtifactPublicationAmbiguous, r.attempt(key, original), err) +} + +// attempt describes one acquisition attempt so an operator can match an ambiguous outcome to the +// claim object it may have left behind: the claim content carries the same owner, operation, and +// lock ID. +func (r *Repository) attempt(key string, doc *claimDoc) string { + return fmt.Sprintf("lock %q attempt by owner %q (operation %q, lock ID %s) at claim object %s", + doc.Resource, doc.Owner, doc.OperationID, doc.Token, r.uri(key)) } // adopt turns a created claim into a handle. A context that ended while the claim was being created diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 3ea837b8392..5fa0d56d096 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -330,6 +330,8 @@ func TestSettleClaimDoesNotRetryBehindForeignOwner(t *testing.T) { before := len(spec.fake.RequestsFor(s3test.OpPutObject)) handle, err := repo.settleClaim(ctx, layout.LockPath(lockResource), &claimDoc{Resource: lockResource, Owner: "original", Token: "original-token"}) require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + assert.ErrorContains(t, err, `owner "original"`, "the error identifies the uncertain attempt") + assert.ErrorContains(t, err, "lock ID original-token", "an operator can match a late claim to this attempt") assert.Nil(t, handle) assert.Len(t, spec.fake.RequestsFor(s3test.OpPutObject), before, "a foreign claim does not prove the original request has finished") } From 6d0c9a9683719dee8d49546ddf8c1800e78aa402 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 22:17:28 -0600 Subject: [PATCH 13/53] fix(artifact): close remaining OCI review findings - An unchanged OCI tag no longer bypasses the compare-and-swap guard. A caller that expected the tag to be absent now gets a conflict, which matches the layout and reference-implementation contract. - A signature referrer whose descriptor names another concrete artifact type is skipped before it costs inspection budget, so unrelated referrers cannot crowd out a valid bundle. Untyped referrers and the registry's empty-config placeholder type are still inspected. - Verify reports an incomplete inspection, not "no evidence", when the candidate budget truncated the scan. - Layout.Untag writes index.json before it untags the store. A failed write leaves the handle unchanged, so the call can be retried. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/oci/attach.go | 16 ++++++++- pkg/artifact/oci/publish.go | 5 +++ pkg/artifact/oci/review_guard_test.go | 48 +++++++++++++++++++++++++++ pkg/artifact/oci/sign.go | 14 ++++++++ pkg/oci/transport_layout.go | 14 +++++--- 5 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/oci/review_guard_test.go diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index 0bb2c378d0a..2ba024770a8 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -51,7 +51,7 @@ func (r *Repository) attachments(ctx context.Context, st store, name string, roo set := &attachmentSet{} inspected := 0 for index := range predecessors { - if !isManifestMediaType(predecessors[index].MediaType) { + if !isManifestMediaType(predecessors[index].MediaType) || !mayBeBundle(&predecessors[index]) { continue } if inspected >= maxAttachments*4 { @@ -107,6 +107,20 @@ func (r *Repository) readAttachment(ctx context.Context, st store, name string, return newAttachment(&manifest, desc, layer, rootDesc.Digest), "", nil } +// mayBeBundle reports whether a referrer descriptor can be a Sigstore bundle. A descriptor that +// names another concrete artifact type is skipped before it costs part of the inspection budget, +// so unrelated referrers cannot crowd out real signature evidence. The descriptor type is only a +// hint: registries derive it from the config media type, so an untyped referrer or one with the +// empty config placeholder is still inspected. +func mayBeBundle(desc *ocispec.Descriptor) bool { + switch desc.ArtifactType { + case "", ocispec.MediaTypeEmptyJSON: + return true + default: + return strings.HasPrefix(desc.ArtifactType, bundleMediaTypePrefix) + } +} + // isBundleReferrer reports whether a manifest is a Sigstore bundle attached to the subject digest. // Referrers of other types and manifests that merely contain the root as a child are not evidence. func isBundleReferrer(manifest *ocispec.Manifest, subject digest.Digest) bool { diff --git a/pkg/artifact/oci/publish.go b/pkg/artifact/oci/publish.go index 34323a2871d..c80ceb86bfa 100644 --- a/pkg/artifact/oci/publish.go +++ b/pkg/artifact/oci/publish.go @@ -100,6 +100,11 @@ func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, op return "", fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest, options.ExpectedPrior.ResolvedDigest) } if state.desc.Digest == incoming.digest { + // Unchanged still honors the compare-and-swap guard, matching the layout contract: a caller + // that expected the tag to be absent gets a conflict, not a silent success. + if options.ExpectedPrior == nil { + return "", fmt.Errorf("%w: %s already exists (%s) but the caller expected it not to", errUtils.ErrArtifactConflict, ref, state.desc.Digest) + } return artifact.OutcomeUnchanged, nil } if state.loaded.class == classPlugin { diff --git a/pkg/artifact/oci/review_guard_test.go b/pkg/artifact/oci/review_guard_test.go new file mode 100644 index 00000000000..7260295cc69 --- /dev/null +++ b/pkg/artifact/oci/review_guard_test.go @@ -0,0 +1,48 @@ +package oci + +import ( + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// An unchanged tag still honors the compare-and-swap guard: a caller that expected the tag absent +// gets a conflict, while the matching expected prior reports unchanged. +func TestDecidePublishUnchangedHonorsExpectedPrior(t *testing.T) { + ref, err := artifact.ParseRef("team/app:1.0.0") + require.NoError(t, err) + current := digest.FromString("current") + state := &tagState{exists: true, desc: ocispec.Descriptor{Digest: current}} + incoming := &incomingRoot{digest: current} + + _, err = decidePublish(ref, state, incoming, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "an absent expected prior must not report unchanged") + + outcome, err := decidePublish(ref, state, incoming, artifact.PublishOptions{ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: current}}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, outcome) + + _, err = decidePublish(ref, state, incoming, artifact.PublishOptions{ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digest.FromString("stale")}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected prior must not report unchanged") +} + +// Referrers that name another artifact type are skipped before they cost inspection budget. +func TestMayBeBundleFiltersByArtifactType(t *testing.T) { + assert.True(t, mayBeBundle(&ocispec.Descriptor{}), "an untyped referrer must still be inspected") + assert.True(t, mayBeBundle(&ocispec.Descriptor{ArtifactType: ocispec.MediaTypeEmptyJSON}), "a registry-derived empty config type is only a hint") + assert.True(t, mayBeBundle(&ocispec.Descriptor{ArtifactType: bundleMediaTypePrefix + "v0.3+json"})) + assert.False(t, mayBeBundle(&ocispec.Descriptor{ArtifactType: "application/vnd.example.sbom+json"})) +} + +// A truncated scan is reported as incomplete rather than as missing evidence. +func TestTruncatedDetectsIncompleteEvidenceScans(t *testing.T) { + assert.False(t, truncated(nil)) + assert.False(t, truncated([]artifact.Diagnostic{{Code: "malformed-evidence"}})) + assert.True(t, truncated([]artifact.Diagnostic{{Code: "malformed-evidence"}, {Code: "evidence-truncated"}})) +} diff --git a/pkg/artifact/oci/sign.go b/pkg/artifact/oci/sign.go index f99fe5f03ed..c4153fd3193 100644 --- a/pkg/artifact/oci/sign.go +++ b/pkg/artifact/oci/sign.go @@ -193,6 +193,10 @@ func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubje return result, err } if len(blobs) == 0 { + if truncated(diagnostics) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "evidence-incomplete", Message: "signature evidence could not be fully inspected; no valid signature was found among the inspected referrers"}) + return result, fmt.Errorf("%w: signature evidence of %s could not be fully inspected", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-evidence", Message: "no signature evidence is attached to the subject"}) return result, fmt.Errorf("%w: no signature evidence is attached to %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) } @@ -200,6 +204,16 @@ func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubje return finishVerification(result, verified, verifyErr) } +// truncated reports whether evidence collection stopped before every candidate was inspected. +func truncated(diagnostics []artifact.Diagnostic) bool { + for index := range diagnostics { + if diagnostics[index].Code == "evidence-truncated" { + return true + } + } + return false +} + // collectEvidence gathers the evidence bound to a subject across its located roots. func (r *Repository) collectEvidence(ctx context.Context, subject artifact.SignatureSubject, roots []subjectRoot) ([]artifact.EvidenceBlob, []artifact.Diagnostic, error) { var ( diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 64488c4ebc9..4f66f644af1 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -329,17 +329,21 @@ func (l *Layout) Untag(ctx context.Context, reference string) error { l.mu.Lock() defer l.mu.Unlock() - if err := l.store.Untag(ctx, reference); err != nil { - return layoutError(err) - } kept := l.index[:0:0] for i := range l.index { if l.index[i].Annotations[ocispec.AnnotationRefName] != reference { kept = append(kept, l.index[i]) } } - l.index = kept - return l.writeIndex() + // Persist index.json before touching the store: a failed write leaves the handle and the file + // unchanged, so the same call can be retried. + if err := l.commitIndex(kept); err != nil { + return err + } + if err := l.store.Untag(ctx, reference); err != nil { + return layoutError(err) + } + return nil } // References returns the named roots in index.json in insertion order. From 8331b70da9265051b0c286c0270e8795219dcd5e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 06:31:38 -0600 Subject: [PATCH 14/53] refactor(artifact): keep OCI publish and attachment checks within complexity limits Code scanning flagged decidePublish and attachments after the previous fix (cyclomatic complexity 11 > 10). The unchanged-tag decision moves into decideUnchanged, and the referrer candidate filter moves into isCandidateReferrer. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/oci/attach.go | 7 ++++++- pkg/artifact/oci/publish.go | 17 +++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index 2ba024770a8..031938cc2b3 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -51,7 +51,7 @@ func (r *Repository) attachments(ctx context.Context, st store, name string, roo set := &attachmentSet{} inspected := 0 for index := range predecessors { - if !isManifestMediaType(predecessors[index].MediaType) || !mayBeBundle(&predecessors[index]) { + if !isCandidateReferrer(&predecessors[index]) { continue } if inspected >= maxAttachments*4 { @@ -107,6 +107,11 @@ func (r *Repository) readAttachment(ctx context.Context, st store, name string, return newAttachment(&manifest, desc, layer, rootDesc.Digest), "", nil } +// isCandidateReferrer reports whether a referrer is a manifest that may hold a Sigstore bundle. +func isCandidateReferrer(desc *ocispec.Descriptor) bool { + return isManifestMediaType(desc.MediaType) && mayBeBundle(desc) +} + // mayBeBundle reports whether a referrer descriptor can be a Sigstore bundle. A descriptor that // names another concrete artifact type is skipped before it costs part of the inspection budget, // so unrelated referrers cannot crowd out real signature evidence. The descriptor type is only a diff --git a/pkg/artifact/oci/publish.go b/pkg/artifact/oci/publish.go index c80ceb86bfa..fe1e77f03d5 100644 --- a/pkg/artifact/oci/publish.go +++ b/pkg/artifact/oci/publish.go @@ -100,12 +100,7 @@ func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, op return "", fmt.Errorf("%w: %s is %s but the caller expected %s", errUtils.ErrArtifactConflict, ref, state.desc.Digest, options.ExpectedPrior.ResolvedDigest) } if state.desc.Digest == incoming.digest { - // Unchanged still honors the compare-and-swap guard, matching the layout contract: a caller - // that expected the tag to be absent gets a conflict, not a silent success. - if options.ExpectedPrior == nil { - return "", fmt.Errorf("%w: %s already exists (%s) but the caller expected it not to", errUtils.ErrArtifactConflict, ref, state.desc.Digest) - } - return artifact.OutcomeUnchanged, nil + return decideUnchanged(ref, state, options) } if state.loaded.class == classPlugin { return "", fmt.Errorf("%w: %s is a legacy plugin release and stays immutable even with force", errUtils.ErrArtifactConflict, ref) @@ -122,6 +117,16 @@ func decidePublish(ref artifact.Ref, state *tagState, incoming *incomingRoot, op return artifact.OutcomeReplaced, nil } +// decideUnchanged reports an unchanged tag. It still honors the compare-and-swap guard, matching +// the layout contract: a caller that expected the tag to be absent gets a conflict, not a silent +// success. +func decideUnchanged(ref artifact.Ref, state *tagState, options artifact.PublishOptions) (artifact.Outcome, error) { + if options.ExpectedPrior == nil { + return "", fmt.Errorf("%w: %s already exists (%s) but the caller expected it not to", errUtils.ErrArtifactConflict, ref, state.desc.Digest) + } + return artifact.OutcomeUnchanged, nil +} + // uploadJob is one package upload and the tag state it started from. type uploadJob struct { pkg *Package From c7bc62daade3ffe4e0b59bd75d89486f7d285b24 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:57:48 -0600 Subject: [PATCH 15/53] fix(artifact): refuse an unusable signer before S3 signing runs The S3 repository called the signer before it checked whether it could store any of the signer's evidence formats. It now refuses such a signer up front with Capabilities.CheckSignerFormats, matching the conformance contract. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/s3/evidence.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index 33d331cb185..bfc733d23d4 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -132,6 +132,10 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject if signer == nil { return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) } + caps := r.Capabilities() + if err := caps.CheckSignerFormats(r.alias, signer); err != nil { + return nil, err + } record, _, err := r.loadRecord(ctx, subject.Digest) if err != nil { return nil, err From 4bb1b468ea098b0c9840c71e664b3ed6988afd9e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 09:33:29 -0600 Subject: [PATCH 16/53] fix(terminal): size the query scan buffer from one length CodeQL flagged make([]byte, 0, len(r.tail)+len(chunk)) as an allocation size that may overflow (go/allocation-size-overflow), because it cannot bound the sum of two lengths. The capacity is only a hint, so it is now sized to the chunk and append grows it for the short carried-over tail. Co-Authored-By: Claude Opus 5.5 --- pkg/terminal/query/query.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/terminal/query/query.go b/pkg/terminal/query/query.go index 71fe4486e4d..b1f9bf37629 100644 --- a/pkg/terminal/query/query.go +++ b/pkg/terminal/query/query.go @@ -76,7 +76,9 @@ func (r *Responder) Scan(chunk []byte) { r.mu.Lock() defer r.mu.Unlock() - data := make([]byte, 0, len(r.tail)+len(chunk)) + // Size to the chunk alone: a sum of two lengths is not a bounded allocation size, and append + // grows the buffer for the short carried-over tail. + data := make([]byte, 0, len(chunk)) data = append(data, r.tail...) data = append(data, chunk...) From 4d8e4a9e0a54a7f3b7212e12a7c9ec23ab3d8bba Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 12:09:28 -0600 Subject: [PATCH 17/53] refactor(artifact): keep delete lock planning within lint limits runDelete reached cyclomatic complexity 13 after digest deletes started locking the canonical reference. The lock targets and the canonical reference are now computed by deleteLockTargets. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/service/mutate.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go index 4189bf49dd6..1fc4436f6e1 100644 --- a/pkg/artifact/service/mutate.go +++ b/pkg/artifact/service/mutate.go @@ -42,9 +42,11 @@ func validateRepositoryRef(repository string, ref artifact.Ref) error { } func (s *Service) runDelete(ctx context.Context, op *operation, request *artifact.DeleteRequest, info *RepositoryInfo, repo artifact.Repository) error { - resource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: request.Ref.String()} - subjectResource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: subjectLockResource(request.Ref.Name)} - locks, err := s.acquireLocks(ctx, op, opDelete, []lockTarget{resource, subjectResource}) + targets, canonical, err := deleteLockTargets(ctx, request, info, repo) + if err != nil { + return err + } + locks, err := s.acquireLocks(ctx, op, opDelete, targets) if err != nil { return err } @@ -71,6 +73,32 @@ func (s *Service) runDelete(ctx context.Context, op *operation, request *artifac return nil } +// deleteLockTargets returns the locks a delete takes and the canonical reference it deletes. A +// digest reference is inspected before locking so the canonical name:version reference is also +// locked; it is resolved again under the locks. +func deleteLockTargets(ctx context.Context, request *artifact.DeleteRequest, info *RepositoryInfo, repo artifact.Repository) ([]lockTarget, string, error) { + resource := lockTarget{alias: request.Repository, repo: repo, info: info, resource: request.Ref.String()} + subject := lockTarget{alias: request.Repository, repo: repo, info: info, resource: subjectLockResource(request.Ref.Name)} + targets := []lockTarget{resource, subject} + canonical := resource.resource + if !request.Ref.IsDigest() { + return targets, canonical, nil + } + prePinned, err := repo.Resolve(ctx, request.Ref) + if err != nil { + return nil, "", err + } + preInspection, err := repo.Inspect(ctx, prePinned) + if err != nil { + return nil, "", err + } + if preInspection.Record != nil { + canonical = preInspection.Record.Name + ":" + preInspection.Record.Version + targets = append(targets, lockTarget{alias: request.Repository, repo: repo, info: info, resource: canonical}) + } + return targets, canonical, nil +} + // Pin implements [artifact.Service]. A pin is immutable reference data for consumer lockfiles; it // acquires no write lock and is not a substitute for signature verification. func (s *Service) Pin(ctx context.Context, request artifact.PinRequest) (*artifact.PinResult, error) { From cbaf2c7c86b1bfd33deabccf7fa3a2111e908856 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:34:45 -0600 Subject: [PATCH 18/53] fix(artifact): never echo endpoint credentials in validation errors Check for embedded credentials before the scheme and host checks, and report only the underlying cause of a URL parse failure, never the raw endpoint. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/options.go | 18 +++++++++++++----- pkg/artifact/options_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 4ad3b35e5e7..8c3d3dcf69e 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -173,21 +173,29 @@ func (o *S3Options) Validate() error { } // validateEndpoint checks an S3 endpoint override: an absolute http or https URL with a host and -// no embedded credentials. It performs no network access. +// no embedded credentials. It performs no network access. No diagnostic repeats the configured +// endpoint until it is known to carry no credentials: the credential check runs first, and a parse +// failure reports only the underlying cause, never the raw URL. func validateEndpoint(endpoint string) error { if endpoint == "" { return nil } parsed, err := url.Parse(endpoint) + if err != nil { + // url.Error embeds the raw URL, which can include credentials; keep only the cause. + var urlErr *url.Error + if errors.As(err, &urlErr) { + err = urlErr.Err + } + return fmt.Errorf("%w: endpoint is not a valid URL: %w", errUtils.ErrArtifactConfigInvalid, err) + } switch { - case err != nil: - return fmt.Errorf("%w: endpoint %q is not a valid URL: %w", errUtils.ErrArtifactConfigInvalid, endpoint, err) + case parsed.User != nil: + return fmt.Errorf("%w: endpoint must not embed credentials; reference an auth identity with `identity` instead", errUtils.ErrArtifactConfigInvalid) case parsed.Scheme != "http" && parsed.Scheme != "https": return fmt.Errorf("%w: endpoint %q must use the http or https scheme", errUtils.ErrArtifactConfigInvalid, endpoint) case parsed.Host == "": return fmt.Errorf("%w: endpoint %q has no host", errUtils.ErrArtifactConfigInvalid, endpoint) - case parsed.User != nil: - return fmt.Errorf("%w: endpoint must not embed credentials; reference an auth identity with `identity` instead", errUtils.ErrArtifactConfigInvalid) } return nil } diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index 7b715b41e71..2f02282d5de 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -231,3 +231,28 @@ func TestDecodeOptions_S3EndpointAndPathStyle(t *testing.T) { err := DecodeOptions(map[string]any{"bucket": "acme-artifacts", "path_style": "yes please"}, &S3Options{}) require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) } + +// TestValidateEndpoint_NeverEchoesCredentials guards that no endpoint diagnostic, including scheme +// and parse failures, repeats credentials embedded in the configured endpoint. +func TestValidateEndpoint_NeverEchoesCredentials(t *testing.T) { + const secret = "s3cr3t-password" + tests := []struct { + name string + endpoint string + wantErr string + }{ + {name: "wrong scheme with credentials", endpoint: "ftp://user:" + secret + "@example.com", wantErr: "credentials"}, + {name: "parse failure with credentials", endpoint: "http://user:" + secret + "@exa mple.com", wantErr: "not a valid URL"}, + {name: "bad port with credentials", endpoint: "http://user:" + secret + "@example.com:port", wantErr: "not a valid URL"}, + {name: "no host with credentials", endpoint: "https://user:" + secret + "@", wantErr: "credentials"}, + {name: "valid scheme with credentials", endpoint: "https://user:" + secret + "@example.com", wantErr: "credentials"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateEndpoint(tt.endpoint) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), secret) + }) + } +} From 0fb243860f6f4d735a953cf46482186fff2ae99f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:04:48 -0600 Subject: [PATCH 19/53] test(artifact): clear ambient AWS profiles in the S3 test isolation helper isolateAWS pointed the SDK at temporary credential files but left AWS_PROFILE and AWS_DEFAULT_PROFILE alone. A developer with a profile selected that the temporary files do not define failed configuration loading, which broke every SDK-backed test that uses the helper. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/identity_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/artifact/s3/identity_test.go b/pkg/artifact/s3/identity_test.go index 3e469758d23..49a69426e76 100644 --- a/pkg/artifact/s3/identity_test.go +++ b/pkg/artifact/s3/identity_test.go @@ -26,12 +26,33 @@ func isolateAWS(t *testing.T) { t.Setenv("AWS_ACCESS_KEY_ID", "ambient-test-key") t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-test-secret") t.Setenv("AWS_REGION", "us-east-1") + // An ambient profile that the temporary files do not define would fail configuration loading. + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") require.NoError(t, os.WriteFile(filepath.Join(dir, "credentials"), []byte("[default]\naws_access_key_id = test-key\naws_secret_access_key = test-secret\n"), 0o600)) t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) t.Setenv("AWS_EC2_METADATA_DISABLED", "true") } +// A developer's selected profile never leaks into an isolated test: with AWS_PROFILE pointing at a +// profile the temporary files do not define, the SDK-backed default chain must still load. +func TestIsolateAWS_IgnoresAmbientProfiles(t *testing.T) { + for _, name := range []string{"AWS_PROFILE", "AWS_DEFAULT_PROFILE"} { + t.Run(name, func(t *testing.T) { + t.Setenv(name, "developer") + isolateAWS(t) + assert.Empty(t, os.Getenv("AWS_PROFILE")) + assert.Empty(t, os.Getenv("AWS_DEFAULT_PROFILE")) + + fake := s3test.NewServer(t) + repo := identityRepo(t, fake, nil, "") + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + }) + } +} + // resolverStub records calls and returns a canned value. type resolverStub struct { calls atomic.Int32 From c7b572db2e2a9f14d347110a4f0f39971ce6f088 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:05:48 -0600 Subject: [PATCH 20/53] fix(artifact): reject a canceled context before claiming an S3 lock A Lock call whose context was already canceled still reached the claim PUT. The canceled request was settled as ambiguous, its read-back found no claim, and a permanent reconciliation fence was written at the claim key, so later mutations needed operator recovery although the acquisition started after cancellation. Check the context before every attempt. Requests that were already in flight when the context ended are still reconciled. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/lock.go | 6 ++++++ pkg/artifact/s3/lock_test.go | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index d3fc21a7c1c..50c21fb827d 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -92,6 +92,12 @@ func (r *Repository) acquire(ctx context.Context, resource, owner, operationID s key := layout.LockPath(resource) deadline := time.Now().Add(r.settings.lockWait) for { + // An acquisition that starts after cancellation must not reach the claim PUT: the canceled + // request would be settled as ambiguous and could leave a permanent reconciliation fence. + // Only requests that were already in flight when the context ended are reconciled. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } handle, err := r.tryClaim(ctx, key, &doc, raw) if handle != nil || err != nil { return handle, err diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 5fa0d56d096..1e38df9c2ee 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -209,6 +209,28 @@ func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { assert.Empty(t, lockObjects(spec.fake), "only the holder's claim ever existed") }) + t.Run("canceled before the first attempt", func(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + // Initialization writes its own objects; only claim creation after this point is measured. + _, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + before := len(spec.fake.RequestsFor(s3test.OpPutObject)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, handle) + assert.Empty(t, lockObjects(spec.fake), "an acquisition that started after cancellation leaves no claim or fence") + assert.Len(t, spec.fake.RequestsFor(s3test.OpPutObject), before, "no claim create was even attempted") + + // The next live acquisition is not blocked by a reconciliation fence. + handle, err = repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + require.NoError(t, err) + require.NoError(t, handle.Release(context.Background())) + }) + for _, lostResponse := range []bool{false, true} { name := "canceled just after the claim was created" if lostResponse { From 802df010b190dde3dd50723cf8585e411ac25d8c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:08:09 -0600 Subject: [PATCH 21/53] fix(artifact): exclude evidence bound to another Record from S3 exports ReadEvidence exported every decodable evidence file in the Record's signature directory. Evidence bound to another Record decoded cleanly and was exported with the set, which then failed the digest check on import. Filter exports by subject kind and digest exactly as verification does. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/evidence.go | 10 +++---- pkg/artifact/s3/signature_test.go | 43 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index bfc733d23d4..64ad31fce6c 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -322,11 +322,11 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDownloadFailed, diag.Message) } } - blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) - for index := range loaded { - blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) - } - return blobs, nil + // Evidence in this Record's directory can still be bound to another Record. Exporting it would + // make the whole set fail the digest check when it is imported, so it is filtered exactly as + // verification filters it. + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: pinned.RecordDigest} + return evidenceBlobs(subject, loaded, &diags), nil } // WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this diff --git a/pkg/artifact/s3/signature_test.go b/pkg/artifact/s3/signature_test.go index d69fc4927f1..e4312faa068 100644 --- a/pkg/artifact/s3/signature_test.go +++ b/pkg/artifact/s3/signature_test.go @@ -259,3 +259,46 @@ func TestFinishVerificationRejectsUnchecked(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) assert.Equal(t, artifact.VerificationUnchecked, result.Status) } + +// Evidence that decodes cleanly but is bound to another Record is never exported with this +// Record's evidence set: importing that set would fail at the digest check. +func TestSignature_ExportExcludesEvidenceBoundToAnotherRecord(t *testing.T) { + ctx := context.Background() + ownSpec := signedSpec(t, noneSpec()) + own := newRepo(t, ownSpec) + ownPinned := publish(t, own, &prepareSpec{files: map[string]string{"data.txt": "own"}}, artifact.PublishOptions{}).Pinned + ownSubject := recordSubject(t, own, &ownPinned) + _, err := own.Sign(ctx, ownSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + + otherSpec := signedSpec(t, noneSpec()) + other := newRepo(t, otherSpec) + otherPinned := publish(t, other, &prepareSpec{files: map[string]string{"data.txt": "other"}}, artifact.PublishOptions{}).Pinned + otherSubject := recordSubject(t, other, &otherPinned) + require.NotEqual(t, ownSubject.Digest, otherSubject.Digest) + _, err = other.Sign(ctx, otherSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + + // Plant the other Record's valid evidence file in this Record's signature directory. + otherDir := layout.SignatureDir(artifact.SubjectRecord, otherSubject.Digest) + otherKeys := managedKeys(otherSpec.fake, "", otherDir) + require.Len(t, otherKeys, 1) + planted, ok := otherSpec.fake.Object(otherKeys[0]) + require.True(t, ok) + ownDir := layout.SignatureDir(artifact.SubjectRecord, ownSubject.Digest) + plantedKey := ownDir + "/" + otherKeys[0][len(otherDir)+1:] + ownSpec.fake.Seed(plantedKey, planted.Data, nil) + require.Len(t, managedKeys(ownSpec.fake, "", ownDir), 2, "both the signed and the planted file are stored") + + blobs, err := own.ReadEvidence(ctx, &ownPinned) + require.NoError(t, err) + require.Len(t, blobs, 1, "only evidence bound to this Record is exported") + assert.Equal(t, ownSubject.Digest, blobs[0].Evidence.Subject.Digest) + assert.Equal(t, artifact.SubjectRecord, blobs[0].Evidence.Subject.Kind) + + // The exported set always imports cleanly into a repository that holds the same Record. + destination := newRepo(t, signedSpec(t, noneSpec())) + destPinned := publish(t, destination, &prepareSpec{files: map[string]string{"data.txt": "own"}}, artifact.PublishOptions{}).Pinned + _, err = destination.WriteEvidence(ctx, &destPinned, blobs, nil) + require.NoError(t, err) +} From b831cadbcaa5473f98d9b1ca2a1f5b9ff7cd4db0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:09:21 -0600 Subject: [PATCH 22/53] fix(oci): reject an empty reference before filtering the layout index Untag with an empty reference matched every unnamed root, including attachment roots, and persisted that removal to index.json before the store refused the empty reference. Reject the reference first so the persisted index stays unchanged. Co-Authored-By: Claude Sonnet 5.5 --- pkg/oci/transport_layout.go | 5 +++++ pkg/oci/transport_subject_test.go | 36 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 4f66f644af1..01b1d41916c 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -327,6 +327,11 @@ func withReference(entries []ocispec.Descriptor, named ocispec.Descriptor) []oci func (l *Layout) Untag(ctx context.Context, reference string) error { defer perf.Track(nil, "oci.Layout.Untag")() + // Reject an empty reference before filtering: it would match every unnamed root, including + // attachment roots, and index.json would be rewritten before the store refused the call. + if reference == "" { + return layoutError(errdef.ErrMissingReference) + } l.mu.Lock() defer l.mu.Unlock() kept := l.index[:0:0] diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index 9f5becb534e..f1e53ef8f29 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -247,3 +247,39 @@ func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers( require.NoError(t, err) assert.Equal(t, root.Digest, resolved.Digest) } + +// An empty reference is rejected before index.json is filtered: it would otherwise drop every +// unnamed root, including attachment roots, and persist that before the store refuses the call. +func TestLayoutUntag_EmptyReferenceLeavesTheIndexUnchanged(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + attachment := `{"schemaVersion":2,"subject":` + subject + `}` + attachmentDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(attachment)) + require.NoError(t, layout.Push(ctx, attachmentDescriptor, strings.NewReader(attachment))) + named := `{"schemaVersion":2,"layers":[]}` + namedDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(named)) + require.NoError(t, layout.Push(ctx, namedDescriptor, strings.NewReader(named))) + require.NoError(t, layout.Tag(ctx, namedDescriptor, "app:1")) + wantRoots := []string{attachmentDescriptor.Digest.String(), namedDescriptor.Digest.String()} + require.Equal(t, wantRoots, layoutRoots(t, directory), "the unnamed attachment root and the named root are listed") + before, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + + err = layout.Untag(ctx, "") + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + + after, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), "the persisted index is untouched") + assert.Equal(t, wantRoots, layoutRoots(t, directory)) + assert.Equal(t, []string{"app:1"}, layout.References()) + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + assert.Equal(t, []string{"app:1"}, reopened.References()) +} From 23c58b1759419916587d9a8b29f9030d0a348dca Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:13:39 -0600 Subject: [PATCH 23/53] fix(artifact): carry abandoned S3 payloads in the journal until a retry removes them A retry under a different packaging replaces the pending operation journal. The abandoned payload lived only in memory, so a second retry that stopped before cleanup lost its recovery metadata and the objects stayed behind forever. The replacement journal now lists the abandoned payloads, every retry carries them forward, and both the completing publish and an unchanged republish remove them. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/layout/layout_test.go | 25 ++++++++++ pkg/artifact/layout/records.go | 35 +++++++++++++- pkg/artifact/s3/build.go | 16 +++++-- pkg/artifact/s3/edge_test.go | 46 ++++++++++++++++++ pkg/artifact/s3/publish.go | 76 +++++++++++++++++++++++++----- pkg/artifact/s3/reserve.go | 2 +- 6 files changed, 183 insertions(+), 17 deletions(-) diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go index e86a73a53df..e7ea271b472 100644 --- a/pkg/artifact/layout/layout_test.go +++ b/pkg/artifact/layout/layout_test.go @@ -550,3 +550,28 @@ func TestOperationRejectsUnsafePriorLocation(t *testing.T) { op := &Operation{SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op", Name: "team/app", Version: "1", CapturedAt: artifact.NewTimestamp(time.Now()), Location: "safe", Payload: []ocispec.Descriptor{}, Documents: []ocispec.Descriptor{}, Prior: &Prior{RecordDigest: digestA, Location: artifact.ReservedNamespace + "/unsafe"}} require.Error(t, op.Validate()) } + +func TestOperation_AbandonedPayloadsAreJournaledAndValidated(t *testing.T) { + op := sampleOperation() + op.Abandoned = []Retired{ + {Packaging: artifact.PackagingTgz, Location: "team/app/1.0.0.tgz"}, + {Packaging: artifact.PackagingNone, Location: "team/app/1.0.0", Files: []string{"a.txt", "dir/b.txt"}, Docs: []string{"README.md"}}, + } + raw, err := op.Marshal() + require.NoError(t, err) + decoded, err := DecodeOperation(raw) + require.NoError(t, err) + assert.Equal(t, op.Abandoned, decoded.Abandoned, "every abandoned payload survives the journal") + + plain := sampleOperation() + raw, err = plain.Marshal() + require.NoError(t, err) + assert.NotContains(t, string(raw), "abandoned", "a journal without abandoned payloads is unchanged on disk") + + for _, location := range []string{"", "../escape", "/absolute", "a/../../b"} { + broken := sampleOperation() + broken.Abandoned = []Retired{{Packaging: artifact.PackagingTgz, Location: location}} + _, err = broken.Marshal() + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, "abandoned location %q", location) + } +} diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go index eb0d38fb82c..271bbd7c3dd 100644 --- a/pkg/artifact/layout/records.go +++ b/pkg/artifact/layout/records.go @@ -331,6 +331,24 @@ type Prior struct { Packaging artifact.Packaging `json:"packaging,omitempty"` } +// Retired names a payload an earlier attempt of the same reference abandoned: written, never +// completed, and replaced by a later attempt under a different location or packaging. The journal +// that replaces the old one carries it until a retry has removed the objects, so a stopped retry +// never loses the recovery metadata of the payload it superseded. +type Retired struct { + // Packaging is the packaging of the abandoned payload, which decides its object keys. + Packaging artifact.Packaging `json:"packaging,omitempty"` + + // Location is where the abandoned payload was written. + Location string `json:"location"` + + // Files are the per-file payload paths below Location. Archive packaging leaves it empty. + Files []string `json:"files,omitempty"` + + // Docs are the distribution document file names written beside the payload. + Docs []string `json:"docs,omitempty"` +} + // Operation is the publication journal: an implementation recovery record, not a public catalog. // It is written before any payload path changes and records everything a same-intent retry needs // to resume or repair the publication: the captured time, the intent, the prior reference, the @@ -372,6 +390,9 @@ type Operation struct { // Prior describes the state being replaced, if any. Prior *Prior `json:"prior,omitempty"` + // Abandoned lists payloads of superseded attempts that still await removal. + Abandoned []Retired `json:"abandoned,omitempty"` + // Payload and Documents are the prepared representation checksums. Payload []ocispec.Descriptor `json:"payload"` Documents []ocispec.Descriptor `json:"documents"` @@ -409,7 +430,19 @@ func (o *Operation) validateState() error { if o.Payload == nil || o.Documents == nil { return invalid(kindOperation, "payload and documents lists are required (use empty lists)") } - return validatePrior(o.Prior) + if err := validatePrior(o.Prior); err != nil { + return err + } + return validateAbandoned(o.Abandoned) +} + +func validateAbandoned(abandoned []Retired) error { + for index := range abandoned { + if _, err := CleanLocation(abandoned[index].Location); err != nil { + return invalid(kindOperation, "abandoned location %q is not a valid relative location", abandoned[index].Location) + } + } + return nil } func validatePrior(prior *Prior) error { diff --git a/pkg/artifact/s3/build.go b/pkg/artifact/s3/build.go index a47ead42452..ea3a731bfa4 100644 --- a/pkg/artifact/s3/build.go +++ b/pkg/artifact/s3/build.go @@ -77,10 +77,8 @@ func (r *Repository) retire(ctx context.Context, job *publishJob, plan *publishP if err != nil { return } - if plan.abandoned != nil { - if err := r.removePayload(cleanup, plan.abandoned, keep); err != nil { - return - } + if err := r.removePayloads(cleanup, plan.abandoned, keep); err != nil { + return } if plan.prior != nil { if err := r.retirePrior(cleanup, plan.prior, job, keep); err != nil { @@ -110,6 +108,16 @@ func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job * return r.deleteObject(ctx, layout.RecordPath(prior.RecordDigest)) } +// removePayloads deletes the objects of every listed payload that the new publication does not reuse. +func (r *Repository) removePayloads(ctx context.Context, payloads []ownedPayload, keep []string) error { + for index := range payloads { + if err := r.removePayload(ctx, &payloads[index], keep); err != nil { + return err + } + } + return nil +} + // removePayload deletes the objects of a payload that the new publication does not reuse. func (r *Repository) removePayload(ctx context.Context, owned *ownedPayload, keep []string) error { keys, err := owned.set.keys(owned.packaging, owned.location) diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index f26df198d32..32779d8b527 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -65,6 +66,51 @@ func TestResumedPublicationRetiresAnAbandonedLocation(t *testing.T) { } } +// An abandoned payload outlives the retry that discovered it: the journal that replaces the old one +// carries it until a retry actually removes it, however many attempts stop first. +func TestResumedPublicationCarriesAbandonedPayloadsAcrossRetries(t *testing.T) { + tests := []struct { + name string + crashed *rspec + retry *rspec + abandoned []string + // stoppedRetries is how many retries stop before they can retire anything. + stoppedRetries int + }{ + {name: "tgz to zip, one stopped retry", crashed: archiveSpec(artifact.PackagingTgz), retry: archiveSpec(artifact.PackagingZip), abandoned: []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json"}, stoppedRetries: 1}, + {name: "tgz to zip, two stopped retries", crashed: archiveSpec(artifact.PackagingTgz), retry: archiveSpec(artifact.PackagingZip), abandoned: []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json"}, stoppedRetries: 2}, + {name: "archive to files, one stopped retry", crashed: archiveSpec(artifact.PackagingTgz), retry: noneSpec(), abandoned: []string{"team/app/1.0.0.tgz", "team/app/1.0.0.tgz.metadata.json"}, stoppedRetries: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := s3test.NewServer(t) + crashed := newRepo(t, tt.crashed.with(func(s *rspec) { s.fake = fake }).withOpts(withFaults(crashAt(faultAfterMetadata)))) + _, err := crashed.Publish(context.Background(), prepare(t, crashed, &prepareSpec{opID: "op-first"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + for attempt := 0; attempt < tt.stoppedRetries; attempt++ { + stopped := newRepo(t, tt.retry.with(func(s *rspec) { s.fake = fake }).withOpts(withFaults(crashAt(faultBeforePayload)))) + _, err = stopped.Publish(context.Background(), prepare(t, stopped, &prepareSpec{opID: fmt.Sprintf("op-stopped-%d", attempt)}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + for _, key := range tt.abandoned { + _, ok := fake.Object(key) + require.True(t, ok, "%s is still awaiting retirement after stopped retry %d", key, attempt) + } + } + + final := newRepo(t, tt.retry.with(func(s *rspec) { s.fake = fake })) + _, err = final.Publish(context.Background(), prepare(t, final, &prepareSpec{opID: "op-final"}), artifact.PublishOptions{}) + require.NoError(t, err) + for _, key := range tt.abandoned { + _, ok := fake.Object(key) + assert.False(t, ok, "the abandoned object %s was retired by the completing retry, not orphaned", key) + } + assertIntact(t, final, defaultRef()) + assert.Empty(t, managedKeys(fake, "", layout.ManagedRoot+"/operations")) + }) + } +} + // A Record object that already exists with different bytes is corruption, never replaced. func TestPublish_ExistingRecordWithDifferentBytesIsRejected(t *testing.T) { spec := noneSpec().fresh(t) diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index 9ed9e028c30..70b17c57893 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -26,7 +26,7 @@ type publishPlan struct { current *refState prior *layout.Prior owned []ownedPayload - abandoned *ownedPayload + abandoned []ownedPayload oldOp *layout.Operation expected *artifact.PinnedReference scope artifact.LockScope @@ -163,19 +163,31 @@ func (r *Repository) unchanged(ctx context.Context, job *publishJob, plan *publi if err != nil { return nil, err } - if journal != nil && journal.Prior != nil { - keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) - if err != nil { - return nil, err - } - if err := r.retirePrior(ctx, journal.Prior, job, keep); err != nil { - return nil, err - } + if err := r.retireJournaled(ctx, job, journal, record, rep); err != nil { + return nil, err } r.tidyOperation(ctx, current.OperationID) return r.result(record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome(), plan.scope), nil } +// retireJournaled removes what a completed operation's journal still lists for removal: the +// replaced Record and its payload, and the payloads of abandoned attempts. +func (r *Repository) retireJournaled(ctx context.Context, job *publishJob, journal *layout.Operation, record *artifact.Record, rep *layout.Representation) error { + if journal == nil || (journal.Prior == nil && len(journal.Abandoned) == 0) { + return nil + } + keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) + if err != nil { + return err + } + if journal.Prior != nil { + if err := r.retirePrior(ctx, journal.Prior, job, keep); err != nil { + return err + } + } + return r.removePayloads(ctx, carriedPayloads(journal), keep) +} + // tidyOperation removes the journal of a finished operation. It is best-effort: a leftover is // harmless and is retried by the next publication. func (r *Repository) tidyOperation(ctx context.Context, operationID string) { @@ -198,15 +210,18 @@ func (r *Repository) resolveLocation(ctx context.Context, job *publishJob, plan if err := r.loadPendingState(ctx, plan, rep); err != nil { return err } + // Payloads that earlier attempts abandoned stay pending until a retry removes them, whichever + // location this attempt uses. + plan.abandoned = carriedPayloads(plan.oldOp) if plan.admission.Resuming() && plan.oldOp != nil && plan.oldOp.IntentDigest == job.prepared.IntentDigest && plan.oldOp.Packaging == job.prepared.Packaging { job.location = plan.oldOp.Location return nil } if plan.oldOp != nil { - plan.abandoned = &ownedPayload{ + plan.abandoned = appendPayload(plan.abandoned, &ownedPayload{ packaging: plan.oldOp.Packaging, location: plan.oldOp.Location, set: setOfDescriptors(plan.oldOp.Payload, plan.oldOp.Documents), - } + }) } job.location, err = layout.RenderLocation(r.keyPattern, &layout.PatternInput{ Name: job.record.Name, Version: job.record.Version, Kind: job.record.Kind, CreatedAt: job.record.CreatedAt.Time(), @@ -228,6 +243,7 @@ func (r *Repository) loadPendingState(ctx context.Context, plan *publishPlan, re plan.oldOp = oldOp } plan.prior = priorState(plan, rep) + plan.owned = append(plan.owned, carriedPayloads(plan.oldOp)...) if plan.oldOp != nil { plan.owned = append(plan.owned, ownedPayload{packaging: plan.oldOp.Packaging, location: plan.oldOp.Location}) if prior := plan.oldOp.Prior; prior != nil && prior.Location != "" { @@ -264,3 +280,41 @@ func priorFrom(recordDigest digest.Digest, rep *layout.Representation) *layout.P } return prior } + +// carriedPayloads returns the abandoned payloads a journal still carries. +func carriedPayloads(op *layout.Operation) []ownedPayload { + if op == nil { + return nil + } + carried := make([]ownedPayload, 0, len(op.Abandoned)) + for index := range op.Abandoned { + retired := &op.Abandoned[index] + carried = append(carried, ownedPayload{ + packaging: retired.Packaging, location: retired.Location, + set: objectSet{files: retired.Files, docs: retired.Docs}, + }) + } + return carried +} + +// appendPayload adds a payload unless one at the same location and packaging is already listed. +func appendPayload(payloads []ownedPayload, payload *ownedPayload) []ownedPayload { + for index := range payloads { + if payloads[index].packaging == payload.packaging && payloads[index].location == payload.location { + return payloads + } + } + return append(payloads, *payload) +} + +// retiredPayloads converts abandoned payloads to their journal form. +func retiredPayloads(payloads []ownedPayload) []layout.Retired { + var retired []layout.Retired + for index := range payloads { + retired = append(retired, layout.Retired{ + Packaging: payloads[index].packaging, Location: payloads[index].location, + Files: payloads[index].set.files, Docs: payloads[index].set.docs, + }) + } + return retired +} diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index 9c1e469453f..f4ed13507e2 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -48,7 +48,7 @@ func (r *Repository) reserve(ctx context.Context, job *publishJob, plan *publish SchemaVersion: layout.SchemaVersion, RepositoryKind: artifact.RepositoryKindS3, OperationID: job.opID, Name: job.record.Name, Version: job.record.Version, IntentDigest: job.prepared.IntentDigest, RecordDigest: job.recordDigest, CapturedAt: job.record.CreatedAt, Packaging: job.prepared.Packaging, - Location: job.location, Prior: plan.prior, Payload: job.payload, Documents: job.documents, Owner: r.settings.owner, + Location: job.location, Prior: plan.prior, Abandoned: retiredPayloads(plan.abandoned), Payload: job.payload, Documents: job.documents, Owner: r.settings.owner, } if plan.expected != nil { journal.ExpectedPrior = plan.expected.ResolvedDigest From 279e93f280b2614c0e9b4e778521b59dc6bbff1d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:14:14 -0600 Subject: [PATCH 24/53] fix(artifact): bind the S3 conditional-write verdict to the endpoint it probed Named-identity clients are rebuilt per operation, so an identity can resolve to another endpoint during the life of a store, but the probe verdict was cached for the whole store. An endpoint that ignores If-None-Match could then inherit a verdict earned elsewhere and overwrite objects that should have been protected. Record the endpoint with the verdict and probe again when the client about to be used targets a different one. An unchanged endpoint keeps its cached verdict and transient failures are still not cached. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/s3/conditional.go | 61 +++++++++++++++++------- pkg/artifact/blob/s3/conditional_test.go | 53 ++++++++++++++++++++ pkg/artifact/blob/s3/delete.go | 10 ++-- pkg/artifact/blob/s3/put.go | 6 +-- pkg/artifact/blob/s3/store.go | 9 ++-- 5 files changed, 111 insertions(+), 28 deletions(-) diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go index 258613df711..13764079a06 100644 --- a/pkg/artifact/blob/s3/conditional.go +++ b/pkg/artifact/blob/s3/conditional.go @@ -23,14 +23,16 @@ const probeOperation = "probe" const probeCleanupTimeout = 5 * time.Second -// ensureConditional makes sure a conditional operation is safe to send. In probe mode it first -// proves, once per store, that the endpoint enforces If-None-Match and If-Match; an endpoint that silently -// ignores the header would otherwise turn create-if-absent into an unconditional overwrite. -func (s *Store) ensureConditional(ctx context.Context, key string, cond blob.Condition) error { +// ensureConditional makes sure a conditional operation is safe to send through api. In probe mode +// it first proves, once per endpoint, that the endpoint enforces If-None-Match and If-Match; an +// endpoint that silently ignores the header would otherwise turn create-if-absent into an +// unconditional overwrite. The verdict covers the endpoint of api, the client the operation is +// about to use, so a named identity that now resolves to another endpoint is probed again. +func (s *Store) ensureConditional(ctx context.Context, api API, key string, cond blob.Condition) error { if !cond.IsConditional() || s.cfg.conditional != ConditionalProbe { return nil } - if err := s.ProbeConditionalWrites(ctx); err != nil { + if err := s.probeConditional(ctx, api); err != nil { return unsupportedConditionalFrom(key, err) } return nil @@ -48,13 +50,26 @@ func unsupportedConditionalFrom(key string, probeErr error) error { // object twice: the second create must be rejected. It then checks that stale If-Match validators // are rejected on PutObject and DeleteObject. The guarantee covers those single-object requests // only; conditional enforcement on CompleteMultipartUpload is not probed. A definite answer is -// cached for the life of the store; a transient failure is returned without being cached. It +// cached for the endpoint it probed; a different endpoint, which a named identity can resolve to +// at any time, is probed again. A transient failure is returned without being cached. It // returns an error matching ErrBlobConditionalUnsupported when the endpoint ignores or rejects the // condition. func (s *Store) ProbeConditionalWrites(ctx context.Context) error { defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")() - if verdict, cached := s.cachedProbe(); cached { + api, err := s.client(ctx) + if err != nil { + return err + } + return s.probeConditional(ctx, api) +} + +// probeConditional probes the endpoint of api unless its verdict is already cached. One probe runs +// at a time; a waiting operation gives up when its context ends instead of blocking on the probe +// in flight. +func (s *Store) probeConditional(ctx context.Context, api API) error { + endpoint := endpointOf(api) + if verdict, cached := s.cachedProbe(endpoint); cached { return verdict } select { @@ -64,13 +79,9 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error { return blob.NewError(blob.KindOther, probeOperation, "", ctx.Err()) } // Another operation may have finished the probe while this one waited for the slot. - if verdict, cached := s.cachedProbe(); cached { + if verdict, cached := s.cachedProbe(endpoint); cached { return verdict } - api, err := s.client(ctx) - if err != nil { - return err - } verdict, cleanupErr := s.runProbe(ctx, api) if cleanupErr != nil { // A leftover probe object is a transient failure: neither verdict is cached, and both @@ -79,17 +90,35 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error { } if verdict == nil || blob.IsConditionalUnsupported(verdict) { s.probeMu.Lock() - s.probed, s.probeError = true, verdict + s.probed, s.probedEndpoint, s.probeError = true, endpoint, verdict s.probeMu.Unlock() } return verdict } -// cachedProbe returns the cached probe verdict, when a definite one exists. -func (s *Store) cachedProbe() (verdict error, cached bool) { +// cachedProbe returns the cached probe verdict, when a definite one exists for endpoint. +func (s *Store) cachedProbe(endpoint string) (verdict error, cached bool) { s.probeMu.Lock() defer s.probeMu.Unlock() - return s.probeError, s.probed + if !s.probed || s.probedEndpoint != endpoint { + return nil, false + } + return s.probeError, true +} + +// endpointOf names the endpoint a client targets: its base endpoint override, or its region for +// the standard AWS endpoint. A client that does not expose its options, such as a test double or +// an injected wrapper, has one fixed endpoint for the life of the store. +func endpointOf(api API) string { + reporter, ok := api.(interface{ Options() s3.Options }) + if !ok { + return "" + } + options := reporter.Options() + if options.BaseEndpoint != nil && *options.BaseEndpoint != "" { + return *options.BaseEndpoint + } + return "region:" + options.Region } // runProbe performs the double create and removes the probe object. The verdict is the probe diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go index a7eb8bb2453..e641332e3aa 100644 --- a/pkg/artifact/blob/s3/conditional_test.go +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -15,6 +15,7 @@ import ( "github.com/cloudposse/atmos/pkg/artifact/blob" "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/store" ) func putIfAbsent(s *Store, key, body string) (*blob.ObjectInfo, error) { @@ -381,3 +382,55 @@ func TestProbeRejectsIgnoredIfMatch(t *testing.T) { }) } } + +// A probe verdict belongs to the endpoint it probed. A named identity is re-resolved on every +// operation, so its endpoint can change during the life of the store; a verdict from the old +// endpoint must never vouch for the new one. +func TestProbeVerdictIsBoundToTheEndpointItProbed(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + enforcing := newFake(t) + ignoring := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) + ignoring.Seed("lock", []byte("owner-a"), nil) + + resolver := &fakeResolver{result: &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: enforcing.Endpoint()}} + s, err := New(enforcing.Bucket(), WithIdentity("deploy", resolver), WithConditionalMode(ConditionalProbe)) + require.NoError(t, err) + + _, err = putIfAbsent(s, "first", "a") + require.NoError(t, err, "the enforcing endpoint passes its probe") + probes := len(enforcing.RequestsFor(s3test.OpPutObject)) + _, err = putIfAbsent(s, "second", "a") + require.NoError(t, err) + assert.Len(t, enforcing.RequestsFor(s3test.OpPutObject), probes+1, "an unchanged endpoint keeps its cached verdict: no second probe") + + resolver.result = &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: ignoring.Endpoint()} + _, err = putIfAbsent(s, "lock", "owner-b") + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "the new endpoint is probed, not trusted: %v", err) + object, ok := ignoring.Object("lock") + require.True(t, ok) + assert.Equal(t, []byte("owner-a"), object.Data, "the existing object on the ignoring endpoint was never overwritten") + + before := len(ignoring.Requests()) + _, err = putIfAbsent(s, "lock", "owner-c") + assert.True(t, blob.IsConditionalUnsupported(err)) + assert.Len(t, ignoring.Requests(), before, "a definite verdict for the new endpoint is cached for it") + + // Switching back is a different endpoint again; the earlier verdict for it does not apply blindly. + resolver.result = &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: enforcing.Endpoint()} + _, err = putIfAbsent(s, "third", "a") + require.NoError(t, err) +} + +// A transient probe failure is never cached, whatever the endpoint. +func TestProbeTransientFailureIsNotCachedAcrossEndpoints(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusServiceUnavailable, Code: "SlowDown", Times: 1}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + err := s.ProbeConditionalWrites(context.Background()) + require.Error(t, err) + assert.False(t, blob.IsConditionalUnsupported(err)) + require.NoError(t, s.ProbeConditionalWrites(context.Background()), "the transient failure was not cached") +} diff --git a/pkg/artifact/blob/s3/delete.go b/pkg/artifact/blob/s3/delete.go index 4ced8cbda2c..cfda196298f 100644 --- a/pkg/artifact/blob/s3/delete.go +++ b/pkg/artifact/blob/s3/delete.go @@ -21,15 +21,15 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) return err } conditional := opts.IfMatch != "" - if conditional { - if err := s.ensureConditional(ctx, key, blob.IfMatch(opts.IfMatch)); err != nil { - return err - } - } api, err := s.client(ctx) if err != nil { return err } + if conditional { + if err := s.ensureConditional(ctx, api, key, blob.IfMatch(opts.IfMatch)); err != nil { + return err + } + } in := &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), IfMatch: optionalString(opts.IfMatch)} if _, err := api.DeleteObject(ctx, in, retryOptions(conditional)...); err != nil { return classify(request{op: blob.OpDelete, key: key, mutating: true, conditional: conditional}, err) diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index dd3355cf3be..614d9c3f51e 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -83,13 +83,13 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) ( if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { return nil, err } - if err := s.ensureConditional(ctx, key, opts.Condition); err != nil { - return nil, err - } api, err := s.client(ctx) if err != nil { return nil, err } + if err := s.ensureConditional(ctx, api, key, opts.Condition); err != nil { + return nil, err + } return &putPlan{ api: api, key: key, diff --git a/pkg/artifact/blob/s3/store.go b/pkg/artifact/blob/s3/store.go index bc5d8089d32..b9e4ba94f71 100644 --- a/pkg/artifact/blob/s3/store.go +++ b/pkg/artifact/blob/s3/store.go @@ -24,10 +24,11 @@ type Store struct { // lets a waiting operation give up when its context ends. probeSlot chan struct{} - // probeMu guards the cached verdict below. - probeMu sync.Mutex - probed bool - probeError error + // probeMu guards the cached verdict below, which is bound to the endpoint it probed. + probeMu sync.Mutex + probed bool + probedEndpoint string + probeError error } // New returns a Store for bucket. No network or credential work happens until the first operation. From 3791054f2fa0ce6090d3d3839c2b37aa936620ee Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:15:34 -0600 Subject: [PATCH 25/53] fix(oci): treat an oversized referrer as malformed evidence A bundle-typed referrer whose manifest exceeds the manifest bound aborted the whole referrer scan, so one oversized referrer could block verification even when another referrer carried a valid signature. Report the size rejection as malformed evidence like the other unusable referrers and keep every other fetch failure fatal. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/attach.go | 6 ++ pkg/artifact/oci/attach_test.go | 113 ++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 pkg/artifact/oci/attach_test.go diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index 031938cc2b3..bf95dc4d692 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -4,6 +4,7 @@ package oci import ( "context" "encoding/json" + "errors" "fmt" "sort" "strings" @@ -88,6 +89,11 @@ func (r *Repository) readAttachment(ctx context.Context, st store, name string, if ctx.Err() != nil { return nil, "", ctx.Err() } + // A referrer that declares more than the manifest bound is malformed evidence, like any + // other unusable referrer: it must not block a valid signature carried by another one. + if errors.Is(err, errUtils.ErrArtifactTooLarge) { + return nil, fmt.Sprintf("referrer %s is too large to inspect: %v", desc.Digest, err), nil + } return nil, "", fmt.Errorf("%w: read referrer %s: %w", errUtils.ErrArtifactDownloadFailed, desc.Digest, err) } var manifest ocispec.Manifest diff --git a/pkg/artifact/oci/attach_test.go b/pkg/artifact/oci/attach_test.go new file mode 100644 index 00000000000..90b9c733e73 --- /dev/null +++ b/pkg/artifact/oci/attach_test.go @@ -0,0 +1,113 @@ +//nolint:gocritic // OCI descriptors are passed by value, mirroring the ORAS content contracts and the fixed artifact.Repository signatures. +package oci + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// referrerStore serves a fixed referrer list and the manifests of the referrers that have content. +// Fetching anything else fails with fetchErr, standing in for a registry that is down. +type referrerStore struct { + store + referrers []ocispec.Descriptor + content map[digest.Digest][]byte + fetchErr error +} + +func (s *referrerStore) Predecessors(context.Context, ocispec.Descriptor) ([]ocispec.Descriptor, error) { + return s.referrers, nil +} + +func (s *referrerStore) Fetch(_ context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + raw, ok := s.content[target.Digest] + if !ok { + return nil, s.fetchErr + } + return io.NopCloser(bytes.NewReader(raw)), nil +} + +// bundleReferrer builds a valid bundle-typed referrer of root and its descriptor. +func bundleReferrer(t *testing.T, root ocispec.Descriptor) (ocispec.Descriptor, []byte) { + t.Helper() + bundleType := bundleMediaTypePrefix + "v0.3+json" + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispec.MediaTypeImageManifest, + ArtifactType: bundleType, + Config: ocispec.DescriptorEmptyJSON, + Layers: []ocispec.Descriptor{{MediaType: bundleType, Digest: digest.FromString("bundle"), Size: 6}}, + Subject: &root, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, ArtifactType: bundleType, Digest: digest.FromBytes(raw), Size: int64(len(raw))}, raw +} + +// An oversized referrer is malformed evidence: it is reported, and it cannot block a valid +// signature from another referrer. Any other fetch failure stays fatal. +func TestAttachments_OversizedReferrerIsMalformedEvidence(t *testing.T) { + root := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString("root"), Size: 100} + valid, validRaw := bundleReferrer(t, root) + oversized := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, ArtifactType: bundleMediaTypePrefix + "v0.3+json", + Digest: digest.FromString("oversized"), Size: maxManifestSize + 1, + } + tests := []struct { + name string + referrers []ocispec.Descriptor + fetchErr error + wantValid []digest.Digest + wantProblem string + wantErr error + }{ + { + name: "oversized referrer beside a valid one", referrers: []ocispec.Descriptor{oversized, valid}, + wantValid: []digest.Digest{valid.Digest}, wantProblem: "too large", + }, + { + name: "only an oversized referrer", referrers: []ocispec.Descriptor{oversized}, + wantProblem: "too large", + }, + { + name: "a registry failure stays fatal", referrers: []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageManifest, ArtifactType: bundleMediaTypePrefix + "v0.3+json", Digest: digest.FromString("down"), Size: 10, + }}, + fetchErr: errors.New("registry unavailable"), wantErr: errUtils.ErrArtifactDownloadFailed, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := &referrerStore{referrers: tt.referrers, content: map[digest.Digest][]byte{valid.Digest: validRaw}, fetchErr: tt.fetchErr} + set, err := (&Repository{cache: map[string][]byte{}}).attachments(context.Background(), st, "team/app", root) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + var got []digest.Digest + for _, item := range set.valid { + got = append(got, item.manifestDesc.Digest) + } + assert.Equal(t, tt.wantValid, got) + require.Len(t, set.diagnostics, 1) + assert.Equal(t, "malformed-evidence", set.diagnostics[0].Code) + assert.Equal(t, artifact.SeverityWarning, set.diagnostics[0].Severity) + assert.Contains(t, set.diagnostics[0].Message, tt.wantProblem) + assert.Contains(t, set.diagnostics[0].Message, oversized.Digest.String()) + }) + } +} From bc9f286d715e9e8d54ce37c6b65e19b4e5c23e5e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:19:44 -0600 Subject: [PATCH 26/53] fix(oci): build the copy result from the copied digest CopyFrom resolved the destination tag again after copying, so a writer that moved the tag in between made the result and the lockfile pin describe content this operation never copied. Read the root back by the copied digest and pin that instead. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 23 +++++++++---- pkg/artifact/oci/resolve.go | 7 ++++ pkg/artifact/oci/transfer_test.go | 54 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 98684e75828..726e179e42a 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -100,16 +100,27 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto return nil, fmt.Errorf("%w: copy %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) } } - return r.transferred(ctx, ref, outcome, options) + pinned, err := r.pinCopied(ctx, dst, loaded, ref) + if err != nil { + return nil, err + } + return r.transferred(ctx, &pinned, ref, outcome, options) } -// transferred builds the result by inspecting what the destination now holds. -func (r *Repository) transferred(ctx context.Context, ref artifact.Ref, outcome artifact.Outcome, options artifact.PublishOptions) (*artifact.PublishResult, error) { - pinned, err := r.Pin(ctx, ref) +// pinCopied pins the digest that was copied, read back from the destination. The destination tag +// is deliberately not resolved again: another writer can move it after the copy, and the pin must +// describe the content this operation published. +func (r *Repository) pinCopied(ctx context.Context, dst store, copied *root, ref artifact.Ref) (artifact.PinnedReference, error) { + landed, err := r.loadCachedRoot(ctx, dst, ref.Name, copied.desc) if err != nil { - return nil, err + return artifact.PinnedReference{}, err } - inspection, err := r.Inspect(ctx, pinned) + return r.pinRoot(ctx, dst, ref, landed) +} + +// transferred builds the result by inspecting the pinned content the destination now holds. +func (r *Repository) transferred(ctx context.Context, pinned *artifact.PinnedReference, ref artifact.Ref, outcome artifact.Outcome, options artifact.PublishOptions) (*artifact.PublishResult, error) { + inspection, err := r.Inspect(ctx, *pinned) if err != nil { return nil, err } diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go index 4149282ddcc..65a306c355e 100644 --- a/pkg/artifact/oci/resolve.go +++ b/pkg/artifact/oci/resolve.go @@ -40,6 +40,13 @@ func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.Pinned if err != nil { return artifact.PinnedReference{}, err } + return r.pinRoot(ctx, st, ref, loaded) +} + +// pinRoot builds the pin of an already resolved root. A caller that must describe one specific +// digest, such as a copy that just wrote it, passes that root instead of resolving the mutable tag +// again, so a tag moved in between can never change what the pin names. +func (r *Repository) pinRoot(ctx context.Context, st store, ref artifact.Ref, loaded *root) (artifact.PinnedReference, error) { pinned := r.pinnedFor(ref, loaded) if loaded.class == classAtmos { record, recordErr := r.readRecord(ctx, st, ref.Name, loaded) diff --git a/pkg/artifact/oci/transfer_test.go b/pkg/artifact/oci/transfer_test.go index 9c7c9062a07..b735e17bc07 100644 --- a/pkg/artifact/oci/transfer_test.go +++ b/pkg/artifact/oci/transfer_test.go @@ -361,3 +361,57 @@ func TestExportRejectsAnotherRootAndImportPrefersNamedRoot(t *testing.T) { require.NoError(t, err) assert.Equal(t, first.ResolvedDigest, root.Digest) } + +// movedTagBackend opens destination stores whose copy tag is moved by a concurrent writer the +// moment the copy has tagged it. +type movedTagBackend struct { + backend + tag string + moved ocispec.Descriptor +} + +func (b *movedTagBackend) open(ctx context.Context, name string, write bool) (store, error) { + opened, err := b.backend.open(ctx, name, write) + if err != nil || !write { + return opened, err + } + return &movedTagStore{store: opened, tag: b.tag, moved: b.moved}, nil +} + +type movedTagStore struct { + store + tag string + moved ocispec.Descriptor +} + +func (s *movedTagStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + if err := s.store.Tag(ctx, desc, reference); err != nil { + return err + } + if reference == s.tag { + return s.store.Tag(ctx, s.moved, reference) + } + return nil +} + +// The copy result describes the digest that was copied. A writer that moves the destination tag +// after the copy must not make the result, and the lockfile pin built from it, name content this +// operation never copied. +func TestCopyFromResultNamesTheCopiedDigestNotTheMovedTag(t *testing.T) { + ctx := context.Background() + source := newLayoutRepo(t) + published := publish(t, source, publishSpec{content: "copied content"}) + destination := newLayoutRepo(t) + other := publish(t, destination, publishSpec{content: "someone else's content"}) + otherDesc, ok := destination.descriptorFor(appName, other.Pinned.ResolvedDigest) + require.True(t, ok) + require.NotEqual(t, published.Pinned.ResolvedDigest, other.Pinned.ResolvedDigest) + + destination.be = &movedTagBackend{backend: destination.be, tag: destination.be.reference(appName, appVersion), moved: otherDesc} + result, err := destination.CopyFrom(ctx, source, published.Pinned, artifact.CopyFromOptions{ExpectedPrior: &other.Pinned, Force: true}) + require.NoError(t, err) + assert.Equal(t, published.Pinned.ResolvedDigest, result.Pinned.ResolvedDigest, "the pin is the copied digest, not whatever the tag names now") + assert.Equal(t, published.Pinned.RecordDigest, result.Pinned.RecordDigest) + assert.Equal(t, published.Pinned.ContentDigest, result.Pinned.ContentDigest) + assert.NotEmpty(t, result.Location) +} From 7f263958d98e967fc5c9b733bd3ecad5079b5e85 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:21:21 -0600 Subject: [PATCH 27/53] fix(oci): recheck sibling tags right before a manifest-scoped delete A registry removes every tag of a manifest it deletes, and a tag another client adds after the first sibling scan was removed unreported. Scan again immediately before the removal so the delete is refused, or the affected references are reported, from the state at removal time. OCI registries have no conditional delete, so this narrows the window rather than closing it. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/delete.go | 28 ++++++++++++--- pkg/artifact/oci/delete_test.go | 64 +++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/pkg/artifact/oci/delete.go b/pkg/artifact/oci/delete.go index f4274ec5c5f..02b095ab0d2 100644 --- a/pkg/artifact/oci/delete.go +++ b/pkg/artifact/oci/delete.go @@ -50,13 +50,29 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference return nil, err } if pinned.Ref.IsDigest() { - return r.deleteDigest(ctx, loaded, name, holders) + return r.deleteDigest(ctx, st, loaded, name, holders) } - return r.deleteTag(ctx, loaded, pinned.Ref, holders) + return r.deleteTag(ctx, st, loaded, pinned.Ref, holders) +} + +// recheckHolders rescans the tags of a manifest immediately before a manifest-scoped removal, which +// also removes every tag of the manifest. OCI registries have no conditional delete and no lock +// that other clients honor, so this narrows the window in which a tag added after the first scan +// is removed unreported, but cannot close it. A reference-scoped backend removes only the named +// reference and keeps the first scan. +func (r *Repository) recheckHolders(ctx context.Context, st store, name string, target digest.Digest, holders []artifact.Ref) ([]artifact.Ref, error) { + if r.be.deleteScope() == artifact.DeleteReference { + return holders, nil + } + return r.referencesTo(ctx, st, name, target) } // deleteTag removes one tag, refusing when that would also remove a sibling reference. -func (r *Repository) deleteTag(ctx context.Context, loaded *root, ref artifact.Ref, holders []artifact.Ref) (*artifact.DeleteResult, error) { +func (r *Repository) deleteTag(ctx context.Context, st store, loaded *root, ref artifact.Ref, holders []artifact.Ref) (*artifact.DeleteResult, error) { + holders, err := r.recheckHolders(ctx, st, ref.Name, loaded.desc.Digest, holders) + if err != nil { + return nil, err + } var siblings []artifact.Ref named := false for _, holder := range holders { @@ -80,7 +96,7 @@ func (r *Repository) deleteTag(ctx context.Context, loaded *root, ref artifact.R } // deleteDigest removes the manifest and every reference to it, reporting each affected one. -func (r *Repository) deleteDigest(ctx context.Context, loaded *root, name string, holders []artifact.Ref) (*artifact.DeleteResult, error) { +func (r *Repository) deleteDigest(ctx context.Context, st store, loaded *root, name string, holders []artifact.Ref) (*artifact.DeleteResult, error) { result := &artifact.DeleteResult{Digest: loaded.desc.Digest} if r.be.deleteScope() == artifact.DeleteReference { if len(holders) == 0 { @@ -95,6 +111,10 @@ func (r *Repository) deleteDigest(ctx context.Context, loaded *root, name string } return result, nil } + holders, err := r.recheckHolders(ctx, st, name, loaded.desc.Digest, holders) + if err != nil { + return nil, err + } if err := r.be.remove(ctx, name, "", loaded.desc); err != nil { return nil, err } diff --git a/pkg/artifact/oci/delete_test.go b/pkg/artifact/oci/delete_test.go index ced1aa14b23..a61f7720898 100644 --- a/pkg/artifact/oci/delete_test.go +++ b/pkg/artifact/oci/delete_test.go @@ -3,6 +3,7 @@ package oci import ( "context" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -134,3 +135,66 @@ func TestRegistryRejectsReferenceScope(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) assert.True(t, resolves(repo, appName, appVersion)) } + +// scanHookBackend runs a hook right after the backend lists the versions of a name, which is the +// moment a concurrent client can add a tag between the sibling scan and the removal. +type scanHookBackend struct { + backend + mu sync.Mutex + scans int + after func(scan int) +} + +func (b *scanHookBackend) versions(ctx context.Context, name string) ([]string, error) { + versions, err := b.backend.versions(ctx, name) + b.mu.Lock() + b.scans++ + scan, hook := b.scans, b.after + b.mu.Unlock() + if hook != nil { + hook(scan) + } + return versions, err +} + +// A tag another client adds after the first sibling scan must still stop a manifest-scoped tag +// delete: the siblings are checked again immediately before the manifest is removed. +func TestRegistryTagDeleteRechecksSiblingsRightBeforeRemoving(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + first := publish(t, repo, publishSpec{content: "shared manifest"}) + pinned := mustResolve(t, repo, appVersion) + + hooked := &scanHookBackend{backend: repo.be} + repo.be = hooked + hooked.after = func(scan int) { + if scan == 1 { + alias(t, repo, first.Pinned, "stable") + } + } + + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteManifest}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a tag added after the first scan is found by the second") + assert.Contains(t, err.Error(), "stable") + assert.True(t, resolves(repo, appName, appVersion), "nothing was removed") + assert.True(t, resolves(repo, appName, "stable"), "the tag added concurrently survives") +} + +// A digest delete reports the references that exist when the manifest is removed, not the ones the +// first scan happened to see. +func TestRegistryDigestDeleteReportsTagsAddedAfterTheFirstScan(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + first := publish(t, repo, publishSpec{content: "shared manifest"}) + hooked := &scanHookBackend{backend: repo.be} + repo.be = hooked + hooked.after = func(scan int) { + if scan == 1 { + alias(t, repo, first.Pinned, "stable") + } + } + + byDigest := artifact.PinnedReference{Ref: artifact.Ref{Name: appName, Digest: first.Pinned.ResolvedDigest}, ResolvedDigest: first.Pinned.ResolvedDigest} + result, err := repo.Delete(context.Background(), byDigest, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + assert.ElementsMatch(t, []artifact.Ref{{Name: appName, Version: appVersion}, {Name: appName, Version: "stable"}}, result.Affected, "the tag added after the first scan is reported") + assert.False(t, resolves(repo, appName, "stable")) +} From c5c4e5baef5867f0c488586478752ac0918e368e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:23:33 -0600 Subject: [PATCH 28/53] fix(oci): keep pinned layout reads working after reopening the repository A layout lists the media type only of roots that still hold a tag. After the tag moved and the repository was reopened, a read of the old pin resolved the manifest as an untyped blob and was rejected, although the bytes still existed. Recover the type from the digest-verified manifest itself and never accept a blob that is not a manifest or index. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/resolve.go | 51 ++++++++++++++++++++++++++ pkg/artifact/oci/roundtrip_test.go | 59 ++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go index 65a306c355e..f085e2e002c 100644 --- a/pkg/artifact/oci/resolve.go +++ b/pkg/artifact/oci/resolve.go @@ -3,11 +3,13 @@ package oci import ( "context" + "encoding/json" "errors" "fmt" "slices" "sort" + "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" @@ -124,6 +126,9 @@ func (r *Repository) rootOf(ctx context.Context, pinned *artifact.PinnedReferenc desc, known := r.descriptorFor(pinned.Ref.Name, pinned.ResolvedDigest) if !known { desc, err = st.Resolve(ctx, pinned.ResolvedDigest.String()) + if err == nil { + desc, err = recoverRootType(ctx, st, desc) + } if err != nil { return nil, nil, resolveError(pinned.Ref, err) } @@ -135,6 +140,52 @@ func (r *Repository) rootOf(ctx context.Context, pinned *artifact.PinnedReferenc return st, loaded, nil } +// recoverRootType restores the media type of a root resolved by digest alone. A layout lists the +// media type only for roots that still hold a reference; a root whose tag has moved resolves as an +// untyped blob, although its bytes are still there. The type is read from the manifest itself after +// the bounded, digest-verified fetch, so a tampered blob is rejected and nothing is guessed. +func recoverRootType(ctx context.Context, st store, desc ocispec.Descriptor) (ocispec.Descriptor, error) { + if isManifestMediaType(desc.MediaType) || isIndexMediaType(desc.MediaType) { + return desc, nil + } + raw, err := fetchBounded(ctx, st, desc, maxManifestSize) + if err != nil { + return ocispec.Descriptor{}, err + } + mediaType, err := manifestTypeOf(desc.Digest, raw) + if err != nil { + return ocispec.Descriptor{}, err + } + if mediaType != "" { + desc.MediaType = mediaType + } + return desc, nil +} + +// manifestTypeOf reads the media type a manifest or index declares, or infers it from the shape of +// one that declares none. It returns an empty type for anything else, which stays rejected. +func manifestTypeOf(dgst digest.Digest, raw []byte) (string, error) { + var shape struct { + MediaType string `json:"mediaType"` + Manifests json.RawMessage `json:"manifests"` + Config json.RawMessage `json:"config"` + } + if err := json.Unmarshal(raw, &shape); err != nil { + return "", fmt.Errorf("%w: %s is not a manifest: %w", errUtils.ErrArtifactRecordInvalid, dgst, err) + } + switch { + case isManifestMediaType(shape.MediaType) || isIndexMediaType(shape.MediaType): + return shape.MediaType, nil + case shape.MediaType != "": + return "", nil + case shape.Manifests != nil: + return ocispec.MediaTypeImageIndex, nil + case shape.Config != nil: + return ocispec.MediaTypeImageManifest, nil + } + return "", nil +} + // resolveDigest resolves name@digest. A registry answers a direct manifest lookup in its // repository path; a layout also requires a live reference to the digest, so a deleted artifact // does not stay addressable. When the digest is not a manifest, it may be the Record digest of diff --git a/pkg/artifact/oci/roundtrip_test.go b/pkg/artifact/oci/roundtrip_test.go index 2d0b5390e80..14a167447ad 100644 --- a/pkg/artifact/oci/roundtrip_test.go +++ b/pkg/artifact/oci/roundtrip_test.go @@ -159,3 +159,62 @@ func TestRejectsMismatchedRecordForItsTag(t *testing.T) { _, err = repo.Inspect(context.Background(), pinned) require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a tag serving another version's Record is tampering") } + +// A pin taken before the tag moved stays readable from a fresh Repository: reopening drops the +// in-memory descriptor cache, and the superseded manifest is no longer listed under any tag, so its +// media type has to be recovered from the verified manifest itself. +func TestMovedTagDoesNotChangeAPinnedReadAfterReopening(t *testing.T) { + reopened := map[string]func(t *testing.T) (first, second *Repository){ + "registry": func(t *testing.T) (*Repository, *Repository) { + reg := newRegistry(t, nil) + return newRegistryRepo(t, reg), newRegistryRepo(t, reg) + }, + "layout": func(t *testing.T) (*Repository, *Repository) { + path := filepath.Join(t.TempDir(), "layout") + return newLayoutRepoAt(t, path), newLayoutRepoAt(t, path) + }, + } + for name, open := range reopened { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + writer, reader := open(t) + first := publish(t, writer, publishSpec{content: "first"}) + pinned, err := writer.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + _, err = writer.Publish(ctx, prepare(t, publishSpec{content: "second"}), artifact.PublishOptions{ExpectedPrior: &first.Pinned, Force: true}) + require.NoError(t, err) + + content, err := reader.Read(ctx, pinned) + require.NoError(t, err, "the old manifest and blobs still exist, so the pinned read works from a fresh handle") + defer content.Close() + recordDigest, err := content.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, pinned.RecordDigest, recordDigest) + + inspection, err := reader.Inspect(ctx, pinned) + require.NoError(t, err) + assert.Equal(t, pinned.ResolvedDigest, inspection.Pinned.ResolvedDigest) + }) + } +} + +// Recovering the type of an unlisted root never turns another blob into a root: a payload layer or +// a Record config named as a pin stays rejected. +func TestPinnedReadAfterReopeningNeverAcceptsANonManifest(t *testing.T) { + path := filepath.Join(t.TempDir(), "layout") + writer := newLayoutRepoAt(t, path) + result := publish(t, writer, publishSpec{content: "x"}) + reader := newLayoutRepoAt(t, path) + + for name, blob := range map[string]digest.Digest{ + "payload layer": payloadDigest(t, result), + "record config": result.Pinned.RecordDigest, + } { + t.Run(name, func(t *testing.T) { + forged := result.Pinned + forged.ResolvedDigest = blob + _, _, err := reader.rootOf(context.Background(), &forged) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) + } +} From f6744c7df00785d5dddee54790601647c173e659 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:24:22 -0600 Subject: [PATCH 29/53] fix(oci): propagate operational failures during digest scans A Record-digest lookup treated every failed root load as a nonmatch, so a registry that answered 503 for the only matching tag produced not-found. Keep treating unverifiable or unusable roots as nonmatches, but return fetch failures to the caller. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/resolve.go | 12 ++- pkg/artifact/oci/resolve_scan_test.go | 110 ++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 pkg/artifact/oci/resolve_scan_test.go diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go index f085e2e002c..641932158f9 100644 --- a/pkg/artifact/oci/resolve.go +++ b/pkg/artifact/oci/resolve.go @@ -236,7 +236,10 @@ func (r *Repository) digestAt(ctx context.Context, st store, ref artifact.Ref, v } loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) if err != nil { - if ctx.Err() != nil { + // A root that cannot be verified or classified is simply not the one being looked for, but + // a registry that fails to answer says nothing about the digest: reporting "not found" + // would turn an outage into a false negative. + if ctx.Err() != nil || isOperationalFailure(err) { return ocispec.Descriptor{}, false, err } return ocispec.Descriptor{}, false, nil @@ -244,6 +247,13 @@ func (r *Repository) digestAt(ctx context.Context, st store, ref artifact.Ref, v return desc, loaded.class == classAtmos && loaded.manifest.Config.Digest == ref.Digest, nil } +// isOperationalFailure reports whether a failed root load says something about the store rather than +// about the root: the fetch itself failed, as opposed to returning content that is absent, fails +// digest verification, or is not a usable manifest. +func isOperationalFailure(err error) bool { + return errors.Is(err, errUtils.ErrArtifactDownloadFailed) && !errors.Is(err, errUtils.ErrOCIIntegrity) && !isNotFound(err) +} + // resolveError classifies a store failure: absent content is not found, a canceled context is // passed through, anything else is a download failure that keeps its cause. func resolveError(ref artifact.Ref, err error) error { diff --git a/pkg/artifact/oci/resolve_scan_test.go b/pkg/artifact/oci/resolve_scan_test.go new file mode 100644 index 00000000000..58bd9bf5f48 --- /dev/null +++ b/pkg/artifact/oci/resolve_scan_test.go @@ -0,0 +1,110 @@ +//nolint:gocritic // OCI descriptors are passed by value, mirroring the ORAS content contracts. +package oci + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// unavailableBackend opens stores whose blob reads fail like a registry that answers 503, while +// tag resolution keeps working. +type unavailableBackend struct { + backend +} + +func (b *unavailableBackend) open(ctx context.Context, name string, write bool) (store, error) { + opened, err := b.backend.open(ctx, name, write) + if err != nil { + return nil, err + } + return &unavailableStore{store: opened}, nil +} + +type unavailableStore struct { + store +} + +var errServiceUnavailable = errors.New("503 service unavailable") + +func (s *unavailableStore) Fetch(context.Context, ocispec.Descriptor) (io.ReadCloser, error) { + return nil, errServiceUnavailable +} + +// tamperedBackend serves wrong bytes for one manifest digest, like a registry returning content +// that fails digest verification. Everything else reads normally. +type tamperedBackend struct { + backend + target digest.Digest +} + +func (b *tamperedBackend) open(ctx context.Context, name string, write bool) (store, error) { + opened, err := b.backend.open(ctx, name, write) + if err != nil { + return nil, err + } + return &tamperedStore{store: opened, target: b.target}, nil +} + +type tamperedStore struct { + store + target digest.Digest +} + +func (s *tamperedStore) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + if desc.Digest == s.target { + return io.NopCloser(bytes.NewReader([]byte("tampered"))), nil + } + return s.store.Fetch(ctx, desc) +} + +// A Record-digest lookup scans the tags of a name. A tag that resolves but whose manifest cannot be +// fetched because the registry is failing is an operational failure, not proof that the digest is +// absent; a root that is merely unusable stays a nonmatch. +func TestScanForDigestPropagatesOperationalFetchFailures(t *testing.T) { + reg := newRegistry(t, nil) + writer := newRegistryRepo(t, reg) + published := publish(t, writer, publishSpec{content: "scan target"}) + byRecord := artifact.Ref{Name: appName, Digest: published.Pinned.RecordDigest} + + t.Run("a failing registry is not a missing digest", func(t *testing.T) { + reader := newRegistryRepo(t, reg) + reader.be = &unavailableBackend{backend: reader.be} + _, err := reader.Resolve(context.Background(), byRecord) + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) + require.NotErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.ErrorContains(t, err, errServiceUnavailable.Error()) + }) + + t.Run("a healthy registry still finds the Record", func(t *testing.T) { + reader := newRegistryRepo(t, reg) + pinned, err := reader.Resolve(context.Background(), byRecord) + require.NoError(t, err) + assert.Equal(t, published.Pinned.ResolvedDigest, pinned.ResolvedDigest) + }) + + t.Run("an unrelated tag that fails verification does not hide the match", func(t *testing.T) { + other := publish(t, writer, publishSpec{version: "0.9.0", content: "different content"}) + reader := newRegistryRepo(t, reg) + reader.be = &tamperedBackend{backend: reader.be, target: other.Pinned.ResolvedDigest} + pinned, err := reader.Resolve(context.Background(), byRecord) + require.NoError(t, err, "a tag whose manifest fails verification is a nonmatch, not a failure of the lookup") + assert.Equal(t, published.Pinned.ResolvedDigest, pinned.ResolvedDigest) + }) + + t.Run("an absent digest is still not found", func(t *testing.T) { + reader := newRegistryRepo(t, reg) + _, err := reader.Resolve(context.Background(), artifact.Ref{Name: appName, Digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000"}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) +} From 4ca1f9b24f896b14d53b12807334b9bc7e7af418 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 14:26:11 -0600 Subject: [PATCH 30/53] fix(oci): serialize layout writes across processes with a layout-wide guard The host lock was keyed by reference, so two processes publishing a/app:1 and b/app:1 took different locks, opened the same index.json, and could overwrite each other's update. Publish, copy, delete, and attachment writes now take a file lock scoped to the layout root, which also excludes other goroutines and Repository values. The attachment write reopens the layout under the guard so it never commits a stale index. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/attach.go | 15 +++ pkg/artifact/oci/backend.go | 47 +++++++-- pkg/artifact/oci/copy.go | 6 +- pkg/artifact/oci/delete.go | 6 +- pkg/artifact/oci/layout_guard_test.go | 137 ++++++++++++++++++++++++++ pkg/artifact/oci/publish.go | 6 +- 6 files changed, 201 insertions(+), 16 deletions(-) create mode 100644 pkg/artifact/oci/layout_guard_test.go diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index bf95dc4d692..30635d99296 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -213,6 +213,21 @@ type attachRequest struct { // signature additionally carries the Record digest and the config descriptor digest of the // addressed manifest. Writing is idempotent: identical evidence yields an identical manifest. func (r *Repository) storeAttachment(ctx context.Context, st store, request *attachRequest) (artifact.SignatureEvidence, string, error) { + // A layout records the attachment root in the shared index.json, so the write joins the same + // layout-wide guard as publication, and the store is reopened under it: a handle opened before + // the guard was taken could hold an index that another writer has since replaced. + release, err := r.guardLayout(ctx) + if err != nil { + return artifact.SignatureEvidence{}, "", err + } + defer release() + if layout, isLayout := r.be.(*layoutBackend); isLayout { + fresh, openErr := layout.layout(ctx, true) + if openErr != nil { + return artifact.SignatureEvidence{}, "", openErr + } + st = fresh + } rootDesc, subject, payload, configDigest := request.root, request.subject, request.payload, request.configDigest layer := ocispec.Descriptor{ MediaType: payload.MediaType, diff --git a/pkg/artifact/oci/backend.go b/pkg/artifact/oci/backend.go index a273cb505c5..b6443d9695d 100644 --- a/pkg/artifact/oci/backend.go +++ b/pkg/artifact/oci/backend.go @@ -16,6 +16,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/filelock" atmosoci "github.com/cloudposse/atmos/pkg/oci" ) @@ -201,15 +202,41 @@ func (b *layoutBackend) deleteScope() artifact.DeleteSemantics { return artifact func (b *layoutBackend) enumeratesCollection() bool { return true } -// layoutPathLocks serializes in-process writers to one layout; cross-process writers are excluded -// by the host guard the service takes. -var layoutPathLocks sync.Map - -func lockLayout(root string) func() { - value, _ := layoutPathLocks.LoadOrStore(root, &sync.Mutex{}) - mu, _ := value.(*sync.Mutex) - mu.Lock() - return mu.Unlock +// layoutGuardResource names the layout-wide write guard. It starts with a NUL byte, which no +// artifact reference can contain, so it can never collide with a reference lock. +const layoutGuardResource = "\x00layout" + +// guardLayout serializes one read-modify-write of a layout's index.json against every other writer +// of the same layout: other goroutines, other Repository values, and other Atmos processes on this +// host. The scope is the layout root, not the artifact reference, because every reference lives in +// the same index.json and two writers of different references would otherwise overwrite each +// other's successful update. The returned release function must be called when the write is done. +// A registry has no shared index file to protect, so its guard is a no-op. +func (r *Repository) guardLayout(ctx context.Context) (func(), error) { + if _, isLayout := r.be.(*layoutBackend); !isLayout { + return func() {}, nil + } + path, err := r.lockPath(layoutGuardResource) + if err != nil { + return nil, err + } + held, release, done := make(chan struct{}), make(chan struct{}), make(chan error, 1) + go func() { + done <- filelock.New(path).WithExclusive(ctx, func() error { + close(held) + <-release + return nil + }) + }() + select { + case <-held: + return func() { + close(release) + <-done + }, nil + case err := <-done: + return nil, fmt.Errorf("waiting for the write guard of layout %q in repository %q: %w", r.lockNamespace(), r.alias, err) + } } // exists reports whether the layout has been created. @@ -292,7 +319,7 @@ func (b *layoutBackend) names(ctx context.Context) ([]string, error) { } // remove untags one reference. Blobs stay: they may be shared with other references. The caller -// holds the in-process layout lock. +// holds the layout write guard. func (b *layoutBackend) remove(ctx context.Context, name, version string, _ ocispec.Descriptor) error { opened, err := b.layout(ctx, true) if err != nil { diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 726e179e42a..0f9b7ee281d 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -75,9 +75,11 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto if err := ref.Validate(); err != nil { return nil, err } - if layout, ok := r.be.(*layoutBackend); ok { - defer lockLayout(layout.root)() + release, err := r.guardLayout(ctx) + if err != nil { + return nil, err } + defer release() dst, err := r.be.open(ctx, ref.Name, true) if err != nil { return nil, err diff --git a/pkg/artifact/oci/delete.go b/pkg/artifact/oci/delete.go index 02b095ab0d2..89a71b9cfe4 100644 --- a/pkg/artifact/oci/delete.go +++ b/pkg/artifact/oci/delete.go @@ -32,9 +32,11 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference if err := artifact.ValidateName(name); err != nil { return nil, err } - if layout, ok := r.be.(*layoutBackend); ok { - defer lockLayout(layout.root)() + release, err := r.guardLayout(ctx) + if err != nil { + return nil, err } + defer release() st, loaded, err := r.rootOf(ctx, &pinned) if err != nil { return nil, err diff --git a/pkg/artifact/oci/layout_guard_test.go b/pkg/artifact/oci/layout_guard_test.go new file mode 100644 index 00000000000..406d7f9b641 --- /dev/null +++ b/pkg/artifact/oci/layout_guard_test.go @@ -0,0 +1,137 @@ +package oci + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/filelock" +) + +// layoutRepoWithGuardDir opens a layout repository whose write guards live in lockDir. Repositories +// that share a layout path and a lock directory behave like separate Atmos processes on one host. +func layoutRepoWithGuardDir(t *testing.T, path, lockDir string) *Repository { + t.Helper() + repo, err := NewRepository(context.Background(), "layout", layoutConfig(path), WithOwner(testOwner), WithLockDir(lockDir)) + require.NoError(t, err) + return repo +} + +// The layout write guard is scoped to the layout root: a writer of a/app and a writer of b/app share +// one index.json, so they must exclude each other even though their reference locks differ. +func TestLayoutGuardExcludesWritersOfDifferentReferences(t *testing.T) { + path, lockDir := filepath.Join(t.TempDir(), "layout"), t.TempDir() + holder := layoutRepoWithGuardDir(t, path, lockDir) + writer := layoutRepoWithGuardDir(t, path, lockDir) + existing := publish(t, holder, publishSpec{name: "a/app", content: "first"}) + + release, err := holder.guardLayout(context.Background()) + require.NoError(t, err) + + // Every layout write path waits for the guard: publish, copy, delete, and attachments. + timeout := func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), 100*time.Millisecond) + } + t.Run("publish", func(t *testing.T) { + ctx, cancel := timeout() + defer cancel() + _, err := writer.Publish(ctx, prepare(t, publishSpec{name: "b/app", content: "second"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, filelock.ErrAcquire) + assert.False(t, resolves(writer, "b/app", appVersion), "nothing was written while the guard was held elsewhere") + }) + t.Run("copy", func(t *testing.T) { + source := newLayoutRepo(t) + copied := publish(t, source, publishSpec{name: "c/app", content: "copied"}) + ctx, cancel := timeout() + defer cancel() + _, err := writer.CopyFrom(ctx, source, copied.Pinned, artifact.CopyFromOptions{}) + require.ErrorIs(t, err, filelock.ErrAcquire) + assert.False(t, resolves(writer, "c/app", appVersion)) + }) + t.Run("delete", func(t *testing.T) { + ctx, cancel := timeout() + defer cancel() + _, err := writer.Delete(ctx, existing.Pinned, artifact.DeleteOptions{Scope: artifact.DeleteReference}) + require.ErrorIs(t, err, filelock.ErrAcquire) + assert.True(t, resolves(writer, "a/app", appVersion)) + }) + t.Run("attachment", func(t *testing.T) { + st, loaded, err := writer.rootOf(context.Background(), &existing.Pinned) + require.NoError(t, err) + subject := artifact.SignatureSubject{Kind: artifact.SubjectOCIRoot, Digest: loaded.desc.Digest, MediaType: loaded.desc.MediaType, Size: loaded.desc.Size} + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: bundleMediaTypePrefix + "v0.3+json", Data: []byte("bundle")} + ctx, cancel := timeout() + defer cancel() + _, _, err = writer.storeAttachment(ctx, st, &attachRequest{root: loaded.desc, subject: subject, payload: payload}) + require.ErrorIs(t, err, filelock.ErrAcquire) + }) + + // A waiting writer proceeds as soon as the guard is released. + done := make(chan error, 1) + go func() { + _, err := writer.Publish(context.Background(), prepare(t, publishSpec{name: "b/app", content: "second"}), artifact.PublishOptions{}) + done <- err + }() + select { + case err := <-done: + t.Fatalf("publish finished while the guard was held: %v", err) + case <-time.After(150 * time.Millisecond): + } + release() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("publish did not proceed after the guard was released") + } + assert.True(t, resolves(writer, "a/app", appVersion)) + assert.True(t, resolves(writer, "b/app", appVersion)) +} + +// Concurrent writers of different references through separate Repository values never lose each +// other's index update: every successful publication is listed afterwards. +func TestLayoutConcurrentWritersKeepEveryReference(t *testing.T) { + const writers = 8 + path, lockDir := filepath.Join(t.TempDir(), "layout"), t.TempDir() + prepared := make([]*artifact.PreparedArtifact, writers) + repos := make([]*Repository, writers) + for index := range writers { + repos[index] = layoutRepoWithGuardDir(t, path, lockDir) + prepared[index] = prepare(t, publishSpec{name: fmt.Sprintf("team%d/app", index), content: fmt.Sprintf("content %d", index)}) + } + + var wait sync.WaitGroup + errs := make([]error, writers) + for index := range writers { + wait.Add(1) + go func() { + defer wait.Done() + _, errs[index] = repos[index].Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wait.Wait() + for index, err := range errs { + require.NoError(t, err, "writer %d", index) + } + + reader := layoutRepoWithGuardDir(t, path, lockDir) + for index := range writers { + name := fmt.Sprintf("team%d/app", index) + assert.True(t, resolves(reader, name, appVersion), "%s was lost by a concurrent writer", name) + } +} + +// A registry has no shared index file, so its guard never touches the lock directory. +func TestRegistryGuardIsANoOp(t *testing.T) { + repo := newRegistryRepo(t, newRegistry(t, nil)) + release, err := repo.guardLayout(context.Background()) + require.NoError(t, err) + release() +} diff --git a/pkg/artifact/oci/publish.go b/pkg/artifact/oci/publish.go index fe1e77f03d5..c0742658d90 100644 --- a/pkg/artifact/oci/publish.go +++ b/pkg/artifact/oci/publish.go @@ -41,9 +41,11 @@ func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArt if err := ref.Validate(); err != nil { return nil, err } - if layout, ok := r.be.(*layoutBackend); ok { - defer lockLayout(layout.root)() + release, err := r.guardLayout(ctx) + if err != nil { + return nil, err } + defer release() st, err := r.be.open(ctx, ref.Name, true) if err != nil { return nil, err From e5ad7607b2e0d20e38e65fbf3f1d45b262ceeddc Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 15:22:20 -0600 Subject: [PATCH 31/53] fix(artifact): redact registry userinfo in OCI option validation errors OCIRegistryOptions.Validate repeated the configured registry verbatim, so user:password@host leaked into the error. Redact embedded userinfo before echoing the value. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/options.go | 23 ++++++++++++++++++++++- pkg/artifact/options_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 8c3d3dcf69e..156e087d9fb 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -226,7 +226,7 @@ func (o *OCIRegistryOptions) Validate() error { if o.Registry == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) } else if !validRegistryHost(o.Registry) { - errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Registry))) + errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, redactRegistry(o.Registry))) } errs = append(errs, validatePrefix(o.Prefix)) if o.Prefix != "" { @@ -242,6 +242,27 @@ func (o *OCIRegistryOptions) Validate() error { const registryPortBits = 16 +// redactRegistry returns registry safe to repeat in a diagnostic: when its authority embeds +// userinfo (user:password@host), the userinfo is replaced with a placeholder so a credential +// typed into the registry option never reaches an error message. +func redactRegistry(registry string) string { + rest := registry + prefix := "" + if scheme, after, found := strings.Cut(registry, "://"); found { + prefix, rest = scheme+"://", after + } + authority, tail, _ := strings.Cut(rest, "/") + at := strings.LastIndex(authority, "@") + if at < 0 { + return registry + } + redacted := prefix + "REDACTED@" + authority[at+1:] + if strings.Contains(rest, "/") { + redacted += "/" + tail + } + return redacted +} + func validRegistryHost(host string) bool { if !registryHostPattern.MatchString(host) { return false diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index 2f02282d5de..d8d7fe134d8 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -256,3 +256,28 @@ func TestValidateEndpoint_NeverEchoesCredentials(t *testing.T) { }) } } + +// TestOCIRegistryOptionsValidate_NeverEchoesCredentials guards that a registry diagnostic repeats +// the configured value only with any embedded userinfo redacted. +func TestOCIRegistryOptionsValidate_NeverEchoesCredentials(t *testing.T) { + const secret = "s3cr3t-password" + tests := []struct { + name string + registry string + wantErr string + }{ + {name: "userinfo only", registry: "user:" + secret + "@registry.example.com", wantErr: "registry.example.com"}, + {name: "scheme and userinfo", registry: "https://user:" + secret + "@registry.example.com", wantErr: "https://REDACTED@registry.example.com"}, + {name: "userinfo and path", registry: "https://user:" + secret + "@registry.example.com/acme/atmos", wantErr: "https://REDACTED@registry.example.com/acme/atmos"}, + {name: "token without password", registry: secret + "@registry.example.com:5000", wantErr: "REDACTED@registry.example.com:5000"}, + {name: "scheme without credentials stays visible", registry: "https://registry.example.com", wantErr: "https://registry.example.com"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := (&OCIRegistryOptions{Registry: tt.registry}).Validate() + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), secret) + }) + } +} From 737936550dacc89b34b0074b8734395187161274 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:08:21 -0600 Subject: [PATCH 32/53] fix(oci): treat an integrity-failed referrer as malformed evidence A referrer manifest that fails digest verification or whose blob is missing no longer aborts the whole attachment scan. It is reported as malformed evidence, like an oversized referrer, so a valid signature carried by another referrer is still evaluated. Transport failures stay fatal. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/attach.go | 5 ++++ pkg/artifact/oci/attach_test.go | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/pkg/artifact/oci/attach.go b/pkg/artifact/oci/attach.go index 30635d99296..f17ab017b80 100644 --- a/pkg/artifact/oci/attach.go +++ b/pkg/artifact/oci/attach.go @@ -94,6 +94,11 @@ func (r *Repository) readAttachment(ctx context.Context, st store, name string, if errors.Is(err, errUtils.ErrArtifactTooLarge) { return nil, fmt.Sprintf("referrer %s is too large to inspect: %v", desc.Digest, err), nil } + // A referrer manifest that fails digest verification, or whose blob is gone, is unusable + // evidence in the same way; only transport failures stay fatal. + if errors.Is(err, errUtils.ErrArtifactIntegrityFailed) { + return nil, fmt.Sprintf("referrer %s failed integrity verification: %v", desc.Digest, err), nil + } return nil, "", fmt.Errorf("%w: read referrer %s: %w", errUtils.ErrArtifactDownloadFailed, desc.Digest, err) } var manifest ocispec.Manifest diff --git a/pkg/artifact/oci/attach_test.go b/pkg/artifact/oci/attach_test.go index 90b9c733e73..0732566601b 100644 --- a/pkg/artifact/oci/attach_test.go +++ b/pkg/artifact/oci/attach_test.go @@ -14,6 +14,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/errdef" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -111,3 +112,55 @@ func TestAttachments_OversizedReferrerIsMalformedEvidence(t *testing.T) { }) } } + +// A referrer whose manifest bytes fail digest verification, or whose manifest blob is gone, is +// malformed evidence like an oversized one: it is reported and cannot block a valid signature +// carried by another referrer. +func TestAttachments_IntegrityFailedReferrerIsMalformedEvidence(t *testing.T) { + root := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString("root"), Size: 100} + valid, validRaw := bundleReferrer(t, root) + bundleType := bundleMediaTypePrefix + "v0.3+json" + corrupt := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, ArtifactType: bundleType, + Digest: digest.FromString("corrupt"), Size: int64(len("tampered bytes")), + } + missing := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, ArtifactType: bundleType, + Digest: digest.FromString("missing"), Size: 10, + } + tests := []struct { + name string + referrers []ocispec.Descriptor + bad ocispec.Descriptor + wantValid []digest.Digest + wantText string + }{ + {name: "corrupt referrer beside a valid one", referrers: []ocispec.Descriptor{corrupt, valid}, bad: corrupt, wantValid: []digest.Digest{valid.Digest}, wantText: "integrity"}, + {name: "only a corrupt referrer", referrers: []ocispec.Descriptor{corrupt}, bad: corrupt, wantText: "integrity"}, + {name: "missing referrer blob beside a valid one", referrers: []ocispec.Descriptor{missing, valid}, bad: missing, wantValid: []digest.Digest{valid.Digest}, wantText: "integrity"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := &referrerStore{ + referrers: tt.referrers, + content: map[digest.Digest][]byte{ + valid.Digest: validRaw, + corrupt.Digest: []byte("tampered bytes"), + }, + fetchErr: errdef.ErrNotFound, + } + set, err := (&Repository{cache: map[string][]byte{}}).attachments(context.Background(), st, "team/app", root) + require.NoError(t, err) + var got []digest.Digest + for _, item := range set.valid { + got = append(got, item.manifestDesc.Digest) + } + assert.Equal(t, tt.wantValid, got) + require.Len(t, set.diagnostics, 1) + assert.Equal(t, "malformed-evidence", set.diagnostics[0].Code) + assert.Equal(t, artifact.SeverityWarning, set.diagnostics[0].Severity) + assert.Contains(t, set.diagnostics[0].Message, tt.wantText) + assert.Contains(t, set.diagnostics[0].Message, tt.bad.Digest.String()) + }) + } +} From b11efb2b90ecb9f56e73b2b6a4480534b217999a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:08:54 -0600 Subject: [PATCH 33/53] fix(oci): preserve stat errors when checking whether a layout exists Only a missing index.json means the layout is absent. Any other stat failure (for example a permission error) is returned to both references and layout instead of being reported as an empty collection or a missing layout. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/backend.go | 32 +++++++++++--- pkg/artifact/oci/layout_exists_test.go | 58 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/oci/layout_exists_test.go diff --git a/pkg/artifact/oci/backend.go b/pkg/artifact/oci/backend.go index b6443d9695d..942ae1c0421 100644 --- a/pkg/artifact/oci/backend.go +++ b/pkg/artifact/oci/backend.go @@ -239,10 +239,20 @@ func (r *Repository) guardLayout(ctx context.Context) (func(), error) { } } -// exists reports whether the layout has been created. -func (b *layoutBackend) exists() bool { +// exists reports whether the layout has been created. Only a missing index means absence: any +// other failure to inspect it (a permission error, a path component that is not a directory) +// describes an inaccessible layout and is returned so callers do not mistake it for an empty or +// absent repository. +func (b *layoutBackend) exists() (bool, error) { info, err := os.Stat(filepath.Join(b.root, ocispec.ImageIndexFile)) - return err == nil && info.Mode().IsRegular() + switch { + case err == nil: + return info.Mode().IsRegular(), nil + case errors.Is(err, os.ErrNotExist): + return false, nil + default: + return false, err + } } // open returns a fresh handle on the layout so a long-lived Repository never serves a stale @@ -252,8 +262,14 @@ func (b *layoutBackend) open(ctx context.Context, _ string, write bool) (store, } func (b *layoutBackend) layout(ctx context.Context, write bool) (*atmosoci.Layout, error) { - if !write && !b.exists() { - return nil, fmt.Errorf("%w: the OCI layout %q does not exist", errUtils.ErrArtifactNotFound, b.root) + if !write { + present, err := b.exists() + if err != nil { + return nil, fmt.Errorf("%w: inspect layout %q: %w", errUtils.ErrArtifactDownloadFailed, b.root, err) + } + if !present { + return nil, fmt.Errorf("%w: the OCI layout %q does not exist", errUtils.ErrArtifactNotFound, b.root) + } } opened, err := atmosoci.OpenLayout(ctx, b.root) if err != nil { @@ -265,7 +281,11 @@ func (b *layoutBackend) layout(ctx context.Context, write bool) (*atmosoci.Layou // references returns every named root as name and version, skipping references that are not // artifact references (for example a layout shared with other tools). func (b *layoutBackend) references(ctx context.Context) ([]artifact.Ref, error) { - if !b.exists() { + present, err := b.exists() + if err != nil { + return nil, fmt.Errorf("%w: inspect layout %q: %w", errUtils.ErrArtifactListFailed, b.root, err) + } + if !present { return nil, nil } opened, err := b.layout(ctx, false) diff --git a/pkg/artifact/oci/layout_exists_test.go b/pkg/artifact/oci/layout_exists_test.go new file mode 100644 index 00000000000..381df480521 --- /dev/null +++ b/pkg/artifact/oci/layout_exists_test.go @@ -0,0 +1,58 @@ +package oci + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// A layout whose index cannot be inspected is an inaccessible repository, not an absent one: +// listing must report the failure instead of an empty collection, and reading must not claim the +// layout does not exist. +func TestLayoutBackend_UninspectableLayoutIsNotAbsent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("a file used as a directory reports a not-found error on Windows") + } + dir := t.TempDir() + blocker := filepath.Join(dir, "not-a-directory") + require.NoError(t, os.WriteFile(blocker, []byte("x"), 0o600)) + backend := newLayoutBackend(blocker) + + tests := []struct { + name string + call func() error + wantErr error + }{ + {name: "references", call: func() error { _, err := backend.references(context.Background()); return err }, wantErr: errUtils.ErrArtifactListFailed}, + {name: "versions", call: func() error { _, err := backend.versions(context.Background(), "team/app"); return err }, wantErr: errUtils.ErrArtifactListFailed}, + {name: "names", call: func() error { _, err := backend.names(context.Background()); return err }, wantErr: errUtils.ErrArtifactListFailed}, + {name: "layout read", call: func() error { _, err := backend.layout(context.Background(), false); return err }, wantErr: errUtils.ErrArtifactDownloadFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.call() + require.Error(t, err) + assert.NotErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.ErrorIs(t, err, tt.wantErr) + }) + } +} + +// An absent layout stays an empty collection for listing and not-found for reading. +func TestLayoutBackend_AbsentLayoutStaysAbsent(t *testing.T) { + backend := newLayoutBackend(filepath.Join(t.TempDir(), "missing")) + + refs, err := backend.references(context.Background()) + require.NoError(t, err) + assert.Empty(t, refs) + + _, err = backend.layout(context.Background(), false) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} From 917b1157e4dc75d7909d96c0ba9d35d2305a3710 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:09:28 -0600 Subject: [PATCH 34/53] fix(artifact): redact credentials in scheme-relative registry values A registry value of the form //user:secret@host has its authority after the leading slashes; redactRegistry now parses it so the userinfo never reaches a validation diagnostic. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/options.go | 3 +++ pkg/artifact/options_test.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 156e087d9fb..fa79fad86a7 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -250,6 +250,9 @@ func redactRegistry(registry string) string { prefix := "" if scheme, after, found := strings.Cut(registry, "://"); found { prefix, rest = scheme+"://", after + } else if after, found := strings.CutPrefix(registry, "//"); found { + // A scheme-relative value (//user:secret@host) carries its authority after the slashes. + prefix, rest = "//", after } authority, tail, _ := strings.Cut(rest, "/") at := strings.LastIndex(authority, "@") diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index d8d7fe134d8..3c0fa455744 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -270,6 +270,10 @@ func TestOCIRegistryOptionsValidate_NeverEchoesCredentials(t *testing.T) { {name: "scheme and userinfo", registry: "https://user:" + secret + "@registry.example.com", wantErr: "https://REDACTED@registry.example.com"}, {name: "userinfo and path", registry: "https://user:" + secret + "@registry.example.com/acme/atmos", wantErr: "https://REDACTED@registry.example.com/acme/atmos"}, {name: "token without password", registry: secret + "@registry.example.com:5000", wantErr: "REDACTED@registry.example.com:5000"}, + {name: "scheme-relative userinfo", registry: "//user:" + secret + "@registry.example.com", wantErr: "//REDACTED@registry.example.com"}, + {name: "scheme-relative userinfo and path", registry: "//user:" + secret + "@registry.example.com/acme/atmos", wantErr: "//REDACTED@registry.example.com/acme/atmos"}, + {name: "scheme-relative token without password", registry: "//" + secret + "@registry.example.com:5000", wantErr: "//REDACTED@registry.example.com:5000"}, + {name: "scheme-relative without credentials stays visible", registry: "//registry.example.com", wantErr: "//registry.example.com"}, {name: "scheme without credentials stays visible", registry: "https://registry.example.com", wantErr: "https://registry.example.com"}, } for _, tt := range tests { From 21ebc8e14ec59b85137eb52c1e4a50e8c5bebe89 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:10:16 -0600 Subject: [PATCH 35/53] fix(s3): stop polling when the configured lock wait expires The poll sleep is now limited to the remaining wait, so WithLockWait(20ms) no longer sleeps the default poll interval and attempts the claim after the configured wait has expired. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/lock.go | 23 ++++++++++++++++++++++ pkg/artifact/s3/lock_test.go | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 50c21fb827d..a0957aaa9d2 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -114,6 +114,29 @@ func (r *Repository) acquire(ctx context.Context, resource, owner, operationID s } } +// awaitRetry waits one poll interval before the next claim attempt. It returns the held-claim +// diagnostic when the context ends or the wait deadline passes, and nil when another attempt is due. +func (r *Repository) awaitRetry(ctx context.Context, key, resource string, deadline time.Time) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return r.heldError(ctx, key, resource, ctxErr) + } + remaining := time.Until(deadline) + if remaining <= 0 { + return r.heldError(ctx, key, resource, nil) + } + // The sleep never outlasts the configured wait, so the last attempt happens at the deadline + // instead of one full poll interval after it. + if err := sleep(ctx, boundedPoll(r.settings.lockPoll, remaining)); err != nil { + return r.heldError(ctx, key, resource, err) + } + return nil +} + +// boundedPoll limits one poll interval to the wait that remains. +func boundedPoll(poll, remaining time.Duration) time.Duration { + return min(poll, remaining) +} + // sleep waits for d or until ctx ends. func sleep(ctx context.Context, d time.Duration) error { timer := time.NewTimer(d) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 1e38df9c2ee..daa43cea91d 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -170,6 +170,43 @@ func TestLock_WaitIsBoundedByTheContext(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactConflict) } +// The configured wait bounds the poll sleep too: a poll interval longer than the wait must not +// carry the acquisition past its deadline into another attempt. +func TestLock_PollSleepIsLimitedToTheRemainingWait(t *testing.T) { + spec := noneSpec().fresh(t) + holder := newRepo(t, spec) + _, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) + require.NoError(t, err) + + // The poll interval is far longer than the wait and the context, so only a sleep clamped to + // the remaining wait lets the conflict arrive before the context ends. + peer := peerOf(t, spec, WithLockWait(20*time.Millisecond), withLockPoll(time.Hour)) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err = peer.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NoError(t, ctx.Err(), "the wait, not the context, ended the acquisition") + assert.NotErrorIs(t, err, context.DeadlineExceeded) +} + +func TestBoundedPoll(t *testing.T) { + tests := []struct { + name string + poll time.Duration + remaining time.Duration + want time.Duration + }{ + {name: "poll shorter than the wait", poll: 10 * time.Millisecond, remaining: time.Second, want: 10 * time.Millisecond}, + {name: "poll longer than the remaining wait", poll: 250 * time.Millisecond, remaining: 20 * time.Millisecond, want: 20 * time.Millisecond}, + {name: "poll equal to the remaining wait", poll: time.Second, remaining: time.Second, want: time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, boundedPoll(tt.poll, tt.remaining)) + }) + } +} + // A crash leaves the claim; nothing expires or steals it, and removing exactly that object recovers. func TestLock_CrashLeftClaimBlocksUntilTheOperatorRemovesIt(t *testing.T) { spec := noneSpec().fresh(t) From 65cb29309fe81f9493c7592ebbeb85e222a62e45 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:12:44 -0600 Subject: [PATCH 36/53] fix(s3): keep claim-release errors beside a failed mutation A failed mutation under an acquired claim used to discard the error from releasing the claim, leaving a permanent claim the caller never heard about. Sign, WriteEvidence, Publish, and Delete now join both errors. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/delete.go | 12 +--- pkg/artifact/s3/evidence.go | 8 +-- pkg/artifact/s3/lock.go | 14 +++++ pkg/artifact/s3/publish.go | 4 +- pkg/artifact/s3/release_errors_test.go | 83 ++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 pkg/artifact/s3/release_errors_test.go diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go index 971349b766a..40f019c17df 100644 --- a/pkg/artifact/s3/delete.go +++ b/pkg/artifact/s3/delete.go @@ -42,9 +42,7 @@ func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference return nil, err } defer func() { - if releaseErr := release(); err == nil { - err = releaseErr - } + err = joinRelease(err, release()) }() return r.deleteLocked(ctx, &pinned, &options) } @@ -63,9 +61,7 @@ func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedRe return nil, err } defer func() { - if releaseErr := claims.Release(ctx); err == nil { - err = releaseErr - } + err = joinRelease(err, claims.Release(ctx)) }() id := deletionPrefix + uuid.NewString() intent := deletionIntent(id) @@ -143,9 +139,7 @@ func (r *Repository) recoverDeletion(ctx context.Context, state *refState) (err return err } defer func() { - if releaseErr := claims.Release(ctx); err == nil { - err = releaseErr - } + err = joinRelease(err, claims.Release(ctx)) }() op, err := r.loadOperation(ctx, state.ref.OperationID) if err != nil { diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index 64ad31fce6c..7c23fd17fb9 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -145,9 +145,7 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return nil, err } defer func() { - if releaseErr := release(); err == nil { - err = releaseErr - } + err = joinRelease(err, release()) }() return r.signLocked(ctx, subject, signer) } @@ -347,9 +345,7 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR return nil, err } defer func() { - if releaseErr := release(); err == nil { - err = releaseErr - } + err = joinRelease(err, release()) }() raw, err := r.requireCurrentRecord(ctx, pinned.RecordDigest) if err != nil { diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index a0957aaa9d2..527187eee4e 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -356,6 +356,20 @@ func (l *claimHandle) goneAfter(ctx context.Context, err error) bool { return readErr == nil && current == nil } +// joinRelease combines the outcome of a claimed mutation with the outcome of releasing its claim, +// so a release failure after a failed mutation stays visible instead of being dropped. It returns +// err unchanged when the release succeeded and the release error alone when err is nil. +func joinRelease(err, releaseErr error) error { + switch { + case releaseErr == nil: + return err + case err == nil: + return releaseErr + default: + return errors.Join(err, releaseErr) + } +} + // holdClaim validates an existing claim or acquires one, and keeps it held through a mutation. func (r *Repository) holdClaim(ctx context.Context, supplied artifact.LockHandle, resource, operationID string) (func() error, error) { handle, local := supplied.(*claimHandle) diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index 70b17c57893..4d726f7b5ec 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -62,9 +62,7 @@ func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArt return nil, err } defer func() { - if releaseErr := release(); err == nil { - err = releaseErr - } + err = joinRelease(err, release()) }() return r.publishLocked(ctx, job, &options, scope) } diff --git a/pkg/artifact/s3/release_errors_test.go b/pkg/artifact/s3/release_errors_test.go new file mode 100644 index 00000000000..01d865ecc7f --- /dev/null +++ b/pkg/artifact/s3/release_errors_test.go @@ -0,0 +1,83 @@ +package s3 + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// errTestSigner is the mutation failure the failing signer reports. +var errTestSigner = errors.New("signer unavailable") + +// failingSigner fails to sign, so the mutation under the claim fails. +type failingSigner struct{ *artifacttest.FakeSigner } + +func (failingSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + return nil, errTestSigner +} + +// A mutation that fails while the claim cannot be released reports both failures: the caller must +// learn about the claim that remains held, not only about the mutation error. +func TestClaimedMutations_KeepTheReleaseErrorBesideTheMutationError(t *testing.T) { + tests := []struct { + name string + mutate func(t *testing.T, repo *Repository, pinned *artifact.PinnedReference) error + wantErr error + }{ + { + name: "sign", + mutate: func(t *testing.T, repo *Repository, pinned *artifact.PinnedReference) error { + signer := failingSigner{newSigner()} + _, err := repo.Sign(context.Background(), recordSubject(t, repo, pinned), signer, artifact.SignOptions{}) + return err + }, + wantErr: errTestSigner, + }, + { + name: "write evidence", + mutate: func(t *testing.T, repo *Repository, pinned *artifact.PinnedReference) error { + blobs := []artifact.EvidenceBlob{{Evidence: artifact.SignatureEvidence{Subject: artifact.SignatureSubject{Kind: artifact.SubjectRecord}}}} + _, err := repo.WriteEvidence(context.Background(), pinned, blobs, nil) + return err + }, + wantErr: errUtils.ErrArtifactRefInvalid, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := signedSpec(t, noneSpec()) + seed := newRepo(t, spec) + publish(t, seed, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, seed, defaultRef()) + + // Once the claim is created, remove it behind the holder's back: the release then + // finds nothing to release. + store := &hookStore{Store: newStoreFor(t, spec)} + store.afterPut = func(key string, info *blob.ObjectInfo, err error) (*blob.ObjectInfo, error) { + if err == nil && strings.Contains(key, "/locks/") { + require.NoError(t, store.Delete(context.Background(), key, blob.DeleteOptions{})) + } + return info, err + } + repo, err := newRepository("test", spec.config(spec.fake.Bucket()), + []Option{WithStore(store), WithVerifier(&artifacttest.FakeVerifier{}), withLockPoll(5 * time.Millisecond)}, &artifact.RepositoryOptions{}) + require.NoError(t, err) + + err = tt.mutate(t, repo, &pinned) + require.Error(t, err) + assert.ErrorIs(t, err, tt.wantErr, "the mutation error survives") + assert.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the release failure survives") + assert.ErrorContains(t, err, "is not held") + }) + } +} From 6800689fa72280a0d0d758d2d728dbf8d019ef9f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:14:32 -0600 Subject: [PATCH 37/53] fix(s3): include abandoned payload locations in the reservation claim claimOf listed only a reference's pending location, so another reference could reserve an abandoned location while it was empty and then lose its archive and sidecar when the original reference retired that payload. The claim now also covers every abandoned location in the journal, for pending and completed references, until retirement removes it. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/reserve.go | 29 +++++++++--- pkg/artifact/s3/reserve_abandoned_test.go | 55 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 pkg/artifact/s3/reserve_abandoned_test.go diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index f4ed13507e2..62d035a5f1d 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -25,9 +25,7 @@ func (r *Repository) begin(ctx context.Context, job *publishJob, plan *publishPl return nil, err } pending, beginErr := r.reserve(ctx, job, plan) - if releaseErr := claims.Release(ctx); beginErr == nil { - beginErr = releaseErr - } + beginErr = joinRelease(beginErr, claims.Release(ctx)) if beginErr != nil { return nil, beginErr } @@ -129,7 +127,9 @@ func (r *Repository) collectClaims(ctx context.Context, ownKey string) ([]layout } // claimOf returns the claim a reference holds: its representation when completed, its journal when -// pending. A reference whose state is missing holds no claim. +// pending. A reference whose state is missing holds no claim. Locations that earlier attempts +// abandoned stay claimed until a retry retires their payload, even while they hold no objects: the +// retirement deletes by the journaled payload and does not check who owns the location by then. func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layout.Claim, error) { base := layout.Claim{ReferenceKey: ref.Key(), Name: ref.Name, Version: ref.Version} if ref.Available() { @@ -138,6 +138,13 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou return nil, err } base.Paths = claimPaths(rep.Packaging, rep.Location) + // A completed reference keeps its journal until the cleanup that follows completion has + // removed everything the journal still lists. + journal, err := r.loadOperation(ctx, ref.OperationID) + if err != nil { + return nil, err + } + base.Paths = append(base.Paths, abandonedClaimPaths(journal)...) return &base, nil } op, err := r.loadOperation(ctx, ref.OperationID) @@ -145,10 +152,22 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou return nil, err } base.OperationID = op.OperationID - base.Paths = claimPaths(op.Packaging, op.Location) + base.Paths = append(claimPaths(op.Packaging, op.Location), abandonedClaimPaths(op)...) return &base, nil } +// abandonedClaimPaths lists the paths of every payload a journal still awaits removing. +func abandonedClaimPaths(op *layout.Operation) []string { + if op == nil { + return nil + } + var paths []string + for index := range op.Abandoned { + paths = append(paths, claimPaths(op.Abandoned[index].Packaging, op.Abandoned[index].Location)...) + } + return paths +} + // checkUnmanaged fails when a path the publication would write already holds objects and this // reference does not own it. func (r *Repository) checkUnmanaged(ctx context.Context, candidate *layout.Claim, plan *publishPlan) error { diff --git a/pkg/artifact/s3/reserve_abandoned_test.go b/pkg/artifact/s3/reserve_abandoned_test.go new file mode 100644 index 00000000000..5b0297c3654 --- /dev/null +++ b/pkg/artifact/s3/reserve_abandoned_test.go @@ -0,0 +1,55 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// A location an interrupted attempt abandoned stays claimed by its reference until a retry retires +// the payload. Until then another reference must not reserve it, even while it holds no objects: +// the retiring attempt deletes by the journaled payload and would destroy the newcomer's archive +// and metadata sidecar. +func TestAbandonedLocationStaysClaimedUntilRetired(t *testing.T) { + const abandonedArchive = "team/app/1.0.0.tgz" + fake := s3test.NewServer(t) + onFake := func(s *rspec) { s.fake = fake } + + // The first attempt stops before uploading anything, then a retry that needs a different + // packaging stops too: its journal lists the tgz location as abandoned although it is empty. + first := newRepo(t, archiveSpec(artifact.PackagingTgz).with(onFake).withOpts(withFaults(crashAt(faultBeforePayload)))) + _, err := first.Publish(context.Background(), prepare(t, first, &prepareSpec{opID: "op-first"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + stopped := newRepo(t, archiveSpec(artifact.PackagingZip).with(onFake).withOpts(withFaults(crashAt(faultBeforePayload)))) + _, err = stopped.Publish(context.Background(), prepare(t, stopped, &prepareSpec{opID: "op-stopped"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + _, present := fake.Object(abandonedArchive) + require.False(t, present, "the abandoned location holds no object yet") + + newcomer := newRepo(t, archiveSpec(artifact.PackagingTgz).withPattern("team/app/{{ .Version }}.tgz").with(onFake)) + newcomerRef := ref("other/app", testVersion) + before := bucketKeys(fake) + _, err = newcomer.Publish(context.Background(), prepare(t, newcomer, &prepareSpec{name: "other/app", files: map[string]string{"a.txt": "newcomer"}}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.ErrorContains(t, err, abandonedArchive) + assert.Equal(t, before, bucketKeys(fake), "the refused reservation changed nothing") + _, err = newcomer.Resolve(context.Background(), newcomerRef) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + + // The retry that completes the first reference retires the abandoned payload and releases the + // claim: the location is free again. + final := newRepo(t, archiveSpec(artifact.PackagingZip).with(onFake)) + _, err = final.Publish(context.Background(), prepare(t, final, &prepareSpec{opID: "op-final"}), artifact.PublishOptions{}) + require.NoError(t, err) + assertIntact(t, final, defaultRef()) + _, err = newcomer.Publish(context.Background(), prepare(t, newcomer, &prepareSpec{name: "other/app", files: map[string]string{"a.txt": "newcomer"}}), artifact.PublishOptions{}) + require.NoError(t, err, "a retired location is claimable again") + assertIntact(t, newcomer, newcomerRef) + assertIntact(t, final, defaultRef()) +} From 135e20dcabddbf3cda8f5eca959aac7a09f23f6b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:16:25 -0600 Subject: [PATCH 38/53] fix(oci): count one root per digest when selecting a layout root Two reference names for the same root digest matched twice, so importing by that digest was rejected as ambiguous. Matching entries are now deduplicated by digest; selections that identify different roots are still rejected. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 4 +- pkg/artifact/oci/copy_select_test.go | 57 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 pkg/artifact/oci/copy_select_test.go diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 0f9b7ee281d..093703f1a92 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -241,7 +242,8 @@ func layoutRoot(directory, selector string) (ocispec.Descriptor, error) { } var matches []ocispec.Descriptor for _, entry := range index.Manifests { - if (selector == "" && entry.Annotations[ocispec.AnnotationRefName] != "") || (selector != "" && (entry.Digest.String() == selector || entry.Annotations[ocispec.AnnotationRefName] == selector)) { + if matchesLayoutSelector(entry, selector) && !slices.ContainsFunc(matches, func(seen ocispec.Descriptor) bool { return seen.Digest == entry.Digest }) { + // Two reference names for one root digest select one root, not two. matches = append(matches, entry) } } diff --git a/pkg/artifact/oci/copy_select_test.go b/pkg/artifact/oci/copy_select_test.go new file mode 100644 index 00000000000..7250ba41266 --- /dev/null +++ b/pkg/artifact/oci/copy_select_test.go @@ -0,0 +1,57 @@ +package oci + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Two reference names for one root digest select that root once; different digests stay ambiguous. +func TestLayoutRoot_DeduplicatesNamesOfOneDigest(t *testing.T) { + first := digest.FromString("first root") + second := digest.FromString("second root") + entry := func(dgst digest.Digest, name string) ocispec.Descriptor { + return ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, Digest: dgst, Size: 10, + Annotations: map[string]string{ocispec.AnnotationRefName: name}, + } + } + tests := []struct { + name string + entries []ocispec.Descriptor + selector string + want digest.Digest + wantErr error + }{ + {name: "digest selector over two names", entries: []ocispec.Descriptor{entry(first, "a:1"), entry(first, "b:1")}, selector: first.String(), want: first}, + {name: "no selector over two names", entries: []ocispec.Descriptor{entry(first, "a:1"), entry(first, "b:1")}, want: first}, + {name: "name selector", entries: []ocispec.Descriptor{entry(first, "a:1"), entry(second, "b:1")}, selector: "b:1", want: second}, + {name: "two digests stay ambiguous", entries: []ocispec.Descriptor{entry(first, "a:1"), entry(second, "b:1")}, wantErr: errUtils.ErrArtifactRefInvalid}, + {name: "a selector matching nothing", entries: []ocispec.Descriptor{entry(first, "a:1")}, selector: "missing:1", wantErr: errUtils.ErrArtifactRefInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + raw, err := json.Marshal(ocispec.Index{Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: ocispec.MediaTypeImageIndex, Manifests: tt.entries}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, ocispec.ImageIndexFile), raw, 0o600)) + + got, err := layoutRoot(dir, tt.selector) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got.Digest) + }) + } +} From 913eacd074d999a6c75b6bca6d01f958636aaff6 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:16:47 -0600 Subject: [PATCH 39/53] fix(oci): check the Record identity before ImportLayout writes the reference ImportLayout handed the caller's reference to the transfer without reading the Record, so a layout whose Record names b/app:2 imported as a/app:1 wrote that tag and only then failed the identity check, leaving the mismatched tag published (or replacing a valid one). The Record is now checked against the reference before any destination write. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 19 +++++++++ pkg/artifact/oci/copy_identity_test.go | 56 ++++++++++++++++++++++++++ pkg/artifact/oci/fetch.go | 2 +- pkg/artifact/oci/repository.go | 3 +- 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 pkg/artifact/oci/copy_identity_test.go diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 093703f1a92..397582e215e 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -171,9 +171,28 @@ func (r *Repository) ImportLayout(ctx context.Context, directory string, ref art if err != nil { return nil, err } + // The Record binds a package to its own name and version. Checking it before the transfer + // keeps a mismatched import from publishing a tag that every later read would reject. + if err := r.requireImportIdentity(ctx, src, loaded, ref); err != nil { + return nil, err + } return r.transfer(ctx, src, loaded, ref, artifact.PublishOptions{ExpectedPrior: options.ExpectedPrior, Force: options.Force, Lock: options.Lock}) } +// requireImportIdentity rejects an import of an Atmos package whose Record names another +// reference than the one it would be published under. A foreign graph carries no Record and is +// published under the requested reference as it is. +func (r *Repository) requireImportIdentity(ctx context.Context, src atmosoci.Fetcher, loaded *root, ref artifact.Ref) error { + if loaded.class != classAtmos { + return nil + } + record, err := r.readRecord(ctx, src, ref.Name, loaded) + if err != nil { + return err + } + return requireSubjectOf(record, ref) +} + // ExportLayout exports the pinned graph, with every referrer, to a standard OCI image layout // directory holding exactly one root, so it can travel as a payload and be imported again with its // digest intact. It returns the exported root descriptor. diff --git a/pkg/artifact/oci/copy_identity_test.go b/pkg/artifact/oci/copy_identity_test.go new file mode 100644 index 00000000000..9cd750f824a --- /dev/null +++ b/pkg/artifact/oci/copy_identity_test.go @@ -0,0 +1,56 @@ +package oci + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// An import whose destination reference disagrees with the package's own Record is rejected before +// anything is written: no mismatched tag is published, and an existing valid tag stays intact. +func TestImportLayout_RejectsAMismatchedRecordIdentityBeforeWriting(t *testing.T) { + ctx := context.Background() + source := newLayoutRepo(t) + published := publish(t, source, publishSpec{content: "exported content"}).Pinned + dir := t.TempDir() + _, err := source.ExportLayout(ctx, published, dir) + require.NoError(t, err) + + tests := []struct { + name string + ref artifact.Ref + }{ + {name: "another name", ref: artifact.Ref{Name: "other/app", Version: appVersion}}, + {name: "another version", ref: artifact.Ref{Name: appName, Version: "9.9.9"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + destination := newLayoutRepo(t) + existing := publish(t, destination, publishSpec{name: tt.ref.Name, version: tt.ref.Version, content: "already published"}).Pinned + before, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + + _, err = destination.ImportLayout(ctx, dir, tt.ref, ImportOptions{Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + + after, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + assert.Equal(t, before, after, "the rejected import published no tag") + intact, err := destination.Resolve(ctx, tt.ref) + require.NoError(t, err) + assert.Equal(t, existing.ResolvedDigest, intact.ResolvedDigest, "the existing reference is untouched") + }) + } + + t.Run("matching identity imports", func(t *testing.T) { + destination := newLayoutRepo(t) + result, err := destination.ImportLayout(ctx, dir, artifact.Ref{Name: appName, Version: appVersion}, ImportOptions{}) + require.NoError(t, err) + assert.Equal(t, published.ResolvedDigest, result.Pinned.ResolvedDigest) + }) +} diff --git a/pkg/artifact/oci/fetch.go b/pkg/artifact/oci/fetch.go index 41f4f48650c..1d882f8e5b1 100644 --- a/pkg/artifact/oci/fetch.go +++ b/pkg/artifact/oci/fetch.go @@ -25,7 +25,7 @@ const errFormatDigestCause = "%w: %s: %w" // fetchBounded reads one small blob with the declared size and digest verified; a larger or // tampered blob never reaches the caller. Oversize content is reported as too large, everything // else that fails verification as an integrity failure. -func fetchBounded(ctx context.Context, st store, desc ocispec.Descriptor, limit int64) ([]byte, error) { +func fetchBounded(ctx context.Context, st atmosoci.Fetcher, desc ocispec.Descriptor, limit int64) ([]byte, error) { if desc.Size > limit || desc.Size < 0 { return nil, fmt.Errorf("%w: %s declares %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, desc.Digest, desc.Size, limit) } diff --git a/pkg/artifact/oci/repository.go b/pkg/artifact/oci/repository.go index 773e79ef067..6e1853f4956 100644 --- a/pkg/artifact/oci/repository.go +++ b/pkg/artifact/oci/repository.go @@ -11,6 +11,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + atmosoci "github.com/cloudposse/atmos/pkg/oci" "github.com/cloudposse/atmos/pkg/perf" ) @@ -204,7 +205,7 @@ func (r *Repository) loadCachedRoot(ctx context.Context, st store, name string, } // readRecord fetches the Record config blob of an Atmos package with a bounded, verified read. -func (r *Repository) readRecord(ctx context.Context, st store, name string, loaded *root) (*artifact.Record, error) { +func (r *Repository) readRecord(ctx context.Context, st atmosoci.Fetcher, name string, loaded *root) (*artifact.Record, error) { config := loaded.manifest.Config if raw, ok := r.cached(name, config); ok { return artifact.DecodeRecord(raw) From 4077b9366f69256b680cee2d91dc742ad5571922 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:18:54 -0600 Subject: [PATCH 40/53] fix(oci): serialize the registry copy's read-decide-write sequence transfer read the destination tag and applied the reference policy before ExtendedCopy wrote the tag unconditionally, and a caller that passed no Lock ran the whole sequence unprotected, so a writer that moved the tag in between was overwritten without the policy check. A registry destination now takes the reference lock itself, before the tag read, when the caller holds none; a caller-held lock is used as is. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 29 ++++- pkg/artifact/oci/copy_serialize_test.go | 135 ++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 pkg/artifact/oci/copy_serialize_test.go diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 397582e215e..c93e8a5146d 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -4,6 +4,7 @@ package oci import ( "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -72,7 +73,7 @@ func (r *Repository) destinationRef(ctx context.Context, st store, loaded *root, // transfer copies a root and its referrers from src into this repository under ref after the // reference contract (expected prior, force, plugin immutability) allows it. -func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphStore, loaded *root, ref artifact.Ref, options artifact.PublishOptions) (*artifact.PublishResult, error) { +func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphStore, loaded *root, ref artifact.Ref, options artifact.PublishOptions) (result *artifact.PublishResult, err error) { if err := ref.Validate(); err != nil { return nil, err } @@ -81,6 +82,15 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto return nil, err } defer release() + releaseReference, err := r.guardReference(ctx, ref, options.Lock) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := releaseReference(); releaseErr != nil { + err = errors.Join(err, releaseErr) + } + }() dst, err := r.be.open(ctx, ref.Name, true) if err != nil { return nil, err @@ -110,6 +120,23 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto return r.transferred(ctx, &pinned, ref, outcome, options) } +// guardReference keeps the registry copy's read-decide-write sequence under one lock. A registry +// tag has no compare-and-swap, and the tag is written unconditionally once the graph has been +// transferred, so the policy decision made from the tag read is only as good as the exclusion that +// protects the tag until the write. A caller that holds the reference lock already has that +// exclusion; a caller that passes none gets the reference lock taken here, before the tag is +// read. A layout is protected by its write guard instead, so it takes no reference lock. +func (r *Repository) guardReference(ctx context.Context, ref artifact.Ref, held artifact.LockHandle) (func() error, error) { + if r.be.kind() != artifact.RepositoryKindOCIRegistry || held != nil { + return func() error { return nil }, nil + } + handle, err := r.Lock(ctx, artifact.LockRequest{Resource: ref.String()}) + if err != nil { + return nil, fmt.Errorf("lock %s before copying into repository %q: %w", ref, r.alias, err) + } + return func() error { return handle.Release(ctx) }, nil +} + // pinCopied pins the digest that was copied, read back from the destination. The destination tag // is deliberately not resolved again: another writer can move it after the copy, and the pin must // describe the content this operation published. diff --git a/pkg/artifact/oci/copy_serialize_test.go b/pkg/artifact/oci/copy_serialize_test.go new file mode 100644 index 00000000000..66cb5f46328 --- /dev/null +++ b/pkg/artifact/oci/copy_serialize_test.go @@ -0,0 +1,135 @@ +package oci + +import ( + "context" + "sync" + "testing" + "time" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// rendezvous lets the first two readers of the destination tag meet, so a test can make two +// concurrent copies read the tag before either writes it. A reader that cannot meet its peer +// within the window (because the peer is blocked behind a lock) carries on alone. +type rendezvous struct { + mu sync.Mutex + arrived int + met chan struct{} + window time.Duration +} + +func newRendezvous(window time.Duration) *rendezvous { + return &rendezvous{met: make(chan struct{}), window: window} +} + +func (r *rendezvous) arrive() { + r.mu.Lock() + r.arrived++ + if r.arrived == 2 { + close(r.met) + } + r.mu.Unlock() + select { + case <-r.met: + case <-time.After(r.window): + } +} + +// rendezvousBackend opens write stores whose tag reads meet at the rendezvous. +type rendezvousBackend struct { + backend + tag string + point *rendezvous +} + +func (b *rendezvousBackend) open(ctx context.Context, name string, write bool) (store, error) { + opened, err := b.backend.open(ctx, name, write) + if err != nil || !write { + return opened, err + } + return &rendezvousStore{store: opened, tag: b.tag, point: b.point}, nil +} + +type rendezvousStore struct { + store + tag string + point *rendezvous +} + +func (s *rendezvousStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + desc, err := s.store.Resolve(ctx, reference) + if reference == s.tag { + s.point.arrive() + } + return desc, err +} + +// Two copies of different content into one registry tag, with no caller-held lock, must not both +// pass the "tag is absent" decision: the copy takes the reference lock itself, so the second one +// reads the tag the first wrote and loses with a conflict instead of silently overwriting it. +func TestCopyFrom_RegistryDestinationSerializesCallersWithoutALock(t *testing.T) { + ctx := context.Background() + registry := newRegistry(t, nil) + destination := newRegistryRepo(t, registry) + destination.be = &rendezvousBackend{backend: destination.be, tag: destination.be.reference(appName, appVersion), point: newRendezvous(500 * time.Millisecond)} + + sources := make([]*Repository, 2) + pins := make([]artifact.PinnedReference, 2) + for index, content := range []string{"first writer", "second writer"} { + sources[index] = newLayoutRepo(t) + pins[index] = publish(t, sources[index], publishSpec{content: content}).Pinned + } + require.NotEqual(t, pins[0].ResolvedDigest, pins[1].ResolvedDigest) + + errs := make([]error, len(sources)) + results := make([]*artifact.PublishResult, len(sources)) + var wg sync.WaitGroup + for index := range sources { + wg.Add(1) + go func() { + defer wg.Done() + results[index], errs[index] = destination.CopyFrom(ctx, sources[index], pins[index], artifact.CopyFromOptions{}) + }() + } + wg.Wait() + + winner := -1 + for index, err := range errs { + if err == nil { + require.Equal(t, -1, winner, "only one copy may create the tag") + winner = index + continue + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + require.NotEqual(t, -1, winner, "one copy creates the tag") + assert.Equal(t, artifact.OutcomeCreated, results[winner].Outcome) + current, err := destination.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.NoError(t, err) + assert.Equal(t, pins[winner].ResolvedDigest, current.ResolvedDigest, "the tag still names the winner's content") +} + +// A caller that already holds the writer lock keeps it: the copy never waits on a second claim for +// the same reference, which would deadlock behind the caller's own guard. +func TestCopyFrom_RegistryDestinationUsesTheCallersLock(t *testing.T) { + registry := newRegistry(t, nil) + destination := newRegistryRepo(t, registry) + source := newLayoutRepo(t) + pinned := publish(t, source, publishSpec{content: "held lock"}).Pinned + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + held, err := destination.Lock(ctx, artifact.LockRequest{Resource: artifact.Ref{Name: appName, Version: appVersion}.String()}) + require.NoError(t, err) + result, err := destination.CopyFrom(ctx, source, pinned, artifact.CopyFromOptions{Lock: held}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + require.NoError(t, held.Release(ctx), "the copy left the caller's lock held") +} From 1c5d5a2464e093f215abecfc84396c386149408e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:44:00 -0600 Subject: [PATCH 41/53] fix(oci): reject malformed descriptor digests before reading them A registry manifest can list a layer or child with an empty digest. The digest library panics when asked for the algorithm of a digest without a separator, and classify accepted such a descriptor, so the first read terminated the process. classify now rejects a root that references a malformed digest, and the lazy reader validates its descriptor before asking for the algorithm. Both report an integrity failure. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/classify.go | 20 +++++++++ pkg/artifact/oci/fetch.go | 6 +++ pkg/artifact/oci/fetch_digest_test.go | 65 +++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 pkg/artifact/oci/fetch_digest_test.go diff --git a/pkg/artifact/oci/classify.go b/pkg/artifact/oci/classify.go index 1c35c4220e6..e9b06b2d414 100644 --- a/pkg/artifact/oci/classify.go +++ b/pkg/artifact/oci/classify.go @@ -104,6 +104,9 @@ func classify(desc ocispec.Descriptor, raw []byte) (*root, error) { if err := json.Unmarshal(raw, &loaded.index); err != nil { return nil, fmt.Errorf("%w: index %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) } + if err := requireDescriptorDigests(desc, loaded.index.Manifests); err != nil { + return nil, err + } loaded.class = classIndex if hasPluginAnnotations(loaded.index.Annotations) { loaded.class = classPlugin @@ -112,6 +115,9 @@ func classify(desc ocispec.Descriptor, raw []byte) (*root, error) { if err := json.Unmarshal(raw, &loaded.manifest); err != nil { return nil, fmt.Errorf("%w: manifest %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) } + if err := requireDescriptorDigests(desc, []ocispec.Descriptor{loaded.manifest.Config}, loaded.manifest.Layers); err != nil { + return nil, err + } loaded.class = classImage switch loaded.manifest.Config.MediaType { case artifact.MediaTypeConfig: @@ -125,6 +131,20 @@ func classify(desc ocispec.Descriptor, raw []byte) (*root, error) { return loaded, nil } +// requireDescriptorDigests rejects a root that references a descriptor whose digest is malformed. A +// registry serves whatever bytes it holds, so an empty or truncated digest is an integrity failure +// of the root, caught here before a traversal or a read can ask the digest library for it. +func requireDescriptorDigests(root ocispec.Descriptor, groups ...[]ocispec.Descriptor) error { + for _, group := range groups { + for index := range group { + if err := group[index].Digest.Validate(); err != nil { + return fmt.Errorf("%w: %s references a descriptor with the malformed digest %q: %w", errUtils.ErrArtifactIntegrityFailed, root.Digest, group[index].Digest, err) + } + } + } + return nil +} + // hasPluginAnnotations reports whether annotations identify a legacy plugin release. func hasPluginAnnotations(annotations map[string]string) bool { if annotations[pluginAnnotationName] != "" { diff --git a/pkg/artifact/oci/fetch.go b/pkg/artifact/oci/fetch.go index 1d882f8e5b1..af01a926c1f 100644 --- a/pkg/artifact/oci/fetch.go +++ b/pkg/artifact/oci/fetch.go @@ -94,6 +94,12 @@ func (l *lazyReader) Read(p []byte) (int, error) { } func (l *lazyReader) open() { + // The digest library panics on a digest without an algorithm, so a descriptor from a registry + // manifest is validated before anything asks for its algorithm. + if err := l.desc.Digest.Validate(); err != nil { + l.err = fmt.Errorf(errFormatDigestCause, errUtils.ErrArtifactIntegrityFailed, l.desc.Digest, err) + return + } if !l.desc.Digest.Algorithm().Available() { l.err = fmt.Errorf("%w: unavailable digest algorithm for %s", errUtils.ErrArtifactIntegrityFailed, l.desc.Digest) return diff --git a/pkg/artifact/oci/fetch_digest_test.go b/pkg/artifact/oci/fetch_digest_test.go new file mode 100644 index 00000000000..01467104280 --- /dev/null +++ b/pkg/artifact/oci/fetch_digest_test.go @@ -0,0 +1,65 @@ +package oci + +import ( + "context" + "encoding/json" + "io" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// A registry manifest can list a layer or child with an empty or malformed digest. The reader must +// report it as an integrity failure; the digest library panics on a digest without an algorithm. +func TestMalformedDescriptorDigestsAreIntegrityFailures(t *testing.T) { + ctx := context.Background() + malformed := []digest.Digest{"", "sha256", "sha256:zz", digest.Digest("unknown:" + digest.FromString("x").Encoded())} + + for _, bad := range malformed { + t.Run("lazy reader "+string(bad), func(t *testing.T) { + reader := newLazyReader(ctx, nil, ocispec.Descriptor{MediaType: artifact.MediaTypeTar, Digest: bad, Size: 3}) + _, err := io.ReadAll(reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + + t.Run("manifest layer "+string(bad), func(t *testing.T) { + raw, err := json.Marshal(ocispec.Manifest{ + Config: ocispec.Descriptor{MediaType: artifact.MediaTypeConfig, Digest: digest.FromString("c"), Size: 1}, + Layers: []ocispec.Descriptor{{MediaType: artifact.MediaTypeTar, Digest: bad, Size: 3}}, + }) + require.NoError(t, err) + desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(raw), Size: int64(len(raw))} + + _, err = classify(desc, raw) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = successorsOf(desc, raw) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + + t.Run("index child "+string(bad), func(t *testing.T) { + raw, err := json.Marshal(ocispec.Index{Manifests: []ocispec.Descriptor{{MediaType: ocispec.MediaTypeImageManifest, Digest: bad, Size: 3}}}) + require.NoError(t, err) + desc := ocispec.Descriptor{MediaType: ocispec.MediaTypeImageIndex, Digest: digest.FromBytes(raw), Size: int64(len(raw))} + + _, err = classify(desc, raw) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } + + t.Run("well-formed digests classify", func(t *testing.T) { + raw, err := json.Marshal(ocispec.Manifest{ + Config: ocispec.Descriptor{MediaType: artifact.MediaTypeConfig, Digest: digest.FromString("c"), Size: 1}, + Layers: []ocispec.Descriptor{{MediaType: artifact.MediaTypeTar, Digest: digest.FromString("l"), Size: 1}}, + }) + require.NoError(t, err) + loaded, err := classify(ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromBytes(raw), Size: int64(len(raw))}, raw) + require.NoError(t, err) + assert.Equal(t, classAtmos, loaded.class) + }) +} From 05adcd6c406ecb7836df6cbe2930090753c1cf9a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:44:36 -0600 Subject: [PATCH 42/53] fix(oci): validate an import's destination and package before publishing An import or native copy published the tag once the graph had been transferred, so a layout holding a canonical Record without its archive payload layer replaced a valid package with one that every later read rejected, and a digest-only destination was stored under an empty version that listing never exposes. The shared transfer now requires a version and checks the package structure (Record identity, exactly one archive payload, documents matching the Record) before any lock is taken or the destination is opened. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 47 +++++-- pkg/artifact/oci/copy_validate_test.go | 181 +++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 pkg/artifact/oci/copy_validate_test.go diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index c93e8a5146d..0dbd20aec95 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -74,7 +74,12 @@ func (r *Repository) destinationRef(ctx context.Context, st store, loaded *root, // transfer copies a root and its referrers from src into this repository under ref after the // reference contract (expected prior, force, plugin immutability) allows it. func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphStore, loaded *root, ref artifact.Ref, options artifact.PublishOptions) (result *artifact.PublishResult, err error) { - if err := ref.Validate(); err != nil { + if err := requireDestinationVersion(ref); err != nil { + return nil, err + } + // A package is checked as a whole before any lock is taken or the destination is touched: the + // tag moves once the graph has been copied, so a malformed package must never get that far. + if err := r.requireImportable(ctx, src, loaded, ref); err != nil { return nil, err } release, err := r.guardLayout(ctx) @@ -186,6 +191,9 @@ type ImportOptions struct { func (r *Repository) ImportLayout(ctx context.Context, directory string, ref artifact.Ref, options ImportOptions) (*artifact.PublishResult, error) { defer perf.Track(nil, "oci.Repository.ImportLayout")() + if err := requireDestinationVersion(ref); err != nil { + return nil, err + } desc, err := layoutRoot(directory, options.Select) if err != nil { return nil, err @@ -198,18 +206,28 @@ func (r *Repository) ImportLayout(ctx context.Context, directory string, ref art if err != nil { return nil, err } - // The Record binds a package to its own name and version. Checking it before the transfer - // keeps a mismatched import from publishing a tag that every later read would reject. - if err := r.requireImportIdentity(ctx, src, loaded, ref); err != nil { - return nil, err - } return r.transfer(ctx, src, loaded, ref, artifact.PublishOptions{ExpectedPrior: options.ExpectedPrior, Force: options.Force, Lock: options.Lock}) } -// requireImportIdentity rejects an import of an Atmos package whose Record names another -// reference than the one it would be published under. A foreign graph carries no Record and is -// published under the requested reference as it is. -func (r *Repository) requireImportIdentity(ctx context.Context, src atmosoci.Fetcher, loaded *root, ref artifact.Ref) error { +// requireDestinationVersion rejects a destination without a version. A reference may name a digest +// to resolve with, but a copy or import publishes a name:version tag: a digest-only destination +// would be stored under an empty version, which artifact listing never exposes. +func requireDestinationVersion(ref artifact.Ref) error { + if err := ref.Validate(); err != nil { + return err + } + if ref.Version == "" { + return fmt.Errorf("%w: %s needs a version to be published under, not only a digest", errUtils.ErrArtifactRefInvalid, ref) + } + return nil +} + +// requireImportable rejects a package that every later read would reject, before the transfer can +// publish it. The Record binds an Atmos package to its own name and version, the manifest must +// carry exactly the archive payload and the documents the Record describes, and every descriptor +// must be well formed. A foreign graph carries no Record and is published under the requested +// reference as it is. +func (r *Repository) requireImportable(ctx context.Context, src atmosoci.Fetcher, loaded *root, ref artifact.Ref) error { if loaded.class != classAtmos { return nil } @@ -217,7 +235,14 @@ func (r *Repository) requireImportIdentity(ctx context.Context, src atmosoci.Fet if err != nil { return err } - return requireSubjectOf(record, ref) + if err := requireSubjectOf(record, ref); err != nil { + return err + } + layers, err := splitAtmosLayers(&loaded.manifest) + if err != nil { + return err + } + return checkDocuments(record, layers) } // ExportLayout exports the pinned graph, with every referrer, to a standard OCI image layout diff --git a/pkg/artifact/oci/copy_validate_test.go b/pkg/artifact/oci/copy_validate_test.go new file mode 100644 index 00000000000..8840b8bb72b --- /dev/null +++ b/pkg/artifact/oci/copy_validate_test.go @@ -0,0 +1,181 @@ +package oci + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// rewriteLayoutManifest replaces the single root manifest of an exported layout with a mutated +// copy, writing the new manifest blob and pointing the index at it. It returns the new root digest. +func rewriteLayoutManifest(t *testing.T, dir string, mutate func(*ocispec.Manifest)) digest.Digest { + t.Helper() + indexPath := filepath.Join(dir, ocispec.ImageIndexFile) + rawIndex, err := os.ReadFile(indexPath) + require.NoError(t, err) + var index ocispec.Index + require.NoError(t, json.Unmarshal(rawIndex, &index)) + require.Len(t, index.Manifests, 1) + entry := &index.Manifests[0] + + blobDir := filepath.Join(dir, ocispec.ImageBlobsDir, string(entry.Digest.Algorithm())) + rawManifest, err := os.ReadFile(filepath.Join(blobDir, entry.Digest.Encoded())) + require.NoError(t, err) + var manifest ocispec.Manifest + require.NoError(t, json.Unmarshal(rawManifest, &manifest)) + mutate(&manifest) + rewritten, err := json.Marshal(manifest) + require.NoError(t, err) + + entry.Digest = digest.FromBytes(rewritten) + entry.Size = int64(len(rewritten)) + require.NoError(t, os.WriteFile(filepath.Join(blobDir, entry.Digest.Encoded()), rewritten, 0o600)) + rawIndex, err = json.Marshal(index) + require.NoError(t, err) + require.NoError(t, os.WriteFile(indexPath, rawIndex, 0o600)) + return entry.Digest +} + +// exportedPackage publishes a package to a layout and exports it, returning the export directory. +func exportedPackage(t *testing.T) (dir string, published artifact.PinnedReference) { + t.Helper() + source := newLayoutRepo(t) + published = publish(t, source, publishSpec{content: "exported content"}).Pinned + dir = t.TempDir() + _, err := source.ExportLayout(context.Background(), published, dir) + require.NoError(t, err) + return dir, published +} + +// A layout holds a canonical, correctly named Record yet may lack the archive payload layer. A +// forced import of it must be rejected before the destination tag moves: the existing package +// stays readable instead of being replaced by one that every later read rejects. +func TestImportLayout_RejectsAMalformedPackageBeforeReplacingTheTag(t *testing.T) { + ctx := context.Background() + tests := []struct { + name string + mutate func(*ocispec.Manifest) + }{ + {name: "no archive payload layer", mutate: func(m *ocispec.Manifest) { + var kept []ocispec.Descriptor + for _, layer := range m.Layers { + if _, isPayload := packagingOf(layer.MediaType); !isPayload { + kept = append(kept, layer) + } + } + m.Layers = kept + }}, + {name: "two archive payload layers", mutate: func(m *ocispec.Manifest) { + for _, layer := range m.Layers { + if _, isPayload := packagingOf(layer.MediaType); isPayload { + m.Layers = append(m.Layers, layer) + return + } + } + }}, + {name: "a document the Record does not describe", mutate: func(m *ocispec.Manifest) { + m.Layers = m.Layers[:len(m.Layers)-1] + }}, + {name: "a layer that is neither payload nor document", mutate: func(m *ocispec.Manifest) { + m.Layers = append(m.Layers, ocispec.Descriptor{MediaType: "application/x-unknown", Digest: digest.FromString("x"), Size: 1}) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, _ := exportedPackage(t) + rewriteLayoutManifest(t, dir, tt.mutate) + + destination := newLayoutRepo(t) + existing := publish(t, destination, publishSpec{content: "already published"}).Pinned + ref := artifact.Ref{Name: appName, Version: appVersion} + before, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + + _, err = destination.ImportLayout(ctx, dir, ref, ImportOptions{Force: true, ExpectedPrior: &existing}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + + after, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + assert.Equal(t, before, after, "the rejected import changed no tag") + intact, err := destination.Resolve(ctx, ref) + require.NoError(t, err) + assert.Equal(t, existing.ResolvedDigest, intact.ResolvedDigest, "the existing package is still the one served") + _, err = destination.Inspect(ctx, intact) + require.NoError(t, err, "the existing package still inspects") + }) + } +} + +// An import destination is a name:version tag. A digest-only reference is valid to resolve with, +// but a layout would store it under an empty version that artifact listing never exposes, so the +// import refuses it before anything is written to the destination. +func TestImportLayout_RequiresAVersionForTheDestination(t *testing.T) { + ctx := context.Background() + dir, published := exportedPackage(t) + + tests := []struct { + name string + ref artifact.Ref + }{ + {name: "digest only", ref: artifact.Ref{Name: appName, Digest: published.ResolvedDigest}}, + {name: "neither version nor digest", ref: artifact.Ref{Name: appName}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + destinationPath := filepath.Join(t.TempDir(), "layout") + destination := newLayoutRepoAt(t, destinationPath) + + _, err := destination.ImportLayout(ctx, dir, tt.ref, ImportOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + + _, statErr := os.Stat(destinationPath) + assert.True(t, os.IsNotExist(statErr), "the destination layout was never opened for writing") + }) + } + + t.Run("a version imports and lists", func(t *testing.T) { + destination := newLayoutRepo(t) + _, err := destination.ImportLayout(ctx, dir, artifact.Ref{Name: appName, Version: appVersion}, ImportOptions{}) + require.NoError(t, err) + refs, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + assert.Equal(t, []artifact.Ref{{Name: appName, Version: appVersion}}, refs) + }) +} + +// A native copy between repositories shares the transfer with an import, so it too refuses a +// package without its archive payload layer before the destination tag moves. +func TestCopyFrom_RejectsAMalformedPackageBeforeReplacingTheTag(t *testing.T) { + ctx := context.Background() + dir, _ := exportedPackage(t) + rewritten := rewriteLayoutManifest(t, dir, func(m *ocispec.Manifest) { + var kept []ocispec.Descriptor + for _, layer := range m.Layers { + if _, isPayload := packagingOf(layer.MediaType); !isPayload { + kept = append(kept, layer) + } + } + m.Layers = kept + }) + source := newLayoutRepoAt(t, dir) + destination := newLayoutRepo(t) + existing := publish(t, destination, publishSpec{content: "already published"}).Pinned + ref := artifact.Ref{Name: appName, Version: appVersion} + + _, err := destination.CopyFrom(ctx, source, artifact.PinnedReference{Ref: ref, ResolvedDigest: rewritten}, artifact.CopyFromOptions{Force: true, ExpectedPrior: &existing}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + + intact, err := destination.Resolve(ctx, ref) + require.NoError(t, err) + assert.Equal(t, existing.ResolvedDigest, intact.ResolvedDigest, "the existing package is still the one served") +} From 1bafd615b6315e1a4e634ed5cba6ed9b29fa8a2c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:00:27 -0600 Subject: [PATCH 43/53] fix(s3): keep every location a publication journal owes cleanup on claimed, settled, and retired once A completed publication keeps its journal until the cleanup that follows completion has removed what it still lists: the replaced Record with its payload and evidence (Prior) and the payloads of abandoned attempts. Three paths lost that work or the protection around it: - A completed or pending reference claimed its representation and its abandoned paths but not its Prior. After a retirement stopped past the payload removal, another reference could reserve the empty location and the retry then deleted its objects by the journaled keys. - A forced replacement never loaded the completed operation's journal, and a deletion wrote a journal listing only the current payload and then removed the original. Both orphaned the older payload, Record, and journal. - Retiring a Prior whose Record was already gone failed on the missing Record, so a journal whose retirement had finished could never be cleared. The journal's Prior now stays claimed while the journal exists. A forced replacement and a deletion settle the completed journal first, and a failure while settling stops them with the journal and its claims intact. Retirement treats a missing Record as already retired, since the Record is removed last. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/build.go | 30 +- pkg/artifact/s3/delete.go | 5 + pkg/artifact/s3/journal_lifecycle_test.go | 395 ++++++++++++++++++++++ pkg/artifact/s3/keys.go | 18 + pkg/artifact/s3/publish.go | 31 +- pkg/artifact/s3/reserve.go | 21 +- pkg/artifact/s3/settle.go | 117 +++++++ 7 files changed, 580 insertions(+), 37 deletions(-) create mode 100644 pkg/artifact/s3/journal_lifecycle_test.go create mode 100644 pkg/artifact/s3/settle.go diff --git a/pkg/artifact/s3/build.go b/pkg/artifact/s3/build.go index ea3a731bfa4..3c3fc75bb36 100644 --- a/pkg/artifact/s3/build.go +++ b/pkg/artifact/s3/build.go @@ -2,10 +2,12 @@ package s3 import ( "context" + "errors" "slices" "github.com/opencontainers/go-digest" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/artifact/blob" "github.com/cloudposse/atmos/pkg/artifact/layout" @@ -81,24 +83,27 @@ func (r *Repository) retire(ctx context.Context, job *publishJob, plan *publishP return } if plan.prior != nil { - if err := r.retirePrior(cleanup, plan.prior, job, keep); err != nil { + if err := r.retirePrior(cleanup, plan.prior, keep, job.recordDigest); err != nil { return } } _ = r.deleteObject(cleanup, layout.OperationPath(job.opID)) } -// retirePrior removes the payload, Record, and evidence of the replaced publication. -func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job *publishJob, keep []string) error { - if prior.RecordDigest == "" || prior.RecordDigest == job.recordDigest { +// retirePrior removes the payload, Record, and evidence of the replaced publication, never a Record +// listed as protected. The Record goes last, so a Record that is already gone proves the payload +// and the evidence are too: a retry after a stopped retirement then has nothing left to remove +// instead of failing on the missing Record forever. +func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, keep []string, protected ...digest.Digest) error { + if prior.RecordDigest == "" || slices.Contains(protected, prior.RecordDigest) { return nil } if prior.Location != "" { - record, _, err := r.loadRecord(ctx, prior.RecordDigest) + set, err := r.priorSet(ctx, prior) if err != nil { return err } - if err := r.removePayload(ctx, &ownedPayload{packaging: prior.Packaging, location: prior.Location, set: setOfRecord(record)}, keep); err != nil { + if err := r.removePayload(ctx, &ownedPayload{packaging: prior.Packaging, location: prior.Location, set: set}, keep); err != nil { return err } } @@ -108,6 +113,19 @@ func (r *Repository) retirePrior(ctx context.Context, prior *layout.Prior, job * return r.deleteObject(ctx, layout.RecordPath(prior.RecordDigest)) } +// priorSet returns the objects the replaced publication wrote, from its Record. A Record that no +// longer exists was retired together with its payload, so nothing is left to name. +func (r *Repository) priorSet(ctx context.Context, prior *layout.Prior) (objectSet, error) { + record, _, err := r.loadRecord(ctx, prior.RecordDigest) + if errors.Is(err, errUtils.ErrArtifactNotFound) { + return objectSet{}, nil + } + if err != nil { + return objectSet{}, err + } + return setOfRecord(record), nil +} + // removePayloads deletes the objects of every listed payload that the new publication does not reuse. func (r *Repository) removePayloads(ctx context.Context, payloads []ownedPayload, keep []string) error { for index := range payloads { diff --git a/pkg/artifact/s3/delete.go b/pkg/artifact/s3/delete.go index 40f019c17df..30ad2ecddff 100644 --- a/pkg/artifact/s3/delete.go +++ b/pkg/artifact/s3/delete.go @@ -56,6 +56,11 @@ func (r *Repository) deleteLocked(ctx context.Context, pinned *artifact.PinnedRe if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != stored.RecordDigest { return nil, fmt.Errorf("%w: %s:%s is %s but the delete expected %s", errUtils.ErrArtifactConflict, stored.Name, stored.Version, stored.RecordDigest, options.ExpectedCurrent.ResolvedDigest) } + // The deletion journal replaces the publication's, and it lists only the current payload, so + // what the publication's journal still owes is retired before the deletion begins. + if err := r.settle(ctx, stored, v.record, v.rep, ""); err != nil { + return nil, deleteFailed(err) + } claims, err := r.acquire(ctx, claimsResource, r.settings.owner, "") if err != nil { return nil, err diff --git a/pkg/artifact/s3/journal_lifecycle_test.go b/pkg/artifact/s3/journal_lifecycle_test.go new file mode 100644 index 00000000000..f8fc06784ab --- /dev/null +++ b/pkg/artifact/s3/journal_lifecycle_test.go @@ -0,0 +1,395 @@ +package s3 + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// These tests pin one invariant of the publication journal: every location a journal still owes +// cleanup on stays claimed, survives a replacement and a deletion, and is retired exactly once. +// A journal owes cleanup when a completed publication replaced an earlier one (the Prior) or +// carried the payloads of abandoned attempts, and its retirement did not finish. + +// errDeleteDenied is what the gate answers for a delete it was told to refuse. +var errDeleteDenied = errors.New("delete denied") + +// gateStore decorates a store so a test can refuse chosen deletes, which is how a retirement fails +// part way: an interrupted retirement is a delete that did not happen. +type gateStore struct { + blob.Store + mu sync.Mutex + refuse func(key string) bool +} + +func (g *gateStore) Delete(ctx context.Context, key string, opts blob.DeleteOptions) error { + g.mu.Lock() + denied := g.refuse != nil && g.refuse(key) + g.mu.Unlock() + if denied { + return errDeleteDenied + } + return g.Store.Delete(ctx, key, opts) +} + +// refuseKeys makes the gate refuse the deletes the predicate selects; nil lets every delete through. +func (g *gateStore) refuseKeys(refuse func(key string) bool) { + g.mu.Lock() + defer g.mu.Unlock() + g.refuse = refuse +} + +// shape is one packaging of the payloads a lifecycle test replaces. +type shape struct { + name string + packaging artifact.Packaging + suffix string +} + +var lifecycleShapes = []shape{ + {name: "files", packaging: artifact.PackagingNone}, + {name: "archive", packaging: artifact.PackagingTgz, suffix: ".tgz"}, +} + +// lifecycle is one reference replaced once, so its completed journal lists the first publication +// as its Prior. Each generation lives at its own location, as a key_pattern with a date would +// place it, so the locations the journal owes cleanup on are distinct from the live one. +type lifecycle struct { + t *testing.T + shape shape + fake *s3test.Server + gate *gateStore + fault armedFault + repos map[string]*Repository + base *artifact.PublishResult + current *artifact.PublishResult +} + +// newLifecycle publishes the first generation under old/. Later generations go through repo. +func newLifecycle(t *testing.T, s shape) *lifecycle { + t.Helper() + l := &lifecycle{t: t, shape: s, fake: s3test.NewServer(t), repos: map[string]*Repository{}} + l.gate = &gateStore{Store: newStoreFor(t, archiveSpec(s.packaging).with(func(c *rspec) { c.fake = l.fake }))} + l.base = publish(t, l.repo("old"), l.spec("old", "base"), artifact.PublishOptions{}) + return l +} + +// repo returns the repository whose key_pattern places the generation under its own directory. +// Every repository shares the bucket and the gate, so a refused delete affects all of them. +func (l *lifecycle) repo(generation string) *Repository { + l.t.Helper() + if repo, ok := l.repos[generation]; ok { + return repo + } + spec := archiveSpec(l.shape.packaging).withPattern(generation + "/{{ .Name }}" + l.shape.suffix).with(func(c *rspec) { c.fake = l.fake }) + repo := newRepo(l.t, spec.withOpts(WithStore(l.gate), l.fault.option())) + l.repos[generation] = repo + return repo +} + +// spec describes one generation's content. The operation ID is stable per generation, so +// publishing it again is the same intent. +func (l *lifecycle) spec(generation, content string) *prepareSpec { + return &prepareSpec{files: map[string]string{"data.txt": content, generation + ".txt": content}, opID: "op-" + generation} +} + +// replace force-publishes a generation over the current one. +func (l *lifecycle) replace(generation string) (*artifact.PublishResult, error) { + l.t.Helper() + prior := l.base + if l.current != nil { + prior = l.current + } + repo := l.repo(generation) + result, err := repo.Publish(context.Background(), prepare(l.t, repo, l.spec(generation, generation)), artifact.PublishOptions{Force: true, ExpectedPrior: &prior.Pinned}) + if err == nil { + l.current = result + } + return result, err +} + +// retirementLeftBehind replaces the first generation while the retirement stops after the replaced +// payload is gone, as a transient delete failure on the Record would. +func (l *lifecycle) retirementLeftBehind() { + l.t.Helper() + l.gate.refuseKeys(func(key string) bool { return key == layout.RecordPath(l.base.Pinned.RecordDigest) }) + _, err := l.replace("new") + require.NoError(l.t, err, "the reference completed; retirement is best effort") + l.gate.refuseKeys(nil) + require.Empty(l.t, l.oldPayload(), "the replaced payload was removed before the retirement stopped") + require.Len(l.t, l.journals(), 1, "the journal keeps the Prior it still owes") +} + +// crashedAfterCommit replaces the first generation and stops before any retirement ran. +func (l *lifecycle) crashedAfterCommit() { + l.t.Helper() + l.fault.hook = crashAt(faultAfterCommit) + _, err := l.replace("new") + require.ErrorIs(l.t, err, errCrash) + l.fault.hook = nil + repo := l.repo("new") + pinned, err := repo.Resolve(context.Background(), defaultRef()) + require.NoError(l.t, err, "a crash after the commit leaves a completed reference") + l.current = &artifact.PublishResult{Pinned: pinned} + require.NotEmpty(l.t, l.oldPayload(), "nothing of the replaced payload was retired yet") +} + +func (l *lifecycle) journals() []string { + return managedKeys(l.fake, "", layout.ManagedRoot+"/operations") +} + +func (l *lifecycle) records() []string { + return managedKeys(l.fake, "", layout.ManagedRoot+"/records") +} + +// oldPayload lists what is still stored under the first generation's location. +func (l *lifecycle) oldPayload() []string { + var out []string + for _, key := range payloadKeys(l.fake, "") { + if strings.HasPrefix(key, "old/") { + out = append(out, key) + } + } + return out +} + +func (l *lifecycle) payloadUnder(dir string) []string { + var out []string + for _, key := range payloadKeys(l.fake, "") { + if strings.HasPrefix(key, dir+"/") { + out = append(out, key) + } + } + return out +} + +// newcomer is a second reference whose key_pattern resolves onto the first generation's location. +func (l *lifecycle) newcomer() (*Repository, *prepareSpec) { + l.t.Helper() + spec := archiveSpec(l.shape.packaging).withPattern("old/team/app" + l.shape.suffix).with(func(c *rspec) { c.fake = l.fake }) + return newRepo(l.t, spec.withOpts(WithStore(l.gate))), &prepareSpec{name: "other/app", files: map[string]string{"a.txt": "newcomer"}} +} + +// A retirement that stops after the replaced payload is gone leaves the Prior in the journal while +// its location holds nothing. The location must stay claimed: a newcomer that took it would have +// its objects deleted by the retry, which retires by the journaled keys and cannot tell whose they are. +func TestJournalPrior_LocationStaysClaimedUntilRetired(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := newLifecycle(t, s) + l.retirementLeftBehind() + + newcomer, newcomerSpec := l.newcomer() + before := bucketKeys(l.fake) + _, err := newcomer.Publish(context.Background(), prepare(t, newcomer, newcomerSpec), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.ErrorContains(t, err, "old/team/app") + assert.Equal(t, before, bucketKeys(l.fake), "the refused reservation changed nothing") + + // The retry finishes the retirement exactly once and releases the claim. + retry := publish(t, l.repo("new"), l.spec("new", "new"), artifact.PublishOptions{ExpectedPrior: &l.current.Pinned}) + assert.Equal(t, artifact.OutcomeUnchanged, retry.Outcome) + assert.Empty(t, l.journals(), "the journal is removed once nothing is owed") + _, err = l.repo("new").Read(context.Background(), l.base.Pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the replaced Record is retired") + + published, err := newcomer.Publish(context.Background(), prepare(t, newcomer, newcomerSpec), artifact.PublishOptions{}) + require.NoError(t, err, "a retired location is claimable again") + assertIntact(t, newcomer, ref("other/app", testVersion)) + assertIntact(t, l.repo("new"), defaultRef()) + assert.NotEmpty(t, published.Location) + }) + } +} + +// A pending replacement still serves the replaced payload at its old location until the new one +// completes, so that location stays claimed by the reference's journal. +func TestPendingJournal_PriorLocationStaysClaimed(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := newLifecycle(t, s) + l.fault.hook = crashAt(faultBeforePayload) + _, err := l.replace("new") + require.ErrorIs(t, err, errCrash) + l.fault.hook = nil + require.NotEmpty(t, l.oldPayload(), "the replaced payload is still stored") + + newcomer, newcomerSpec := l.newcomer() + _, err = newcomer.Publish(context.Background(), prepare(t, newcomer, newcomerSpec), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.NotEmpty(t, l.oldPayload(), "the replaced payload was not touched") + + _, err = l.replace("new") + require.NoError(t, err, "the same-intent retry completes the replacement") + require.Empty(t, l.oldPayload()) + _, err = newcomer.Publish(context.Background(), prepare(t, newcomer, newcomerSpec), artifact.PublishOptions{}) + require.NoError(t, err) + }) + } +} + +// A forced replacement of a completed publication whose retirement did not finish must not drop what +// that journal still owes: the earlier payload, Record, and the journal itself are retired. +func TestForceReplacement_SettlesTheReplacedJournal(t *testing.T) { + setups := []struct { + name string + leave func(*lifecycle) + }{ + {name: "the retirement stopped", leave: (*lifecycle).retirementLeftBehind}, + {name: "the retirement never ran", leave: (*lifecycle).crashedAfterCommit}, + } + for _, s := range lifecycleShapes { + for _, setup := range setups { + t.Run(s.name+"/"+setup.name, func(t *testing.T) { + l := newLifecycle(t, s) + setup.leave(l) + firstGeneration := l.base.Pinned + + _, err := l.replace("third") + require.NoError(t, err) + + assert.Empty(t, l.oldPayload(), "the payload the first journal owed is gone") + assert.Empty(t, l.payloadUnder("new"), "the replaced generation's payload is gone") + assert.NotEmpty(t, l.payloadUnder("third")) + assert.Empty(t, l.journals(), "no journal is orphaned") + assert.Len(t, l.records(), 1, "only the live Record remains") + _, err = l.repo("third").Read(context.Background(), firstGeneration) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assertIntact(t, l.repo("third"), defaultRef()) + }) + } + } +} + +// Deleting a completed publication whose retirement did not finish removes everything the journal +// still owed, along with the artifact itself. +func TestDelete_SettlesTheJournalBeforeDeleting(t *testing.T) { + setups := []struct { + name string + leave func(*lifecycle) + }{ + {name: "the retirement stopped", leave: (*lifecycle).retirementLeftBehind}, + {name: "the retirement never ran", leave: (*lifecycle).crashedAfterCommit}, + } + for _, s := range lifecycleShapes { + for _, setup := range setups { + t.Run(s.name+"/"+setup.name, func(t *testing.T) { + l := newLifecycle(t, s) + setup.leave(l) + + _, err := l.repo("new").Delete(context.Background(), l.current.Pinned, artifact.DeleteOptions{ExpectedCurrent: &l.current.Pinned}) + require.NoError(t, err) + + assert.Empty(t, payloadKeys(l.fake, ""), "no payload object survives") + assert.Empty(t, l.records(), "no Record survives") + assert.Empty(t, l.journals(), "no journal survives") + }) + } + } +} + +// A failure while settling stops the replacement and the deletion, and keeps the journal and its +// claim, so the retry still finds what is owed. +func TestSettlingFailureKeepsTheJournal(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := newLifecycle(t, s) + l.crashedAfterCommit() + l.gate.refuseKeys(func(key string) bool { return key == layout.RecordPath(l.base.Pinned.RecordDigest) }) + + _, err := l.replace("third") + require.ErrorIs(t, err, errUtils.ErrArtifactDeleteFailed) + _, err = l.repo("new").Delete(context.Background(), l.current.Pinned, artifact.DeleteOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDeleteFailed) + assertIntact(t, l.repo("new"), defaultRef()) + assert.Len(t, l.journals(), 1, "the journal still lists what is owed") + + l.gate.refuseKeys(nil) + _, err = l.replace("third") + require.NoError(t, err, "the retry settles and replaces") + assert.Empty(t, l.journals()) + assert.Len(t, l.records(), 1) + }) + } +} + +// A journal whose Prior is already gone entirely (the Record is deleted last) must still be +// retirable: the retry has nothing left to remove and clears the journal instead of failing forever. +func TestRetirement_IsIdempotentOnceTheRecordIsGone(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := newLifecycle(t, s) + l.gate.refuseKeys(func(key string) bool { return strings.HasPrefix(key, layout.ManagedRoot+"/operations/") }) + _, err := l.replace("new") + require.NoError(t, err) + l.gate.refuseKeys(nil) + require.Empty(t, l.oldPayload()) + require.Len(t, l.journals(), 1, "only the journal delete failed") + _, err = l.repo("new").Read(context.Background(), l.base.Pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the replaced Record is already retired") + + retry := publish(t, l.repo("new"), l.spec("new", "new"), artifact.PublishOptions{ExpectedPrior: &l.current.Pinned}) + assert.Equal(t, artifact.OutcomeUnchanged, retry.Outcome) + assert.Empty(t, l.journals()) + assertIntact(t, l.repo("new"), defaultRef()) + }) + } +} + +// A newcomer racing the retry that finishes a retirement either is refused while the journal owes +// the location, or takes it once the retirement is done. It is never admitted in between, which +// would let the retry delete the newcomer's objects under the same keys. +func TestJournalPrior_RetirementRacingANewcomerNeverDeletesTheNewcomer(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := newLifecycle(t, s) + l.retirementLeftBehind() + newcomer, _ := l.newcomer() + // The newcomer writes the very keys the journal's Prior lists. + content := &prepareSpec{name: "other/app", files: map[string]string{"data.txt": "newcomer", "old.txt": "newcomer"}} + + // Preparation fails the test, which only the test goroutine may do. + retry := prepare(t, l.repo("new"), l.spec("new", "new")) + incoming := prepare(t, newcomer, content) + + var wg sync.WaitGroup + errs := make(chan error, 2) + wg.Add(2) + go func() { + defer wg.Done() + _, err := l.repo("new").Publish(context.Background(), retry, artifact.PublishOptions{ExpectedPrior: &l.current.Pinned}) + errs <- err + }() + go func() { + defer wg.Done() + var err error + for range 200 { + if _, err = newcomer.Publish(context.Background(), incoming, artifact.PublishOptions{}); !errors.Is(err, errUtils.ErrArtifactConflict) { + break + } + } + errs <- err + }() + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + assertIntact(t, newcomer, ref("other/app", testVersion)) + assertIntact(t, l.repo("new"), defaultRef()) + assert.Empty(t, l.journals()) + }) + } +} diff --git a/pkg/artifact/s3/keys.go b/pkg/artifact/s3/keys.go index 57ac598026e..ee2773bf859 100644 --- a/pkg/artifact/s3/keys.go +++ b/pkg/artifact/s3/keys.go @@ -84,3 +84,21 @@ func claimPaths(packaging artifact.Packaging, location string) []string { } return []string{location} } + +// shapesOf returns the packagings a recorded packaging may mean. A payload written before the +// packaging was recorded may be either shape. +func shapesOf(packaging artifact.Packaging) []artifact.Packaging { + if packaging == "" { + return []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz} + } + return []artifact.Packaging{packaging} +} + +// anyShapeClaimPaths lists the claim paths of a payload whose packaging may be unrecorded. +func anyShapeClaimPaths(packaging artifact.Packaging, location string) []string { + var paths []string + for _, shape := range shapesOf(packaging) { + paths = append(paths, claimPaths(shape, location)...) + } + return paths +} diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index 4d726f7b5ec..ab7bfe53a5b 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -77,6 +77,13 @@ func (r *Repository) publishLocked(ctx context.Context, job *publishJob, options if plan.admission == layout.AdmitUnchanged { return r.unchanged(ctx, job, plan) } + if plan.admission == layout.AdmitReplace { + // The journal of the publication being replaced is dropped with it, so what it still owes + // is retired first. + if err := r.settleReplaced(ctx, plan.current.ref); err != nil { + return nil, err + } + } if err := r.resolveLocation(ctx, job, plan); err != nil { return nil, err } @@ -157,35 +164,13 @@ func (r *Repository) unchanged(ctx context.Context, job *publishJob, plan *publi if rep == nil || rep.RecordDigest != current.RecordDigest { return nil, fmt.Errorf("%w: %s:%s is completed but has no matching representation", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) } - journal, err := r.loadOperation(ctx, current.OperationID) - if err != nil { - return nil, err - } - if err := r.retireJournaled(ctx, job, journal, record, rep); err != nil { + if err := r.settle(ctx, current, record, rep, job.recordDigest); err != nil { return nil, err } r.tidyOperation(ctx, current.OperationID) return r.result(record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome(), plan.scope), nil } -// retireJournaled removes what a completed operation's journal still lists for removal: the -// replaced Record and its payload, and the payloads of abandoned attempts. -func (r *Repository) retireJournaled(ctx context.Context, job *publishJob, journal *layout.Operation, record *artifact.Record, rep *layout.Representation) error { - if journal == nil || (journal.Prior == nil && len(journal.Abandoned) == 0) { - return nil - } - keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) - if err != nil { - return err - } - if journal.Prior != nil { - if err := r.retirePrior(ctx, journal.Prior, job, keep); err != nil { - return err - } - } - return r.removePayloads(ctx, carriedPayloads(journal), keep) -} - // tidyOperation removes the journal of a finished operation. It is best-effort: a leftover is // harmless and is retried by the next publication. func (r *Repository) tidyOperation(ctx context.Context, operationID string) { diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index 62d035a5f1d..12fb4a925be 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -127,9 +127,8 @@ func (r *Repository) collectClaims(ctx context.Context, ownKey string) ([]layout } // claimOf returns the claim a reference holds: its representation when completed, its journal when -// pending. A reference whose state is missing holds no claim. Locations that earlier attempts -// abandoned stay claimed until a retry retires their payload, even while they hold no objects: the -// retirement deletes by the journaled payload and does not check who owns the location by then. +// pending, and in both states the locations its journal still awaits removing. A reference whose +// state is missing holds no claim. func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layout.Claim, error) { base := layout.Claim{ReferenceKey: ref.Key(), Name: ref.Name, Version: ref.Version} if ref.Available() { @@ -144,7 +143,7 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou if err != nil { return nil, err } - base.Paths = append(base.Paths, abandonedClaimPaths(journal)...) + base.Paths = append(base.Paths, journalClaimPaths(journal)...) return &base, nil } op, err := r.loadOperation(ctx, ref.OperationID) @@ -152,18 +151,24 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou return nil, err } base.OperationID = op.OperationID - base.Paths = append(claimPaths(op.Packaging, op.Location), abandonedClaimPaths(op)...) + base.Paths = append(claimPaths(op.Packaging, op.Location), journalClaimPaths(op)...) return &base, nil } -// abandonedClaimPaths lists the paths of every payload a journal still awaits removing. -func abandonedClaimPaths(op *layout.Operation) []string { +// journalClaimPaths lists the paths of every payload a journal still awaits removing: the replaced +// payload (Prior) and the payloads of abandoned attempts. They stay claimed until the journal is +// retired, even while they hold no objects, because the retirement deletes by the journaled keys +// and does not check who owns the location by then. +func journalClaimPaths(op *layout.Operation) []string { if op == nil { return nil } var paths []string + if op.Prior != nil && op.Prior.Location != "" { + paths = append(paths, anyShapeClaimPaths(op.Prior.Packaging, op.Prior.Location)...) + } for index := range op.Abandoned { - paths = append(paths, claimPaths(op.Abandoned[index].Packaging, op.Abandoned[index].Location)...) + paths = append(paths, anyShapeClaimPaths(op.Abandoned[index].Packaging, op.Abandoned[index].Location)...) } return paths } diff --git a/pkg/artifact/s3/settle.go b/pkg/artifact/s3/settle.go new file mode 100644 index 00000000000..3538bbc321b --- /dev/null +++ b/pkg/artifact/s3/settle.go @@ -0,0 +1,117 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// A completed publication keeps its journal until the cleanup that follows completion has removed +// everything the journal still lists: the replaced Record with its payload and evidence (Prior), +// and the payloads of abandoned attempts. Until then the journal is the only record of that work, +// so whatever ends the journal must first finish the work or carry it on: +// +// - the journal's locations stay claimed (see claimOf), so no other reference can reserve one; +// - a same-intent republication retires it (see unchanged); +// - a forced replacement and a deletion settle it before they write anything (see settle). +// +// Every removal is by exact journaled key and is idempotent, so a retirement that stops part way is +// finished by the next settlement, which removes each object once. + +// retireJournaled removes what a completed operation's journal still lists for removal: the +// replaced Record and its payload, and the payloads of abandoned attempts. The keys of the live +// payload are kept, and the Records listed as protected are never retired. +func (r *Repository) retireJournaled(ctx context.Context, journal *layout.Operation, keep []string, protected ...digest.Digest) error { + if journal == nil { + return nil + } + if journal.Prior != nil { + if err := r.retirePrior(ctx, journal.Prior, keep, protected...); err != nil { + return err + } + } + return r.removePayloads(ctx, carriedPayloads(journal), keep) +} + +// owes reports whether a journal still lists cleanup. +func owes(journal *layout.Operation) bool { + return journal != nil && (journal.Prior != nil || len(journal.Abandoned) > 0) +} + +// settle finishes the cleanup a completed reference's journal still owes and then removes the +// journal. The live payload of the reference is kept; the Records it and the incoming publication +// use are never retired. A journal that owes nothing is left for its caller to tidy. +func (r *Repository) settle(ctx context.Context, current *layout.Reference, record *artifact.Record, rep *layout.Representation, incoming digest.Digest) error { + if current.OperationID == "" { + return nil + } + journal, err := r.loadOperation(ctx, current.OperationID) + if err != nil || !owes(journal) { + return err + } + return r.retireAndTidy(ctx, journal, record, rep, current.RecordDigest, incoming) +} + +// retireAndTidy retires what the journal owes, keeping the live payload and never retiring the +// Records listed, and then removes the journal. +func (r *Repository) retireAndTidy(ctx context.Context, journal *layout.Operation, record *artifact.Record, rep *layout.Representation, protected ...digest.Digest) error { + keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) + if err != nil { + return err + } + if err := r.retireJournaled(ctx, journal, keep, protected...); err != nil { + return err + } + r.tidyOperation(ctx, journal.OperationID) + return nil +} + +// settleReplaced settles the journal of the completed reference a forced replacement is about to +// replace. The reference may be what a repair replaces, so a representation that no longer matches +// is not an obstacle: the journal then names the live location. +func (r *Repository) settleReplaced(ctx context.Context, current *layout.Reference) error { + if current.OperationID == "" { + return nil + } + journal, err := r.loadOperation(ctx, current.OperationID) + if err != nil { + return err + } + if !owes(journal) { + r.tidyOperation(ctx, current.OperationID) + return nil + } + record, _, err := r.loadRecord(ctx, current.RecordDigest) + if err != nil { + return err + } + rep, err := r.liveRepresentation(ctx, current, journal) + if err != nil { + return err + } + if err := r.retireAndTidy(ctx, journal, record, rep, current.RecordDigest); err != nil { + return fmt.Errorf("finish the cleanup owed by operation %s before replacing %s:%s: %w", current.OperationID, current.Name, current.Version, err) + } + return nil +} + +// liveRepresentation returns where the completed reference's payload lives: its representation, or +// the journal that wrote it when the representation is missing or describes another Record. +func (r *Repository) liveRepresentation(ctx context.Context, current *layout.Reference, journal *layout.Operation) (*layout.Representation, error) { + rep, err := r.loadRepresentation(ctx, current.Key()) + if err != nil { + return nil, err + } + if rep != nil && rep.RecordDigest == current.RecordDigest { + return rep, nil + } + if journal.RecordDigest == current.RecordDigest && journal.Location != "" { + return &layout.Representation{Packaging: journal.Packaging, Location: journal.Location}, nil + } + return nil, fmt.Errorf("%w: %s:%s is completed but its payload location is unknown", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) +} From 51f51bdaa10e68535e1da9e807321337e3e1397e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:05:19 -0600 Subject: [PATCH 44/53] fix(s3): check every key a publication writes against the keys its reference owns For per-file packaging the unmanaged-object check exempted a whole owned location: claimPaths returns only the location, so descendant keys were never looked at. A user object added below an owned location, for example team/app/1.0.0/two.txt, passed the reservation when a later publication included that file, and the unconditional upload then replaced it, which the reservation promises never to do. The reservation now carries the exact keys each owned payload consists of (the representation, the interrupted operation, its replaced prior, and carried abandoned payloads) and checks every key the publication writes below an owned location: an owned key is exempt, any other must be absent. The check runs under the claims lock before the journal or any payload object is written. Archive document sidecars are checked the same way. A location no reference owns is still refused whole. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/doc.go | 11 +++ pkg/artifact/s3/keys.go | 13 +++ pkg/artifact/s3/publish.go | 17 ++-- pkg/artifact/s3/reserve.go | 89 ++++++++++++++----- pkg/artifact/s3/unmanaged_keys_test.go | 116 +++++++++++++++++++++++++ 5 files changed, 219 insertions(+), 27 deletions(-) create mode 100644 pkg/artifact/s3/unmanaged_keys_test.go diff --git a/pkg/artifact/s3/doc.go b/pkg/artifact/s3/doc.go index aa35056e007..47994fb0aa3 100644 --- a/pkg/artifact/s3/doc.go +++ b/pkg/artifact/s3/doc.go @@ -37,6 +37,17 @@ // readers, until a retry with the same intent repairs it; there is no rollback. A pending // operation of a different intent is never stolen. // +// A completed publication keeps its journal until the cleanup that follows completion has removed +// everything the journal lists: the replaced Record with its payload and evidence, and the payloads +// of abandoned attempts. Until then those locations stay claimed, so no other reference can reserve +// one, and each removal is by exact journaled key and safe to repeat. A republication with the same +// intent finishes the cleanup; a forced replacement or a deletion finishes it before it writes +// anything, and stops with the journal intact when it cannot. +// +// A reference owns the exact keys its payloads consist of, not the location as a prefix. Before it +// writes anything, a publication refuses a key below an owned location that already holds an object +// the reference does not own, so an object a user added there is never overwritten. +// // # Locks and recovery // // Lock provides repository-scope mutation coordination through owner claim objects under the diff --git a/pkg/artifact/s3/keys.go b/pkg/artifact/s3/keys.go index ee2773bf859..89f516f0700 100644 --- a/pkg/artifact/s3/keys.go +++ b/pkg/artifact/s3/keys.go @@ -102,3 +102,16 @@ func anyShapeClaimPaths(packaging artifact.Packaging, location string) []string } return paths } + +// anyShapeKeys lists the object keys of a payload whose packaging may be unrecorded. +func (s objectSet) anyShapeKeys(packaging artifact.Packaging, location string) ([]string, error) { + var keys []string + for _, shape := range shapesOf(packaging) { + shapeKeys, err := s.keys(shape, location) + if err != nil { + return nil, err + } + keys = append(keys, shapeKeys...) + } + return keys, nil +} diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index ab7bfe53a5b..96032bbaddc 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -216,7 +216,7 @@ func (r *Repository) resolveLocation(ctx context.Context, job *publishJob, plan // publication replaces. The interrupted operation's journal is read when one is pending. func (r *Repository) loadPendingState(ctx context.Context, plan *publishPlan, rep *layout.Representation) error { if rep != nil { - plan.owned = append(plan.owned, ownedPayload{packaging: rep.Packaging, location: rep.Location}) + plan.owned = append(plan.owned, ownedPayload{packaging: rep.Packaging, location: rep.Location, set: setOfDescriptors(rep.Payload, rep.Documents)}) } if plan.current != nil && !plan.current.ref.Available() { oldOp, err := r.loadOperation(ctx, plan.current.ref.OperationID) @@ -227,11 +227,18 @@ func (r *Repository) loadPendingState(ctx context.Context, plan *publishPlan, re } plan.prior = priorState(plan, rep) plan.owned = append(plan.owned, carriedPayloads(plan.oldOp)...) - if plan.oldOp != nil { - plan.owned = append(plan.owned, ownedPayload{packaging: plan.oldOp.Packaging, location: plan.oldOp.Location}) - if prior := plan.oldOp.Prior; prior != nil && prior.Location != "" { - plan.owned = append(plan.owned, ownedPayload{packaging: prior.Packaging, location: prior.Location}) + if plan.oldOp == nil { + return nil + } + plan.owned = append(plan.owned, ownedPayload{ + packaging: plan.oldOp.Packaging, location: plan.oldOp.Location, set: setOfDescriptors(plan.oldOp.Payload, plan.oldOp.Documents), + }) + if prior := plan.oldOp.Prior; prior != nil && prior.Location != "" { + set, err := r.priorSet(ctx, prior) + if err != nil { + return err } + plan.owned = append(plan.owned, ownedPayload{packaging: prior.Packaging, location: prior.Location, set: set}) } return nil } diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index 12fb4a925be..d667ea7929b 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "slices" + "strings" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -85,7 +86,7 @@ func (r *Repository) checkCollision(ctx context.Context, job *publishJob, plan * return fmt.Errorf("%w: location %q for %s:%s is already claimed by %s:%s%s; give the repository a key_pattern that keeps references apart", errUtils.ErrArtifactConflict, job.location, job.record.Name, job.record.Version, other.Name, other.Version, pendingSuffix(other)) } - return r.checkUnmanaged(ctx, &candidate, plan) + return r.checkUnmanaged(ctx, job, &candidate, plan) } func pendingSuffix(claim *layout.Claim) string { @@ -173,38 +174,82 @@ func journalClaimPaths(op *layout.Operation) []string { return paths } -// checkUnmanaged fails when a path the publication would write already holds objects and this -// reference does not own it. -func (r *Repository) checkUnmanaged(ctx context.Context, candidate *layout.Claim, plan *publishPlan) error { - var owned []string - for index := range plan.owned { - for _, path := range claimPaths(plan.owned[index].packaging, plan.owned[index].location) { - owned = append(owned, layout.FoldPath(path)) - } - // A prior written before packaging was recorded may be either shape. - if plan.owned[index].packaging == "" { - for _, packaging := range []artifact.Packaging{artifact.PackagingNone, artifact.PackagingTgz} { - for _, path := range claimPaths(packaging, plan.owned[index].location) { - owned = append(owned, layout.FoldPath(path)) - } - } - } +// checkUnmanaged fails when an object the publication would overwrite exists and this reference +// does not own it. A location no reference owns must hold nothing at all. An owned location is owned +// only through the exact keys its payloads consist of, so a user's object below it is never +// overwritten: every key the publication writes there must be one the reference already owns, or +// be absent. It runs before the journal and any payload object are written. +func (r *Repository) checkUnmanaged(ctx context.Context, job *publishJob, candidate *layout.Claim, plan *publishPlan) error { + ownedPaths, ownedKeys, err := ownership(plan.owned) + if err != nil { + return err } for _, path := range candidate.Paths { - if slices.Contains(owned, layout.FoldPath(path)) { + if slices.Contains(ownedPaths, layout.FoldPath(path)) { continue } - occupied, err := r.occupied(ctx, path) - if err != nil { + if err := r.requireUnoccupied(ctx, path); err != nil { return err } - if occupied { - return fmt.Errorf("%w: %q already holds objects that are not managed by this repository; remove them or choose another key_pattern", errUtils.ErrArtifactConflict, path) + } + return r.checkWriteKeys(ctx, job, ownedPaths, ownedKeys) +} + +// ownership lists the paths the owned payloads claim, in folded form, and the exact object keys they +// consist of. +func ownership(owned []ownedPayload) (paths, keys []string, err error) { + for index := range owned { + for _, path := range anyShapeClaimPaths(owned[index].packaging, owned[index].location) { + paths = append(paths, layout.FoldPath(path)) + } + payloadKeys, keyErr := owned[index].set.anyShapeKeys(owned[index].packaging, owned[index].location) + if keyErr != nil { + return nil, nil, keyErr + } + keys = append(keys, payloadKeys...) + } + return paths, keys, nil +} + +// checkWriteKeys requires every key the publication writes below an owned path to be one the +// reference owns or to be absent. Keys below a path no reference owns are covered by the whole-path +// check. +func (r *Repository) checkWriteKeys(ctx context.Context, job *publishJob, ownedPaths, ownedKeys []string) error { + items, err := job.plan() + if err != nil { + return err + } + for index := range items { + key := items[index].place(job.location) + if slices.Contains(ownedKeys, key) || !coveredBy(ownedPaths, key) { + continue + } + if err := r.requireUnoccupied(ctx, key); err != nil { + return err } } return nil } +// coveredBy reports whether an owned path contains the key. Owned paths are in folded form, +// matching how claims are compared. +func coveredBy(ownedPaths []string, key string) bool { + folded := layout.FoldPath(key) + return slices.ContainsFunc(ownedPaths, func(path string) bool { return folded == path || strings.HasPrefix(folded, path+"/") }) +} + +// requireUnoccupied fails with a conflict when the path holds an object, or objects below it. +func (r *Repository) requireUnoccupied(ctx context.Context, path string) error { + occupied, err := r.occupied(ctx, path) + if err != nil { + return err + } + if occupied { + return fmt.Errorf("%w: %q already holds objects that are not managed by this repository; remove them or choose another key_pattern", errUtils.ErrArtifactConflict, path) + } + return nil +} + // occupied reports whether an object, or any object below the path as a prefix, exists. func (r *Repository) occupied(ctx context.Context, path string) (bool, error) { info, err := r.stat(ctx, path) diff --git a/pkg/artifact/s3/unmanaged_keys_test.go b/pkg/artifact/s3/unmanaged_keys_test.go new file mode 100644 index 00000000000..d993c4ecda6 --- /dev/null +++ b/pkg/artifact/s3/unmanaged_keys_test.go @@ -0,0 +1,116 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// A reference owns the exact keys its payload consists of, not the whole location. An object a user +// added below an owned location is never overwritten by a later publication that writes that key; +// the reservation refuses before the journal or any payload object is written. +func TestReservation_RejectsAnUnmanagedKeyBelowAnOwnedLocation(t *testing.T) { + const ( + perFileKey = "team/app/1.0.0/two.txt" + sidecarKey = "team/app/1.0.0.tgz.docs/README.md" + ) + tests := []struct { + name string + spec *rspec + baseFiles map[string]string + userKey string + replace map[string]string + wantKey string + }{ + { + name: "per-file payload key", spec: noneSpec(), userKey: perFileKey, + baseFiles: map[string]string{"one.txt": "1"}, replace: map[string]string{"one.txt": "1b", "two.txt": "2"}, wantKey: perFileKey, + }, + { + name: "per-file distribution document", spec: noneSpec(), userKey: "team/app/1.0.0/.atmos/artifact/README.md", + baseFiles: map[string]string{"one.txt": "1"}, replace: map[string]string{"one.txt": "1b", ".atmos/artifact/README.md": "doc"}, + wantKey: "team/app/1.0.0/.atmos/artifact/README.md", + }, + { + name: "archive document sidecar", spec: archiveSpec(artifact.PackagingTgz), userKey: sidecarKey, + baseFiles: map[string]string{"one.txt": "1"}, replace: map[string]string{"one.txt": "1b", ".atmos/artifact/README.md": "doc"}, wantKey: sidecarKey, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec.fresh(t) + repo := newRepo(t, spec) + base := publish(t, repo, &prepareSpec{files: tt.baseFiles}, artifact.PublishOptions{}) + spec.fake.Seed(tt.userKey, []byte("mine"), nil) + before := bucketKeys(spec.fake) + + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{files: tt.replace, opID: "op-replace"}), + artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.ErrorContains(t, err, tt.wantKey) + + object, ok := spec.fake.Object(tt.userKey) + require.True(t, ok) + assert.Equal(t, []byte("mine"), object.Data, "the user's object was not overwritten") + assert.Equal(t, before, bucketKeys(spec.fake), "nothing was written: no journal, no pending reference, no payload") + assertIntact(t, repo, defaultRef()) + }) + } +} + +// The reference's own keys stay exempt, and an extra user object the publication does not write is +// left alone: only a key the publication would write can conflict. +func TestReservation_AllowsOwnedKeysAndLeavesUnrelatedUserObjects(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + base := publish(t, repo, &prepareSpec{files: map[string]string{"one.txt": "1", "gone.txt": "g"}}, artifact.PublishOptions{}) + spec.fake.Seed("team/app/1.0.0/user.txt", []byte("mine"), nil) + + replaced, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{files: map[string]string{"one.txt": "1b"}, opID: "op-replace"}), + artifact.PublishOptions{Force: true, ExpectedPrior: &base.Pinned}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, replaced.Outcome) + assertIntact(t, repo, defaultRef()) + keys := payloadKeys(spec.fake, "") + assert.Contains(t, keys, "team/app/1.0.0/user.txt", "an unmanaged object the publication does not write stays") + assert.NotContains(t, keys, "team/app/1.0.0/gone.txt", "the replaced payload's own key was retired") + object, _ := spec.fake.Object("team/app/1.0.0/user.txt") + assert.Equal(t, []byte("mine"), object.Data) +} + +// A resumed publication owns the keys its interrupted attempt already wrote, so the retry is not +// refused by its own partial upload, while a user's key it did not write still is. +func TestReservation_ResumeOwnsItsOwnPartialUpload(t *testing.T) { + var fault armedFault + spec := noneSpec().fresh(t) + repo := newRepo(t, spec.withOpts(fault.option())) + prepared := prepare(t, repo, &prepareSpec{files: map[string]string{"one.txt": "1", "two.txt": "2"}, opID: "op-first"}) + + fault.hook = crashAt(faultDuringPayload) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + require.NotEmpty(t, payloadKeys(spec.fake, ""), "the interrupted attempt wrote part of the payload") + fault.hook = nil + + result, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err, "the retry owns the keys it wrote") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) +} + +// A location no reference owns is still refused whole, whichever key the user's object has. +func TestReservation_RejectsAnyObjectAtAnUnownedLocation(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + spec.fake.Seed("team/app/1.0.0/unrelated.txt", []byte("mine"), nil) + + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{files: map[string]string{"one.txt": "1"}}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.ErrorContains(t, err, "not managed by this repository") + assert.Equal(t, []string{"team/app/1.0.0/unrelated.txt"}, payloadKeys(spec.fake, "")) +} From a84664dc2325e006e44cd29b60b001e1bdd1b3a5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:03:52 -0600 Subject: [PATCH 45/53] test(artifacts): outlast the first claim attempt in the lock wait test TestLock_WaitIsBoundedByTheContext gave Lock 100ms. On a loaded machine the first claim attempt can outlast that, and an acquisition whose context already ended returns the context error alone, without the held-claim conflict the test asserts. Use a 2s deadline, which is far longer than one attempt and far shorter than the default wait. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/lock_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index daa43cea91d..6ca1279a569 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -163,9 +163,15 @@ func TestLock_WaitIsBoundedByTheContext(t *testing.T) { _, err := holder.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: "owner-a"}) require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + // Build the peer before starting the clock: under load, repository setup must not consume the + // deadline, or Lock would reject the already-expired context before ever contending for the claim. + peer := peerOf(t, spec) + // The deadline must outlast the first claim attempt on a loaded machine: an attempt that starts + // after the context ended returns the context error alone, without the held-claim conflict this + // test asserts. Two seconds is far longer than one attempt and far shorter than the default wait. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - _, err = peerOf(t, spec).Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + _, err = peer.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) require.ErrorIs(t, err, context.DeadlineExceeded) require.ErrorIs(t, err, errUtils.ErrArtifactConflict) } @@ -240,7 +246,7 @@ func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = peerOf(t, spec).Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + _, err = peer.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) require.ErrorIs(t, err, context.Canceled) require.NoError(t, held.Release(context.Background())) assert.Empty(t, lockObjects(spec.fake), "only the holder's claim ever existed") From 1875575ab27231e53c6424675dcc1e4eebbcca68 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:25:20 -0600 Subject: [PATCH 46/53] fix(artifact/s3): keep the recovered payload location until the replacement's journal names it A completed reference whose representation is missing or describes another Record locates its payload only through the journal that wrote it. A forced replacement settled that journal and then removed it, so the replacement's plan never owned the payload: a repair at the same location failed as unmanaged content, and a repair at another location orphaned the old payload. settleReplaced now returns the location it recovered and keeps the journal that is the only record of it. The replacement owns the recovered payload, records it as the content it replaces, and removes the settled journal only after its own journal is durable. A crash between the steps leaves the journal where the retry finds it. A completed reference also keeps claiming the journal-recorded location while its representation does not name the payload. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/publish.go | 20 +- pkg/artifact/s3/reserve.go | 53 ++-- pkg/artifact/s3/settle.go | 96 ++++++-- pkg/artifact/s3/settle_recovery_test.go | 311 ++++++++++++++++++++++++ 4 files changed, 437 insertions(+), 43 deletions(-) create mode 100644 pkg/artifact/s3/settle_recovery_test.go diff --git a/pkg/artifact/s3/publish.go b/pkg/artifact/s3/publish.go index 96032bbaddc..05b0946fe67 100644 --- a/pkg/artifact/s3/publish.go +++ b/pkg/artifact/s3/publish.go @@ -28,6 +28,11 @@ type publishPlan struct { owned []ownedPayload abandoned []ownedPayload oldOp *layout.Operation + // settled is the completed publication's journal a replacement kept because it is the only + // record of the replaced payload's location (see settleReplaced). It is removed once the + // replacement's own journal names that location. + settled string + recovered *layout.Representation expected *artifact.PinnedReference scope artifact.LockScope } @@ -80,9 +85,13 @@ func (r *Repository) publishLocked(ctx context.Context, job *publishJob, options if plan.admission == layout.AdmitReplace { // The journal of the publication being replaced is dropped with it, so what it still owes // is retired first. - if err := r.settleReplaced(ctx, plan.current.ref); err != nil { + recovered, err := r.settleReplaced(ctx, plan.current.ref) + if err != nil { return nil, err } + if recovered != nil { + plan.recovered, plan.settled = recovered, plan.current.ref.OperationID + } } if err := r.resolveLocation(ctx, job, plan); err != nil { return nil, err @@ -218,6 +227,15 @@ func (r *Repository) loadPendingState(ctx context.Context, plan *publishPlan, re if rep != nil { plan.owned = append(plan.owned, ownedPayload{packaging: rep.Packaging, location: rep.Location, set: setOfDescriptors(rep.Payload, rep.Documents)}) } + if plan.recovered != nil { + // The stored representation does not name the live payload; the settled journal did. The + // payload belongs to this reference, and it is the content the replacement replaces. + plan.owned = append(plan.owned, ownedPayload{ + packaging: plan.recovered.Packaging, location: plan.recovered.Location, + set: setOfDescriptors(plan.recovered.Payload, plan.recovered.Documents), + }) + rep = plan.recovered + } if plan.current != nil && !plan.current.ref.Available() { oldOp, err := r.loadOperation(ctx, plan.current.ref.OperationID) if err != nil { diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index d667ea7929b..a5d9ddb3c33 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -33,6 +33,11 @@ func (r *Repository) begin(ctx context.Context, job *publishJob, plan *publishPl if plan.oldOp != nil && plan.oldOp.OperationID != job.opID { r.tidyOperation(ctx, plan.oldOp.OperationID) } + if plan.settled != "" && plan.settled != job.opID { + // The new journal now names the replaced payload's location (Prior), so the journal that + // held the only other record of it has nothing left to carry. + r.tidyOperation(ctx, plan.settled) + } return pending, r.settings.fault(faultBeforePayload) } @@ -127,25 +132,13 @@ func (r *Repository) collectClaims(ctx context.Context, ownKey string) ([]layout return claims, nil } -// claimOf returns the claim a reference holds: its representation when completed, its journal when -// pending, and in both states the locations its journal still awaits removing. A reference whose -// state is missing holds no claim. +// claimOf returns the claim a reference holds: its representation when completed (or its journal, +// when that is the only record of the live payload), its journal when pending, and in both states +// the locations its journal still awaits removing. A reference whose state is missing holds no claim. func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layout.Claim, error) { base := layout.Claim{ReferenceKey: ref.Key(), Name: ref.Name, Version: ref.Version} if ref.Available() { - rep, err := r.loadRepresentation(ctx, ref.Key()) - if err != nil || rep == nil { - return nil, err - } - base.Paths = claimPaths(rep.Packaging, rep.Location) - // A completed reference keeps its journal until the cleanup that follows completion has - // removed everything the journal still lists. - journal, err := r.loadOperation(ctx, ref.OperationID) - if err != nil { - return nil, err - } - base.Paths = append(base.Paths, journalClaimPaths(journal)...) - return &base, nil + return r.completedClaim(ctx, ref, &base) } op, err := r.loadOperation(ctx, ref.OperationID) if err != nil || op == nil { @@ -156,6 +149,34 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou return &base, nil } +// completedClaim returns the claim of a completed reference: the location its representation names, +// the location its journal records when the representation does not name the live payload (the +// journal is then the only record of it), and the locations the journal still awaits removing. A +// completed reference with none of them holds no claim. +func (r *Repository) completedClaim(ctx context.Context, ref *layout.Reference, base *layout.Claim) (*layout.Claim, error) { + rep, err := r.loadRepresentation(ctx, ref.Key()) + if err != nil { + return nil, err + } + // A completed reference keeps its journal until the cleanup that follows completion has + // removed everything the journal still lists. + journal, err := r.loadOperation(ctx, ref.OperationID) + if err != nil { + return nil, err + } + if rep != nil { + base.Paths = claimPaths(rep.Packaging, rep.Location) + } + if recovered := recoveredRepresentation(ref, journal); recovered != nil && (rep == nil || rep.RecordDigest != ref.RecordDigest) { + base.Paths = append(base.Paths, claimPaths(recovered.Packaging, recovered.Location)...) + } + base.Paths = append(base.Paths, journalClaimPaths(journal)...) + if len(base.Paths) == 0 { + return nil, nil + } + return base, nil +} + // journalClaimPaths lists the paths of every payload a journal still awaits removing: the replaced // payload (Prior) and the payloads of abandoned attempts. They stay claimed until the journal is // retired, even while they hold no objects, because the retirement deletes by the journaled keys diff --git a/pkg/artifact/s3/settle.go b/pkg/artifact/s3/settle.go index 3538bbc321b..e6ada087cec 100644 --- a/pkg/artifact/s3/settle.go +++ b/pkg/artifact/s3/settle.go @@ -60,58 +60,102 @@ func (r *Repository) settle(ctx context.Context, current *layout.Reference, reco // retireAndTidy retires what the journal owes, keeping the live payload and never retiring the // Records listed, and then removes the journal. func (r *Repository) retireAndTidy(ctx context.Context, journal *layout.Operation, record *artifact.Record, rep *layout.Representation, protected ...digest.Digest) error { - keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) - if err != nil { - return err - } - if err := r.retireJournaled(ctx, journal, keep, protected...); err != nil { + if err := r.retireOwed(ctx, journal, record, rep, protected...); err != nil { return err } r.tidyOperation(ctx, journal.OperationID) return nil } +// retireOwed retires what the journal owes, keeping the live payload and never retiring the Records +// listed. It leaves the journal in place. +func (r *Repository) retireOwed(ctx context.Context, journal *layout.Operation, record *artifact.Record, rep *layout.Representation, protected ...digest.Digest) error { + keep, err := setOfRecord(record).keys(rep.Packaging, rep.Location) + if err != nil { + return err + } + return r.retireJournaled(ctx, journal, keep, protected...) +} + // settleReplaced settles the journal of the completed reference a forced replacement is about to // replace. The reference may be what a repair replaces, so a representation that no longer matches // is not an obstacle: the journal then names the live location. -func (r *Repository) settleReplaced(ctx context.Context, current *layout.Reference) error { +// +// When the journal is the only record of the live location, it is the one thing the settlement must +// not remove: the representation that would name the location is missing or describes another +// Record. The settlement then retires what the journal owes, keeps the journal, and returns the +// location it recovered. The replacement carries it into its own journal (see resolveLocation) and +// removes the settled journal only once that journal is durable (see begin). A crash anywhere in +// between leaves the journal, and so the location, where the retry finds it. A representation that +// names the live payload needs no recovery: the settlement returns nil and removes the journal. +func (r *Repository) settleReplaced(ctx context.Context, current *layout.Reference) (*layout.Representation, error) { if current.OperationID == "" { - return nil + return nil, nil } journal, err := r.loadOperation(ctx, current.OperationID) if err != nil { - return err + return nil, err + } + live, recovered, err := r.locateLive(ctx, current, journal) + if err != nil { + return nil, err } - if !owes(journal) { + if owes(journal) { + if err := r.retireLive(ctx, current, journal, live); err != nil { + return nil, err + } + } + if recovered == nil { r.tidyOperation(ctx, current.OperationID) - return nil } - record, _, err := r.loadRecord(ctx, current.RecordDigest) + return recovered, nil +} + +// locateLive finds where the completed reference's payload lives. The stored representation names +// it when it describes the reference's Record; otherwise the journal that wrote the payload does, +// and that representation is also returned as recovered. Neither locates it when both are nil. +func (r *Repository) locateLive(ctx context.Context, current *layout.Reference, journal *layout.Operation) (live, recovered *layout.Representation, err error) { + rep, err := r.loadRepresentation(ctx, current.Key()) if err != nil { - return err + return nil, nil, err } - rep, err := r.liveRepresentation(ctx, current, journal) + if rep != nil && rep.RecordDigest == current.RecordDigest { + return rep, nil, nil + } + recovered = recoveredRepresentation(current, journal) + return recovered, recovered, nil +} + +// retireLive retires what the journal owes, keeping the live payload. +func (r *Repository) retireLive(ctx context.Context, current *layout.Reference, journal *layout.Operation, live *layout.Representation) error { + record, _, err := r.loadRecord(ctx, current.RecordDigest) if err != nil { return err } - if err := r.retireAndTidy(ctx, journal, record, rep, current.RecordDigest); err != nil { + if live == nil { + return unknownLocation(current) + } + if err := r.retireOwed(ctx, journal, record, live, current.RecordDigest); err != nil { return fmt.Errorf("finish the cleanup owed by operation %s before replacing %s:%s: %w", current.OperationID, current.Name, current.Version, err) } return nil } -// liveRepresentation returns where the completed reference's payload lives: its representation, or -// the journal that wrote it when the representation is missing or describes another Record. -func (r *Repository) liveRepresentation(ctx context.Context, current *layout.Reference, journal *layout.Operation) (*layout.Representation, error) { - rep, err := r.loadRepresentation(ctx, current.Key()) - if err != nil { - return nil, err - } - if rep != nil && rep.RecordDigest == current.RecordDigest { - return rep, nil +// recoveredRepresentation describes the payload a completed reference points at from the journal +// that wrote it, or returns nil when the journal is missing or wrote another Record. The journal +// lists the payload and the documents, so the recovered representation owns exactly the keys the +// stored one did. +func recoveredRepresentation(current *layout.Reference, journal *layout.Operation) *layout.Representation { + if journal == nil || journal.RecordDigest != current.RecordDigest || journal.Location == "" { + return nil } - if journal.RecordDigest == current.RecordDigest && journal.Location != "" { - return &layout.Representation{Packaging: journal.Packaging, Location: journal.Location}, nil + return &layout.Representation{ + Name: current.Name, Version: current.Version, RecordDigest: current.RecordDigest, OperationID: journal.OperationID, + Packaging: journal.Packaging, Location: journal.Location, Payload: journal.Payload, Documents: journal.Documents, } - return nil, fmt.Errorf("%w: %s:%s is completed but its payload location is unknown", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) +} + +// unknownLocation is the failure for a completed reference whose payload location no record names. +func unknownLocation(current *layout.Reference) error { + return fmt.Errorf("%w: %s:%s is completed but its payload location is unknown", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) } diff --git a/pkg/artifact/s3/settle_recovery_test.go b/pkg/artifact/s3/settle_recovery_test.go new file mode 100644 index 00000000000..58baf7c73e4 --- /dev/null +++ b/pkg/artifact/s3/settle_recovery_test.go @@ -0,0 +1,311 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// These tests pin the recovery of a completed reference whose representation no longer names its +// payload. The journal that wrote the payload is then the only record of where the payload lives, so +// a replacement may not remove that journal before the location is carried into the new journal. + +// representationKey is the stored key of the lifecycle reference's representation. +func representationKey() string { + return layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)) +} + +// rawRepresentation returns the stored representation bytes of the lifecycle reference. +func (l *lifecycle) rawRepresentation(repo *Repository) []byte { + l.t.Helper() + raw, _, err := repo.readObject(context.Background(), representationKey(), layout.MaxDocumentBytes) + require.NoError(l.t, err) + require.NotEmpty(l.t, raw) + return raw +} + +// orphanedRepresentation is how a completed reference loses the record of its payload location. +type orphanedRepresentation struct { + name string + apply func(l *lifecycle, staleRepresentation []byte) +} + +var orphanings = []orphanedRepresentation{ + {name: "the representation is missing", apply: func(l *lifecycle, _ []byte) { + require.NoError(l.t, l.repo("new").deleteObject(context.Background(), representationKey())) + }}, + {name: "the representation describes the replaced Record", apply: func(l *lifecycle, stale []byte) { + l.fake.Seed(representationKey(), stale, nil) + }}, +} + +// completedWithOrphanedRepresentation leaves a completed reference at the "new" location whose +// journal still owes the replaced payload and whose representation does not name the live payload. +func completedWithOrphanedRepresentation(t *testing.T, s shape, orphaning orphanedRepresentation) *lifecycle { + t.Helper() + l := newLifecycle(t, s) + stale := l.rawRepresentation(l.repo("old")) + l.crashedAfterCommit() + orphaning.apply(l, stale) + require.Len(t, l.journals(), 1, "the journal is the only record of the live location") + require.NotEmpty(t, l.payloadUnder("new")) + return l +} + +// currentReference returns the stored reference of the lifecycle artifact. +func (l *lifecycle) currentReference() *layout.Reference { + l.t.Helper() + state, err := l.repo("new").loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(l.t, err) + require.NotNil(l.t, state) + return state.ref +} + +// journalID is the operation the completed reference points at. +func (l *lifecycle) journalID() string { return l.currentReference().OperationID } + +// pendingJournal returns the journal of the pending reference, read through the generation's repository. +func (l *lifecycle) pendingJournal(generation string) *layout.Operation { + l.t.Helper() + state, err := l.repo(generation).loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(l.t, err) + require.NotNil(l.t, state) + require.False(l.t, state.ref.Available(), "the reference is pending") + journal, err := l.repo(generation).loadOperation(context.Background(), state.ref.OperationID) + require.NoError(l.t, err) + require.NotNil(l.t, journal) + return journal +} + +// newcomerAt is a second reference whose key_pattern resolves onto the given location. +func (l *lifecycle) newcomerAt(location string) (*Repository, *prepareSpec) { + l.t.Helper() + spec := archiveSpec(l.shape.packaging).withPattern(location + l.shape.suffix).with(func(c *rspec) { c.fake = l.fake }) + return newRepo(l.t, spec.withOpts(WithStore(l.gate))), &prepareSpec{name: "other/app", files: map[string]string{"a.txt": "newcomer"}} +} + +// repair force-publishes different content for the generation over the current reference. +func (l *lifecycle) repair(generation string) (*artifact.PublishResult, error) { + l.t.Helper() + repo := l.repo(generation) + spec := &prepareSpec{files: map[string]string{"repaired.txt": "repaired-" + generation}, opID: "op-repair-" + generation} + return repo.Publish(context.Background(), prepare(l.t, repo, spec), artifact.PublishOptions{Force: true, ExpectedPrior: &l.current.Pinned}) +} + +// A forced repair of a reference whose representation does not name its payload settles the journal +// without losing the live location: the repair at the same location owns its payload, and a repair +// at another location retires the payload that was left behind. +func TestForceRepair_KeepsTheRecoveredPayloadLocation(t *testing.T) { + targets := []struct { + name string + generation string + wantAt string + }{ + {name: "at the same location", generation: "new", wantAt: "new"}, + {name: "at another location", generation: "third", wantAt: "third"}, + } + for _, s := range lifecycleShapes { + for _, orphaning := range orphanings { + for _, target := range targets { + t.Run(s.name+"/"+orphaning.name+"/"+target.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphaning) + + _, err := l.repair(target.generation) + require.NoError(t, err, "the live payload is owned by the reference being repaired") + + assert.Empty(t, l.oldPayload(), "the payload the journal owed is retired") + assert.NotEmpty(t, l.payloadUnder(target.wantAt)) + if target.generation != "new" { + assert.Empty(t, l.payloadUnder("new"), "the payload the journal located is retired, not orphaned") + } + assert.Empty(t, l.journals(), "no journal is orphaned") + assert.Len(t, l.records(), 1, "only the live Record remains") + assertIntact(t, l.repo(target.generation), defaultRef()) + }) + } + } + } +} + +// A completed reference whose journal owes nothing still has that journal as the only record of +// its payload when the representation is missing, so the repair keeps it until the new journal +// names the location. +func TestForceRepair_KeepsAJournalThatOwesNothing(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphanings[0]) + repo := l.repo("new") + journal, err := repo.loadOperation(context.Background(), l.journalID()) + require.NoError(t, err) + journal.Prior, journal.Abandoned = nil, nil + require.NoError(t, repo.writeOperation(context.Background(), journal)) + require.False(t, owes(journal)) + + recovered, err := repo.settleReplaced(context.Background(), l.currentReference()) + require.NoError(t, err) + require.NotNil(t, recovered, "the journal locates the payload") + assert.Equal(t, journal.Location, recovered.Location) + assert.Len(t, l.journals(), 1, "the journal is not removed while it is the only record of the location") + + _, err = l.repair("third") + require.NoError(t, err) + assert.Empty(t, l.payloadUnder("new"), "the located payload is retired") + assert.Empty(t, l.journals()) + assertIntact(t, l.repo("third"), defaultRef()) + }) + } +} + +// settleReplaced hands back what it recovered and keeps the journal that is the only record of it. +// A representation that names the live payload needs no recovery, and its journal is removed. +func TestSettleReplaced_ReturnsTheRecoveredRepresentation(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name+"/recovered", func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphanings[0]) + repo := l.repo("new") + journal, err := repo.loadOperation(context.Background(), l.journalID()) + require.NoError(t, err) + + recovered, err := repo.settleReplaced(context.Background(), l.currentReference()) + require.NoError(t, err) + require.NotNil(t, recovered) + assert.Equal(t, journal.Location, recovered.Location) + assert.Equal(t, journal.Packaging, recovered.Packaging) + assert.Equal(t, journal.RecordDigest, recovered.RecordDigest) + assert.Equal(t, journal.Payload, recovered.Payload) + assert.Equal(t, journal.Documents, recovered.Documents) + assert.Empty(t, l.oldPayload(), "what the journal owed is retired") + assert.Len(t, l.journals(), 1, "the journal still locates the live payload") + }) + t.Run(s.name+"/intact", func(t *testing.T) { + l := newLifecycle(t, s) + l.crashedAfterCommit() + + recovered, err := l.repo("new").settleReplaced(context.Background(), l.currentReference()) + require.NoError(t, err) + assert.Nil(t, recovered, "the stored representation already names the payload") + assert.Empty(t, l.journals(), "the journal is removed once settled") + }) + } +} + +// A crash between the settlement and the new journal leaves the reference as it was, so the retry +// finds the location again. +func TestForceRepair_SurvivesACrashAfterSettling(t *testing.T) { + for _, s := range lifecycleShapes { + for _, orphaning := range orphanings { + t.Run(s.name+"/"+orphaning.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphaning) + _, err := l.repo("new").settleReplaced(context.Background(), l.currentReference()) + require.NoError(t, err) + require.Len(t, l.journals(), 1) + + _, err = l.repair("third") + require.NoError(t, err) + assert.Empty(t, l.payloadUnder("new")) + assert.Empty(t, l.journals()) + assertIntact(t, l.repo("third"), defaultRef()) + }) + } + } +} + +// A crash once the repair is pending leaves the old location in the new journal's Prior: it stays +// claimed, and the same-intent retry completes the repair and retires it. +func TestForceRepair_SurvivesACrashAfterTheNewJournal(t *testing.T) { + for _, s := range lifecycleShapes { + for _, orphaning := range orphanings { + t.Run(s.name+"/"+orphaning.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphaning) + l.fault.hook = crashAt(faultBeforePayload) + _, err := l.repair("third") + require.ErrorIs(t, err, errCrash) + l.fault.hook = nil + + assert.NotEmpty(t, l.payloadUnder("new"), "the live payload is untouched while the repair is pending") + assert.Len(t, l.journals(), 1, "the pending journal replaced the settled one") + + _, err = l.repair("third") + require.NoError(t, err, "the same-intent retry completes the repair") + assert.Empty(t, l.payloadUnder("new"), "the located payload is retired") + assert.Empty(t, l.journals()) + assertIntact(t, l.repo("third"), defaultRef()) + }) + } + } +} + +// The recovered location is claimed by the pending repair, so no other reference reserves it. +func TestForceRepair_PendingRepairKeepsTheLocationClaimed(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphanings[0]) + l.fault.hook = crashAt(faultBeforePayload) + _, err := l.repair("third") + require.ErrorIs(t, err, errCrash) + l.fault.hook = nil + + journal := l.pendingJournal("third") + require.NotNil(t, journal.Prior) + assert.NotEmpty(t, journal.Prior.Location, "the new journal names the located payload") + + // A newcomer whose key_pattern resolves onto the located payload is refused. + newcomer, spec := l.newcomerAt("new/team/app") + before := bucketKeys(l.fake) + _, err = newcomer.Publish(context.Background(), prepare(t, newcomer, spec), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, before, bucketKeys(l.fake), "the refused reservation changed nothing") + }) + } +} + +// Negative path: nothing recovers a payload whose location no record names. The repair does not +// invent one, and it does not fail the replacement that cannot reach the unknown payload. +func TestSettleReplaced_NoRecoveryWithoutAJournalForTheRecord(t *testing.T) { + for _, s := range lifecycleShapes { + t.Run(s.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphanings[0]) + repo := l.repo("new") + journal, err := repo.loadOperation(context.Background(), l.journalID()) + require.NoError(t, err) + journal.RecordDigest = l.base.Pinned.RecordDigest + require.NoError(t, repo.writeOperation(context.Background(), journal)) + + _, err = repo.settleReplaced(context.Background(), l.currentReference()) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.Len(t, l.journals(), 1, "a failed settlement keeps the journal") + }) + } +} + +// A completed reference whose representation does not name its payload still claims the location the +// journal records: a newcomer that took it would have its objects deleted by the retirement or the +// repair, which act by the journaled keys. +func TestClaim_LocationRecordedOnlyByTheJournalStaysClaimed(t *testing.T) { + for _, s := range lifecycleShapes { + for _, orphaning := range orphanings { + t.Run(s.name+"/"+orphaning.name, func(t *testing.T) { + l := completedWithOrphanedRepresentation(t, s, orphaning) + // With the payload objects gone, the claim is the only thing keeping the location. + for _, key := range l.payloadUnder("new") { + require.NoError(t, l.gate.Store.Delete(context.Background(), key, blob.DeleteOptions{})) + } + require.Empty(t, l.payloadUnder("new")) + newcomer, spec := l.newcomerAt("new/team/app") + before := bucketKeys(l.fake) + + _, err := newcomer.Publish(context.Background(), prepare(t, newcomer, spec), artifact.PublishOptions{}) + + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, before, bucketKeys(l.fake), "the refused reservation changed nothing") + }) + } + } +} From a8179edde1421f39168079e3f017309b44872b52 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:25:51 -0600 Subject: [PATCH 47/53] test(artifact/s3): bound the newcomer race by the retirement, not an attempt count The newcomer was required to win within 200 attempts, but a refusal while the retirement still owes the location is the correct answer, so a late retirement could fail the test with no payload deleted incorrectly. The attempts now end when the retirement finishes (or the context ends), a refusal is accepted, and the newcomer is published once the retirement is over. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/journal_lifecycle_test.go | 64 ++++++++++++++++++----- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/pkg/artifact/s3/journal_lifecycle_test.go b/pkg/artifact/s3/journal_lifecycle_test.go index f8fc06784ab..6fd6f95bd9c 100644 --- a/pkg/artifact/s3/journal_lifecycle_test.go +++ b/pkg/artifact/s3/journal_lifecycle_test.go @@ -6,6 +6,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -349,7 +350,9 @@ func TestRetirement_IsIdempotentOnceTheRecordIsGone(t *testing.T) { // A newcomer racing the retry that finishes a retirement either is refused while the journal owes // the location, or takes it once the retirement is done. It is never admitted in between, which -// would let the retry delete the newcomer's objects under the same keys. +// would let the retry delete the newcomer's objects under the same keys. A refusal during the +// retirement is correct, so the newcomer's attempts are bounded by the retirement finishing, never +// by a count, and it is published once the retirement is over. func TestJournalPrior_RetirementRacingANewcomerNeverDeletesTheNewcomer(t *testing.T) { for _, s := range lifecycleShapes { t.Run(s.name, func(t *testing.T) { @@ -363,28 +366,38 @@ func TestJournalPrior_RetirementRacingANewcomerNeverDeletesTheNewcomer(t *testin retry := prepare(t, l.repo("new"), l.spec("new", "new")) incoming := prepare(t, newcomer, content) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + retired := make(chan struct{}) var wg sync.WaitGroup - errs := make(chan error, 2) + retryErr := make(chan error, 1) + type raced struct { + admitted bool + err error + } + raceResult := make(chan raced, 1) wg.Add(2) go func() { defer wg.Done() - _, err := l.repo("new").Publish(context.Background(), retry, artifact.PublishOptions{ExpectedPrior: &l.current.Pinned}) - errs <- err + defer close(retired) + _, err := l.repo("new").Publish(ctx, retry, artifact.PublishOptions{ExpectedPrior: &l.current.Pinned}) + retryErr <- err }() go func() { defer wg.Done() - var err error - for range 200 { - if _, err = newcomer.Publish(context.Background(), incoming, artifact.PublishOptions{}); !errors.Is(err, errUtils.ErrArtifactConflict) { - break - } - } - errs <- err + admitted, err := raceUntilRetired(ctx, newcomer, incoming, retired) + raceResult <- raced{admitted: admitted, err: err} }() wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) + require.NoError(t, <-retryErr) + race := <-raceResult + require.NoError(t, race.err, "a refusal while the retirement ran is correct; anything else is not") + + // The retirement is over, so the location is claimable: a newcomer that was refused + // throughout takes it now. + if !race.admitted { + _, err := newcomer.Publish(ctx, incoming, artifact.PublishOptions{}) + require.NoError(t, err, "a retired location is claimable again") } assertIntact(t, newcomer, ref("other/app", testVersion)) @@ -393,3 +406,26 @@ func TestJournalPrior_RetirementRacingANewcomerNeverDeletesTheNewcomer(t *testin }) } } + +// raceUntilRetired publishes the newcomer repeatedly until it is admitted or the retirement it races +// is over, whichever comes first. A conflict is the correct answer while the retirement still owes +// the location, so it is not an error. The attempts end on the observable condition, not on a count. +// It reports whether the newcomer was admitted. +func raceUntilRetired(ctx context.Context, newcomer *Repository, incoming *artifact.PreparedArtifact, retired <-chan struct{}) (admitted bool, err error) { + for { + _, err := newcomer.Publish(ctx, incoming, artifact.PublishOptions{}) + if err == nil { + return true, nil + } + if !errors.Is(err, errUtils.ErrArtifactConflict) { + return false, err + } + select { + case <-retired: + return false, nil + case <-ctx.Done(): + return false, ctx.Err() + default: + } + } +} From cc79d636a60e61b45e42ca037fa4790fb8e40273 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:28:29 -0600 Subject: [PATCH 48/53] fix(artifact/oci): bind the source Record to its reference before a native copy chooses the destination CopyFrom resolved the source without reading its Record and then took the destination name and version from the Record, so a tag serving another artifact's package selected that artifact's reference, and the destination check compared the Record with itself. The Record is now checked against the reference the source was resolved by first, and a mismatch fails with the integrity error before either destination tag is written. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/copy.go | 7 ++++ pkg/artifact/oci/copy_identity_test.go | 52 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index 0dbd20aec95..c240b7769ad 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -63,6 +63,13 @@ func (r *Repository) destinationRef(ctx context.Context, st store, loaded *root, if err != nil { return artifact.Ref{}, err } + // The source was resolved without reading its Record, so the Record is bound to the + // reference it was resolved by before it chooses the destination. Otherwise a tag that + // serves another artifact's package would pick that artifact's reference here, and the + // destination check would compare the Record with itself. + if err := requireSubjectOf(record, pinned.Ref); err != nil { + return artifact.Ref{}, err + } return artifact.Ref{Name: record.Name, Version: record.Version}, nil } if pinned.Ref.Version == "" { diff --git a/pkg/artifact/oci/copy_identity_test.go b/pkg/artifact/oci/copy_identity_test.go index 9cd750f824a..d039db49977 100644 --- a/pkg/artifact/oci/copy_identity_test.go +++ b/pkg/artifact/oci/copy_identity_test.go @@ -54,3 +54,55 @@ func TestImportLayout_RejectsAMismatchedRecordIdentityBeforeWriting(t *testing.T assert.Equal(t, published.ResolvedDigest, result.Pinned.ResolvedDigest) }) } + +// A native copy publishes an Atmos package under the name and version its Record carries, so the +// Record must first be bound to the reference the source was resolved by. Resolve does not check the +// Record, so a tag in one repository path that serves another artifact's package would otherwise +// choose the destination, and the later identity check would compare the Record with itself. +func TestCopyFrom_RejectsASourceWhoseRecordNamesAnotherReference(t *testing.T) { + ctx := context.Background() + tests := []struct { + name string + ref artifact.Ref + }{ + {name: "another name", ref: artifact.Ref{Name: "other/app", Version: appVersion}}, + {name: "another version", ref: artifact.Ref{Name: appName, Version: "9.9.9"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := newLayoutRepo(t) + published := publish(t, source, publishSpec{content: "tampered source"}).Pinned + // Serve the package under a tag of another artifact, as a malicious writer could. + st, err := source.be.open(ctx, tt.ref.Name, true) + require.NoError(t, err) + desc, ok := source.descriptorFor(appName, published.ResolvedDigest) + require.True(t, ok) + require.NoError(t, st.Tag(ctx, desc, source.be.reference(tt.ref.Name, tt.ref.Version))) + served, err := source.Resolve(ctx, tt.ref) + require.NoError(t, err, "Resolve does not read the Record") + + destination := newLayoutRepo(t) + before, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + + _, err = destination.CopyFrom(ctx, source, served, artifact.CopyFromOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + + after, err := destination.be.(*layoutBackend).references(ctx) + require.NoError(t, err) + assert.Equal(t, before, after, "neither the requested nor the Record's reference was written") + _, err = destination.Resolve(ctx, artifact.Ref{Name: appName, Version: appVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } + + t.Run("a source whose Record matches copies", func(t *testing.T) { + source := newLayoutRepo(t) + published := publish(t, source, publishSpec{content: "genuine source"}).Pinned + destination := newLayoutRepo(t) + + result, err := destination.CopyFrom(ctx, source, published, artifact.CopyFromOptions{}) + require.NoError(t, err) + assert.Equal(t, published.ResolvedDigest, result.Pinned.ResolvedDigest) + }) +} From 19141d69b0dd9af90c522d200be1fcee0dd186b5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:28:55 -0600 Subject: [PATCH 49/53] fix(artifact/oci): classify only the root media types the copy transport can traverse A media type that merely ended in .manifest.v2+json or .manifest.list.v2+json passed classification, but the copy transport follows a root to its config, layers, and children only for the exact OCI and Docker types. An import of such a root would have tagged it without its content. Classification now accepts exactly those four types, so the graph source and the copy transport agree on where a root ends and anything else is rejected as an unsupported media type. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/oci/classify.go | 16 ++- pkg/artifact/oci/classify_mediatype_test.go | 126 ++++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 pkg/artifact/oci/classify_mediatype_test.go diff --git a/pkg/artifact/oci/classify.go b/pkg/artifact/oci/classify.go index e9b06b2d414..f2b8c253d96 100644 --- a/pkg/artifact/oci/classify.go +++ b/pkg/artifact/oci/classify.go @@ -78,12 +78,24 @@ func (r *root) original() *ocispec.Descriptor { // isIndex reports whether the root is an index or manifest list. func (r *root) isIndex() bool { return isIndexMediaType(r.desc.MediaType) } +// The Docker media types of a manifest and of a manifest list. Together with the OCI types they are the +// whole set of roots the copy transport follows to their config, layers, and children. +const ( + mediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + mediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" +) + +// isIndexMediaType reports whether the media type is an OCI index or a Docker manifest list. The match +// is exact: the copy transport follows a root's children only for these types, so a type that merely +// ends the same way would be classified here but copied as a leaf, without its content. func isIndexMediaType(mediaType string) bool { - return mediaType == ocispec.MediaTypeImageIndex || strings.HasSuffix(mediaType, ".manifest.list.v2+json") + return mediaType == ocispec.MediaTypeImageIndex || mediaType == mediaTypeDockerManifestList } +// isManifestMediaType reports whether the media type is an OCI image manifest or a Docker manifest. +// The match is exact for the reason isIndexMediaType gives. func isManifestMediaType(mediaType string) bool { - return mediaType == ocispec.MediaTypeImageManifest || strings.HasSuffix(mediaType, ".manifest.v2+json") + return mediaType == ocispec.MediaTypeImageManifest || mediaType == mediaTypeDockerManifest } // loadRoot fetches a root by its resolved descriptor with a bounded, digest-verified read and diff --git a/pkg/artifact/oci/classify_mediatype_test.go b/pkg/artifact/oci/classify_mediatype_test.go new file mode 100644 index 00000000000..3361a595b30 --- /dev/null +++ b/pkg/artifact/oci/classify_mediatype_test.go @@ -0,0 +1,126 @@ +package oci + +import ( + "context" + "encoding/json" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// The media types a root is classified by are the ones the copy transport can traverse. ORAS follows +// a manifest to its config and layers only for these exact types, so a type that merely ends the way +// one of them does would be copied as a leaf: an import would tag the root without its content. +var traversableRoots = []struct { + name string + mediaType string + index bool +}{ + {name: "OCI image manifest", mediaType: ocispec.MediaTypeImageManifest}, + {name: "Docker image manifest", mediaType: "application/vnd.docker.distribution.manifest.v2+json"}, + {name: "OCI image index", mediaType: ocispec.MediaTypeImageIndex, index: true}, + {name: "Docker manifest list", mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", index: true}, +} + +var lookalikeRoots = []struct { + name string + mediaType string +}{ + {name: "custom manifest suffix", mediaType: "application/x-custom.manifest.v2+json"}, + {name: "custom manifest list suffix", mediaType: "application/x-custom.manifest.list.v2+json"}, + {name: "vendor docker-like manifest", mediaType: "application/vnd.example.distribution.manifest.v2+json"}, + {name: "traversable type with a suffix", mediaType: ocispec.MediaTypeImageManifest + "+x"}, + {name: "traversable type in another case", mediaType: "Application/Vnd.Oci.Image.Manifest.V1+Json"}, + {name: "empty", mediaType: ""}, +} + +func TestRootMediaTypes_AreExactlyTheTraversableOnes(t *testing.T) { + for _, tt := range traversableRoots { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.index, isIndexMediaType(tt.mediaType)) + assert.Equal(t, !tt.index, isManifestMediaType(tt.mediaType)) + }) + } + for _, tt := range lookalikeRoots { + t.Run(tt.name, func(t *testing.T) { + assert.False(t, isIndexMediaType(tt.mediaType), "the copy transport would not traverse it") + assert.False(t, isManifestMediaType(tt.mediaType), "the copy transport would not traverse it") + }) + } +} + +func TestClassify_RejectsMediaTypesTheCopyTransportCannotTraverse(t *testing.T) { + raw, err := json.Marshal(ocispec.Manifest{ + Config: ocispec.Descriptor{MediaType: artifact.MediaTypeConfig, Digest: digest.FromString("c"), Size: 1}, + Layers: []ocispec.Descriptor{{MediaType: artifact.MediaTypeTar, Digest: digest.FromString("l"), Size: 1}}, + }) + require.NoError(t, err) + rawIndex, err := json.Marshal(ocispec.Index{Manifests: []ocispec.Descriptor{{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.FromString("m"), Size: 1}}}) + require.NoError(t, err) + + for _, tt := range traversableRoots { + t.Run("classifies "+tt.name, func(t *testing.T) { + body := raw + if tt.index { + body = rawIndex + } + loaded, err := classify(ocispec.Descriptor{MediaType: tt.mediaType, Digest: digest.FromBytes(body), Size: int64(len(body))}, body) + require.NoError(t, err) + assert.Equal(t, tt.index, loaded.isIndex()) + successors, err := successorsOf(loaded.desc, body) + require.NoError(t, err) + assert.NotEmpty(t, successors) + }) + } + for _, tt := range lookalikeRoots { + t.Run("rejects "+tt.name, func(t *testing.T) { + _, err := classify(ocispec.Descriptor{MediaType: tt.mediaType, Digest: digest.FromBytes(raw), Size: int64(len(raw))}, raw) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) + } +} + +// A root by digest alone has no media type until its bytes say which it is. A manifest that declares a +// type the copy transport cannot traverse is not recovered as one. +func TestManifestTypeOf_RecoversOnlyTraversableTypes(t *testing.T) { + for _, tt := range traversableRoots { + t.Run(tt.name, func(t *testing.T) { + raw, err := json.Marshal(map[string]any{"mediaType": tt.mediaType}) + require.NoError(t, err) + got, err := manifestTypeOf(digest.FromBytes(raw), raw) + require.NoError(t, err) + assert.Equal(t, tt.mediaType, got) + }) + } + for _, tt := range lookalikeRoots { + if tt.mediaType == "" { + continue + } + t.Run(tt.name, func(t *testing.T) { + raw, err := json.Marshal(map[string]any{"mediaType": tt.mediaType, "config": map[string]any{}}) + require.NoError(t, err) + got, err := manifestTypeOf(digest.FromBytes(raw), raw) + require.NoError(t, err) + assert.Empty(t, got, "a declared type that is not traversable stays rejected") + }) + } +} + +// The graph source the signing and reading paths use follows the same set of types, so it ends where +// the copy transport ends. +func TestGraphSource_DoesNotFollowLookalikeRoots(t *testing.T) { + for _, tt := range lookalikeRoots { + t.Run(tt.name, func(t *testing.T) { + source := &graphSource{} + successors, err := source.Successors(context.Background(), ocispec.Descriptor{MediaType: tt.mediaType, Digest: digest.FromString("x"), Size: 1}) + require.NoError(t, err) + assert.Empty(t, successors) + }) + } +} From 0bf5c4dabd31c8d9a4cf95203b00ef54d239b87d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:32:46 -0600 Subject: [PATCH 50/53] test(artifact): isolate every S3 config test from the machine's shared AWS files A named-identity test pointed the SDK at a temporary credentials file but left the shared config file to the default chain. A developer default profile with role_arn and source_profile was merged in and selected AssumeRole, so the test reached the real STS service and failed. s3test gains IsolateAWSConfig, which points both shared files at empty temporary files and clears the variables that select a profile or another credential source. The identity tests of the blob store, the S3 repository, and the CI artifact store use it, and the named-identity test also names an empty config file explicitly. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/s3/client_test.go | 16 +++++- pkg/artifact/blob/s3/conditional_test.go | 2 +- pkg/artifact/blob/s3/s3test/isolate.go | 59 +++++++++++++++++++++ pkg/artifact/blob/s3/s3test/isolate_test.go | 57 ++++++++++++++++++++ pkg/artifact/s3/identity_test.go | 13 ++--- pkg/ci/artifact/s3/endpoint_request_test.go | 2 + pkg/ci/artifact/s3/store_test.go | 6 +++ 7 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 pkg/artifact/blob/s3/s3test/isolate.go create mode 100644 pkg/artifact/blob/s3/s3test/isolate_test.go diff --git a/pkg/artifact/blob/s3/client_test.go b/pkg/artifact/blob/s3/client_test.go index fc690bd913d..3b74e78886b 100644 --- a/pkg/artifact/blob/s3/client_test.go +++ b/pkg/artifact/blob/s3/client_test.go @@ -20,10 +20,12 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" "github.com/cloudposse/atmos/pkg/store" ) func TestIdentityIsResolvedLazily(t *testing.T) { + s3test.IsolateAWSConfig(t) resolver := &fakeResolver{err: errors.New("sso expired")} s, err := New("bucket", WithIdentity("deploy", resolver), WithRegion("us-east-1")) require.NoError(t, err, "constructing the store must not touch credentials") @@ -35,6 +37,7 @@ func TestIdentityIsResolvedLazily(t *testing.T) { } func TestUnresolvableIdentityFailsClosed(t *testing.T) { + s3test.IsolateAWSConfig(t) t.Setenv("AWS_ACCESS_KEY_ID", "ambient-key") t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret") t.Setenv("AWS_REGION", "us-east-1") @@ -68,6 +71,7 @@ func TestUnresolvableIdentityFailsClosed(t *testing.T) { } func TestLegacyClientKeepsAmbientFallback(t *testing.T) { + s3test.IsolateAWSConfig(t) t.Setenv("AWS_REGION", "us-east-1") t.Setenv("AWS_ACCESS_KEY_ID", "k") t.Setenv("AWS_SECRET_ACCESS_KEY", "s") @@ -82,6 +86,7 @@ func TestLegacyClientKeepsAmbientFallback(t *testing.T) { } func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) { + s3test.IsolateAWSConfig(t) requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { @@ -186,6 +191,7 @@ func TestBuildClientOptFnsMatchesLegacyEndpointBehavior(t *testing.T) { } func TestHTTPClientInjectionReachesTheSDK(t *testing.T) { + s3test.IsolateAWSConfig(t) fake := newFake(t) var used bool client := httpClientFunc(func(r *http.Request) (*http.Response, error) { @@ -208,19 +214,25 @@ type httpClientFunc func(*http.Request) (*http.Response, error) func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { return f(r) } func TestNamedIdentityReloadsRotatedCredentials(t *testing.T) { + isolated := s3test.IsolateAWSConfig(t) headers := make(chan string, 2) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { headers <- r.Header.Get("Authorization") w.WriteHeader(http.StatusOK) })) defer server.Close() - t.Setenv("AWS_EC2_METADATA_DISABLED", "true") creds := filepath.Join(t.TempDir(), "credentials") write := func(key string) { require.NoError(t, os.WriteFile(creds, []byte("[default]\naws_access_key_id = "+key+"\naws_secret_access_key = secret\n"), 0o600)) } write("first-key") - resolver := &fakeResolver{result: &store.AWSAuthConfig{CredentialsFile: creds, Profile: "default", Region: "us-east-1"}} + resolver := &fakeResolver{result: &store.AWSAuthConfig{ + CredentialsFile: creds, + // The shared config is named too, so the developer's own default profile is never merged in. + ConfigFile: filepath.Join(isolated, "config"), + Profile: "default", + Region: "us-east-1", + }} s, err := New("artifacts", WithIdentity("deploy", resolver), WithEndpoint(server.URL)) require.NoError(t, err) _, err = s.Stat(context.Background(), "file", blob.StatOptions{}) diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go index e641332e3aa..0936ef2cc9b 100644 --- a/pkg/artifact/blob/s3/conditional_test.go +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -387,9 +387,9 @@ func TestProbeRejectsIgnoredIfMatch(t *testing.T) { // operation, so its endpoint can change during the life of the store; a verdict from the old // endpoint must never vouch for the new one. func TestProbeVerdictIsBoundToTheEndpointItProbed(t *testing.T) { + s3test.IsolateAWSConfig(t) t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") - t.Setenv("AWS_EC2_METADATA_DISABLED", "true") enforcing := newFake(t) ignoring := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) ignoring.Seed("lock", []byte("owner-a"), nil) diff --git a/pkg/artifact/blob/s3/s3test/isolate.go b/pkg/artifact/blob/s3/s3test/isolate.go new file mode 100644 index 00000000000..186719c59a5 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/isolate.go @@ -0,0 +1,59 @@ +package s3test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// ambientAWSVariables are the variables through which the developer's or the runner's AWS setup +// reaches the SDK's default chain: the profile selection, the web identity and container credential +// sources, and the endpoint, FIPS, and dual-stack overrides. Static credentials and the region are +// left to each test, which sets the ones it needs. +var ambientAWSVariables = []string{ + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_SESSION_TOKEN", + "AWS_ROLE_ARN", + "AWS_ROLE_SESSION_NAME", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_ENDPOINT_URL", + "AWS_ENDPOINT_URL_S3", + "AWS_USE_FIPS_ENDPOINT", + "AWS_USE_DUALSTACK_ENDPOINT", +} + +// isolatedFileMode is the permission of the empty shared files, which hold no secret. +const isolatedFileMode = 0o600 + +// IsolateAWSConfig keeps a test that builds an AWS SDK configuration away from the AWS setup of the +// machine it runs on. The shared config and credentials files point at empty temporary files, the +// variables that select a profile or another credential source are cleared, and instance metadata is +// disabled. A developer's default profile with a role_arn would otherwise be merged into the test's +// configuration and select AssumeRole, which reaches the real STS service. It returns the directory +// holding the files, so a test can write the profile it wants into them. +func IsolateAWSConfig(t testing.TB) string { + defer perf.Track(nil, "s3test.IsolateAWSConfig")() + + t.Helper() + dir := t.TempDir() + config, credentials := filepath.Join(dir, "config"), filepath.Join(dir, "credentials") + for _, path := range []string{config, credentials} { + if err := os.WriteFile(path, nil, isolatedFileMode); err != nil { + t.Fatalf("create an empty AWS file: %v", err) + } + } + for _, name := range ambientAWSVariables { + t.Setenv(name, "") + } + t.Setenv("AWS_CONFIG_FILE", config) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", credentials) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + return dir +} diff --git a/pkg/artifact/blob/s3/s3test/isolate_test.go b/pkg/artifact/blob/s3/s3test/isolate_test.go new file mode 100644 index 00000000000..f3bd3de244c --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/isolate_test.go @@ -0,0 +1,57 @@ +package s3test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// poisonedConfig is a developer's shared config whose default profile assumes a role, the setup that +// reaches the real STS service from a test that does not isolate it. +const poisonedConfig = "[default]\nrole_arn = arn:aws:iam::123456789012:role/developer\nsource_profile = base\nregion = eu-west-1\n\n" + + "[profile base]\naws_access_key_id = AKIAPOISONED\naws_secret_access_key = poisoned\n" + +func TestIsolateAWSConfig_IgnoresTheMachinesAWSSetup(t *testing.T) { + ambient := t.TempDir() + poisoned := filepath.Join(ambient, "config") + require.NoError(t, os.WriteFile(poisoned, []byte(poisonedConfig), 0o600)) + t.Setenv("AWS_CONFIG_FILE", poisoned) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", poisoned) + t.Setenv("AWS_PROFILE", "developer") + t.Setenv("AWS_DEFAULT_PROFILE", "developer") + t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/ambient") + t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", filepath.Join(ambient, "missing-token")) + t.Setenv("AWS_ENDPOINT_URL", "http://ambient.invalid") + + dir := IsolateAWSConfig(t) + + for _, name := range ambientAWSVariables { + assert.Empty(t, os.Getenv(name), name) + } + assert.Equal(t, filepath.Join(dir, "config"), os.Getenv("AWS_CONFIG_FILE")) + assert.Equal(t, filepath.Join(dir, "credentials"), os.Getenv("AWS_SHARED_CREDENTIALS_FILE")) + assert.Equal(t, "true", os.Getenv("AWS_EC2_METADATA_DISABLED")) + for _, name := range []string{"config", "credentials"} { + info, err := os.Stat(filepath.Join(dir, name)) + require.NoError(t, err) + assert.Zero(t, info.Size(), "the isolated %s file is empty", name) + } + + // The default chain loads without the profile, the role, or the endpoint of the machine, and + // the credentials it resolves are the ones the test sets. + // The region is the one thing the helper leaves to the test, so this test pins it too. + t.Setenv("AWS_REGION", "") + t.Setenv("AWS_DEFAULT_REGION", "") + t.Setenv("AWS_ACCESS_KEY_ID", "test-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + loaded, err := config.LoadDefaultConfig(t.Context()) + require.NoError(t, err) + assert.Empty(t, loaded.Region, "the machine's region is not merged in") + credentials, err := loaded.Credentials.Retrieve(t.Context()) + require.NoError(t, err) + assert.Equal(t, "test-key", credentials.AccessKeyID) +} diff --git a/pkg/artifact/s3/identity_test.go b/pkg/artifact/s3/identity_test.go index 49a69426e76..b2835ae5ca6 100644 --- a/pkg/artifact/s3/identity_test.go +++ b/pkg/artifact/s3/identity_test.go @@ -18,21 +18,16 @@ import ( "github.com/cloudposse/atmos/pkg/store" ) -// isolateAWS gives the SDK static environment credentials and no ambient profile, so a test proves -// what the repository does with an identity rather than what the developer machine happens to have. +// isolateAWS gives the SDK static environment credentials and none of the machine's AWS setup, so a +// test proves what the repository does with an identity rather than what the developer machine +// happens to have: no profile, no shared config, no other credential source. func isolateAWS(t *testing.T) { t.Helper() - dir := t.TempDir() + dir := s3test.IsolateAWSConfig(t) t.Setenv("AWS_ACCESS_KEY_ID", "ambient-test-key") t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-test-secret") t.Setenv("AWS_REGION", "us-east-1") - // An ambient profile that the temporary files do not define would fail configuration loading. - t.Setenv("AWS_PROFILE", "") - t.Setenv("AWS_DEFAULT_PROFILE", "") require.NoError(t, os.WriteFile(filepath.Join(dir, "credentials"), []byte("[default]\naws_access_key_id = test-key\naws_secret_access_key = test-secret\n"), 0o600)) - t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) - t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) - t.Setenv("AWS_EC2_METADATA_DISABLED", "true") } // A developer's selected profile never leaks into an isolated test: with AWS_PROFILE pointing at a diff --git a/pkg/ci/artifact/s3/endpoint_request_test.go b/pkg/ci/artifact/s3/endpoint_request_test.go index 8bd73858e0e..0ebeabb8b17 100644 --- a/pkg/ci/artifact/s3/endpoint_request_test.go +++ b/pkg/ci/artifact/s3/endpoint_request_test.go @@ -12,6 +12,7 @@ import ( sdk "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/stretchr/testify/require" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" "github.com/cloudposse/atmos/pkg/ci/artifact" ) @@ -45,6 +46,7 @@ func TestIdentityEndpointUsesBucketPath(t *testing.T) { } func TestAdapterIdentityResolverRoutesToEndpoint(t *testing.T) { + s3test.IsolateAWSConfig(t) requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { diff --git a/pkg/ci/artifact/s3/store_test.go b/pkg/ci/artifact/s3/store_test.go index 2a58c34804e..e5495c3ff4c 100644 --- a/pkg/ci/artifact/s3/store_test.go +++ b/pkg/ci/artifact/s3/store_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" "github.com/cloudposse/atmos/pkg/ci/artifact" "github.com/cloudposse/atmos/pkg/store" ) @@ -224,6 +225,7 @@ func (f *fakeAuthContextResolver) ResolveGCPAuthContext(_ context.Context, _ str // TestNewStore_DeferredInitWhenIdentitySet verifies the client is not // initialized at construction time when an identity is configured. func TestNewStore_DeferredInitWhenIdentitySet(t *testing.T) { + s3test.IsolateAWSConfig(t) backend, err := NewStore(artifact.StoreOptions{ Identity: "deploy", Options: map[string]any{ @@ -245,6 +247,7 @@ func TestNewStore_DeferredInitWhenIdentitySet(t *testing.T) { // at construction time when no identity is configured. Pin AWS env vars so // LoadDefaultConfig takes the success path regardless of host credentials. func TestNewStore_EagerInitWithoutIdentity(t *testing.T) { + s3test.IsolateAWSConfig(t) t.Setenv("AWS_REGION", "us-east-1") t.Setenv("AWS_ACCESS_KEY_ID", "") t.Setenv("AWS_SECRET_ACCESS_KEY", "") @@ -414,6 +417,7 @@ func TestStore_BuildClientOptFns(t *testing.T) { // identity-configured backend without an injected resolver falls back to // the default credential chain rather than failing. func TestStore_InitIdentityClient_FallsBackWhenNoResolver(t *testing.T) { + s3test.IsolateAWSConfig(t) s := &Store{ bucket: "test-bucket", region: "us-east-1", @@ -428,6 +432,7 @@ func TestStore_InitIdentityClient_FallsBackWhenNoResolver(t *testing.T) { // TestStore_InitIdentityClient_PropagatesResolverError verifies resolver // errors surface through ensureClient with the expected error sentinel. func TestStore_InitIdentityClient_PropagatesResolverError(t *testing.T) { + s3test.IsolateAWSConfig(t) resolver := &fakeAuthContextResolver{ awsErr: errors.New("identity chain failed"), } @@ -449,6 +454,7 @@ func TestStore_InitIdentityClient_PropagatesResolverError(t *testing.T) { // TestStore_EnsureClient_CallsResolverOnceAcrossOperations verifies the // sync.Once gate: the resolver is invoked once per backend instance. func TestStore_EnsureClient_CallsResolverOnceAcrossOperations(t *testing.T) { + s3test.IsolateAWSConfig(t) resolver := &fakeAuthContextResolver{ awsResult: &artifact.AWSAuthConfig{ Region: "us-east-1", From 98d95072dabaeeaa8f08798775509f92fd0ec632 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 15:01:47 -0600 Subject: [PATCH 51/53] fix(s3): keep the held-claim conflict when the context ends between lock attempts If the context expired after a claim attempt found the claim held but before the next attempt began, Lock returned the bare context error and dropped the conflict that callers (and TestLock_WaitIsBoundedByTheContext) rely on. Report the held claim in that case, as the wait-expiry path already does. A context that is already done before any attempt still returns its error alone. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/lock.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index 527187eee4e..a957502e7ef 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -91,25 +91,26 @@ func (r *Repository) acquire(ctx context.Context, resource, owner, operationID s } key := layout.LockPath(resource) deadline := time.Now().Add(r.settings.lockWait) + contended := false for { // An acquisition that starts after cancellation must not reach the claim PUT: the canceled // request would be settled as ambiguous and could leave a permanent reconciliation fence. // Only requests that were already in flight when the context ended are reconciled. if ctxErr := ctx.Err(); ctxErr != nil { + if contended { + // The context ended between two attempts, after a claim was found held: keep that + // conflict visible beside the context error, as the wait-expiry path does. + return nil, r.heldError(ctx, key, resource, ctxErr) + } return nil, ctxErr } handle, err := r.tryClaim(ctx, key, &doc, raw) if handle != nil || err != nil { return handle, err } - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, r.heldError(ctx, key, resource, ctxErr) - } - if !time.Now().Before(deadline) { - return nil, r.heldError(ctx, key, resource, nil) - } - if err := sleep(ctx, r.settings.lockPoll); err != nil { - return nil, r.heldError(ctx, key, resource, err) + contended = true + if err := r.awaitRetry(ctx, key, resource, deadline); err != nil { + return nil, err } } } From 4455fa9e435190a4ed257967630ec031c3cc5c8b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 21:10:21 -0600 Subject: [PATCH 52/53] fix(artifact): address the CodeRabbit findings on the S3 and OCI repositories - Refuse a lock claim larger than readClaim can read before the conditional PUT, so every claim that is created can be read back and recovered. - Reload a reference whose journal is missing and derive its claim from its current state, so a completed reference still blocks an overlapping reservation. - Keep an attachment's unnamed root when its last name is removed, as when a tag moves. Each change has a regression test. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/s3/edge_test.go | 31 +++++++++++++++++++++++++++++++ pkg/artifact/s3/lock.go | 5 +++++ pkg/artifact/s3/lock_test.go | 13 +++++++++++++ pkg/artifact/s3/reserve.go | 18 +++++++++++++++++- pkg/oci/transport_layout.go | 11 +++++++++++ pkg/oci/transport_subject_test.go | 25 +++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/s3/edge_test.go b/pkg/artifact/s3/edge_test.go index 32779d8b527..22c5e12a6c1 100644 --- a/pkg/artifact/s3/edge_test.go +++ b/pkg/artifact/s3/edge_test.go @@ -273,6 +273,37 @@ func TestDelete_PartialFailureLeavesGarbageNotADanglingReference(t *testing.T) { assert.False(t, exists, "recovery deletes leftovers at the old location") } +// A reference listed as pending whose journal is gone may have completed in the meantime: its current state +// decides, so the completed reference still claims its location and blocks an overlapping reservation. +func TestClaimOf_ReloadsAPendingReferenceWhoseJournalIsGone(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + state, err := repo.loadReference(context.Background(), layout.ReferenceKey(testName, testVersion)) + require.NoError(t, err) + stale := *state.ref + stale.State = layout.StatePending + require.NoError(t, repo.deleteObject(context.Background(), layout.OperationPath(stale.OperationID))) + + claim, err := repo.claimOf(context.Background(), &stale) + + require.NoError(t, err) + require.NotNil(t, claim, "the completed reference still holds its location") + assert.Contains(t, claim.Paths, "team/app/1.0.0") +} + +// A reference that no longer exists at all holds no claim. +func TestClaimOf_AVanishedReferenceHoldsNoClaim(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + gone := layout.Reference{Name: "gone/app", Version: "1.0.0", State: layout.StatePending, OperationID: "op-gone"} + + claim, err := repo.claimOf(context.Background(), &gone) + + require.NoError(t, err) + assert.Nil(t, claim) +} + // A lost response to the reference delete is settled by reading the reference. func TestDelete_LostResponseIsReconciled(t *testing.T) { spec := noneSpec().fresh(t) diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index a957502e7ef..c0870fbc8c9 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -89,6 +89,11 @@ func (r *Repository) acquire(ctx context.Context, resource, owner, operationID s if err != nil { return nil, fmt.Errorf("%w: encode lock claim: %w", errUtils.ErrArtifactUploadFailed, err) } + // readClaim refuses a claim object larger than maxClaimBytes, so a larger one would be created and then + // be impossible to read, diagnose, or recover. Refuse it before the conditional PUT. + if int64(len(raw)) > maxClaimBytes { + return nil, fmt.Errorf("%w: the lock claim for %q is %d bytes, over the %d byte limit", errUtils.ErrArtifactTooLarge, resource, len(raw), maxClaimBytes) + } key := layout.LockPath(resource) deadline := time.Now().Add(r.settings.lockWait) contended := false diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 6ca1279a569..fcc059b5c13 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -3,6 +3,7 @@ package s3 import ( "context" "encoding/json" + "strings" "sync" "sync/atomic" "testing" @@ -417,3 +418,15 @@ func TestFenceAdoptsOriginalClaimThatWonTheRace(t *testing.T) { assert.Equal(t, doc.Token, handle.Token()) require.NoError(t, handle.Release(ctx)) } + +// A claim that readClaim would refuse as too large is refused before it is written, so every claim that is +// created can be read back and recovered. +func TestLock_RefusesAClaimLargerThanItCanBeRead(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + + _, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: lockResource, Owner: strings.Repeat("o", int(maxClaimBytes)+1)}) + + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + assert.Empty(t, lockObjects(spec.fake), "no claim object was created") +} diff --git a/pkg/artifact/s3/reserve.go b/pkg/artifact/s3/reserve.go index a5d9ddb3c33..a375223bd12 100644 --- a/pkg/artifact/s3/reserve.go +++ b/pkg/artifact/s3/reserve.go @@ -141,14 +141,30 @@ func (r *Repository) claimOf(ctx context.Context, ref *layout.Reference) (*layou return r.completedClaim(ctx, ref, &base) } op, err := r.loadOperation(ctx, ref.OperationID) - if err != nil || op == nil { + if err != nil { return nil, err } + if op == nil { + // The reference was listed as pending, but a missing journal is also what a completed reference + // leaves behind once its cleanup finished. Its current state decides: a completed reference still + // claims its location, so an overlapping reservation cannot take it. + return r.claimOfCurrent(ctx, ref, &base) + } base.OperationID = op.OperationID base.Paths = append(claimPaths(op.Packaging, op.Location), journalClaimPaths(op)...) return &base, nil } +// claimOfCurrent reloads a reference whose journal is missing and derives its claim from the state it has +// now. A reference that is gone, or still pending without a journal, holds no claim. +func (r *Repository) claimOfCurrent(ctx context.Context, listed *layout.Reference, base *layout.Claim) (*layout.Claim, error) { + current, err := r.loadReference(ctx, listed.Key()) + if err != nil || current == nil || !current.ref.Available() { + return nil, err + } + return r.completedClaim(ctx, current.ref, base) +} + // completedClaim returns the claim of a completed reference: the location its representation names, // the location its journal records when the representation does not name the live payload (the // journal is then the only record of it), and the locations the journal still awaits removing. A diff --git a/pkg/oci/transport_layout.go b/pkg/oci/transport_layout.go index 01b1d41916c..93fd0127b31 100644 --- a/pkg/oci/transport_layout.go +++ b/pkg/oci/transport_layout.go @@ -335,9 +335,20 @@ func (l *Layout) Untag(ctx context.Context, reference string) error { l.mu.Lock() defer l.mu.Unlock() kept := l.index[:0:0] + var removed []ocispec.Descriptor for i := range l.index { if l.index[i].Annotations[ocispec.AnnotationRefName] != reference { kept = append(kept, l.index[i]) + } else { + removed = append(removed, l.index[i]) + } + } + // A root that loses its only name stays listed when it is an attachment, as when a tag moves away + // from it: without its own entry it would no longer be a referrer of its subject after a reopen. + for i := range removed { + var err error + if kept, err = l.keepDisplacedAttachment(ctx, kept, removed[i]); err != nil { + return err } } // Persist index.json before touching the store: a failed write leaves the handle and the file diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index f1e53ef8f29..8fb87b4495b 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -153,6 +153,31 @@ func TestResolveDoesNotTrustATagThatWasNotPersisted(t *testing.T) { assert.Equal(t, root.Digest, resolved.Digest) } +// Removing an attachment's last name must not drop the attachment from the layout: it stays an unnamed +// root, so reopening the layout still finds it as a referrer of its subject. +func TestUntagPreservesTheAttachmentRoot(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + subject := pushSubjectManifest(t, layout, nil, "subject") + attachment := pushSubjectManifest(t, layout, &subject, "attachment") + require.NoError(t, layout.Tag(ctx, attachment, "named")) + + require.NoError(t, layout.Untag(ctx, "named")) + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + assert.NotContains(t, reopened.References(), "named", "the name is gone") + referrers, err := reopened.Predecessors(ctx, subject) + require.NoError(t, err) + digests := make([]string, 0, len(referrers)) + for _, referrer := range referrers { + digests = append(digests, referrer.Digest.String()) + } + assert.Contains(t, digests, attachment.Digest.String(), "the attachment is still discoverable after reopening") +} + // A relative layout root is anchored when the layout is opened: changing the working directory afterwards // must not send index commits to a different directory than the one holding the blobs. func TestOpenLayoutAnchorsARelativeRoot(t *testing.T) { From e953f75f23f2151152ee180eef9f4645a0c3a955 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 18:40:36 -0600 Subject: [PATCH 53/53] fix(artifact): preserve remote backend merge resolutions after rebase --- pkg/artifact/oci/backend.go | 18 +++++--- pkg/artifact/oci/classify.go | 54 +++++++++++++++-------- pkg/artifact/oci/copy.go | 58 ++++++++++++++++++------- pkg/artifact/oci/delete.go | 6 ++- pkg/artifact/oci/delete_test.go | 16 +++---- pkg/artifact/oci/list.go | 14 ------ pkg/artifact/oci/repository.go | 37 +++++++++------- pkg/artifact/oci/sign.go | 7 ++- pkg/artifact/options.go | 26 +---------- pkg/artifact/options_test.go | 26 +++++++---- pkg/artifact/redact_test.go | 10 +++++ pkg/artifact/s3/evidence.go | 39 ++++++++++------- pkg/artifact/s3/lock.go | 22 +++++++--- pkg/artifact/s3/lock_test.go | 2 +- pkg/artifact/s3/resolve.go | 6 ++- pkg/artifact/s3/service_test.go | 1 + pkg/artifact/service/mutate.go | 11 +++-- pkg/oci/transport_subject_test.go | 72 +++++++++++++++---------------- 18 files changed, 247 insertions(+), 178 deletions(-) diff --git a/pkg/artifact/oci/backend.go b/pkg/artifact/oci/backend.go index 942ae1c0421..8e3f714e3a9 100644 --- a/pkg/artifact/oci/backend.go +++ b/pkg/artifact/oci/backend.go @@ -154,13 +154,7 @@ func (b *registryBackend) names(ctx context.Context) ([]string, error) { } repositories, err := catalog.Repositories(ctx, b.registry) if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - if atmosoci.IsCatalogUnavailable(err) { - return nil, b.unsupportedEnumeration(fmt.Sprintf("the registry did not provide a repository catalog (%v)", err)) - } - return nil, fmt.Errorf("%w: list registry catalog: %w", errUtils.ErrArtifactListFailed, err) + return nil, b.catalogError(ctx, err) } scope := "" if b.prefix != "" { @@ -178,6 +172,16 @@ func (b *registryBackend) names(ctx context.Context) ([]string, error) { return names, nil } +func (b *registryBackend) catalogError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + if atmosoci.IsCatalogUnavailable(err) { + return b.unsupportedEnumeration(fmt.Sprintf("the registry did not provide a repository catalog (%v)", err)) + } + return fmt.Errorf("%w: list registry catalog: %w", errUtils.ErrArtifactListFailed, err) +} + func (b *registryBackend) unsupportedEnumeration(reason string) error { return artifact.NewUnsupportedCapability(b.registry+"/"+b.prefix, "list", "enumerate/collection", reason+"; listing the versions of a named artifact and every other direct named operation still works without a catalog, or configure an artifact catalog for discovery") diff --git a/pkg/artifact/oci/classify.go b/pkg/artifact/oci/classify.go index f2b8c253d96..7542a759ab6 100644 --- a/pkg/artifact/oci/classify.go +++ b/pkg/artifact/oci/classify.go @@ -113,36 +113,52 @@ func classify(desc ocispec.Descriptor, raw []byte) (*root, error) { loaded := &root{desc: desc, raw: raw} switch { case isIndexMediaType(desc.MediaType): - if err := json.Unmarshal(raw, &loaded.index); err != nil { - return nil, fmt.Errorf("%w: index %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) - } - if err := requireDescriptorDigests(desc, loaded.index.Manifests); err != nil { + if err := classifyIndex(loaded); err != nil { return nil, err } - loaded.class = classIndex - if hasPluginAnnotations(loaded.index.Annotations) { - loaded.class = classPlugin - } case isManifestMediaType(desc.MediaType): - if err := json.Unmarshal(raw, &loaded.manifest); err != nil { - return nil, fmt.Errorf("%w: manifest %s: %w", errUtils.ErrArtifactIntegrityFailed, desc.Digest, err) - } - if err := requireDescriptorDigests(desc, []ocispec.Descriptor{loaded.manifest.Config}, loaded.manifest.Layers); err != nil { + if err := classifyManifest(loaded); err != nil { return nil, err } - loaded.class = classImage - switch loaded.manifest.Config.MediaType { - case artifact.MediaTypeConfig: - loaded.class = classAtmos - case pluginConfigMediaType: - loaded.class = classPlugin - } default: return nil, fmt.Errorf("%w: %s has unsupported media type %q", errUtils.ErrArtifactRecordInvalid, desc.Digest, desc.MediaType) } return loaded, nil } +// classifyIndex decodes an index root and sets its class from the release annotations. +func classifyIndex(loaded *root) error { + if err := json.Unmarshal(loaded.raw, &loaded.index); err != nil { + return fmt.Errorf("%w: index %s: %w", errUtils.ErrArtifactIntegrityFailed, loaded.desc.Digest, err) + } + if err := requireDescriptorDigests(loaded.desc, loaded.index.Manifests); err != nil { + return err + } + loaded.class = classIndex + if hasPluginAnnotations(loaded.index.Annotations) { + loaded.class = classPlugin + } + return nil +} + +// classifyManifest decodes an image manifest root and sets its class from the config media type. +func classifyManifest(loaded *root) error { + if err := json.Unmarshal(loaded.raw, &loaded.manifest); err != nil { + return fmt.Errorf("%w: manifest %s: %w", errUtils.ErrArtifactIntegrityFailed, loaded.desc.Digest, err) + } + if err := requireDescriptorDigests(loaded.desc, []ocispec.Descriptor{loaded.manifest.Config}, loaded.manifest.Layers); err != nil { + return err + } + loaded.class = classImage + switch loaded.manifest.Config.MediaType { + case artifact.MediaTypeConfig: + loaded.class = classAtmos + case pluginConfigMediaType: + loaded.class = classPlugin + } + return nil +} + // requireDescriptorDigests rejects a root that references a descriptor whose digest is malformed. A // registry serves whatever bytes it holds, so an empty or truncated digest is an integrity failure // of the root, caught here before a traversal or a read can ask the digest library for it. diff --git a/pkg/artifact/oci/copy.go b/pkg/artifact/oci/copy.go index c240b7769ad..a5a12768afa 100644 --- a/pkg/artifact/oci/copy.go +++ b/pkg/artifact/oci/copy.go @@ -89,17 +89,12 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto if err := r.requireImportable(ctx, src, loaded, ref); err != nil { return nil, err } - release, err := r.guardLayout(ctx) - if err != nil { - return nil, err - } - defer release() - releaseReference, err := r.guardReference(ctx, ref, options.Lock) + release, err := r.lockTransfer(ctx, ref, options.Lock) if err != nil { return nil, err } defer func() { - if releaseErr := releaseReference(); releaseErr != nil { + if releaseErr := release(); releaseErr != nil { err = errors.Join(err, releaseErr) } }() @@ -116,14 +111,8 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto if err != nil { return nil, err } - { - _, err = atmosoci.ExtendedCopy(ctx, - atmosoci.GraphSource{Store: src, Reference: loaded.desc.Digest.String()}, - atmosoci.Destination{Store: dst, Reference: reference}, - atmosoci.WithCopyConcurrency(r.settings.concurrency)) - if err != nil { - return nil, fmt.Errorf("%w: copy %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) - } + if err := r.copyGraph(ctx, src, dst, loaded, ref); err != nil { + return nil, err } pinned, err := r.pinCopied(ctx, dst, loaded, ref) if err != nil { @@ -132,6 +121,37 @@ func (r *Repository) transfer(ctx context.Context, src atmosoci.ReadOnlyGraphSto return r.transferred(ctx, &pinned, ref, outcome, options) } +// lockTransfer takes the layout write guard and then the reference lock, and returns one release +// that frees them in the reverse order. When the reference lock cannot be taken the layout guard +// is released before the error is returned. +func (r *Repository) lockTransfer(ctx context.Context, ref artifact.Ref, held artifact.LockHandle) (func() error, error) { + releaseLayout, err := r.guardLayout(ctx) + if err != nil { + return nil, err + } + releaseReference, err := r.guardReference(ctx, ref, held) + if err != nil { + releaseLayout() + return nil, err + } + return func() error { + defer releaseLayout() + return releaseReference() + }, nil +} + +// copyGraph copies the root and its referrers from src into dst under ref. +func (r *Repository) copyGraph(ctx context.Context, src atmosoci.ReadOnlyGraphStore, dst store, loaded *root, ref artifact.Ref) error { + _, err := atmosoci.ExtendedCopy(ctx, + atmosoci.GraphSource{Store: src, Reference: loaded.desc.Digest.String()}, + atmosoci.Destination{Store: dst, Reference: r.be.reference(ref.Name, ref.Version)}, + atmosoci.WithCopyConcurrency(r.settings.concurrency)) + if err != nil { + return fmt.Errorf("%w: copy %s: %w", errUtils.ErrArtifactUploadFailed, ref, err) + } + return nil +} + // guardReference keeps the registry copy's read-decide-write sequence under one lock. A registry // tag has no compare-and-swap, and the tag is written unconditionally once the graph has been // transferred, so the policy decision made from the tag read is only as good as the exclusion that @@ -333,3 +353,11 @@ func layoutRoot(directory, selector string) (ocispec.Descriptor, error) { } return ocispec.Descriptor{MediaType: matches[0].MediaType, Digest: matches[0].Digest, Size: matches[0].Size}, nil } + +func matchesLayoutSelector(entry ocispec.Descriptor, selector string) bool { + name := entry.Annotations[ocispec.AnnotationRefName] + if selector == "" { + return name != "" + } + return entry.Digest.String() == selector || name == selector +} diff --git a/pkg/artifact/oci/delete.go b/pkg/artifact/oci/delete.go index 89a71b9cfe4..bed003a7d4c 100644 --- a/pkg/artifact/oci/delete.go +++ b/pkg/artifact/oci/delete.go @@ -25,7 +25,7 @@ import ( func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { defer perf.Track(nil, "oci.Repository.Delete")() - if options.Scope == artifact.DeleteReference && r.be.deleteScope() != artifact.DeleteReference { + if referenceScopeUnsupported(options.Scope, r.be.deleteScope()) { return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/tag", "the registry can only delete a manifest") } name := pinned.Ref.Name @@ -69,6 +69,10 @@ func (r *Repository) recheckHolders(ctx context.Context, st store, name string, return r.referencesTo(ctx, st, name, target) } +func referenceScopeUnsupported(requested, supported artifact.DeleteSemantics) bool { + return requested == artifact.DeleteReference && supported != artifact.DeleteReference +} + // deleteTag removes one tag, refusing when that would also remove a sibling reference. func (r *Repository) deleteTag(ctx context.Context, st store, loaded *root, ref artifact.Ref, holders []artifact.Ref) (*artifact.DeleteResult, error) { holders, err := r.recheckHolders(ctx, st, ref.Name, loaded.desc.Digest, holders) diff --git a/pkg/artifact/oci/delete_test.go b/pkg/artifact/oci/delete_test.go index a61f7720898..fc989c0e877 100644 --- a/pkg/artifact/oci/delete_test.go +++ b/pkg/artifact/oci/delete_test.go @@ -13,14 +13,14 @@ import ( "github.com/cloudposse/atmos/pkg/artifact" ) -// alias tags an existing manifest with another version, creating a sibling reference. -func alias(t *testing.T, repo *Repository, pinned artifact.PinnedReference, version string) { +// aliasStable tags an existing manifest as "stable", creating a sibling reference. +func aliasStable(t *testing.T, repo *Repository, pinned artifact.PinnedReference) { t.Helper() st, err := repo.be.open(context.Background(), pinned.Ref.Name, true) require.NoError(t, err) desc, ok := repo.descriptorFor(pinned.Ref.Name, pinned.ResolvedDigest) require.True(t, ok) - require.NoError(t, st.Tag(context.Background(), desc, repo.be.reference(pinned.Ref.Name, version))) + require.NoError(t, st.Tag(context.Background(), desc, repo.be.reference(pinned.Ref.Name, "stable"))) } func resolves(repo *Repository, name, version string) bool { @@ -31,7 +31,7 @@ func resolves(repo *Repository, name, version string) bool { func TestRegistryTagDeleteRefusesWhenItWouldRemoveASiblingTag(t *testing.T) { repo := newRegistryRepo(t, newRegistry(t, nil)) first := publish(t, repo, publishSpec{content: "shared manifest"}) - alias(t, repo, first.Pinned, "stable") + aliasStable(t, repo, first.Pinned) pinned, err := repo.Resolve(context.Background(), artifact.Ref{Name: appName, Version: appVersion}) require.NoError(t, err) @@ -45,7 +45,7 @@ func TestRegistryTagDeleteRefusesWhenItWouldRemoveASiblingTag(t *testing.T) { func TestRegistryDigestDeleteIsManifestScopedAndReportsAffectedReferences(t *testing.T) { repo := newRegistryRepo(t, newRegistry(t, nil)) first := publish(t, repo, publishSpec{content: "shared manifest"}) - alias(t, repo, first.Pinned, "stable") + aliasStable(t, repo, first.Pinned) publish(t, repo, publishSpec{version: "2.0.0", content: "unrelated"}) byDigest := artifact.PinnedReference{Ref: artifact.Ref{Name: appName, Digest: first.Pinned.ResolvedDigest}, ResolvedDigest: first.Pinned.ResolvedDigest} @@ -87,7 +87,7 @@ func mustResolve(t *testing.T, repo *Repository, version string) artifact.Pinned func TestLayoutTagDeleteRemovesOnlyTheNamedReference(t *testing.T) { repo := newLayoutRepo(t) first := publish(t, repo, publishSpec{content: "shared"}) - alias(t, repo, first.Pinned, "stable") + aliasStable(t, repo, first.Pinned) pinned := mustResolve(t, repo, appVersion) result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteReference}) @@ -168,7 +168,7 @@ func TestRegistryTagDeleteRechecksSiblingsRightBeforeRemoving(t *testing.T) { repo.be = hooked hooked.after = func(scan int) { if scan == 1 { - alias(t, repo, first.Pinned, "stable") + aliasStable(t, repo, first.Pinned) } } @@ -188,7 +188,7 @@ func TestRegistryDigestDeleteReportsTagsAddedAfterTheFirstScan(t *testing.T) { repo.be = hooked hooked.after = func(scan int) { if scan == 1 { - alias(t, repo, first.Pinned, "stable") + aliasStable(t, repo, first.Pinned) } } diff --git a/pkg/artifact/oci/list.go b/pkg/artifact/oci/list.go index 0011bced11f..b5fbde30018 100644 --- a/pkg/artifact/oci/list.go +++ b/pkg/artifact/oci/list.go @@ -70,20 +70,6 @@ func (c listCursor) skipsVersion(name, version string) bool { return c.set && name == c.name && version <= c.version } -// remaining returns the entries of name that follow the cursor. -func (c listCursor) remaining(name string, items []artifact.ListEntry) []artifact.ListEntry { - if !c.set || name != c.name { - return items - } - var rest []artifact.ListEntry - for index := range items { - if items[index].Version > c.version { - rest = append(rest, items[index]) - } - } - return rest -} - // namesFor returns the names to list: the requested one, or the enumerated collection. func (r *Repository) namesFor(ctx context.Context, name string) ([]string, error) { if name != "" { diff --git a/pkg/artifact/oci/repository.go b/pkg/artifact/oci/repository.go index 6e1853f4956..5dc056a9245 100644 --- a/pkg/artifact/oci/repository.go +++ b/pkg/artifact/oci/repository.go @@ -140,22 +140,7 @@ func (r *Repository) track(name string, loaded *root) { return } if len(r.tracked) >= manifestCacheEntries { - old := r.tracked[0] - r.tracked = r.tracked[1:] - delete(r.pinned, cacheKey(old.name, old.desc)) - for _, key := range old.keys { - items := r.roots[key][:0] - for _, item := range r.roots[key] { - if item.name != old.name || item.desc.Digest != old.desc.Digest { - items = append(items, item) - } - } - if len(items) == 0 { - delete(r.roots, key) - } else { - r.roots[key] = items - } - } + r.evictTrackedRoot() } r.tracked = append(r.tracked, trackedRoot{name: name, desc: loaded.desc, keys: keys}) for _, key := range keys { @@ -171,6 +156,26 @@ func (r *Repository) track(name string, loaded *root) { r.pinned[keyName] = loaded.desc } +// evictTrackedRoot removes the oldest root and all of its lookup entries; caller holds r.mu. +func (r *Repository) evictTrackedRoot() { + old := r.tracked[0] + r.tracked = r.tracked[1:] + delete(r.pinned, cacheKey(old.name, old.desc)) + for _, key := range old.keys { + items := r.roots[key][:0] + for _, item := range r.roots[key] { + if item.name != old.name || item.desc.Digest != old.desc.Digest { + items = append(items, item) + } + } + if len(items) == 0 { + delete(r.roots, key) + } else { + r.roots[key] = items + } + } +} + // descriptorFor returns the descriptor remembered for a resolved digest of a name. func (r *Repository) descriptorFor(name string, dgst digest.Digest) (ocispec.Descriptor, bool) { r.mu.Lock() diff --git a/pkg/artifact/oci/sign.go b/pkg/artifact/oci/sign.go index c4153fd3193..a8fb4bbfdd9 100644 --- a/pkg/artifact/oci/sign.go +++ b/pkg/artifact/oci/sign.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -99,7 +100,7 @@ func bindSubject(subject artifact.SignatureSubject, loaded *root) error { return fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, loaded.manifest.Config.Size) } case artifact.SubjectOCIRoot: - if loaded.desc.Digest != subject.Digest || loaded.desc.MediaType != subject.MediaType || (subject.Size != 0 && loaded.desc.Size != subject.Size) { + if !matchesRootSubject(subject, loaded.desc) { return fmt.Errorf("%w: %s is not the manifest or index digest of the located root", errUtils.ErrArtifactRefInvalid, subject.Digest) } default: @@ -108,6 +109,10 @@ func bindSubject(subject artifact.SignatureSubject, loaded *root) error { return nil } +func matchesRootSubject(subject artifact.SignatureSubject, desc ocispec.Descriptor) bool { + return desc.Digest == subject.Digest && desc.MediaType == subject.MediaType && (subject.Size == 0 || desc.Size == subject.Size) +} + // Sign implements [artifact.Repository]. The signer produces evidence over the subject; the // repository stores it as a referrer of the addressed manifest. A Record signature is bound to // the manifest's config descriptor; an image or plugin signature is bound to the original diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index fa79fad86a7..8c3d3dcf69e 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -226,7 +226,7 @@ func (o *OCIRegistryOptions) Validate() error { if o.Registry == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) } else if !validRegistryHost(o.Registry) { - errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, redactRegistry(o.Registry))) + errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Registry))) } errs = append(errs, validatePrefix(o.Prefix)) if o.Prefix != "" { @@ -242,30 +242,6 @@ func (o *OCIRegistryOptions) Validate() error { const registryPortBits = 16 -// redactRegistry returns registry safe to repeat in a diagnostic: when its authority embeds -// userinfo (user:password@host), the userinfo is replaced with a placeholder so a credential -// typed into the registry option never reaches an error message. -func redactRegistry(registry string) string { - rest := registry - prefix := "" - if scheme, after, found := strings.Cut(registry, "://"); found { - prefix, rest = scheme+"://", after - } else if after, found := strings.CutPrefix(registry, "//"); found { - // A scheme-relative value (//user:secret@host) carries its authority after the slashes. - prefix, rest = "//", after - } - authority, tail, _ := strings.Cut(rest, "/") - at := strings.LastIndex(authority, "@") - if at < 0 { - return registry - } - redacted := prefix + "REDACTED@" + authority[at+1:] - if strings.Contains(rest, "/") { - redacted += "/" + tail - } - return redacted -} - func validRegistryHost(host string) bool { if !registryHostPattern.MatchString(host) { return false diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index 3c0fa455744..434cccf2783 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -260,19 +260,24 @@ func TestValidateEndpoint_NeverEchoesCredentials(t *testing.T) { // TestOCIRegistryOptionsValidate_NeverEchoesCredentials guards that a registry diagnostic repeats // the configured value only with any embedded userinfo redacted. func TestOCIRegistryOptionsValidate_NeverEchoesCredentials(t *testing.T) { - const secret = "s3cr3t-password" + const ( + secret = "s3cr3t-password" + user = "svc-deploy" + ) tests := []struct { name string registry string wantErr string }{ - {name: "userinfo only", registry: "user:" + secret + "@registry.example.com", wantErr: "registry.example.com"}, - {name: "scheme and userinfo", registry: "https://user:" + secret + "@registry.example.com", wantErr: "https://REDACTED@registry.example.com"}, - {name: "userinfo and path", registry: "https://user:" + secret + "@registry.example.com/acme/atmos", wantErr: "https://REDACTED@registry.example.com/acme/atmos"}, - {name: "token without password", registry: secret + "@registry.example.com:5000", wantErr: "REDACTED@registry.example.com:5000"}, - {name: "scheme-relative userinfo", registry: "//user:" + secret + "@registry.example.com", wantErr: "//REDACTED@registry.example.com"}, - {name: "scheme-relative userinfo and path", registry: "//user:" + secret + "@registry.example.com/acme/atmos", wantErr: "//REDACTED@registry.example.com/acme/atmos"}, - {name: "scheme-relative token without password", registry: "//" + secret + "@registry.example.com:5000", wantErr: "//REDACTED@registry.example.com:5000"}, + {name: "userinfo only", registry: user + ":" + secret + "@registry.example.com", wantErr: RedactedValue}, + {name: "scheme and userinfo", registry: "https://" + user + ":" + secret + "@registry.example.com", wantErr: "https://registry.example.com"}, + {name: "userinfo and path", registry: "https://" + user + ":" + secret + "@registry.example.com/acme/atmos", wantErr: "https://registry.example.com/acme/atmos"}, + {name: "token without password", registry: secret + "@registry.example.com:5000", wantErr: RedactedValue}, + {name: "scheme-relative userinfo", registry: "//" + user + ":" + secret + "@registry.example.com", wantErr: "//registry.example.com"}, + {name: "scheme-relative userinfo and path", registry: "//" + user + ":" + secret + "@registry.example.com/acme/atmos", wantErr: "//registry.example.com/acme/atmos"}, + {name: "scheme-relative token without password", registry: "//" + secret + "@registry.example.com:5000", wantErr: "//registry.example.com:5000"}, + {name: "malformed escape with userinfo", registry: "https://" + user + ":" + secret + "@registry.example.com/%zz", wantErr: RedactedValue}, + {name: "unparsable host with userinfo", registry: "https://" + user + ":" + secret + "@exa mple.com", wantErr: RedactedValue}, {name: "scheme-relative without credentials stays visible", registry: "//registry.example.com", wantErr: "//registry.example.com"}, {name: "scheme without credentials stays visible", registry: "https://registry.example.com", wantErr: "https://registry.example.com"}, } @@ -281,7 +286,10 @@ func TestOCIRegistryOptionsValidate_NeverEchoesCredentials(t *testing.T) { err := (&OCIRegistryOptions{Registry: tt.registry}).Validate() require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) assert.Contains(t, err.Error(), tt.wantErr) - assert.NotContains(t, err.Error(), secret) + // The security property holds whatever the redacted format is: neither the secret nor + // the user name appears anywhere in the error text. + assert.False(t, strings.Contains(err.Error(), secret), "the secret must never be echoed: %q", err.Error()) + assert.False(t, strings.Contains(err.Error(), user), "the user name must never be echoed: %q", err.Error()) }) } } diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go index dc0dae78518..b47883181ba 100644 --- a/pkg/artifact/redact_test.go +++ b/pkg/artifact/redact_test.go @@ -38,12 +38,22 @@ func TestRedactURL(t *testing.T) { {name: "scheme-less userinfo", raw: "user:hunter2@ghcr.io", want: RedactedValue}, {name: "scheme-less userinfo and port", raw: "user:hunter2@ghcr.io:5000", want: RedactedValue}, {name: "malformed without an at sign is not credential bearing", raw: "oci://ghcr.io/%zz", want: "oci://ghcr.io/%zz"}, + {name: "scheme and userinfo", raw: "https://svc-deploy:hunter2@registry.example.com", want: "https://registry.example.com"}, + {name: "scheme and userinfo with path", raw: "https://svc-deploy:hunter2@registry.example.com/acme/atmos", want: "https://registry.example.com/acme/atmos"}, + {name: "scheme-relative userinfo", raw: "//svc-deploy:hunter2@registry.example.com", want: "//registry.example.com"}, + {name: "scheme-relative userinfo with path", raw: "//svc-deploy:hunter2@registry.example.com/acme/atmos", want: "//registry.example.com/acme/atmos"}, + {name: "scheme-relative token without password", raw: "//hunter2@registry.example.com:5000", want: "//registry.example.com:5000"}, + {name: "token without password and port", raw: "hunter2@registry.example.com:5000", want: RedactedValue}, + {name: "malformed escape with scheme-relative userinfo", raw: "//svc-deploy:hunter2@registry.example.com/%zz", want: RedactedValue}, + {name: "unparsable host with userinfo", raw: "https://svc-deploy:hunter2@exa mple.com", want: RedactedValue}, + {name: "scheme-relative without credentials", raw: "//registry.example.com", want: "//registry.example.com"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := RedactURL(tt.raw) assert.Equal(t, tt.want, got) assert.NotContains(t, got, "hunter2") + assert.NotContains(t, got, "svc-deploy") }) } } diff --git a/pkg/artifact/s3/evidence.go b/pkg/artifact/s3/evidence.go index 7c23fd17fb9..9b1e6794978 100644 --- a/pkg/artifact/s3/evidence.go +++ b/pkg/artifact/s3/evidence.go @@ -39,11 +39,8 @@ func (r *Repository) loadEvidence(ctx context.Context, subject digest.Digest) ([ ) attempted := 0 for index := range objects { - stem, ok := strings.CutSuffix(strings.TrimPrefix(objects[index].Key, dir), ".json") - if !ok || strings.Contains(stem, "/") { - continue - } - if digest.NewDigestFromEncoded(digest.SHA256, stem).Validate() != nil { + stem, ok := evidenceObjectStem(objects[index].Key, dir) + if !ok { continue } if attempted >= maxEvidenceFiles { @@ -67,6 +64,11 @@ func (r *Repository) loadEvidence(ctx context.Context, subject digest.Digest) ([ return loaded, diags, nil } +func evidenceObjectStem(key, dir string) (string, bool) { + stem, ok := strings.CutSuffix(strings.TrimPrefix(key, dir), ".json") + return stem, ok && !strings.Contains(stem, "/") && digest.NewDigestFromEncoded(digest.SHA256, stem).Validate() == nil +} + func (r *Repository) readEvidenceObject(ctx context.Context, key, hexDigest string) (*layout.Evidence, error, error) { want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) if err := want.Validate(); err != nil { @@ -192,18 +194,9 @@ func (r *Repository) storeEvidence(ctx context.Context, subject artifact.Signatu return artifact.SignatureEvidence{}, err } if existing == nil { - loaded, diagnostics, err := r.loadEvidence(ctx, subject.Digest) - if err != nil { + if err := r.checkEvidenceCapacity(ctx, subject.Digest); err != nil { return artifact.SignatureEvidence{}, err } - if len(loaded) >= maxEvidenceFiles { - return artifact.SignatureEvidence{}, fmt.Errorf("%w: evidence limit reached", errUtils.ErrArtifactTooLarge) - } - for _, diagnostic := range diagnostics { - if diagnostic.Code == "evidence-truncated" { - return artifact.SignatureEvidence{}, fmt.Errorf("%w: %s", errUtils.ErrArtifactTooLarge, diagnostic.Message) - } - } } if err := r.putImmutable(ctx, key, raw, contentTypeJSON, layout.MaxEvidenceFileBytes); err != nil { return artifact.SignatureEvidence{}, wrapWrite("store evidence", err) @@ -211,6 +204,22 @@ func (r *Repository) storeEvidence(ctx context.Context, subject artifact.Signatu return evidence.Reference(r.uri(key)), nil } +func (r *Repository) checkEvidenceCapacity(ctx context.Context, subject digest.Digest) error { + loaded, diagnostics, err := r.loadEvidence(ctx, subject) + if err != nil { + return err + } + if len(loaded) >= maxEvidenceFiles { + return fmt.Errorf("%w: evidence limit reached", errUtils.ErrArtifactTooLarge) + } + for _, diagnostic := range diagnostics { + if diagnostic.Code == "evidence-truncated" { + return fmt.Errorf("%w: %s", errUtils.ErrArtifactTooLarge, diagnostic.Message) + } + } + return nil +} + // deleteEvidence removes every evidence object stored for a Record subject. func (r *Repository) deleteEvidence(ctx context.Context, subject digest.Digest) error { objects, err := r.listPrefix(ctx, layout.SignatureDir(artifact.SubjectRecord, subject)+"/") diff --git a/pkg/artifact/s3/lock.go b/pkg/artifact/s3/lock.go index c0870fbc8c9..30512db9a62 100644 --- a/pkg/artifact/s3/lock.go +++ b/pkg/artifact/s3/lock.go @@ -85,14 +85,9 @@ func (r *Repository) acquire(ctx context.Context, resource, owner, operationID s SchemaVersion: layout.SchemaVersion, Resource: resource, Owner: owner, OperationID: operationID, Token: uuid.NewString(), AcquiredAt: time.Now().UTC().Format(time.RFC3339), } - raw, err := json.Marshal(&doc) + raw, err := encodeClaim(resource, &doc) if err != nil { - return nil, fmt.Errorf("%w: encode lock claim: %w", errUtils.ErrArtifactUploadFailed, err) - } - // readClaim refuses a claim object larger than maxClaimBytes, so a larger one would be created and then - // be impossible to read, diagnose, or recover. Refuse it before the conditional PUT. - if int64(len(raw)) > maxClaimBytes { - return nil, fmt.Errorf("%w: the lock claim for %q is %d bytes, over the %d byte limit", errUtils.ErrArtifactTooLarge, resource, len(raw), maxClaimBytes) + return nil, err } key := layout.LockPath(resource) deadline := time.Now().Add(r.settings.lockWait) @@ -243,6 +238,19 @@ func (r *Repository) adopt(ctx context.Context, key string, doc *claimDoc) (*cla return handle, nil } +// encodeClaim serializes a claim. The reader (readClaim) refuses a claim object larger than maxClaimBytes, so a larger one +// would be created and then be impossible to read, diagnose, or recover. Such a claim is refused before the conditional PUT. +func encodeClaim(resource string, doc *claimDoc) ([]byte, error) { + raw, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("%w: encode lock claim: %w", errUtils.ErrArtifactUploadFailed, err) + } + if int64(len(raw)) > maxClaimBytes { + return nil, fmt.Errorf("%w: the lock claim for %q is %d bytes, over the %d byte limit", errUtils.ErrArtifactTooLarge, resource, len(raw), maxClaimBytes) + } + return raw, nil +} + // readClaim reads a claim object. A missing claim returns (nil, "", nil); an unreadable one is // returned as a claim with no token so diagnostics can still name the object. func (r *Repository) readClaim(ctx context.Context, key string) (*claimDoc, string, error) { diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index fcc059b5c13..5b9e353809b 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -247,7 +247,7 @@ func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = peer.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) + _, err = peerOf(t, spec).Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "owner-b"}) require.ErrorIs(t, err, context.Canceled) require.NoError(t, held.Release(context.Background())) assert.Empty(t, lockObjects(spec.fake), "only the holder's claim ever existed") diff --git a/pkg/artifact/s3/resolve.go b/pkg/artifact/s3/resolve.go index d401b9e2180..800b0a4e7e6 100644 --- a/pkg/artifact/s3/resolve.go +++ b/pkg/artifact/s3/resolve.go @@ -177,7 +177,7 @@ func (r *Repository) listEntry(ctx context.Context, candidate *listCandidate, la if err != nil { return artifact.ListEntry{}, false, err } - if current == nil || !current.ref.Available() || current.ref.Generation != candidate.generation || current.ref.RecordDigest != candidate.digest { + if !candidateCurrent(current, candidate) { return artifact.ListEntry{}, false, nil } if recordErr != nil { @@ -203,6 +203,10 @@ func (r *Repository) listEntry(ctx context.Context, candidate *listCandidate, la }, true, nil } +func candidateCurrent(current *refState, candidate *listCandidate) bool { + return current != nil && current.ref.Available() && current.ref.Generation == candidate.generation && current.ref.RecordDigest == candidate.digest +} + // descriptorsOf lists evidence descriptors. func descriptorsOf(evidence []artifact.SignatureEvidence) []ocispec.Descriptor { out := make([]ocispec.Descriptor, 0, len(evidence)) diff --git a/pkg/artifact/s3/service_test.go b/pkg/artifact/s3/service_test.go index 8930889e569..c604f71b3b6 100644 --- a/pkg/artifact/s3/service_test.go +++ b/pkg/artifact/s3/service_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" "github.com/cloudposse/atmos/pkg/artifact/fixtures" "github.com/cloudposse/atmos/pkg/artifact/local" diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go index 1fc4436f6e1..9d3900d9c5a 100644 --- a/pkg/artifact/service/mutate.go +++ b/pkg/artifact/service/mutate.go @@ -4,12 +4,14 @@ import ( "context" "fmt" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/perf" ) -// Delete implements [artifact.Service]. The reference is resolved once; deletion is scoped to the -// pinned subject within the repository's declared delete capability and guarded by the writer lock. +// Delete implements [artifact.Service]. Deletion is scoped to the pinned subject within the +// repository's declared delete capability and guarded by the writer locks. Digest references are +// inspected before locking to identify the canonical reference, then resolved again under lock. func (s *Service) Delete(ctx context.Context, request artifact.DeleteRequest) (*artifact.OperationResult, error) { defer perf.Track(nil, "service.Service.Delete")() @@ -56,7 +58,10 @@ func (s *Service) runDelete(ctx context.Context, op *operation, request *artifac if err != nil { return err } - result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleForResource(request.Repository, resource.resource)}) + if request.Ref.IsDigest() && inspection.Record != nil && canonical != inspection.Record.Name+":"+inspection.Record.Version { + return fmt.Errorf("%w: reference changed while acquiring the delete locks", errUtils.ErrArtifactConflict) + } + result, err := repo.Delete(ctx, pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: info.Capabilities.Delete, Lock: locks.handleForResource(request.Repository, canonical)}) if err != nil { return fmt.Errorf("delete %s in repository %q: %w", request.Ref, request.Repository, err) } diff --git a/pkg/oci/transport_subject_test.go b/pkg/oci/transport_subject_test.go index 8fb87b4495b..8a82ba464d9 100644 --- a/pkg/oci/transport_subject_test.go +++ b/pkg/oci/transport_subject_test.go @@ -81,6 +81,42 @@ func TestLayoutPush_RecordsOnlyManifestsThatNameASubject(t *testing.T) { } } +// An empty reference is rejected before index.json is filtered: it would otherwise drop every +// unnamed root, including attachment roots, and persist that before the store refuses the call. +func TestLayoutUntag_EmptyReferenceLeavesTheIndexUnchanged(t *testing.T) { + ctx := t.Context() + directory := filepath.Join(t.TempDir(), "layout") + layout, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + + subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` + attachment := `{"schemaVersion":2,"subject":` + subject + `}` + attachmentDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(attachment)) + require.NoError(t, layout.Push(ctx, attachmentDescriptor, strings.NewReader(attachment))) + named := `{"schemaVersion":2,"layers":[]}` + namedDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(named)) + require.NoError(t, layout.Push(ctx, namedDescriptor, strings.NewReader(named))) + require.NoError(t, layout.Tag(ctx, namedDescriptor, "app:1")) + wantRoots := []string{attachmentDescriptor.Digest.String(), namedDescriptor.Digest.String()} + require.Equal(t, wantRoots, layoutRoots(t, directory), "the unnamed attachment root and the named root are listed") + before, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + + err = layout.Untag(ctx, "") + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrOCILayout) + + after, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), "the persisted index is untouched") + assert.Equal(t, wantRoots, layoutRoots(t, directory)) + assert.Equal(t, []string{"app:1"}, layout.References()) + + reopened, err := oci.OpenLayout(ctx, directory) + require.NoError(t, err) + assert.Equal(t, []string{"app:1"}, reopened.References()) +} + // pushSubjectManifest pushes an image manifest that optionally names a subject and returns its descriptor. func pushSubjectManifest(t *testing.T, layout *oci.Layout, subject *ocispec.Descriptor, note string) ocispec.Descriptor { t.Helper() @@ -272,39 +308,3 @@ func TestExtendedCopyWithoutSubjectTraversalKeepsTheSelectedRootAndItsReferrers( require.NoError(t, err) assert.Equal(t, root.Digest, resolved.Digest) } - -// An empty reference is rejected before index.json is filtered: it would otherwise drop every -// unnamed root, including attachment roots, and persist that before the store refuses the call. -func TestLayoutUntag_EmptyReferenceLeavesTheIndexUnchanged(t *testing.T) { - ctx := t.Context() - directory := filepath.Join(t.TempDir(), "layout") - layout, err := oci.OpenLayout(ctx, directory) - require.NoError(t, err) - - subject := `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:` + strings.Repeat("a", 64) + `","size":1}` - attachment := `{"schemaVersion":2,"subject":` + subject + `}` - attachmentDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(attachment)) - require.NoError(t, layout.Push(ctx, attachmentDescriptor, strings.NewReader(attachment))) - named := `{"schemaVersion":2,"layers":[]}` - namedDescriptor := oci.Descriptor(ocispec.MediaTypeImageManifest, []byte(named)) - require.NoError(t, layout.Push(ctx, namedDescriptor, strings.NewReader(named))) - require.NoError(t, layout.Tag(ctx, namedDescriptor, "app:1")) - wantRoots := []string{attachmentDescriptor.Digest.String(), namedDescriptor.Digest.String()} - require.Equal(t, wantRoots, layoutRoots(t, directory), "the unnamed attachment root and the named root are listed") - before, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) - require.NoError(t, err) - - err = layout.Untag(ctx, "") - require.Error(t, err) - require.ErrorIs(t, err, errUtils.ErrOCILayout) - - after, err := os.ReadFile(filepath.Join(directory, ocispec.ImageIndexFile)) - require.NoError(t, err) - assert.Equal(t, string(before), string(after), "the persisted index is untouched") - assert.Equal(t, wantRoots, layoutRoots(t, directory)) - assert.Equal(t, []string{"app:1"}, layout.References()) - - reopened, err := oci.OpenLayout(ctx, directory) - require.NoError(t, err) - assert.Equal(t, []string{"app:1"}, reopened.References()) -}