From afcffcfd2016faf42b3d42a610f808d74acac879 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 10:40:01 -0500 Subject: [PATCH 01/43] feat(artifacts): extract blob transport and back CI stores with it (A05a) Co-Authored-By: Claude Sonnet 5.5 --- .golangci.yml | 4 + errors/errors.go | 25 ++ pkg/artifact/blob/blob_test.go | 179 +++++++++ pkg/artifact/blob/blobtest/cases_basic.go | 242 ++++++++++++ .../blob/blobtest/cases_conditional.go | 183 +++++++++ pkg/artifact/blob/blobtest/cases_list.go | 105 +++++ pkg/artifact/blob/blobtest/conformance.go | 116 ++++++ pkg/artifact/blob/blobtest/doc.go | 4 + pkg/artifact/blob/boundary_test.go | 45 +++ pkg/artifact/blob/capabilities.go | 62 +++ pkg/artifact/blob/doc.go | 21 + pkg/artifact/blob/errors.go | 164 ++++++++ pkg/artifact/blob/key.go | 62 +++ pkg/artifact/blob/list.go | 30 ++ pkg/artifact/blob/local/attrs.go | 45 +++ pkg/artifact/blob/local/delete.go | 51 +++ pkg/artifact/blob/local/doc.go | 18 + pkg/artifact/blob/local/errors.go | 65 ++++ pkg/artifact/blob/local/etag.go | 44 +++ pkg/artifact/blob/local/fsutil.go | 85 ++++ pkg/artifact/blob/local/get.go | 116 ++++++ pkg/artifact/blob/local/list.go | 165 ++++++++ pkg/artifact/blob/local/lock.go | 59 +++ pkg/artifact/blob/local/options.go | 38 ++ pkg/artifact/blob/local/put.go | 223 +++++++++++ pkg/artifact/blob/local/store.go | 105 +++++ pkg/artifact/blob/local/store_test.go | 144 +++++++ pkg/artifact/blob/mock_store.go | 144 +++++++ pkg/artifact/blob/s3/api.go | 41 ++ pkg/artifact/blob/s3/classify.go | 147 +++++++ pkg/artifact/blob/s3/client.go | 176 +++++++++ pkg/artifact/blob/s3/client_test.go | 198 ++++++++++ pkg/artifact/blob/s3/conditional.go | 94 +++++ pkg/artifact/blob/s3/conditional_test.go | 259 ++++++++++++ pkg/artifact/blob/s3/delete.go | 38 ++ pkg/artifact/blob/s3/doc.go | 15 + pkg/artifact/blob/s3/emulator_test.go | 35 ++ pkg/artifact/blob/s3/helpers_test.go | 96 +++++ pkg/artifact/blob/s3/list.go | 61 +++ pkg/artifact/blob/s3/multipart.go | 257 ++++++++++++ pkg/artifact/blob/s3/options.go | 176 +++++++++ pkg/artifact/blob/s3/put.go | 181 +++++++++ pkg/artifact/blob/s3/put_test.go | 262 +++++++++++++ pkg/artifact/blob/s3/read.go | 118 ++++++ pkg/artifact/blob/s3/s3test/chunked.go | 57 +++ pkg/artifact/blob/s3/s3test/cond.go | 75 ++++ pkg/artifact/blob/s3/s3test/doc.go | 18 + pkg/artifact/blob/s3/s3test/emulator.go | 49 +++ pkg/artifact/blob/s3/s3test/fault.go | 138 +++++++ pkg/artifact/blob/s3/s3test/list.go | 163 ++++++++ pkg/artifact/blob/s3/s3test/multipart.go | 196 ++++++++++ pkg/artifact/blob/s3/s3test/objects.go | 217 +++++++++++ pkg/artifact/blob/s3/s3test/options.go | 74 ++++ pkg/artifact/blob/s3/s3test/record.go | 145 +++++++ pkg/artifact/blob/s3/s3test/server.go | 239 ++++++++++++ pkg/artifact/blob/s3/s3test/server_test.go | 368 ++++++++++++++++++ pkg/artifact/blob/s3/s3test/state.go | 157 ++++++++ pkg/artifact/blob/s3/s3test/util.go | 50 +++ pkg/artifact/blob/s3/store.go | 129 ++++++ pkg/artifact/blob/s3/store_test.go | 147 +++++++ pkg/artifact/blob/store.go | 197 ++++++++++ pkg/artifact/blob/tags.go | 64 +++ pkg/ci/artifact/local/store.go | 229 +++++------ .../artifact/local/transport_parity_test.go | 178 +++++++++ pkg/ci/artifact/s3/store.go | 266 +++++-------- pkg/ci/artifact/s3/transport_parity_test.go | 203 ++++++++++ 66 files changed, 7789 insertions(+), 268 deletions(-) create mode 100644 pkg/artifact/blob/blob_test.go create mode 100644 pkg/artifact/blob/blobtest/cases_basic.go create mode 100644 pkg/artifact/blob/blobtest/cases_conditional.go create mode 100644 pkg/artifact/blob/blobtest/cases_list.go create mode 100644 pkg/artifact/blob/blobtest/conformance.go create mode 100644 pkg/artifact/blob/blobtest/doc.go create mode 100644 pkg/artifact/blob/boundary_test.go create mode 100644 pkg/artifact/blob/capabilities.go create mode 100644 pkg/artifact/blob/doc.go create mode 100644 pkg/artifact/blob/errors.go create mode 100644 pkg/artifact/blob/key.go create mode 100644 pkg/artifact/blob/list.go create mode 100644 pkg/artifact/blob/local/attrs.go create mode 100644 pkg/artifact/blob/local/delete.go create mode 100644 pkg/artifact/blob/local/doc.go create mode 100644 pkg/artifact/blob/local/errors.go create mode 100644 pkg/artifact/blob/local/etag.go create mode 100644 pkg/artifact/blob/local/fsutil.go create mode 100644 pkg/artifact/blob/local/get.go create mode 100644 pkg/artifact/blob/local/list.go create mode 100644 pkg/artifact/blob/local/lock.go create mode 100644 pkg/artifact/blob/local/options.go create mode 100644 pkg/artifact/blob/local/put.go create mode 100644 pkg/artifact/blob/local/store.go create mode 100644 pkg/artifact/blob/local/store_test.go create mode 100644 pkg/artifact/blob/mock_store.go create mode 100644 pkg/artifact/blob/s3/api.go create mode 100644 pkg/artifact/blob/s3/classify.go create mode 100644 pkg/artifact/blob/s3/client.go create mode 100644 pkg/artifact/blob/s3/client_test.go create mode 100644 pkg/artifact/blob/s3/conditional.go create mode 100644 pkg/artifact/blob/s3/conditional_test.go create mode 100644 pkg/artifact/blob/s3/delete.go create mode 100644 pkg/artifact/blob/s3/doc.go create mode 100644 pkg/artifact/blob/s3/emulator_test.go create mode 100644 pkg/artifact/blob/s3/helpers_test.go create mode 100644 pkg/artifact/blob/s3/list.go create mode 100644 pkg/artifact/blob/s3/multipart.go create mode 100644 pkg/artifact/blob/s3/options.go create mode 100644 pkg/artifact/blob/s3/put.go create mode 100644 pkg/artifact/blob/s3/put_test.go create mode 100644 pkg/artifact/blob/s3/read.go create mode 100644 pkg/artifact/blob/s3/s3test/chunked.go create mode 100644 pkg/artifact/blob/s3/s3test/cond.go create mode 100644 pkg/artifact/blob/s3/s3test/doc.go create mode 100644 pkg/artifact/blob/s3/s3test/emulator.go create mode 100644 pkg/artifact/blob/s3/s3test/fault.go create mode 100644 pkg/artifact/blob/s3/s3test/list.go create mode 100644 pkg/artifact/blob/s3/s3test/multipart.go create mode 100644 pkg/artifact/blob/s3/s3test/objects.go create mode 100644 pkg/artifact/blob/s3/s3test/options.go create mode 100644 pkg/artifact/blob/s3/s3test/record.go create mode 100644 pkg/artifact/blob/s3/s3test/server.go create mode 100644 pkg/artifact/blob/s3/s3test/server_test.go create mode 100644 pkg/artifact/blob/s3/s3test/state.go create mode 100644 pkg/artifact/blob/s3/s3test/util.go create mode 100644 pkg/artifact/blob/s3/store.go create mode 100644 pkg/artifact/blob/s3/store_test.go create mode 100644 pkg/artifact/blob/store.go create mode 100644 pkg/artifact/blob/tags.go create mode 100644 pkg/ci/artifact/local/transport_parity_test.go create mode 100644 pkg/ci/artifact/s3/transport_parity_test.go diff --git a/.golangci.yml b/.golangci.yml index a820f830cd8..334edfaed82 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -104,6 +104,10 @@ linters: # pkg/ci/artifact/s3 is the S3-backed CI artifact store; it legitimately # depends on the AWS SDK, same rationale as pkg/ci/planfile/s3 above. - "!**/pkg/ci/artifact/s3/**" + # pkg/artifact/blob/s3 is the shared AWS SDK object transport behind the artifact S3 + # repository and the CI artifact S3 adapter (including its in-process S3 test fake). + # The provider-neutral contract is pkg/artifact/blob, which imports no SDK. + - "!**/pkg/artifact/blob/s3/**" # The S3 downloader signs source requests with request-scoped AWS credentials. # Only this cloud-specific leaf imports the SDK; the parent downloader stays generic. - "!**/pkg/downloader/s3/**" diff --git a/errors/errors.go b/errors/errors.go index f7cd45d8f15..34d1c43393a 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2140,6 +2140,31 @@ var ( ErrArtifactStreamFailed = errors.New("artifact stream failed") ) +// Blob transport sentinels. They classify outcomes of the transport-only object I/O in +// pkg/artifact/blob so callers can branch with errors.Is without depending on a backend SDK. +var ( + // ErrBlobNotFound is returned when the addressed object does not exist. + ErrBlobNotFound = errors.New("blob object not found") + + // ErrBlobPreconditionFailed is returned when a conditional write or delete (if-absent or if-match) lost its race. + ErrBlobPreconditionFailed = errors.New("blob precondition failed") + + // ErrBlobConditionalUnsupported is returned when the endpoint cannot enforce a required condition. + // The operation is rejected instead of silently overwriting. + ErrBlobConditionalUnsupported = errors.New("blob endpoint cannot enforce the required condition") + + // ErrBlobOutcomeUnknown is returned when a mutation was sent but its result was not observed + // (timeout, connection reset, server error). Callers must reconcile by reading the object. + ErrBlobOutcomeUnknown = errors.New("blob mutation outcome is unknown") + + // ErrBlobInvalid is returned for malformed keys, tags, options, sizes, or ranges. + ErrBlobInvalid = errors.New("invalid blob request") + + // ErrBlobIdentityUnresolved is returned when a configured identity cannot be resolved to credentials. + // The generic blob transport fails closed instead of falling back to ambient credentials. + ErrBlobIdentityUnresolved = errors.New("blob identity could not be resolved") +) + // ExitCodeError is a typed error that preserves subcommand exit codes. // This allows the root command to exit with the same code as the subcommand. // When Code is 0, it indicates successful completion that should exit cleanly without printing errors. diff --git a/pkg/artifact/blob/blob_test.go b/pkg/artifact/blob/blob_test.go new file mode 100644 index 00000000000..5291657b905 --- /dev/null +++ b/pkg/artifact/blob/blob_test.go @@ -0,0 +1,179 @@ +package blob + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestErrorKindsMatchSentinelsAndCauses(t *testing.T) { + cause := errors.New("sdk detail") + cases := []struct { + kind Kind + sentinel error + check func(error) bool + }{ + {KindNotFound, errUtils.ErrBlobNotFound, IsNotFound}, + {KindPrecondition, errUtils.ErrBlobPreconditionFailed, IsPreconditionFailed}, + {KindUnsupported, errUtils.ErrBlobConditionalUnsupported, IsConditionalUnsupported}, + {KindAmbiguous, errUtils.ErrBlobOutcomeUnknown, IsAmbiguous}, + {KindInvalid, errUtils.ErrBlobInvalid, func(err error) bool { return errors.Is(err, errUtils.ErrBlobInvalid) }}, + {KindIdentity, errUtils.ErrBlobIdentityUnresolved, func(err error) bool { return errors.Is(err, errUtils.ErrBlobIdentityUnresolved) }}, + } + for _, tc := range cases { + err := fmt.Errorf("wrapped: %w", NewError(tc.kind, "put", "k", cause)) + assert.ErrorIs(t, err, tc.sentinel) + assert.ErrorIs(t, err, cause, "the cause stays visible") + assert.True(t, tc.check(err)) + assert.Contains(t, err.Error(), `blob put "k"`) + assert.Contains(t, err.Error(), "sdk detail") + } + assert.False(t, IsNotFound(NewError(KindPrecondition, "put", "k", nil))) +} + +func TestErrorMessagesDoNotDuplicateTheSentinel(t *testing.T) { + err := NewError(KindNotFound, "get", "k", fmt.Errorf("%w: gone", errUtils.ErrBlobNotFound)) + assert.Equal(t, 1, strings.Count(err.Error(), errUtils.ErrBlobNotFound.Error())) + assert.Equal(t, `blob get "k": blob object not found`, NewError(KindNotFound, "get", "k", nil).Error()) + assert.Equal(t, `blob list: failed`, NewError(KindOther, "list", "", nil).Error()) + assert.Equal(t, `blob list: boom`, NewError(KindOther, "list", "", errors.New("boom")).Error()) +} + +func TestOnlyAmbiguousErrorsAreTemporary(t *testing.T) { + for kind := KindOther; kind <= KindIdentity; kind++ { + assert.Equal(t, kind == KindAmbiguous, NewError(kind, "put", "k", nil).Temporary(), "kind %d", kind) + } + var temporary interface{ Temporary() bool } + require.True(t, errors.As(NewError(KindAmbiguous, "put", "k", nil), &temporary)) +} + +func TestValidateKey(t *testing.T) { + valid := []string{"a", "a/b", "dir/file.tar", "x/y.z/w", strings.Repeat("a", MaxKeyBytes)} + for _, key := range valid { + assert.NoError(t, ValidateKey(key), key) + } + invalid := []string{"", "/a", "a/", "a//b", "a/./b", "a/../b", "..", ".", "a\\b", "a\x00b", "\xff", strings.Repeat("a", MaxKeyBytes+1)} + for _, key := range invalid { + err := ValidateKey(key) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } +} + +func TestValidateOpaqueKeyAllowsLegacyPatterns(t *testing.T) { + for _, key := range []string{"a//b", "/leading", "trailing/", "a/../b", "a\\b"} { + assert.NoError(t, ValidateOpaqueKey(key), key) + } + for _, key := range []string{"", "a\x00b", strings.Repeat("a", MaxKeyBytes+1)} { + assert.Error(t, ValidateOpaqueKey(key), key) + } +} + +func TestMergeTagsNeverAliasesInputs(t *testing.T) { + defaults := map[string]string{"a": "1", "b": "2"} + overrides := map[string]string{"b": "3", "c": "4"} + merged := MergeTags(defaults, overrides) + assert.Equal(t, map[string]string{"a": "1", "b": "3", "c": "4"}, merged) + + merged["a"] = "mutated" + assert.Equal(t, "1", defaults["a"], "result to source isolation") + defaults["z"] = "late" + assert.NotContains(t, merged, "z", "source to result isolation") + assert.Equal(t, "2", map[string]string{"a": "1", "b": "2"}["b"]) + assert.Nil(t, MergeTags(nil, nil)) + assert.Nil(t, CloneMap(nil)) + clone := CloneMap(overrides) + clone["c"] = "x" + assert.Equal(t, "4", overrides["c"]) +} + +func TestValidateTags(t *testing.T) { + assert.NoError(t, ValidateTags(nil)) + assert.NoError(t, ValidateTags(map[string]string{"k": strings.Repeat("v", MaxTagValueChars)})) + + many := map[string]string{} + for i := range MaxTags + 1 { + many[fmt.Sprint("k", i)] = "v" + } + for name, tags := range map[string]map[string]string{ + "too many": many, + "empty key": {"": "v"}, + "long key": {strings.Repeat("k", MaxTagKeyChars+1): "v"}, + "long val": {"k": strings.Repeat("v", MaxTagValueChars+1)}, + } { + assert.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid, name) + } +} + +func TestConditionHelpers(t *testing.T) { + assert.False(t, Condition{}.IsConditional()) + assert.True(t, IfAbsent().IsConditional()) + assert.Equal(t, Condition{Kind: ConditionIfMatch, ETag: `"x"`}, IfMatch(`"x"`)) + assert.False(t, GetOptions{}.IsRange()) + assert.True(t, GetOptions{Offset: 1}.IsRange()) + assert.True(t, GetOptions{Length: 1}.IsRange()) +} + +func TestCapabilityChecksFailClearly(t *testing.T) { + none := Capabilities{} + assert.NoError(t, none.CheckPut("s", "k", Condition{})) + err := none.CheckPut("s", "k", IfAbsent()) + assert.True(t, IsConditionalUnsupported(err)) + assert.Contains(t, err.Error(), "s store cannot enforce conditional writes") + + assert.NoError(t, none.CheckDelete("s", "k", DeleteOptions{})) + assert.True(t, IsConditionalUnsupported(none.CheckDelete("s", "k", DeleteOptions{IfMatch: "x"}))) + + full := Capabilities{ConditionalPut: true, ConditionalDelete: true} + assert.NoError(t, full.CheckPut("s", "k", IfMatch("x"))) + assert.NoError(t, full.CheckDelete("s", "k", DeleteOptions{IfMatch: "x"})) +} + +func TestListAllFollowsCursorsAndMergesPrefixes(t *testing.T) { + store := &listOnlyStore{pages: []ListPage{ + {Objects: []ObjectInfo{{Key: "a"}, {Key: "b"}}, CommonPrefixes: []string{"p/"}, NextCursor: "c1"}, + {Objects: []ObjectInfo{{Key: "c"}}, NextCursor: "c2"}, + {Objects: []ObjectInfo{{Key: "d"}}, CommonPrefixes: []string{"q/"}}, + }} + objects, prefixes, err := ListAll(context.Background(), store, ListOptions{Prefix: "x/", Limit: 2}) + require.NoError(t, err) + assert.Equal(t, []string{"a", "b", "c", "d"}, []string{objects[0].Key, objects[1].Key, objects[2].Key, objects[3].Key}) + assert.Equal(t, []string{"p/", "q/"}, prefixes) + assert.Equal(t, []string{"", "c1", "c2"}, []string{store.calls[0].Cursor, store.calls[1].Cursor, store.calls[2].Cursor}) + for _, call := range store.calls { + assert.Equal(t, "x/", call.Prefix) + assert.Equal(t, 2, call.Limit) + } +} + +func TestListAllStopsOnError(t *testing.T) { + boom := errors.New("boom") + store := &listOnlyStore{err: boom} + _, _, err := ListAll(context.Background(), store, ListOptions{}) + require.ErrorIs(t, err, boom) +} + +// listOnlyStore implements only List; other methods come from the generated mock and would fail +// the test if called. +type listOnlyStore struct { + MockStore + pages []ListPage + calls []ListOptions + err error +} + +func (s *listOnlyStore) List(_ context.Context, opts ListOptions) (*ListPage, error) { + s.calls = append(s.calls, opts) + if s.err != nil { + return nil, s.err + } + page := s.pages[len(s.calls)-1] + return &page, nil +} diff --git a/pkg/artifact/blob/blobtest/cases_basic.go b/pkg/artifact/blob/blobtest/cases_basic.go new file mode 100644 index 00000000000..14176f2bbb8 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_basic.go @@ -0,0 +1,242 @@ +package blobtest + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func (c *conformance) putGetRoundTrip(t *testing.T) { + store := c.newStore(t) + data := []byte("hello blob") + + info := put(t, store, keyNested, data) + assert.Equal(t, keyNested, info.Key) + assert.Equal(t, int64(len(data)), info.Size) + assert.NotEmpty(t, info.ETag) + + assert.Equal(t, data, read(t, store, keyNested)) + stat, err := store.Stat(context.Background(), keyNested, blob.StatOptions{}) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), stat.Size) + assert.Equal(t, info.ETag, stat.ETag, "Put and Stat must report the same concurrency token") +} + +func (c *conformance) zeroLength(t *testing.T) { + store := c.newStore(t) + info := put(t, store, "empty", nil) + assert.Zero(t, info.Size) + assert.NotEmpty(t, info.ETag) + + assert.Empty(t, read(t, store, "empty")) + stat, err := store.Stat(context.Background(), "empty", blob.StatOptions{}) + require.NoError(t, err) + assert.Zero(t, stat.Size) + + // A zero-length body of unknown size is also a valid object. + _, err = store.Put(context.Background(), "empty-unknown", bytes.NewReader(nil), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Empty(t, read(t, store, "empty-unknown")) +} + +func (c *conformance) missing(t *testing.T) { + store := c.newStore(t) + ctx := context.Background() + + _, err := store.Get(ctx, "nope", blob.GetOptions{}) + assert.True(t, blob.IsNotFound(err), "Get: %v", err) + _, err = store.Stat(ctx, "nope", blob.StatOptions{}) + assert.True(t, blob.IsNotFound(err), "Stat: %v", err) + assert.NoError(t, store.Delete(ctx, "nope", blob.DeleteOptions{}), "unconditional delete of a missing object succeeds") +} + +func (c *conformance) overwrite(t *testing.T) { + store := c.newStore(t) + first := put(t, store, keyObject, []byte("one")) + second := put(t, store, keyObject, []byte("two!")) + assert.NotEqual(t, first.ETag, second.ETag, "different content must yield a different token") + assert.Equal(t, []byte("two!"), read(t, store, keyObject)) + + require.NoError(t, store.Delete(context.Background(), keyObject, blob.DeleteOptions{})) + requireAbsent(t, store, keyObject) +} + +func (c *conformance) invalidKeys(t *testing.T) { + store := c.newStore(t) + _, err := store.Put(context.Background(), "", bytes.NewReader([]byte(bodyX)), blob.PutOptions{Size: 1}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = store.Get(context.Background(), "", blob.GetOptions{}) + require.Error(t, err) + _, err = store.Stat(context.Background(), "", blob.StatOptions{}) + require.Error(t, err) +} + +func (c *conformance) attributes(t *testing.T) { + store := c.newStore(t) + caps := store.Capabilities() + ctx := context.Background() + opts := blob.PutOptions{ + Size: 3, + ContentType: "application/json", + Metadata: map[string]string{"owner": "atmos"}, + Tags: map[string]string{"team": "platform", "env": "dev"}, + } + _, err := store.Put(ctx, keyAttrs, bytes.NewReader([]byte("{ }")), opts) + require.NoError(t, err) + + stat, err := store.Stat(ctx, keyAttrs, blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + if caps.Metadata { + assert.Equal(t, "application/json", stat.ContentType) + assert.Equal(t, "atmos", stat.Metadata["owner"]) + } + if caps.Tags { + assert.Equal(t, opts.Tags, stat.Tags) + } + + // Rewriting without attributes must not keep the old ones. + put(t, store, keyAttrs, []byte("{ }")) + stat, err = store.Stat(ctx, keyAttrs, blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Empty(t, stat.Tags) + assert.Empty(t, stat.Metadata) + + tooMany := map[string]string{} + for _, key := range []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} { + tooMany[key] = "v" + } + _, err = store.Put(ctx, "bad-tags", bytes.NewReader(nil), blob.PutOptions{Size: 0, Tags: tooMany}) + require.Error(t, err, "more than the S3 limit of 10 tags must be rejected on every backend") + requireAbsent(t, store, "bad-tags") +} + +func (c *conformance) ranged(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().Range { + t.Skipf("store does not advertise range reads") + } + put(t, store, "r", []byte("0123456789")) + for _, tc := range []struct { + name string + opts blob.GetOptions + want string + }{ + {"middle", blob.GetOptions{Offset: 2, Length: 4}, "2345"}, + {"tail", blob.GetOptions{Offset: tailOffset}, "789"}, + {"clamped", blob.GetOptions{Offset: clampedOffset, Length: listPageGuard}, "89"}, + } { + t.Run(tc.name, func(t *testing.T) { + object, err := store.Get(context.Background(), "r", tc.opts) + require.NoError(t, err) + defer object.Body.Close() + data, err := io.ReadAll(object.Body) + require.NoError(t, err) + assert.Equal(t, tc.want, string(data)) + assert.Equal(t, int64(10), object.Size, "Size is the full object size") + }) + } +} + +func (c *conformance) sizeMismatch(t *testing.T) { + store := c.newStore(t) + for name, body := range map[string]string{"short": "ab", "long": "abcdef"} { + _, err := store.Put(context.Background(), name, bytes.NewReader([]byte(body)), blob.PutOptions{Size: 4}) + require.Error(t, err, name) + requireAbsent(t, store, name) + } +} + +// errSourceBroke is the failure the broken-source case injects. +var errSourceBroke = errors.New("source broke") + +// mockFailingReader yields data and then fails. +type mockFailingReader struct { + data []byte + err error +} + +func (f *mockFailingReader) Read(p []byte) (int, error) { + if len(f.data) == 0 { + return 0, f.err + } + n := copy(p, f.data) + f.data = f.data[n:] + return n, nil +} + +func (c *conformance) failedBody(t *testing.T) { + store := c.newStore(t) + boom := errSourceBroke + _, err := store.Put(context.Background(), "partial", &mockFailingReader{data: bytes.Repeat([]byte(bodyX), sourceChunk), err: boom}, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, boom) + requireAbsent(t, store, "partial") +} + +func (c *conformance) canceled(t *testing.T) { + store := c.newStore(t) + ctx, cancel := context.WithCancel(context.Background()) + body := io.MultiReader(bytes.NewReader(bytes.Repeat([]byte(bodyX), sourceChunk)), &mockCancelingReader{cancel: cancel}) + _, err := store.Put(ctx, "canceled", body, blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + requireAbsent(t, store, "canceled") +} + +// mockCancelingReader cancels its context on first read and then reports the cancellation. +type mockCancelingReader struct{ cancel context.CancelFunc } + +func (r *mockCancelingReader) Read([]byte) (int, error) { + r.cancel() + return 0, context.Canceled +} + +func (c *conformance) largeStream(t *testing.T) { + store := c.newStore(t) + const size = 3<<20 + 123 + unit := []byte("0123456789abcdef") + build := func() io.Reader { + return io.LimitReader(&mockRepeatReader{unit: unit}, size) + } + want := sha256.New() + _, err := io.Copy(want, build()) + require.NoError(t, err) + + for name, declared := range map[string]int64{"known-size": size, "unknown-size": blob.UnknownSize} { + info, err := store.Put(context.Background(), name, build(), blob.PutOptions{Size: declared}) + require.NoError(t, err, name) + assert.Equal(t, int64(size), info.Size, name) + + object, err := store.Get(context.Background(), name, blob.GetOptions{}) + require.NoError(t, err, name) + got := sha256.New() + n, err := io.Copy(got, object.Body) + require.NoError(t, err, name) + require.NoError(t, object.Body.Close()) + assert.Equal(t, int64(size), n, name) + assert.Equal(t, want.Sum(nil), got.Sum(nil), name) + } +} + +// mockRepeatReader yields unit forever. +type mockRepeatReader struct { + unit []byte + off int +} + +func (r *mockRepeatReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = r.unit[(r.off+i)%len(r.unit)] + } + r.off = (r.off + len(p)) % len(r.unit) + return len(p), nil +} diff --git a/pkg/artifact/blob/blobtest/cases_conditional.go b/pkg/artifact/blob/blobtest/cases_conditional.go new file mode 100644 index 00000000000..45d3e50e240 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_conditional.go @@ -0,0 +1,183 @@ +package blobtest + +import ( + "bytes" + "context" + "fmt" + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// putWith stores data under a condition and returns the result. +func putWith(store blob.Store, key string, data []byte, cond blob.Condition) (*blob.ObjectInfo, error) { + return store.Put(context.Background(), key, bytes.NewReader(data), blob.PutOptions{Size: int64(len(data)), Condition: cond}) +} + +func (c *conformance) ifAbsent(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + _, err := putWith(store, keyLock, []byte("a"), blob.IfAbsent()) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "a store without conditional writes must say so: %v", err) + requireAbsent(t, store, keyLock) + return + } + first, err := putWith(store, keyLock, []byte("owner-a"), blob.IfAbsent()) + require.NoError(t, err) + assert.NotEmpty(t, first.ETag) + + _, err = putWith(store, keyLock, []byte("owner-b"), blob.IfAbsent()) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "second create-if-absent must lose: %v", err) + assert.False(t, blob.IsAmbiguous(err)) + assert.Equal(t, []byte("owner-a"), read(t, store, keyLock), "a lost race must not change the object") + + require.NoError(t, store.Delete(context.Background(), keyLock, blob.DeleteOptions{})) + _, err = putWith(store, keyLock, []byte("owner-c"), blob.IfAbsent()) + require.NoError(t, err, "create-if-absent succeeds again after delete") +} + +func (c *conformance) ifMatch(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + first := put(t, store, keyRef, []byte("v1")) + + second, err := putWith(store, keyRef, []byte("v2"), blob.IfMatch(first.ETag)) + require.NoError(t, err) + assert.NotEqual(t, first.ETag, second.ETag) + + _, err = putWith(store, keyRef, []byte("v3"), blob.IfMatch(first.ETag)) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "stale token must lose: %v", err) + assert.Equal(t, []byte("v2"), read(t, store, keyRef)) + + _, err = putWith(store, "ghost", []byte(bodyX), blob.IfMatch(`"abc"`)) + require.Error(t, err) + assert.True(t, blob.IsNotFound(err), "if-match on a missing object: %v", err) + requireAbsent(t, store, "ghost") + + _, err = putWith(store, keyRef, []byte(bodyX), blob.Condition{Kind: blob.ConditionIfMatch}) + require.Error(t, err, "if-match without a token is invalid") +} + +func (c *conformance) conditionalDelete(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalDelete { + put(t, store, keyObject, []byte(bodyX)) + err := store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: `"abc"`}) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Equal(t, []byte(bodyX), read(t, store, keyObject)) + return + } + info := put(t, store, keyObject, []byte(bodyX)) + + err := store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: `"stale"`}) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + assert.Equal(t, []byte(bodyX), read(t, store, keyObject), "a rejected delete must keep the object") + + require.NoError(t, store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: info.ETag})) + requireAbsent(t, store, keyObject) + + err = store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: info.ETag}) + assert.True(t, blob.IsNotFound(err), "conditional delete of a missing object: %v", err) +} + +func (c *conformance) concurrentIfAbsent(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + const writers = 12 + var ( + wg sync.WaitGroup + winners atomic.Int32 + losers atomic.Int32 + start = make(chan struct{}) + ) + for i := range writers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := putWith(store, "claim", []byte("owner-"+strconv.Itoa(i)), blob.IfAbsent()) + switch { + case err == nil: + winners.Add(1) + case blob.IsPreconditionFailed(err): + losers.Add(1) + default: + t.Errorf("writer %d: unexpected error %v", i, err) + } + }() + } + close(start) + wg.Wait() + assert.Equal(t, int32(1), winners.Load(), "exactly one creator wins") + assert.Equal(t, int32(writers-1), losers.Load()) +} + +func (c *conformance) concurrentSwap(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + put(t, store, keyCounter, []byte("0")) + const workers, increments = 6, 5 + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for range increments { + incrementWithRetry(t, store) + } + }() + } + wg.Wait() + assert.Equal(t, strconv.Itoa(workers*increments), string(read(t, store, keyCounter)), "every successful swap must be reflected") +} + +// incrementWithRetry performs a read-modify-write loop guarded by If-Match. +func incrementWithRetry(t *testing.T, store blob.Store) { + t.Helper() + for range maxRetries { + stat, err := store.Stat(context.Background(), keyCounter, blob.StatOptions{}) + if err != nil { + t.Errorf("stat: %v", err) + return + } + object, err := store.Get(context.Background(), keyCounter, blob.GetOptions{}) + if err != nil { + t.Errorf("get: %v", err) + return + } + raw := make([]byte, counterBuffer) + n, _ := object.Body.Read(raw) + _ = object.Body.Close() + current, err := strconv.Atoi(string(raw[:n])) + if err != nil { + continue + } + next := []byte(fmt.Sprint(current + 1)) + _, err = putWith(store, keyCounter, next, blob.IfMatch(stat.ETag)) + if err == nil { + return + } + if !blob.IsPreconditionFailed(err) { + t.Errorf("swap: %v", err) + return + } + } + t.Errorf("swap did not converge") +} diff --git a/pkg/artifact/blob/blobtest/cases_list.go b/pkg/artifact/blob/blobtest/cases_list.go new file mode 100644 index 00000000000..5187a254041 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_list.go @@ -0,0 +1,105 @@ +package blobtest + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func (c *conformance) listPrefix(t *testing.T) { + store := c.newStore(t) + for _, key := range []string{"a/2", "a/1", "b/1", "a/sub/3", "ab"} { + put(t, store, key, []byte(key)) + } + objects, prefixes, err := blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "a/"}) + require.NoError(t, err) + assert.Empty(t, prefixes) + assert.Equal(t, []string{"a/1", "a/2", "a/sub/3"}, keysOf(objects), "lexicographic order, prefix-scoped") + for _, object := range objects { + assert.Equal(t, int64(len(object.Key)), object.Size) + } + + objects, _, err = blob.ListAll(context.Background(), store, blob.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, []string{"a/1", "a/2", "a/sub/3", "ab", "b/1"}, keysOf(objects)) + + objects, _, err = blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "zzz/"}) + require.NoError(t, err) + assert.Empty(t, objects) +} + +func (c *conformance) listPagination(t *testing.T) { + store := c.newStore(t) + const total = listKeyCount + want := make([]string, 0, total) + for i := range total { + key := fmt.Sprintf("page/%04d", i) + want = append(want, key) + put(t, store, key, nil) + } + slices.Sort(want) + + var ( + got []string + cursor string + pages int + ) + for { + page, err := store.List(context.Background(), blob.ListOptions{Prefix: "page/", Cursor: cursor, Limit: listPageSize}) + require.NoError(t, err) + assert.LessOrEqual(t, len(page.Objects), listPageSize, "a page never exceeds the limit") + got = append(got, keysOf(page.Objects)...) + pages++ + require.Less(t, pages, listPageGuard, "pagination must terminate") + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + } + assert.Equal(t, want, got, "every key exactly once, in order") + assert.GreaterOrEqual(t, pages, listMinPages, "53 keys at 10 per page need at least 6 pages") +} + +func (c *conformance) listDelimiter(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ListDelimiter { + t.Skipf("store does not advertise delimiter listing") + } + for _, key := range []string{"root.txt", "x/1", "x/2", "x/deep/3", "y/1", "z"} { + put(t, store, key, nil) + } + objects, prefixes, err := blob.ListAll(context.Background(), store, blob.ListOptions{Delimiter: "/"}) + require.NoError(t, err) + assert.Equal(t, []string{"root.txt", "z"}, keysOf(objects)) + assert.Equal(t, []string{"x/", "y/"}, prefixes) + + // Paging must not split or repeat a rolled-up prefix. + var ( + gotObjects []string + gotPrefixes []string + cursor string + ) + for range delimiterPages { + page, err := store.List(context.Background(), blob.ListOptions{Delimiter: "/", Cursor: cursor, Limit: 1}) + require.NoError(t, err) + gotObjects = append(gotObjects, keysOf(page.Objects)...) + gotPrefixes = append(gotPrefixes, page.CommonPrefixes...) + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + } + assert.Equal(t, []string{"root.txt", "z"}, gotObjects) + assert.Equal(t, []string{"x/", "y/"}, gotPrefixes) + + objects, prefixes, err = blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "x/", Delimiter: "/"}) + require.NoError(t, err) + assert.Equal(t, []string{"x/1", "x/2"}, keysOf(objects)) + assert.Equal(t, []string{"x/deep/"}, prefixes) +} diff --git a/pkg/artifact/blob/blobtest/conformance.go b/pkg/artifact/blob/blobtest/conformance.go new file mode 100644 index 00000000000..ce1c83ba81b --- /dev/null +++ b/pkg/artifact/blob/blobtest/conformance.go @@ -0,0 +1,116 @@ +package blobtest + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Keys and sizes the cases share. +const ( + keyObject = "k" + keyNested = "dir/one.txt" + keyAttrs = "attrs.json" + keyLock = "lock" + keyRef = "ref" + keyCounter = "counter" + bodyX = "x" + + sourceChunk = 1024 + maxRetries = 500 + counterBuffer = 16 + listPageSize = 10 + listPageGuard = 100 + listKeyCount = 53 + listMinPages = 6 + delimiterPages = 20 + tailOffset = 7 + clampedOffset = 8 +) + +// Factory returns a fresh, empty store for one subtest. It must register cleanup with t.Cleanup +// and must not share state with stores returned by earlier calls. +type Factory func(t *testing.T) blob.Store + +// RunStoreConformance runs the transport contract against stores built by factory. +func RunStoreConformance(t *testing.T, factory Factory) { + defer perf.Track(nil, "blobtest.RunStoreConformance")() + + t.Helper() + require.NotNil(t, factory, "a store factory is required") + suite := &conformance{factory: factory} + t.Run("PutGetRoundTrip", suite.putGetRoundTrip) + t.Run("ZeroLengthObject", suite.zeroLength) + t.Run("MissingObjects", suite.missing) + t.Run("OverwriteChangesETag", suite.overwrite) + t.Run("RejectsInvalidKeys", suite.invalidKeys) + t.Run("AttributesRoundTrip", suite.attributes) + t.Run("RangedRead", suite.ranged) + t.Run("SizeMismatchLeavesNothing", suite.sizeMismatch) + t.Run("FailedBodyLeavesNothing", suite.failedBody) + t.Run("CanceledPutLeavesNothing", suite.canceled) + t.Run("LargeStreamedBody", suite.largeStream) + t.Run("ConditionalIfAbsent", suite.ifAbsent) + t.Run("ConditionalIfMatch", suite.ifMatch) + t.Run("ConditionalDelete", suite.conditionalDelete) + t.Run("ConcurrentCreateIfAbsentHasOneWinner", suite.concurrentIfAbsent) + t.Run("ConcurrentCompareAndSwapHasNoLostUpdate", suite.concurrentSwap) + t.Run("ListPrefixAndOrder", suite.listPrefix) + t.Run("ListPaginationVisitsEveryKeyOnce", suite.listPagination) + t.Run("ListDelimiterRollsUpPrefixes", suite.listDelimiter) +} + +// conformance holds the factory shared by the cases. +type conformance struct { + factory Factory +} + +// newStore builds a store for one subtest. +func (c *conformance) newStore(t *testing.T) blob.Store { + t.Helper() + store := c.factory(t) + require.NotNil(t, store) + return store +} + +// put stores data unconditionally and fails the test on error. +func put(t *testing.T, store blob.Store, key string, data []byte) *blob.ObjectInfo { + t.Helper() + info, err := store.Put(context.Background(), key, bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.NoError(t, err) + require.NotNil(t, info) + return info +} + +// read returns the full content of key. +func read(t *testing.T, store blob.Store, key string) []byte { + t.Helper() + object, err := store.Get(context.Background(), key, blob.GetOptions{}) + require.NoError(t, err) + defer object.Body.Close() + data, err := io.ReadAll(object.Body) + require.NoError(t, err) + return data +} + +// requireAbsent asserts the key does not exist. +func requireAbsent(t *testing.T, store blob.Store, key string) { + t.Helper() + _, err := store.Stat(context.Background(), key, blob.StatOptions{}) + require.True(t, blob.IsNotFound(err), "expected %q to be absent, got %v", key, err) +} + +// keysOf collects the keys of listed objects. +func keysOf(objects []blob.ObjectInfo) []string { + keys := make([]string, 0, len(objects)) + for _, object := range objects { + keys = append(keys, object.Key) + } + return keys +} diff --git a/pkg/artifact/blob/blobtest/doc.go b/pkg/artifact/blob/blobtest/doc.go new file mode 100644 index 00000000000..22158d75804 --- /dev/null +++ b/pkg/artifact/blob/blobtest/doc.go @@ -0,0 +1,4 @@ +// Package blobtest is the shared conformance suite for [blob.Store] implementations. Every backend +// runs [RunStoreConformance] from its own tests; behavior a backend does not advertise through +// [blob.Capabilities] is asserted to fail clearly instead of being skipped. +package blobtest diff --git a/pkg/artifact/blob/boundary_test.go b/pkg/artifact/blob/boundary_test.go new file mode 100644 index 00000000000..79802d3f037 --- /dev/null +++ b/pkg/artifact/blob/boundary_test.go @@ -0,0 +1,45 @@ +package blob + +import ( + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestTransportHasNoCIOrCommandDependencies keeps the dependency direction one way: CI adapters +// import the blob transport, never the reverse. The blob packages (including every backend and the +// S3 fake) must not reach for CI, command, hook, plugin, runner, or exec packages. +func TestTransportHasNoCIOrCommandDependencies(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + } + checked := 0 + require.NoError(t, filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + parsed, parseErr := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + require.NoError(t, parseErr) + checked++ + for _, spec := range parsed.Imports { + imported, unquoteErr := strconv.Unquote(spec.Path.Value) + require.NoError(t, unquoteErr) + for _, prefix := range forbidden { + require.Falsef(t, imported == prefix || strings.HasPrefix(imported, prefix+"/"), "%s imports forbidden package %s", path, imported) + } + } + return nil + })) + require.Greater(t, checked, 20, "the boundary check must walk the backends and test helpers") +} diff --git a/pkg/artifact/blob/capabilities.go b/pkg/artifact/blob/capabilities.go new file mode 100644 index 00000000000..143c04de211 --- /dev/null +++ b/pkg/artifact/blob/capabilities.go @@ -0,0 +1,62 @@ +package blob + +import ( + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Capabilities describes what a store can enforce. It is computed without network access. +type Capabilities struct { + // ConditionalPut reports that if-absent and if-match writes are enforced. + ConditionalPut bool + + // ConditionalDelete reports that if-match deletes are enforced. + ConditionalDelete bool + + // Multipart reports that bodies larger than one buffer stream without being held in memory. + Multipart bool + + // Tags reports that object tags are stored and returned. + Tags bool + + // Metadata reports that user metadata and content type are stored and returned. + Metadata bool + + // Range reports that partial reads are supported. + Range bool + + // Listing reports that prefix listing with pagination is supported. + Listing bool + + // ListDelimiter reports that listing rolls keys up into common prefixes. + ListDelimiter bool +} + +// SupportsPut reports whether the store can enforce the put condition. +func (c Capabilities) SupportsPut(cond Condition) bool { + defer perf.Track(nil, "blob.Capabilities.SupportsPut")() + + return !cond.IsConditional() || c.ConditionalPut +} + +// CheckPut returns a clear ErrBlobConditionalUnsupported error when the store cannot enforce cond. +func (c Capabilities) CheckPut(store, key string, cond Condition) error { + defer perf.Track(nil, "blob.Capabilities.CheckPut")() + + if c.SupportsPut(cond) { + return nil + } + return NewError(KindUnsupported, OpPut, key, fmt.Errorf("%w: %s store cannot enforce conditional writes", errUtils.ErrBlobConditionalUnsupported, store)) +} + +// CheckDelete returns a clear ErrBlobConditionalUnsupported error when the store cannot enforce an if-match delete. +func (c Capabilities) CheckDelete(store, key string, opts DeleteOptions) error { + defer perf.Track(nil, "blob.Capabilities.CheckDelete")() + + if opts.IfMatch == "" || c.ConditionalDelete { + return nil + } + return NewError(KindUnsupported, OpDelete, key, fmt.Errorf("%w: %s store cannot enforce conditional deletes", errUtils.ErrBlobConditionalUnsupported, store)) +} diff --git a/pkg/artifact/blob/doc.go b/pkg/artifact/blob/doc.go new file mode 100644 index 00000000000..7011e5c482d --- /dev/null +++ b/pkg/artifact/blob/doc.go @@ -0,0 +1,21 @@ +// Package blob is the transport-only object I/O layer shared by artifact repositories and the +// legacy CI artifact adapters. +// +// A [Store] reads and writes opaque byte streams addressed by slash-separated keys. It knows +// nothing about artifact records, CI metadata, signatures, or key layouts: callers own those and +// pass fully formed keys. The package defines streaming put/get, head, conditional write and +// delete (if-absent and if-match), prefix listing with pagination, object tags, user metadata, +// capability reporting, and a small set of classified errors. +// +// Conditional operations carry an ETag. ETags are opaque concurrency tokens and are never payload +// checksums. A store that cannot enforce a requested condition fails that operation with +// [errors.ErrBlobConditionalUnsupported] instead of silently overwriting. A mutation whose result +// was not observed (for example a connection reset after the request was sent) fails with an error +// for which [IsAmbiguous] is true and whose Temporary method reports true; callers reconcile by +// reading the object back. +// +// Implementations live in subpackages: blob/local (filesystem) and blob/s3 (AWS SDK v2). The +// blobtest subpackage holds the shared conformance suite every implementation must pass. +// +// This package must not import pkg/ci or any CI types. Dependencies point from CI adapters to blob. +package blob diff --git a/pkg/artifact/blob/errors.go b/pkg/artifact/blob/errors.go new file mode 100644 index 00000000000..35f6b4ff0f5 --- /dev/null +++ b/pkg/artifact/blob/errors.go @@ -0,0 +1,164 @@ +package blob + +import ( + "errors" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Operation names carried by [Error.Op] and used by every backend. +const ( + OpPut = "put" + OpGet = "get" + OpStat = "stat" + OpDelete = "delete" + OpList = "list" + OpValidate = "validate" + OpNew = "new" +) + +// Kind classifies a blob error independently of the backend that produced it. +type Kind int + +// Error kinds. +const ( + // KindOther is any failure without a more specific class. Only the cause is reported. + KindOther Kind = iota + + // KindNotFound means the addressed object does not exist. + KindNotFound + + // KindPrecondition means a conditional write or delete lost its race. + KindPrecondition + + // KindUnsupported means the endpoint cannot enforce a required condition. + KindUnsupported + + // KindAmbiguous means a mutation was sent but its result was not observed. + KindAmbiguous + + // KindInvalid means the request itself was malformed. + KindInvalid + + // KindIdentity means a configured identity could not be resolved to credentials. + KindIdentity +) + +// sentinel returns the static error for the kind, or nil for KindOther. +func (k Kind) sentinel() error { + switch k { + case KindNotFound: + return errUtils.ErrBlobNotFound + case KindPrecondition: + return errUtils.ErrBlobPreconditionFailed + case KindUnsupported: + return errUtils.ErrBlobConditionalUnsupported + case KindAmbiguous: + return errUtils.ErrBlobOutcomeUnknown + case KindInvalid: + return errUtils.ErrBlobInvalid + case KindIdentity: + return errUtils.ErrBlobIdentityUnresolved + default: + return nil + } +} + +// Error is a classified blob failure. It matches both the kind sentinel and the cause under +// errors.Is, so context cancellation and SDK errors stay visible. +type Error struct { + // Kind is the failure class. + Kind Kind + + // Op is the transport operation, for example "put" or "stat". + Op string + + // Key is the object key, when the operation addressed one. + Key string + + // Err is the underlying cause. + Err error +} + +// NewError returns a classified error. A nil err is allowed for kinds that have a sentinel. +func NewError(kind Kind, op, key string, err error) *Error { + defer perf.Track(nil, "blob.NewError")() + + return &Error{Kind: kind, Op: op, Key: key, Err: err} +} + +// Error implements error. +func (e *Error) Error() string { + defer perf.Track(nil, "blob.Error.Error")() + + subject := "blob " + e.Op + if e.Key != "" { + subject = fmt.Sprintf("%s %q", subject, e.Key) + } + sentinel := e.Kind.sentinel() + switch { + case sentinel == nil && e.Err == nil: + return subject + ": failed" + case sentinel == nil: + return fmt.Sprintf("%s: %v", subject, e.Err) + case e.Err == nil, errors.Is(e.Err, sentinel): + if e.Err == nil { + return fmt.Sprintf("%s: %v", subject, sentinel) + } + return fmt.Sprintf("%s: %v", subject, e.Err) + default: + return fmt.Sprintf("%s: %v: %v", subject, sentinel, e.Err) + } +} + +// Unwrap exposes the kind sentinel and the cause to errors.Is and errors.As. +func (e *Error) Unwrap() []error { + defer perf.Track(nil, "blob.Error.Unwrap")() + + var chain []error + if sentinel := e.Kind.sentinel(); sentinel != nil { + chain = append(chain, sentinel) + } + if e.Err != nil { + chain = append(chain, e.Err) + } + return chain +} + +// Temporary reports an ambiguous outcome: the request may or may not have been applied, so the +// caller must reconcile by reading the object instead of assuming failure. +func (e *Error) Temporary() bool { + defer perf.Track(nil, "blob.Error.Temporary")() + + return e.Kind == KindAmbiguous +} + +// IsNotFound reports whether err means the object does not exist. +func IsNotFound(err error) bool { + defer perf.Track(nil, "blob.IsNotFound")() + + return errors.Is(err, errUtils.ErrBlobNotFound) +} + +// IsPreconditionFailed reports whether err means a conditional write or delete lost its race. +func IsPreconditionFailed(err error) bool { + defer perf.Track(nil, "blob.IsPreconditionFailed")() + + return errors.Is(err, errUtils.ErrBlobPreconditionFailed) +} + +// IsConditionalUnsupported reports whether err means the endpoint cannot enforce a condition. +func IsConditionalUnsupported(err error) bool { + defer perf.Track(nil, "blob.IsConditionalUnsupported")() + + return errors.Is(err, errUtils.ErrBlobConditionalUnsupported) +} + +// IsAmbiguous reports whether err means a mutation's outcome was not observed. +func IsAmbiguous(err error) bool { + defer perf.Track(nil, "blob.IsAmbiguous")() + + return errors.Is(err, errUtils.ErrBlobOutcomeUnknown) +} diff --git a/pkg/artifact/blob/key.go b/pkg/artifact/blob/key.go new file mode 100644 index 00000000000..8f5f12d28bf --- /dev/null +++ b/pkg/artifact/blob/key.go @@ -0,0 +1,62 @@ +package blob + +import ( + "fmt" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MaxKeyBytes is the longest key any store accepts. It matches the S3 object key limit. +const MaxKeyBytes = 1024 + +// ValidateKey checks that key is a portable relative key: non-empty, valid UTF-8 without NUL or +// backslash, no leading or trailing slash, and no empty, "." or ".." segments. Filesystem stores +// require it; S3 stores apply it by default so a key is valid on every backend. +func ValidateKey(key string) error { + defer perf.Track(nil, "blob.ValidateKey")() + + if err := ValidateOpaqueKey(key); err != nil { + return err + } + if strings.ContainsRune(key, '\\') { + return invalidKey(key, "contains a backslash") + } + if strings.HasPrefix(key, "/") || strings.HasSuffix(key, "/") { + return invalidKey(key, "has a leading or trailing slash") + } + for _, segment := range strings.Split(key, "/") { + switch segment { + case "": + return invalidKey(key, "contains an empty segment") + case ".", "..": + return invalidKey(key, "contains a relative segment") + } + } + return nil +} + +// ValidateOpaqueKey applies only the limits every object store shares: non-empty, at most +// [MaxKeyBytes] bytes, valid UTF-8, and no NUL. It is used for legacy key patterns that are +// opaque strings rather than portable paths. +func ValidateOpaqueKey(key string) error { + defer perf.Track(nil, "blob.ValidateOpaqueKey")() + + switch { + case key == "": + return invalidKey(key, "is empty") + case len(key) > MaxKeyBytes: + return invalidKey(key, "is too long") + case !utf8.ValidString(key): + return invalidKey(key, "is not valid UTF-8") + case strings.ContainsRune(key, 0): + return invalidKey(key, "contains NUL") + } + return nil +} + +func invalidKey(key, reason string) error { + return NewError(KindInvalid, OpValidate, key, fmt.Errorf("%w: key %s", errUtils.ErrBlobInvalid, reason)) +} diff --git a/pkg/artifact/blob/list.go b/pkg/artifact/blob/list.go new file mode 100644 index 00000000000..96780780ec9 --- /dev/null +++ b/pkg/artifact/blob/list.go @@ -0,0 +1,30 @@ +package blob + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// ListAll follows the pagination cursor until the listing is complete and returns every object and +// common prefix. Callers that can stop early should page manually. +func ListAll(ctx context.Context, store Store, opts ListOptions) ([]ObjectInfo, []string, error) { + defer perf.Track(nil, "blob.ListAll")() + + var ( + objects []ObjectInfo + prefixes []string + ) + for { + page, err := store.List(ctx, opts) + if err != nil { + return nil, nil, err + } + objects = append(objects, page.Objects...) + prefixes = append(prefixes, page.CommonPrefixes...) + if page.NextCursor == "" { + return objects, prefixes, nil + } + opts.Cursor = page.NextCursor + } +} diff --git a/pkg/artifact/blob/local/attrs.go b/pkg/artifact/blob/local/attrs.go new file mode 100644 index 00000000000..1ff218480ad --- /dev/null +++ b/pkg/artifact/blob/local/attrs.go @@ -0,0 +1,45 @@ +package local + +import ( + "encoding/json" + "errors" + "os" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// attrs is the on-disk attribute file stored beside an object when any attribute is supplied. +type attrs struct { + ContentType string `json:"content_type,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +func (a attrs) empty() bool { + return a.ContentType == "" && len(a.Metadata) == 0 && len(a.Tags) == 0 +} + +func attrsPath(objectPath string) string { + return objectPath + attrsSuffix +} + +// readAttrs loads the attribute file of objectPath; a missing file yields empty attributes. +func readAttrs(objectPath string) (attrs, error) { + var a attrs + data, err := os.ReadFile(attrsPath(objectPath)) + if errors.Is(err, os.ErrNotExist) { + return a, nil + } + if err != nil { + return a, err + } + if err := json.Unmarshal(data, &a); err != nil { + return attrs{}, err + } + return a, nil +} + +// fromPut builds the attributes requested by a put. +func fromPut(opts blob.PutOptions) attrs { + return attrs{ContentType: opts.ContentType, Metadata: blob.CloneMap(opts.Metadata), Tags: blob.CloneMap(opts.Tags)} +} diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go new file mode 100644 index 00000000000..13ea2e2f9ea --- /dev/null +++ b/pkg/artifact/blob/local/delete.go @@ -0,0 +1,51 @@ +package local + +import ( + "context" + "errors" + "os" + "path/filepath" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [blob.Store]. +func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) error { + defer perf.Track(nil, "blob.local.Delete")() + + if err := s.validate(blob.OpDelete, key); err != nil { + return err + } + if err := checkContext(ctx, blob.OpDelete, key); err != nil { + return err + } + release, err := s.guard(ctx, blob.OpDelete, key, opts.IfMatch != "") + if err != nil { + return err + } + defer release() + + target := s.path(key) + if opts.IfMatch != "" { + if err := checkDeleteETag(key, target, opts.IfMatch); err != nil { + return err + } + } + for _, path := range []string{target, attrsPath(target)} { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return wrapIO(blob.OpDelete, key, err) + } + } + removeEmptyDirs(s.root, filepath.Dir(target)) + return nil +} + +// checkDeleteETag verifies the if-match token of a conditional delete. Callers hold the lock. +func checkDeleteETag(key, target, want string) error { + state, err := inspect(target) + if err != nil { + return wrapIO(blob.OpDelete, key, err) + } + return matchETag(blob.OpDelete, key, state, want) +} diff --git a/pkg/artifact/blob/local/doc.go b/pkg/artifact/blob/local/doc.go new file mode 100644 index 00000000000..6030b3b41ac --- /dev/null +++ b/pkg/artifact/blob/local/doc.go @@ -0,0 +1,18 @@ +// Package local is the filesystem implementation of [blob.Store]. +// +// Objects are plain files under a root directory, so a directory written here is readable with +// ordinary tools and matches the legacy CI "local/dir" layout byte for byte. Writes stream to a +// temporary file in the destination directory and are renamed into place, so readers never see a +// partial object. Content type, user metadata, and tags live in an optional ".atmos-blob.json" +// attribute file that is written only when an attribute is supplied. +// +// Conditional writes and deletes are emulated: the condition is checked and the rename performed +// while holding a per-root lock (an in-process mutex plus an OS file lock), so concurrent +// conditional operations from any process are linearizable. Unconditional writes do not take the +// lock unless [WithLockAllWrites] is set; callers that mix conditional and unconditional writers +// on one key must enable it. +// +// A local object cannot share a path with a key prefix: "a" and "a/b" cannot both exist. ETags are +// opaque tokens: the SHA-256 of the content for small objects and a weak size/mtime token for +// objects over the content-hash limit. Listings do not populate ETag; call Stat for a token. +package local diff --git a/pkg/artifact/blob/local/errors.go b/pkg/artifact/blob/local/errors.go new file mode 100644 index 00000000000..b9ff8e33e5b --- /dev/null +++ b/pkg/artifact/blob/local/errors.go @@ -0,0 +1,65 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// wrapIO classifies an OS error: a missing path is NotFound, everything else is a plain failure. +func wrapIO(op, key string, err error) error { + if errors.Is(err, os.ErrNotExist) { + return blob.NewError(blob.KindNotFound, op, key, err) + } + return blob.NewError(blob.KindOther, op, key, err) +} + +func preconditionFailed(op, key, reason string) error { + return blob.NewError(blob.KindPrecondition, op, key, fmt.Errorf("%w: %s", errUtils.ErrBlobPreconditionFailed, reason)) +} + +// classifyCopy maps a body copy failure: size mismatches are invalid requests, the rest are I/O. +func classifyCopy(key string, err error) error { + if errors.Is(err, errUtils.ErrBlobInvalid) { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, err) + } + return blob.NewError(blob.KindOther, blob.OpPut, key, err) +} + +// classifyCreate explains why a temporary file could not be created beside target. An existing +// object occupying a parent path is a key conflict, which the filesystem cannot represent. +func (s *Store) classifyCreate(key, target string, err error) error { + for dir := filepath.Dir(target); strings.HasPrefix(dir, s.root) && dir != s.root; dir = filepath.Dir(dir) { + if info, statErr := os.Stat(dir); statErr == nil && !info.IsDir() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: a parent path is an existing object", errUtils.ErrBlobInvalid)) + } + } + return wrapIO(blob.OpPut, key, err) +} + +func firstError(errs ...error) error { + return errors.Join(errs...) +} + +// ctxReader fails reads once its context is done so a canceled put stops promptly. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "local.ctxReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go new file mode 100644 index 00000000000..f1fca0c3e39 --- /dev/null +++ b/pkg/artifact/blob/local/etag.go @@ -0,0 +1,44 @@ +package local + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "strconv" +) + +// contentETagLimit is the largest object whose ETag is its SHA-256. Larger objects get a weak +// size/mtime token so a head request never reads gigabytes. +const contentETagLimit = 4 << 20 + +// hexBase formats sizes and times inside weak ETags. +const hexBase = 16 + +// etagOf returns the ETag of the file at path described by info. +func etagOf(path string, info os.FileInfo) (string, error) { + if info.Size() > contentETagLimit { + return weakETag(info), nil + } + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return contentETag(hash.Sum(nil)), nil +} + +// contentETag formats a content hash as an ETag. +func contentETag(sum []byte) string { + return `"` + hex.EncodeToString(sum) + `"` +} + +// weakETag derives a token from size and modification time for objects over the content limit. +func weakETag(info os.FileInfo) string { + return fmt.Sprintf(`W/"%s-%s"`, strconv.FormatInt(info.Size(), hexBase), strconv.FormatInt(info.ModTime().UnixNano(), hexBase)) +} diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go new file mode 100644 index 00000000000..1b480ed877a --- /dev/null +++ b/pkg/artifact/blob/local/fsutil.go @@ -0,0 +1,85 @@ +package local + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "runtime" + "time" +) + +const ( + renameAttempts = 8 + renameBackoff = 5 * time.Millisecond +) + +func mkdirAll(dir string, perm os.FileMode) error { + return os.MkdirAll(dir, perm) +} + +// createTemp creates an exclusive temporary file in dir, creating dir when needed. A concurrent +// delete may remove an empty directory between creation and use, so it retries a few times. +func createTemp(dir string, dirPerm, filePerm os.FileMode) (*os.File, error) { + var lastErr error + for range renameAttempts { + if err := mkdirAll(dir, dirPerm); err != nil { + return nil, err + } + name, err := tempName() + if err != nil { + return nil, err + } + file, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_EXCL, filePerm) + if err == nil { + return file, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + lastErr = err + } + return nil, lastErr +} + +func tempName() (string, error) { + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return tempPrefix + hex.EncodeToString(raw[:]), nil +} + +// renameReplace atomically moves src over dst. Windows rejects a rename while another handle is +// open on dst, so it retries briefly there. +func renameReplace(src, dst string) error { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = renameAttempts + } + var err error + for i := range attempts { + if err = os.Rename(src, dst); err == nil { + return nil + } + if i < attempts-1 { + time.Sleep(renameBackoff) + } + } + return err +} + +// removeEmptyDirs removes empty directories from dir up to, but never including, root. +func removeEmptyDirs(root, dir string) { + for dir != root && dir != "." && dir != string(filepath.Separator) && len(dir) > len(root) { + entries, err := os.ReadDir(dir) + if err != nil || len(entries) > 0 { + return + } + if os.Remove(dir) != nil { + return + } + dir = filepath.Dir(dir) + } +} diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go new file mode 100644 index 00000000000..923b95266ec --- /dev/null +++ b/pkg/artifact/blob/local/get.go @@ -0,0 +1,116 @@ +package local + +import ( + "context" + "fmt" + "io" + "os" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// rangedBody closes the file while reading only a window of it. +type rangedBody struct { + io.Reader + file *os.File +} + +func (r *rangedBody) Close() error { + defer perf.Track(nil, "local.rangedBody.Close")() + + return r.file.Close() +} + +// Get implements [blob.Store]. +func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blob.Object, error) { + defer perf.Track(nil, "blob.local.Get")() + + if err := s.validate(blob.OpGet, key); err != nil { + return nil, err + } + if opts.Offset < 0 { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: negative offset", errUtils.ErrBlobInvalid)) + } + if err := checkContext(ctx, blob.OpGet, key); err != nil { + return nil, err + } + target := s.path(key) + file, err := os.Open(target) + if err != nil { + return nil, wrapIO(blob.OpGet, key, err) + } + info, err := s.describe(key, target, file, false) + if err != nil { + _ = file.Close() + return nil, err + } + body, err := openBody(file, info.Size, opts) + if err != nil { + _ = file.Close() + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, err) + } + return &blob.Object{ObjectInfo: *info, Body: body}, nil +} + +// openBody positions file for a full or ranged read. +func openBody(file *os.File, size int64, opts blob.GetOptions) (io.ReadCloser, error) { + if !opts.IsRange() { + return file, nil + } + if opts.Offset > size { + return nil, fmt.Errorf("%w: offset %d is past the end of a %d byte object", errUtils.ErrBlobInvalid, opts.Offset, size) + } + if _, err := file.Seek(opts.Offset, io.SeekStart); err != nil { + return nil, err + } + var reader io.Reader = file + if opts.Length > 0 { + reader = io.LimitReader(file, opts.Length) + } + return &rangedBody{Reader: reader, file: file}, nil +} + +// Stat implements [blob.Store]. +func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.local.Stat")() + + if err := s.validate(blob.OpStat, key); err != nil { + return nil, err + } + if err := checkContext(ctx, blob.OpStat, key); err != nil { + return nil, err + } + target := s.path(key) + file, err := os.Open(target) + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + defer file.Close() + return s.describe(key, target, file, opts.IncludeTags) +} + +// describe builds ObjectInfo for an open regular file. A directory is not an object. +func (s *Store) describe(key, target string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { + stat, err := file.Stat() + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + if !stat.Mode().IsRegular() { + return nil, blob.NewError(blob.KindNotFound, blob.OpStat, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) + } + etag, err := etagOf(target, stat) + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + a, err := readAttrs(target) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpStat, key, fmt.Errorf("read attributes: %w", err)) + } + info := &blob.ObjectInfo{Key: key, Size: stat.Size(), ETag: etag, ContentType: a.ContentType, LastModified: stat.ModTime(), Metadata: a.Metadata} + if withTags { + info.Tags = a.Tags + } + return info, nil +} diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go new file mode 100644 index 00000000000..4752c999431 --- /dev/null +++ b/pkg/artifact/blob/local/list.go @@ -0,0 +1,165 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// defaultPageSize is the entries per page when ListOptions.Limit is zero, matching S3's maximum. +const defaultPageSize = 1000 + +// entry is one object found during a walk. +type entry struct { + key string + info os.FileInfo +} + +// List implements [blob.Store]. +func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage, error) { + defer perf.Track(nil, "blob.local.List")() + + if err := checkContext(ctx, blob.OpList, opts.Prefix); err != nil { + return nil, err + } + if err := validatePrefix(opts.Prefix); err != nil { + return nil, err + } + entries, err := s.walk(ctx, opts.Prefix) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpList, opts.Prefix, err) + } + return paginate(entries, opts), nil +} + +// walk returns every object whose key starts with prefix, sorted by key. +func (s *Store) walk(ctx context.Context, prefix string) ([]entry, error) { + start := s.root + if idx := strings.LastIndexByte(prefix, '/'); idx >= 0 { + start = s.path(prefix[:idx]) + } + var entries []entry + err := filepath.WalkDir(start, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return ignoreMissing(walkErr) + } + if err := ctx.Err(); err != nil { + return err + } + found, ok, err := s.objectAt(path, d, prefix) + if err != nil || !ok { + return err + } + entries = append(entries, found) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].key < entries[j].key }) + return entries, nil +} + +// ignoreMissing treats a path that vanished during the walk as absent. +func ignoreMissing(err error) error { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +// objectAt returns the object stored at a walked path, or false for directories, bookkeeping +// files, keys outside the prefix, and files that vanished. +func (s *Store) objectAt(path string, d fs.DirEntry, prefix string) (entry, bool, error) { + if d.IsDir() || reserved(path, s.root, d.Name()) { + return entry{}, false, nil + } + rel, err := filepath.Rel(s.root, path) + if err != nil { + return entry{}, false, err + } + key := filepath.ToSlash(rel) + if !strings.HasPrefix(key, prefix) { + return entry{}, false, nil + } + info, err := d.Info() + if err != nil { + return entry{}, false, ignoreMissing(err) + } + return entry{key: key, info: info}, info.Mode().IsRegular(), nil +} + +// reserved reports whether a file is one of the store's own bookkeeping files. +func reserved(path, root, name string) bool { + return strings.HasPrefix(name, tempPrefix) || + strings.HasSuffix(name, attrsSuffix) || + (name == lockName && filepath.Dir(path) == root) +} + +// paginate applies the delimiter roll-up, cursor, and limit to sorted entries. +func paginate(entries []entry, opts blob.ListOptions) *blob.ListPage { + limit := opts.Limit + if limit <= 0 || limit > defaultPageSize { + limit = defaultPageSize + } + page := &blob.ListPage{} + count := 0 + lastToken := "" + for _, e := range entries { + token, rolled := rollUp(e.key, opts) + if opts.Cursor != "" && token <= opts.Cursor { + continue + } + if rolled && token == lastToken { + continue + } + if count == limit { + page.NextCursor = lastToken + return page + } + if rolled { + page.CommonPrefixes = append(page.CommonPrefixes, token) + } else { + page.Objects = append(page.Objects, blob.ObjectInfo{Key: e.key, Size: e.info.Size(), LastModified: e.info.ModTime()}) + } + lastToken = token + count++ + } + return page +} + +// rollUp returns the ordering token of a key: the key itself, or its common prefix when the +// delimiter occurs after the listing prefix. +func rollUp(key string, opts blob.ListOptions) (string, bool) { + if opts.Delimiter == "" { + return key, false + } + rest := key[len(opts.Prefix):] + idx := strings.Index(rest, opts.Delimiter) + if idx < 0 { + return key, false + } + return opts.Prefix + rest[:idx+len(opts.Delimiter)], true +} + +// validatePrefix rejects listing prefixes that could leave the root. +func validatePrefix(prefix string) error { + for _, segment := range strings.Split(prefix, "/") { + if segment == ".." { + return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix contains a relative segment", errUtils.ErrBlobInvalid)) + } + } + if strings.ContainsRune(prefix, '\\') || strings.HasPrefix(prefix, "/") { + return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix must be a relative slash path", errUtils.ErrBlobInvalid)) + } + return nil +} diff --git a/pkg/artifact/blob/local/lock.go b/pkg/artifact/blob/local/lock.go new file mode 100644 index 00000000000..a2c1d34a0ff --- /dev/null +++ b/pkg/artifact/blob/local/lock.go @@ -0,0 +1,59 @@ +package local + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/gofrs/flock" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +const lockRetry = 10 * time.Millisecond + +// rootMutexes holds one in-process mutex per root so separate Store values over one directory +// serialize with each other without relying on the file lock alone. +var rootMutexes sync.Map + +func rootMutex(root string) *sync.Mutex { + key := root + if abs, err := filepath.Abs(root); err == nil { + key = abs + } + mutex, _ := rootMutexes.LoadOrStore(key, &sync.Mutex{}) + return mutex.(*sync.Mutex) +} + +// lockRoot takes the per-root lock (in-process mutex, then OS file lock) and returns its release. +func (s *Store) lockRoot(ctx context.Context, op, key string) (func(), error) { + mutex := rootMutex(s.root) + mutex.Lock() + if err := ctx.Err(); err != nil { + mutex.Unlock() + return nil, blob.NewError(blob.KindOther, op, key, err) + } + if err := mkdirAll(s.root, s.dirPerm); err != nil { + mutex.Unlock() + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("create lock directory: %w", err)) + } + fileLock := flock.New(filepath.Join(s.root, lockName)) + if _, err := fileLock.TryLockContext(ctx, lockRetry); err != nil { + mutex.Unlock() + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("acquire root lock: %w", err)) + } + return func() { + _ = fileLock.Unlock() + mutex.Unlock() + }, nil +} + +// guard returns the lock release for a write or delete, or a no-op when no lock is needed. +func (s *Store) guard(ctx context.Context, op, key string, conditional bool) (func(), error) { + if !conditional && !s.lockAll { + return func() {}, nil + } + return s.lockRoot(ctx, op, key) +} diff --git a/pkg/artifact/blob/local/options.go b/pkg/artifact/blob/local/options.go new file mode 100644 index 00000000000..0b9a231d802 --- /dev/null +++ b/pkg/artifact/blob/local/options.go @@ -0,0 +1,38 @@ +package local + +import ( + "os" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Default permissions for created directories and files. They match the legacy local CI store. +const ( + DefaultDirPerm os.FileMode = 0o755 + DefaultFilePerm os.FileMode = 0o644 +) + +// Option configures a [Store]. +type Option func(*Store) + +// WithDirPerm sets the permission bits of created directories. +func WithDirPerm(perm os.FileMode) Option { + defer perf.Track(nil, "blob.local.WithDirPerm")() + + return func(s *Store) { s.dirPerm = perm } +} + +// WithFilePerm sets the permission bits of created files. +func WithFilePerm(perm os.FileMode) Option { + defer perf.Track(nil, "blob.local.WithFilePerm")() + + return func(s *Store) { s.filePerm = perm } +} + +// WithLockAllWrites serializes unconditional writes and deletes with conditional ones through the +// per-root lock. Enable it when one key is written both conditionally and unconditionally. +func WithLockAllWrites() Option { + defer perf.Track(nil, "blob.local.WithLockAllWrites")() + + return func(s *Store) { s.lockAll = true } +} diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go new file mode 100644 index 00000000000..aabee729e3a --- /dev/null +++ b/pkg/artifact/blob/local/put.go @@ -0,0 +1,223 @@ +package local + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// staged is a fully written temporary object waiting to be renamed into place. +type staged struct { + tempPath string + attrsTemp string + size int64 + etag string +} + +// Put implements [blob.Store]. The body is streamed to a temporary file and renamed into place, +// so a failed or canceled put never leaves a partial object. +func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.local.Put")() + + if err := s.validatePut(ctx, key, opts); err != nil { + return nil, err + } + target := s.path(key) + pending, err := s.stage(ctx, key, target, body, opts) + if err != nil { + return nil, err + } + defer pending.discard() + + release, err := s.guard(ctx, blob.OpPut, key, opts.Condition.IsConditional()) + if err != nil { + return nil, err + } + defer release() + + if err := s.checkCondition(key, target, opts.Condition); err != nil { + return nil, err + } + if err := s.commit(key, target, pending, fromPut(opts).empty()); err != nil { + return nil, err + } + return s.statCommitted(key, target, pending) +} + +func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOptions) error { + if err := s.validate(blob.OpPut, key); err != nil { + return err + } + if err := blob.ValidateTags(opts.Tags); err != nil { + return err + } + if opts.Condition.Kind == blob.ConditionIfMatch && opts.Condition.ETag == "" { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + } + return checkContext(ctx, blob.OpPut, key) +} + +// stage writes the body (and attribute file, when needed) to temporary files next to target. +func (s *Store) stage(ctx context.Context, key, target string, body io.Reader, opts blob.PutOptions) (*staged, error) { + file, err := createTemp(filepath.Dir(target), s.dirPerm, s.filePerm) + if err != nil { + return nil, s.classifyCreate(key, target, err) + } + pending := &staged{tempPath: file.Name()} + hash := sha256.New() + written, copyErr := copyBody(ctx, io.MultiWriter(file, hash), body, opts.Size) + closeErr := file.Close() + if err := firstError(copyErr, closeErr); err != nil { + pending.discard() + return nil, classifyCopy(key, err) + } + pending.size = written + pending.etag = contentETag(hash.Sum(nil)) + if err := s.stageAttrs(pending, target, fromPut(opts)); err != nil { + pending.discard() + return nil, wrapIO(blob.OpPut, key, err) + } + return pending, nil +} + +// stageAttrs writes the attribute file to a temporary path when any attribute is set. +func (s *Store) stageAttrs(pending *staged, target string, a attrs) error { + if a.empty() { + return nil + } + data, err := json.Marshal(a) + if err != nil { + return err + } + file, err := createTemp(filepath.Dir(target), s.dirPerm, s.filePerm) + if err != nil { + return err + } + pending.attrsTemp = file.Name() + _, writeErr := file.Write(data) + return firstError(writeErr, file.Close()) +} + +// copyBody streams body into dst, enforcing a declared size and honoring cancellation. +func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (int64, error) { + reader := io.Reader(&ctxReader{ctx: ctx, r: body}) + if size >= 0 { + reader = io.LimitReader(reader, size+1) + } + written, err := io.Copy(dst, reader) + if err != nil { + return written, err + } + if size >= 0 && written != size { + return written, fmt.Errorf("%w: body is %d bytes, declared %d", errUtils.ErrBlobInvalid, written, size) + } + return written, nil +} + +// checkCondition evaluates the put condition against the current object. Callers hold the lock. +func (s *Store) checkCondition(key, target string, cond blob.Condition) error { + if !cond.IsConditional() { + return nil + } + state, err := inspect(target) + if err != nil { + return wrapIO(blob.OpPut, key, err) + } + switch cond.Kind { + case blob.ConditionIfAbsent: + if state.exists { + return preconditionFailed(blob.OpPut, key, "object already exists") + } + case blob.ConditionIfMatch: + return matchETag(blob.OpPut, key, state, cond.ETag) + } + return nil +} + +// fileState is the current state of the file backing a key. +type fileState struct { + path string + info os.FileInfo + exists bool +} + +// inspect reports whether a regular file exists at path. A missing path is not an error. +func inspect(path string) (fileState, error) { + info, err := os.Stat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fileState{}, err + } + return fileState{path: path, info: info, exists: err == nil && info.Mode().IsRegular()}, nil +} + +// matchETag fails with NotFound when the object is missing and with a precondition failure when +// its ETag differs from want. +func matchETag(op, key string, state fileState, want string) error { + if !state.exists { + return blob.NewError(blob.KindNotFound, op, key, nil) + } + current, err := etagOf(state.path, state.info) + if err != nil { + return wrapIO(op, key, err) + } + if current != want { + return preconditionFailed(op, key, "ETag does not match") + } + return nil +} + +// commit moves the staged files into place: attributes first, so an object is only ever visible +// together with its attributes or with stale ones until the data rename lands. +func (s *Store) commit(key, target string, pending *staged, noAttrs bool) error { + switch { + case pending.attrsTemp != "": + if err := renameReplace(pending.attrsTemp, attrsPath(target)); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.attrsTemp = "" + case noAttrs: + if err := os.Remove(attrsPath(target)); err != nil && !errors.Is(err, os.ErrNotExist) { + return wrapIO(blob.OpPut, key, err) + } + } + if info, err := os.Stat(target); err == nil && info.IsDir() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) + } + if err := renameReplace(pending.tempPath, target); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.tempPath = "" + return nil +} + +// statCommitted describes the object that was just written. +func (s *Store) statCommitted(key, target string, pending *staged) (*blob.ObjectInfo, error) { + info, err := os.Stat(target) + if err != nil { + return nil, wrapIO(blob.OpPut, key, err) + } + etag := pending.etag + if info.Size() > contentETagLimit { + etag = weakETag(info) + } + return &blob.ObjectInfo{Key: key, Size: pending.size, ETag: etag, LastModified: info.ModTime()}, nil +} + +// discard removes any temporary files that were not committed. +func (p *staged) discard() { + if p.tempPath != "" { + _ = os.Remove(p.tempPath) + } + if p.attrsTemp != "" { + _ = os.Remove(p.attrsTemp) + } +} diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go new file mode 100644 index 00000000000..4f032e91ce6 --- /dev/null +++ b/pkg/artifact/blob/local/store.go @@ -0,0 +1,105 @@ +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + storeName = "local" + + // Name prefix of in-flight temporary files, which listings never show. + tempPrefix = ".atmos-blob-tmp-" + + // Name suffix of the optional attribute file stored beside an object. + attrsSuffix = ".atmos-blob.json" + + // Name of the per-root lock file. Only conditional operations create it. + lockName = ".atmos-blob.lock" +) + +// Store is a filesystem [blob.Store]. +type Store struct { + root string + dirPerm os.FileMode + filePerm os.FileMode + lockAll bool +} + +// New returns a Store rooted at root. The directory is created on first write. +func New(root string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.local.New")() + + if root == "" { + return nil, blob.NewError(blob.KindInvalid, "new", "", fmt.Errorf("%w: root directory is required", errUtils.ErrBlobInvalid)) + } + store := &Store{root: filepath.Clean(root), dirPerm: DefaultDirPerm, filePerm: DefaultFilePerm} + for _, opt := range opts { + opt(store) + } + return store, nil +} + +// Name implements [blob.Store]. +func (s *Store) Name() string { + defer perf.Track(nil, "local.Store.Name")() + + return storeName +} + +// Root returns the root directory. +func (s *Store) Root() string { + defer perf.Track(nil, "local.Store.Root")() + + return s.root +} + +// Capabilities implements [blob.Store]. +func (s *Store) Capabilities() blob.Capabilities { + defer perf.Track(nil, "local.Store.Capabilities")() + + return blob.Capabilities{ + ConditionalPut: true, + ConditionalDelete: true, + Multipart: true, + Tags: true, + Metadata: true, + Range: true, + Listing: true, + ListDelimiter: true, + } +} + +// validate rejects keys that are unsafe on a filesystem or collide with the store's own files. +func (s *Store) validate(op, key string) error { + if err := blob.ValidateKey(key); err != nil { + return err + } + last := key[strings.LastIndexByte(key, '/')+1:] + if strings.HasPrefix(last, tempPrefix) || strings.HasSuffix(last, attrsSuffix) || key == lockName { + return blob.NewError(blob.KindInvalid, op, key, fmt.Errorf("%w: key uses a reserved name", errUtils.ErrBlobInvalid)) + } + return nil +} + +// path maps a validated key to its file path. +func (s *Store) path(key string) string { + return filepath.Join(s.root, filepath.FromSlash(key)) +} + +// checkContext fails fast when ctx is already done. +func checkContext(ctx context.Context, op, key string) error { + if err := ctx.Err(); err != nil { + return blob.NewError(blob.KindOther, op, key, err) + } + return nil +} + +var _ blob.Store = (*Store)(nil) diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go new file mode 100644 index 00000000000..19c895eae0c --- /dev/null +++ b/pkg/artifact/blob/local/store_test.go @@ -0,0 +1,144 @@ +package local + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" +) + +func newStore(t *testing.T, opts ...Option) *Store { + t.Helper() + store, err := New(filepath.Join(t.TempDir(), "blobs"), opts...) + require.NoError(t, err) + return store +} + +func TestConformance(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { return newStore(t) }) +} + +func TestConformanceLockAllWrites(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { return newStore(t, WithLockAllWrites()) }) +} + +func TestNewRequiresRoot(t *testing.T) { + _, err := New("") + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestWritesPlainFilesWithoutExtraArtifacts(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "stack/component/sha.tar", bytes.NewReader([]byte("tar")), blob.PutOptions{Size: 3}) + require.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(store.Root(), "stack", "component", "sha.tar")) + require.NoError(t, err) + assert.Equal(t, []byte("tar"), data) + + entries, err := os.ReadDir(filepath.Join(store.Root(), "stack", "component")) + require.NoError(t, err) + require.Len(t, entries, 1, "no temp, attribute, or lock files remain for a plain unconditional put") + assert.NoFileExists(t, filepath.Join(store.Root(), lockName)) +} + +func TestConditionalPutCreatesLockFileOnly(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader([]byte("a")), blob.PutOptions{Size: 1, Condition: blob.IfAbsent()}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(store.Root(), lockName)) + + objects, _, err := blob.ListAll(context.Background(), store, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, objects, 1, "bookkeeping files are not listed") + assert.Equal(t, "k", objects[0].Key) +} + +func TestAttributeFileHiddenAndRemoved(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "d/k", bytes.NewReader([]byte("a")), blob.PutOptions{Size: 1, Tags: map[string]string{"a": "b"}}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(store.Root(), "d", "k"+attrsSuffix)) + + objects, _, err := blob.ListAll(ctx, store, blob.ListOptions{}) + require.NoError(t, err) + assert.Len(t, objects, 1) + + require.NoError(t, store.Delete(ctx, "d/k", blob.DeleteOptions{})) + assert.NoDirExists(t, filepath.Join(store.Root(), "d"), "empty parent directories are cleaned up") +} + +func TestReservedAndUnsafeKeysRejected(t *testing.T) { + store := newStore(t) + for _, key := range []string{"../escape", "a/../../b", "/abs", "a//b", "a/", ".", "d/" + tempPrefix + "x", "d/x" + attrsSuffix, lockName, "a\\b"} { + _, err := store.Put(context.Background(), key, bytes.NewReader(nil), blob.PutOptions{Size: 0}) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } + assert.NoDirExists(t, filepath.Join(filepath.Dir(store.Root()), "escape")) +} + +func TestObjectAndPrefixCannotShareAPath(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "a", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + + _, err = store.Put(ctx, "a/b", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a parent path that is an object") + + _, err = store.Put(ctx, "c/d", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + _, err = store.Put(ctx, "c", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a key that is a prefix of other objects") + + _, err = store.Stat(ctx, "c", blob.StatOptions{}) + assert.True(t, blob.IsNotFound(err), "a prefix is not an object") +} + +func TestLargeObjectGetsWeakETag(t *testing.T) { + store := newStore(t) + big := bytes.Repeat([]byte("x"), contentETagLimit+1) + info, err := store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big))}) + require.NoError(t, err) + assert.Contains(t, info.ETag, `W/"`) + stat, err := store.Stat(context.Background(), "big", blob.StatOptions{}) + require.NoError(t, err) + assert.Equal(t, info.ETag, stat.ETag) +} + +func TestListRejectsEscapingPrefix(t *testing.T) { + store := newStore(t) + _, err := store.List(context.Background(), blob.ListOptions{Prefix: "../"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestGetRejectsNegativeOffsetAndPastEnd(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader([]byte("abc")), blob.PutOptions{Size: 3}) + require.NoError(t, err) + _, err = store.Get(context.Background(), "k", blob.GetOptions{Offset: -1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = store.Get(context.Background(), "k", blob.GetOptions{Offset: 9, Length: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestCanceledContextFailsFast(t *testing.T) { + store := newStore(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := store.Put(ctx, "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, context.Canceled) + _, err = store.Stat(ctx, "k", blob.StatOptions{}) + require.ErrorIs(t, err, context.Canceled) + assert.NoDirExists(t, store.Root(), "nothing is created for a canceled operation") +} diff --git a/pkg/artifact/blob/mock_store.go b/pkg/artifact/blob/mock_store.go new file mode 100644 index 00000000000..fb8a0effbaf --- /dev/null +++ b/pkg/artifact/blob/mock_store.go @@ -0,0 +1,144 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/blob (interfaces: Store) +// +// Generated by this command: +// +// mockgen -package blob -destination pkg/artifact/blob/mock_store.go github.com/cloudposse/atmos/pkg/artifact/blob Store +// + +// Package blob is a generated GoMock package. +package blob + +import ( + context "context" + io "io" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Capabilities mocks base method. +func (m *MockStore) Capabilities() Capabilities { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Capabilities") + ret0, _ := ret[0].(Capabilities) + return ret0 +} + +// Capabilities indicates an expected call of Capabilities. +func (mr *MockStoreMockRecorder) Capabilities() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Capabilities", reflect.TypeOf((*MockStore)(nil).Capabilities)) +} + +// Delete mocks base method. +func (m *MockStore) Delete(ctx context.Context, key string, opts DeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, key, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockStoreMockRecorder) Delete(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), ctx, key, opts) +} + +// Get mocks base method. +func (m *MockStore) Get(ctx context.Context, key string, opts GetOptions) (*Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, key, opts) + ret0, _ := ret[0].(*Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockStoreMockRecorder) Get(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStore)(nil).Get), ctx, key, opts) +} + +// List mocks base method. +func (m *MockStore) List(ctx context.Context, opts ListOptions) (*ListPage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, opts) + ret0, _ := ret[0].(*ListPage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockStoreMockRecorder) List(ctx, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockStore)(nil).List), ctx, opts) +} + +// Name mocks base method. +func (m *MockStore) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockStoreMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockStore)(nil).Name)) +} + +// Put mocks base method. +func (m *MockStore) Put(ctx context.Context, key string, body io.Reader, opts PutOptions) (*ObjectInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", ctx, key, body, opts) + ret0, _ := ret[0].(*ObjectInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockStoreMockRecorder) Put(ctx, key, body, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockStore)(nil).Put), ctx, key, body, opts) +} + +// Stat mocks base method. +func (m *MockStore) Stat(ctx context.Context, key string, opts StatOptions) (*ObjectInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stat", ctx, key, opts) + ret0, _ := ret[0].(*ObjectInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stat indicates an expected call of Stat. +func (mr *MockStoreMockRecorder) Stat(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockStore)(nil).Stat), ctx, key, opts) +} diff --git a/pkg/artifact/blob/s3/api.go b/pkg/artifact/blob/s3/api.go new file mode 100644 index 00000000000..60d7be14ce3 --- /dev/null +++ b/pkg/artifact/blob/s3/api.go @@ -0,0 +1,41 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// API is the subset of the S3 client the store uses. *s3.Client satisfies it; tests may supply a +// recording or faulting implementation. +type API interface { + PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + GetObject(ctx context.Context, in *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + DeleteObject(ctx context.Context, in *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) + GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) + CreateMultipartUpload(ctx context.Context, in *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) + UploadPart(ctx context.Context, in *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) + CompleteMultipartUpload(ctx context.Context, in *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) + AbortMultipartUpload(ctx context.Context, in *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) +} + +var _ API = (*s3.Client)(nil) + +// sdkOptions is the per-request SDK option type, aliased so helpers need not import the SDK. +type sdkOptions = s3.Options + +// noRetry disables SDK retries for one request. +func noRetry(o *s3.Options) { + o.Retryer = aws.NopRetryer{} +} + +// skipOptionalChecksums keeps the SDK from attaching optional per-part checksums to multipart +// requests. An upload created without a checksum algorithm must also be completed without them, +// whatever defaults the injected client was built with. Artifact digests provide integrity. +func skipOptionalChecksums(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired +} diff --git a/pkg/artifact/blob/s3/classify.go b/pkg/artifact/blob/s3/classify.go new file mode 100644 index 00000000000..a21bfd97f1a --- /dev/null +++ b/pkg/artifact/blob/s3/classify.go @@ -0,0 +1,147 @@ +package s3 + +import ( + "errors" + "fmt" + "net" + "net/http" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// request describes the operation that failed so the classifier can judge ambiguity. +type request struct { + op string + key string + mutating bool + conditional bool +} + +// Error codes S3-compatible endpoints return when they cannot honor a conditional header. +var conditionalRejectCodes = map[string]bool{ + "NotImplemented": true, + "XNotImplemented": true, + "InvalidArgument": true, + "InvalidRequest": true, + "UnsupportedHeader": true, + "MalformedHeader": true, + "HeaderNotSupported": true, +} + +// classify turns an SDK error into a [blob.Error]. Nil stays nil and classified errors pass through. +func classify(req request, err error) error { + if err == nil { + return nil + } + var classified *blob.Error + if errors.As(err, &classified) { + return err + } + status, code := responseDetails(err) + if status == 0 { + return classifyTransport(req, err) + } + return blob.NewError(kindForResponse(req, status, code), req.op, req.key, err) +} + +// responseDetails extracts the HTTP status and S3 error code, or zero when no response arrived. +func responseDetails(err error) (int, string) { + var code string + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + code = apiErr.ErrorCode() + } + var respErr *awshttp.ResponseError + if errors.As(err, &respErr) { + return respErr.HTTPStatusCode(), code + } + return 0, code +} + +// kindForResponse maps a service response to an error kind. +func kindForResponse(req request, status int, code string) blob.Kind { + if kind, ok := definiteKind(status, code); ok { + return kind + } + if req.conditional && rejectsCondition(status, code) { + return blob.KindUnsupported + } + if req.mutating && ambiguousStatus(status) { + return blob.KindAmbiguous + } + return blob.KindOther +} + +// definiteKind classifies responses whose meaning does not depend on the request. +func definiteKind(status int, code string) (blob.Kind, bool) { + switch { + case code == "NoSuchBucket" || code == "NoSuchUpload": + // A missing bucket or upload is a configuration or ordering problem, never a missing object. + return blob.KindOther, true + case isMissingObject(status, code): + return blob.KindNotFound, true + case isLostRace(status, code): + return blob.KindPrecondition, true + case status == http.StatusRequestedRangeNotSatisfiable || code == "InvalidRange": + return blob.KindInvalid, true + default: + return blob.KindOther, false + } +} + +func isMissingObject(status int, code string) bool { + return status == http.StatusNotFound || code == "NoSuchKey" || code == "NotFound" +} + +// isLostRace reports a failed condition, including S3's conflict answer for concurrent conditional writes. +func isLostRace(status int, code string) bool { + return status == http.StatusPreconditionFailed || code == "PreconditionFailed" || + (status == http.StatusConflict && code == "ConditionalRequestConflict") +} + +// rejectsCondition reports a response that says the endpoint cannot honor a conditional header. +func rejectsCondition(status int, code string) bool { + return status == http.StatusNotImplemented || + (status == http.StatusBadRequest && conditionalRejectCodes[code]) +} + +// ambiguousStatus reports server errors after which a mutation may or may not have been applied. +// 503 (throttling) and 501 (not implemented) are definite rejections. +func ambiguousStatus(status int) bool { + return status >= http.StatusInternalServerError && + status != http.StatusServiceUnavailable && + status != http.StatusNotImplemented +} + +// classifyTransport judges a failure with no response: a mutation whose request may have reached +// the server is ambiguous, everything else is a plain failure. +func classifyTransport(req request, err error) error { + if req.mutating && requestMayHaveBeenSent(err) { + return blob.NewError(blob.KindAmbiguous, req.op, req.key, err) + } + return blob.NewError(blob.KindOther, req.op, req.key, err) +} + +// requestMayHaveBeenSent distinguishes failures after the request left the client from failures +// before it did (credential, signing, endpoint, and connection-establishment errors). +func requestMayHaveBeenSent(err error) bool { + var send *smithyhttp.RequestSendError + if !errors.As(err, &send) { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + return false + } + return true +} + +// unsupportedConditional builds the clear failure for an endpoint that cannot enforce a condition. +func unsupportedConditional(op, key, reason string) error { + return blob.NewError(blob.KindUnsupported, op, key, fmt.Errorf("%w: %s", errUtils.ErrBlobConditionalUnsupported, reason)) +} diff --git a/pkg/artifact/blob/s3/client.go b/pkg/artifact/blob/s3/client.go new file mode 100644 index 00000000000..01a00805aae --- /dev/null +++ b/pkg/artifact/blob/s3/client.go @@ -0,0 +1,176 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/store" +) + +// AuthResolver resolves an Atmos identity name to AWS credentials configuration. It is the AWS +// slice of store.AuthContextResolver, which satisfies it. +type AuthResolver interface { + ResolveAWSAuthContext(ctx context.Context, identityName string) (*store.AWSAuthConfig, error) +} + +// ClientConfig describes how to build an S3 client. Both the generic [Store] and the legacy CI +// adapter build clients through [NewClient] so credential, identity, and endpoint handling live in +// one place. +type ClientConfig struct { + // Region overrides the region of the identity and the default chain. + Region string + + // Identity is the Atmos identity name. Empty uses the default AWS credential chain. + Identity string + + // Resolver turns Identity into credentials configuration. + Resolver AuthResolver + + // EndpointURL overrides the S3 endpoint (for example an emulator). It wins over the endpoint of + // a resolved identity. + EndpointURL string + + // PathStyle selects path-style addressing. Nil means path-style whenever an endpoint override + // applies, which is what emulators such as Floci require. + PathStyle *bool + + // HTTPClient replaces the SDK HTTP client, for example to trust a private CA. The SDK honors + // AWS_CA_BUNDLE when nil. + HTTPClient aws.HTTPClient + + // Legacy keeps the pre-artifact CI behavior: when an identity is configured but no resolver was + // injected, ambient credentials are used, and the SDK's default checksum behavior is kept. Only + // the legacy CI adapter sets it. The generic store never does, so an unresolvable identity + // fails closed. + Legacy bool +} + +// NewClient builds an S3 client from cfg. With an identity it resolves credentials through the +// resolver; without one it uses the default credential chain. See [ClientConfig.Legacy] for the +// one deliberate difference between the generic and legacy paths. +func NewClient(ctx context.Context, cfg *ClientConfig) (*s3.Client, error) { + defer perf.Track(nil, "blob.s3.NewClient")() + + if cfg.Identity == "" { + return newDefaultClient(ctx, cfg, nil) + } + if cfg.Resolver == nil { + if cfg.Legacy { + return newDefaultClient(ctx, cfg, nil) + } + return nil, identityError(cfg.Identity, errUtils.ErrBlobIdentityUnresolved) + } + auth, err := cfg.Resolver.ResolveAWSAuthContext(ctx, cfg.Identity) + if err != nil { + return nil, identityError(cfg.Identity, err) + } + if auth == nil && !cfg.Legacy { + return nil, identityError(cfg.Identity, errUtils.ErrBlobIdentityUnresolved) + } + loadOpts := BuildAuthConfigOpts(cfg.Region, auth) + awsCfg, err := config.LoadDefaultConfig(ctx, withHTTPClient(loadOpts, cfg.HTTPClient)...) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + } + return s3.NewFromConfig(awsCfg, clientOptFns(cfg, auth)...), nil +} + +// identityError reports an unresolvable identity. It wraps both the configuration-load sentinel the +// legacy callers match and the blob identity sentinel the generic path matches. +func identityError(identity string, cause error) error { + return fmt.Errorf("%w: %w: failed to resolve AWS auth context for identity %q: %w", + errUtils.ErrAWSConfigLoadFailed, errUtils.ErrBlobIdentityUnresolved, identity, cause) +} + +// newDefaultClient builds a client from the default credential chain. +func newDefaultClient(ctx context.Context, cfg *ClientConfig, auth *store.AWSAuthConfig) (*s3.Client, error) { + var loadOpts []func(*config.LoadOptions) error + if cfg.Region != "" { + loadOpts = append(loadOpts, config.WithRegion(cfg.Region)) + } + awsCfg, err := config.LoadDefaultConfig(ctx, withHTTPClient(loadOpts, cfg.HTTPClient)...) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + } + return s3.NewFromConfig(awsCfg, clientOptFns(cfg, auth)...), nil +} + +func withHTTPClient(opts []func(*config.LoadOptions) error, client aws.HTTPClient) []func(*config.LoadOptions) error { + if client == nil { + return opts + } + return append(opts, config.WithHTTPClient(client)) +} + +// clientOptFns assembles the S3 client options for cfg and the resolved identity context. +func clientOptFns(cfg *ClientConfig, auth *store.AWSAuthConfig) []func(*s3.Options) { + var optFns []func(*s3.Options) + endpoint := cfg.EndpointURL + if endpoint == "" && auth != nil { + endpoint = auth.EndpointURL + } + if endpoint != "" { + pathStyle := true + if cfg.PathStyle != nil { + pathStyle = *cfg.PathStyle + } + optFns = append(optFns, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = pathStyle + }) + } else if cfg.PathStyle != nil { + pathStyle := *cfg.PathStyle + optFns = append(optFns, func(o *s3.Options) { o.UsePathStyle = pathStyle }) + } + if !cfg.Legacy { + // Integrity is established by artifact digests. Skipping optional request checksums keeps + // multipart completion and emulator endpoints simple. + optFns = append(optFns, func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) + } + return optFns +} + +// BuildClientOptFns returns the S3 client options an identity's auth context requires: an endpoint +// override (for example an emulator) with path-style addressing. Without it an identity scoped to an +// emulator would still route S3 calls to the real AWS endpoint. +func BuildClientOptFns(auth *store.AWSAuthConfig) []func(*s3.Options) { + defer perf.Track(nil, "blob.s3.BuildClientOptFns")() + + return clientOptFns(&ClientConfig{Legacy: true}, auth) +} + +// BuildAuthConfigOpts returns the AWS SDK config options for a resolved auth context. The +// store-level region wins over the identity's region. +func BuildAuthConfigOpts(region string, auth *store.AWSAuthConfig) []func(*config.LoadOptions) error { + defer perf.Track(nil, "blob.s3.BuildAuthConfigOpts")() + + var cfgOpts []func(*config.LoadOptions) error + if auth == nil { + return cfgOpts + } + if auth.CredentialsFile != "" { + cfgOpts = append(cfgOpts, config.WithSharedCredentialsFiles([]string{auth.CredentialsFile})) + } + if auth.ConfigFile != "" { + cfgOpts = append(cfgOpts, config.WithSharedConfigFiles([]string{auth.ConfigFile})) + } + if auth.Profile != "" { + cfgOpts = append(cfgOpts, config.WithSharedConfigProfile(auth.Profile)) + } + if region == "" { + region = auth.Region + } + if region != "" { + cfgOpts = append(cfgOpts, config.WithRegion(region)) + } + return cfgOpts +} diff --git a/pkg/artifact/blob/s3/client_test.go b/pkg/artifact/blob/s3/client_test.go new file mode 100644 index 00000000000..2bc5531fc88 --- /dev/null +++ b/pkg/artifact/blob/s3/client_test.go @@ -0,0 +1,198 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + sdk "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/store" +) + +func TestIdentityIsResolvedLazily(t *testing.T) { + resolver := &fakeResolver{err: errors.New("sso expired")} + s, err := New("bucket", WithIdentity("deploy", resolver), WithRegion("us-east-1")) + require.NoError(t, err, "constructing the store must not touch credentials") + assert.Zero(t, resolver.calls.Load()) + + _, err = s.Stat(context.Background(), "k", blob.StatOptions{}) + require.Error(t, err) + assert.Equal(t, int32(1), resolver.calls.Load()) +} + +func TestUnresolvableIdentityFailsClosed(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret") + t.Setenv("AWS_REGION", "us-east-1") + + cases := map[string]*fakeResolver{ + "resolver error": {err: errors.New("sso expired")}, + "nil auth context": {}, + "no resolver injected": nil, + } + for name, resolver := range cases { + t.Run(name, func(t *testing.T) { + var opt Option + if resolver == nil { + opt = WithIdentity("deploy", nil) + } else { + opt = WithIdentity("deploy", resolver) + } + s, err := New("bucket", opt) + require.NoError(t, err) + for range 2 { + _, err = s.Get(context.Background(), "k", blob.GetOptions{}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved, "never fall back to ambient credentials") + assert.Contains(t, err.Error(), "deploy", "the error names the identity") + } + if resolver != nil { + assert.Equal(t, int32(2), resolver.calls.Load(), "a failed resolution is retried by the next operation, not cached") + } + }) + } +} + +func TestLegacyClientKeepsAmbientFallback(t *testing.T) { + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_ACCESS_KEY_ID", "k") + t.Setenv("AWS_SECRET_ACCESS_KEY", "s") + + client, err := NewClient(context.Background(), &ClientConfig{Identity: "deploy", Legacy: true}) + require.NoError(t, err, "the legacy CI adapter keeps its historical fallback") + assert.NotNil(t, client) + + _, err = NewClient(context.Background(), &ClientConfig{Identity: "deploy"}) + require.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved, "the generic path never does") + require.ErrorIs(t, err, errUtils.ErrAWSConfigLoadFailed) +} + +func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) { + requests := make(chan *http.Request, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Clone(r.Context()) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + endpoint, err := url.Parse(server.URL) + require.NoError(t, err) + _, port, err := net.SplitHostPort(endpoint.Host) + require.NoError(t, err) + endpoint.Host = net.JoinHostPort("localhost", port) + + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + resolver := &fakeResolver{result: &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: endpoint.String()}} + s, err := New("artifacts", WithIdentity("floci", resolver)) + require.NoError(t, err) + + _, err = s.Stat(context.Background(), "templates/stack.yaml", blob.StatOptions{}) + require.NoError(t, err) + request := <-requests + assert.Equal(t, endpoint.Host, request.Host, "path-style: the bucket is not in the host name") + assert.Equal(t, "/artifacts/templates/stack.yaml", request.URL.Path) +} + +func TestExplicitEndpointWinsOverIdentityEndpoint(t *testing.T) { + cfg := &ClientConfig{EndpointURL: "http://explicit:9000"} + var opts sdk.Options + for _, fn := range clientOptFns(cfg, &store.AWSAuthConfig{EndpointURL: "http://identity:4566"}) { + fn(&opts) + } + assert.Equal(t, aws.String("http://explicit:9000"), opts.BaseEndpoint) + assert.True(t, opts.UsePathStyle, "an endpoint override implies path-style addressing") + + pathStyle := false + cfg.PathStyle = &pathStyle + opts = sdk.Options{} + for _, fn := range clientOptFns(cfg, nil) { + fn(&opts) + } + assert.False(t, opts.UsePathStyle, "path-style can be turned off for a custom endpoint") +} + +func TestGenericClientSkipsOptionalChecksumsButLegacyKeepsSDKDefaults(t *testing.T) { + var generic, legacy sdk.Options + for _, fn := range clientOptFns(&ClientConfig{}, nil) { + fn(&generic) + } + for _, fn := range clientOptFns(&ClientConfig{Legacy: true}, nil) { + fn(&legacy) + } + assert.Equal(t, aws.RequestChecksumCalculationWhenRequired, generic.RequestChecksumCalculation) + assert.Equal(t, aws.RequestChecksumCalculation(0), legacy.RequestChecksumCalculation) +} + +func TestBuildAuthConfigOpts(t *testing.T) { + cases := []struct { + name string + region string + auth *store.AWSAuthConfig + want config.LoadOptions + }{ + {"nil", "", nil, config.LoadOptions{}}, + { + "files and profile", "", &store.AWSAuthConfig{CredentialsFile: "/c", ConfigFile: "/f", Profile: "p"}, + config.LoadOptions{SharedCredentialsFiles: []string{"/c"}, SharedConfigFiles: []string{"/f"}, SharedConfigProfile: "p"}, + }, + {"store region wins", "us-east-1", &store.AWSAuthConfig{Region: "us-west-2"}, config.LoadOptions{Region: "us-east-1"}}, + {"identity region fills in", "", &store.AWSAuthConfig{Region: "eu-west-1"}, config.LoadOptions{Region: "eu-west-1"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got config.LoadOptions + for _, opt := range BuildAuthConfigOpts(tc.region, tc.auth) { + require.NoError(t, opt(&got)) + } + assert.Equal(t, tc.want.Region, got.Region) + assert.Equal(t, tc.want.SharedConfigProfile, got.SharedConfigProfile) + assert.Equal(t, tc.want.SharedCredentialsFiles, got.SharedCredentialsFiles) + assert.Equal(t, tc.want.SharedConfigFiles, got.SharedConfigFiles) + }) + } +} + +func TestBuildClientOptFnsMatchesLegacyEndpointBehavior(t *testing.T) { + assert.Empty(t, BuildClientOptFns(nil)) + assert.Empty(t, BuildClientOptFns(&store.AWSAuthConfig{})) + var opts sdk.Options + for _, fn := range BuildClientOptFns(&store.AWSAuthConfig{EndpointURL: "http://localhost:4566"}) { + fn(&opts) + } + assert.Equal(t, aws.String("http://localhost:4566"), opts.BaseEndpoint) + assert.True(t, opts.UsePathStyle) +} + +func TestHTTPClientInjectionReachesTheSDK(t *testing.T) { + fake := newFake(t) + var used bool + client := httpClientFunc(func(r *http.Request) (*http.Response, error) { + used = true + return http.DefaultClient.Do(r) + }) + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + api, err := NewClient(context.Background(), &ClientConfig{Region: "us-east-1", EndpointURL: fake.Endpoint(), HTTPClient: client}) + require.NoError(t, err) + s, err := NewFromClient(api, fake.Bucket()) + require.NoError(t, err) + _, err = s.Put(context.Background(), "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + assert.True(t, used) +} + +type httpClientFunc func(*http.Request) (*http.Response, error) + +func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { return f(r) } diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go new file mode 100644 index 00000000000..ac22a2b5a1f --- /dev/null +++ b/pkg/artifact/blob/s3/conditional.go @@ -0,0 +1,94 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// probePrefix holds the short-lived objects the conditional-write probe creates. +const probePrefix = ".atmos-blob-probe/" + +// ensureConditional makes sure a conditional operation is safe to send. In probe mode it first +// proves, once per store, that the endpoint enforces If-None-Match; an endpoint that silently +// ignores the header would otherwise turn create-if-absent into an unconditional overwrite. +func (s *Store) ensureConditional(ctx context.Context, key string, cond blob.Condition) error { + if !cond.IsConditional() || s.cfg.conditional != ConditionalProbe { + return nil + } + if err := s.ProbeConditionalWrites(ctx); err != nil { + return unsupportedConditionalFrom(key, err) + } + return nil +} + +// unsupportedConditionalFrom reports a failed probe as the operation's error. +func unsupportedConditionalFrom(key string, probeErr error) error { + if blob.IsConditionalUnsupported(probeErr) { + return blob.NewError(blob.KindUnsupported, blob.OpPut, key, probeErr) + } + return probeErr +} + +// ProbeConditionalWrites verifies that the endpoint enforces If-None-Match by creating a throwaway +// object twice: the second create must be rejected. A definite answer is cached for the life of the +// store; a transient failure is returned without being cached. It returns an error matching +// ErrBlobConditionalUnsupported when the endpoint ignores or rejects the condition. +func (s *Store) ProbeConditionalWrites(ctx context.Context) error { + defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")() + + s.probeMu.Lock() + defer s.probeMu.Unlock() + if s.probed { + return s.probeError + } + api, err := s.client(ctx) + if err != nil { + return err + } + err = s.runProbe(ctx, api) + if err == nil || blob.IsConditionalUnsupported(err) { + s.probed, s.probeError = true, err + } + return err +} + +// runProbe performs the double create and cleans up the probe object. +func (s *Store) runProbe(ctx context.Context, api API) error { + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return blob.NewError(blob.KindOther, "probe", "", err) + } + key := probePrefix + hex.EncodeToString(raw[:]) + objectKey := s.objectKey(key) + create := func() error { + _, err := api.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(objectKey), + Body: bytes.NewReader([]byte("atmos conditional write probe")), + IfNoneMatch: aws.String("*"), + }, noRetry) + return classify(request{op: "probe", key: key, mutating: true, conditional: true}, err) + } + if err := create(); err != nil { + return err + } + defer func() { + _, _ = api.DeleteObject(context.WithoutCancel(ctx), &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}) + }() + err := create() + if blob.IsPreconditionFailed(err) { + return nil + } + if err == nil { + return unsupportedConditional("probe", key, "the endpoint accepted a second create-if-absent for the same key, so it ignores If-None-Match") + } + return err +} diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go new file mode 100644 index 00000000000..414a6528508 --- /dev/null +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -0,0 +1,259 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func putIfAbsent(s *Store, key, body string) (*blob.ObjectInfo, error) { + return s.Put(context.Background(), key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body)), Condition: blob.IfAbsent()}) +} + +func TestConditionalHeadersAreSent(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + + first, err := putIfAbsent(s, "lock", "a") + require.NoError(t, err) + _, err = s.Put(context.Background(), "lock", bytes.NewReader([]byte("b")), blob.PutOptions{Size: 1, Condition: blob.IfMatch(first.ETag)}) + require.NoError(t, err) + + puts := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, puts, 2) + assert.Equal(t, "*", puts[0].Header.Get("If-None-Match")) + assert.Empty(t, puts[0].Header.Get("If-Match")) + assert.Equal(t, first.ETag, puts[1].Header.Get("If-Match")) + assert.Empty(t, puts[1].Header.Get("If-None-Match")) +} + +func TestConditionalDeleteSendsIfMatch(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + info := putBytes(t, s, "k", []byte("x"), blob.PutOptions{}) + + err := s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: `"stale"`}) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + require.NoError(t, s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: info.ETag})) + + deletes := fake.RequestsFor(s3test.OpDeleteObject) + require.Len(t, deletes, 2) + assert.Equal(t, `"stale"`, deletes[0].Header.Get("If-Match")) + assert.Equal(t, info.ETag, deletes[1].Header.Get("If-Match")) +} + +func TestEndpointThatCannotEnforceConditionsFailsClearly(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalNotImplemented)) + s := newTestStore(t, fake) + putBytes(t, s, "existing", []byte("keep"), blob.PutOptions{}) + + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "501 maps to a clear unsupported-condition error: %v", err) + assert.False(t, blob.IsAmbiguous(err)) + _, ok := fake.Object("lock") + assert.False(t, ok, "nothing was written") + + _, err = s.Put(context.Background(), "existing", bytes.NewReader([]byte("new")), blob.PutOptions{Size: 3, Condition: blob.IfMatch(`"x"`)}) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + object, _ := fake.Object("existing") + assert.Equal(t, []byte("keep"), object.Data, "the existing object was not overwritten") +} + +func TestEndpointRejectingTheHeaderWithBadRequestIsUnsupported(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusBadRequest, Code: "InvalidArgument", Times: -1}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + + // The same rejection of an unconditional put is a plain failure, not a capability problem. + _, err = s.Put(context.Background(), "plain", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.Error(t, err) + assert.False(t, blob.IsConditionalUnsupported(err)) +} + +func TestDisabledModeNeverSendsConditions(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithConditionalMode(ConditionalDisabled)) + + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + err = s.Delete(context.Background(), "lock", blob.DeleteOptions{IfMatch: `"x"`}) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Empty(t, fake.Requests(), "an endpoint known not to enforce conditions is never asked to") + + putBytes(t, s, "plain", []byte("x"), blob.PutOptions{}) + assert.Len(t, fake.Requests(), 1, "unconditional writes still work") +} + +func TestProbeDetectsEndpointThatIgnoresConditions(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + fake.Seed("lock", []byte("owner-a"), nil) + + _, err := putIfAbsent(s, "lock", "owner-b") + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "a silently-ignoring endpoint must not be trusted: %v", err) + object, _ := fake.Object("lock") + assert.Equal(t, []byte("owner-a"), object.Data, "the caller's object was never touched") + assert.Len(t, fake.Keys(), 1, "probe objects are removed") + + before := len(fake.Requests()) + _, err = putIfAbsent(s, "lock", "owner-c") + assert.True(t, blob.IsConditionalUnsupported(err)) + assert.Len(t, fake.Requests(), before, "a definite probe verdict is cached") +} + +func TestProbeAcceptsEnforcingEndpointOnce(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + _, err := putIfAbsent(s, "lock", "a") + require.NoError(t, err) + _, err = putIfAbsent(s, "lock2", "a") + require.NoError(t, err) + + puts := fake.RequestsFor(s3test.OpPutObject) + assert.Len(t, puts, 4, "two probe creates plus the two caller puts; the probe runs once") + assert.Equal(t, []string{"lock", "lock2"}, fake.Keys(), "the probe object was cleaned up") + assert.Equal(t, 1, countPrefix(puts, probePrefix)/2, "exactly one probe key was created twice") + + require.NoError(t, s.ProbeConditionalWrites(context.Background())) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 4, "explicit probe reuses the cached verdict") +} + +func countPrefix(requests []s3test.Request, prefix string) int { + count := 0 + for _, req := range requests { + if len(req.Key) >= len(prefix) && req.Key[:len(prefix)] == prefix { + count++ + } + } + return count +} + +func TestResponseDroppedAfterCommitIsAmbiguousAndNotRetried(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + + _, err := putIfAbsent(s, "lock", "mine") + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + var blobErr *blob.Error + require.True(t, errors.As(err, &blobErr)) + assert.True(t, blobErr.Temporary(), "ambiguity is signaled through Temporary()") + assert.False(t, blob.IsPreconditionFailed(err), "an SDK retry must not turn our own commit into a fake conflict") + + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1, "conditional requests are never retried by the SDK") + object, ok := fake.Object("lock") + require.True(t, ok, "the write did commit") + assert.Equal(t, []byte("mine"), object.Data) + + // Reconciliation: reading the object back proves ownership. + stat, statErr := s.Stat(context.Background(), "lock", blob.StatOptions{}) + require.NoError(t, statErr) + assert.Equal(t, object.ETag, stat.ETag) +} + +func TestResponseDroppedBeforeCommitIsAlsoAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropBeforeCommit}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "mine") + assert.True(t, blob.IsAmbiguous(err), "the client cannot tell the two apart: %v", err) + _, ok := fake.Object("lock") + assert.False(t, ok) +} + +func TestServerErrorOnConditionalMutationIsAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusInternalServerError, Code: "InternalError"}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestThrottlingIsNotAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusServiceUnavailable, Code: "SlowDown", Times: -1}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.False(t, blob.IsAmbiguous(err), "503 means the request was rejected, not half applied") +} + +func TestUnconditionalPutRetriesTransientServerError(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusInternalServerError, Code: "InternalError"}) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{}) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 2, "the SDK retried once and succeeded") +} + +func TestConnectionFailureBeforeSendingIsNotAmbiguous(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Close() + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.False(t, blob.IsAmbiguous(err), "the request never reached a server: %v", err) +} + +func TestAmbiguousCompletionLeavesTheUploadForReconciliation(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCompleteMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 9) + + _, err := s.Put(context.Background(), "big", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data)), Condition: blob.IfAbsent()}) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Empty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload), "an upload that may have completed is never aborted") + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) +} + +func TestServiceCodesMapToKinds(t *testing.T) { + cases := []struct { + name string + op string + status int32 + code string + check func(error) bool + }{ + {"missing key", s3test.OpGetObject, 404, "NoSuchKey", blob.IsNotFound}, + {"conflicting concurrent write", s3test.OpPutObject, 409, "ConditionalRequestConflict", blob.IsPreconditionFailed}, + {"explicit precondition", s3test.OpPutObject, 412, "PreconditionFailed", blob.IsPreconditionFailed}, + {"access denied is plain", s3test.OpGetObject, 403, "AccessDenied", func(err error) bool { + return !blob.IsNotFound(err) && !blob.IsPreconditionFailed(err) && !blob.IsAmbiguous(err) + }}, + {"missing bucket is not a missing object", s3test.OpGetObject, 404, "NoSuchBucket", func(err error) bool { return !blob.IsNotFound(err) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: tc.op, Status: tc.status, Code: tc.code, Times: -1}) + s := newTestStore(t, fake) + var err error + if tc.op == s3test.OpGetObject { + _, err = s.Get(context.Background(), "k", blob.GetOptions{}) + } else { + _, err = putIfAbsent(s, "k", "x") + } + require.Error(t, err) + assert.True(t, tc.check(err), "%v", err) + }) + } +} diff --git a/pkg/artifact/blob/s3/delete.go b/pkg/artifact/blob/s3/delete.go new file mode 100644 index 00000000000..4ced8cbda2c --- /dev/null +++ b/pkg/artifact/blob/s3/delete.go @@ -0,0 +1,38 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [blob.Store]. Conditional deletes are never retried by the SDK. +func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) error { + defer perf.Track(nil, "blob.s3.Delete")() + + if err := s.validateKey(key); err != nil { + return err + } + if err := s.Capabilities().CheckDelete(storeName, key, opts); err != nil { + return err + } + conditional := opts.IfMatch != "" + if conditional { + if err := s.ensureConditional(ctx, key, blob.IfMatch(opts.IfMatch)); err != nil { + return err + } + } + api, err := s.client(ctx) + if err != nil { + return err + } + in := &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), IfMatch: optionalString(opts.IfMatch)} + if _, err := api.DeleteObject(ctx, in, retryOptions(conditional)...); err != nil { + return classify(request{op: blob.OpDelete, key: key, mutating: true, conditional: conditional}, err) + } + return nil +} diff --git a/pkg/artifact/blob/s3/doc.go b/pkg/artifact/blob/s3/doc.go new file mode 100644 index 00000000000..e87a1bb0981 --- /dev/null +++ b/pkg/artifact/blob/s3/doc.go @@ -0,0 +1,15 @@ +// Package s3 is the AWS SDK v2 implementation of [blob.Store], plus the shared S3 client +// construction used by the legacy CI artifact adapter. +// +// Writes stream: a body that fits one part is sent with a single PutObject, a larger or +// unknown-length body is sent as a multipart upload whose parts are uploaded by a bounded worker +// pool (default 4) from a bounded buffer pool, so memory use is independent of object size. A +// multipart upload owned by a failed or canceled operation is aborted. Conditional writes use +// If-None-Match and If-Match; conditional requests are never retried by the SDK so a lost response +// surfaces as an ambiguous error instead of a misleading precondition failure. An endpoint that +// cannot enforce conditions fails the operation with a clear error, and an optional probe detects +// endpoints that silently ignore the headers. +// +// Credentials are resolved lazily on first use. The generic store fails closed: a configured +// identity that cannot be resolved is an error, never a silent fall back to ambient credentials. +package s3 diff --git a/pkg/artifact/blob/s3/emulator_test.go b/pkg/artifact/blob/s3/emulator_test.go new file mode 100644 index 00000000000..51c916ff002 --- /dev/null +++ b/pkg/artifact/blob/s3/emulator_test.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "crypto/rand" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// TestEmulatorConformance runs the shared suite against a real endpoint. It is skipped unless +// ATMOS_BLOB_S3_ENDPOINT and ATMOS_BLOB_S3_BUCKET are set (see s3test.EmulatorFromEnv); the bucket +// must already exist. Every subtest uses a unique key prefix, so runs do not collide. +func TestEmulatorConformance(t *testing.T) { + emulator := s3test.EmulatorFromEnv(t) + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { + var raw [6]byte + _, err := rand.Read(raw[:]) + require.NoError(t, err) + s, err := New( + emulator.Bucket, + WithEndpoint(emulator.Endpoint), + WithRegion(emulator.Region), + WithPrefix("atmos-blob-conformance/"+hex.EncodeToString(raw[:])), + WithConditionalMode(ConditionalProbe), + WithPartSize(5<<20), + ) + require.NoError(t, err) + return s + }) +} diff --git a/pkg/artifact/blob/s3/helpers_test.go b/pkg/artifact/blob/s3/helpers_test.go new file mode 100644 index 00000000000..6bf281bddcd --- /dev/null +++ b/pkg/artifact/blob/s3/helpers_test.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/store" +) + +// smallPart is a part size that lets tests exercise multipart with kilobytes. +const smallPart = 16 << 10 + +// newFake starts a fake S3 endpoint. +func newFake(t *testing.T, opts ...s3test.Option) *s3test.Server { + t.Helper() + return s3test.NewServer(t, opts...) +} + +// newTestStore returns a store over the fake with small parts unless overridden. +func newTestStore(t *testing.T, fake *s3test.Server, opts ...Option) *Store { + t.Helper() + all := append([]Option{WithPartSize(smallPart)}, opts...) + s, err := NewFromClient(fake.Client(), fake.Bucket(), all...) + require.NoError(t, err) + return s +} + +// putBytes stores data and fails the test on error. +func putBytes(t *testing.T, s *Store, key string, data []byte, opts blob.PutOptions) *blob.ObjectInfo { + t.Helper() + if opts.Size == 0 && len(data) > 0 { + opts.Size = int64(len(data)) + } + info, err := s.Put(context.Background(), key, bytes.NewReader(data), opts) + require.NoError(t, err) + return info +} + +// patterned returns n deterministic bytes. +func patterned(n int) []byte { + data := make([]byte, n) + for i := range data { + data[i] = byte(i*7 + i/251) + } + return data +} + +// countingReader generates size bytes and counts what the consumer has read. +type countingReader struct { + size int64 + read atomic.Int64 + onRead func(read int64) + exhaust func() + finished atomic.Bool +} + +func (r *countingReader) Read(p []byte) (int, error) { + remaining := r.size - r.read.Load() + if remaining <= 0 { + if r.finished.CompareAndSwap(false, true) && r.exhaust != nil { + r.exhaust() + } + return 0, io.EOF + } + n := int64(len(p)) + if n > remaining { + n = remaining + } + for i := int64(0); i < n; i++ { + p[i] = byte(i) + } + total := r.read.Add(n) + if r.onRead != nil { + r.onRead(total) + } + return int(n), nil +} + +// fakeResolver is an AuthResolver double. +type fakeResolver struct { + result *store.AWSAuthConfig + err error + calls atomic.Int32 +} + +func (f *fakeResolver) ResolveAWSAuthContext(_ context.Context, _ string) (*store.AWSAuthConfig, error) { + f.calls.Add(1) + return f.result, f.err +} diff --git a/pkg/artifact/blob/s3/list.go b/pkg/artifact/blob/s3/list.go new file mode 100644 index 00000000000..5b2ced61987 --- /dev/null +++ b/pkg/artifact/blob/s3/list.go @@ -0,0 +1,61 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxListKeys is the most entries S3 returns in one page. +const maxListKeys = 1000 + +// List implements [blob.Store]. The cursor is the S3 continuation token. +func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage, error) { + defer perf.Track(nil, "blob.s3.List")() + + api, err := s.client(ctx) + if err != nil { + return nil, err + } + in := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.cfg.bucket), + Prefix: optionalString(s.listPrefix(opts.Prefix)), + Delimiter: optionalString(opts.Delimiter), + ContinuationToken: optionalString(opts.Cursor), + } + if opts.Limit > 0 { + in.MaxKeys = aws.Int32(int32(min(opts.Limit, maxListKeys))) + } + out, err := api.ListObjectsV2(ctx, in) + if err != nil { + return nil, classify(request{op: blob.OpList, key: opts.Prefix}, err) + } + page := &blob.ListPage{} + for _, object := range out.Contents { + page.Objects = append(page.Objects, blob.ObjectInfo{ + Key: s.storeKey(aws.ToString(object.Key)), + Size: aws.ToInt64(object.Size), + ETag: aws.ToString(object.ETag), + LastModified: aws.ToTime(object.LastModified), + }) + } + for _, prefix := range out.CommonPrefixes { + page.CommonPrefixes = append(page.CommonPrefixes, s.storeKey(aws.ToString(prefix.Prefix))) + } + if aws.ToBool(out.IsTruncated) { + page.NextCursor = aws.ToString(out.NextContinuationToken) + } + return page, nil +} + +// listPrefix maps a store prefix to the bucket prefix. +func (s *Store) listPrefix(prefix string) string { + if s.cfg.prefix == "" { + return prefix + } + return s.cfg.prefix + "/" + prefix +} diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go new file mode 100644 index 00000000000..ad7d1ddccd6 --- /dev/null +++ b/pkg/artifact/blob/s3/multipart.go @@ -0,0 +1,257 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sort" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "golang.org/x/sync/errgroup" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// abortTimeout bounds the cleanup of an abandoned multipart upload, which must outlive the +// canceled operation context. +const abortTimeout = 30 * time.Second + +// multipart is one in-progress multipart upload. +type multipart struct { + store *Store + plan *putPlan + uploadID string +} + +// putMultipart streams the body as a multipart upload with bounded memory and concurrency. The +// upload is aborted on any definite failure, including cancellation. After an ambiguous +// completion it is left alone, because it may already have become the object. +func (s *Store) putMultipart(ctx context.Context, plan *putPlan, first, carry []byte, body io.Reader) (*blob.ObjectInfo, error) { + in := &s3.CreateMultipartUploadInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(plan.objectKey), + ContentType: optionalString(plan.opts.ContentType), + Metadata: plan.opts.Metadata, + Tagging: optionalString(plan.tags), + } + created, err := plan.api.CreateMultipartUpload(ctx, in, skipOptionalChecksums) + if err != nil { + return nil, classify(plan.request(false), err) + } + upload := &multipart{store: s, plan: plan, uploadID: aws.ToString(created.UploadId)} + + parts, total, err := upload.uploadParts(ctx, first, carry, body) + if err == nil && plan.opts.Size >= 0 && total != plan.opts.Size { + err = sizeMismatch(plan.key, total, plan.opts.Size) + } + if err != nil { + return nil, upload.abort(ctx, asBlobError("put", plan.key, err)) + } + return upload.complete(ctx, parts, total) +} + +// partResult records an uploaded part. +type partResult struct { + mu sync.Mutex + parts []types.CompletedPart +} + +func (r *partResult) add(number int32, etag *string) { + r.mu.Lock() + r.parts = append(r.parts, types.CompletedPart{ETag: etag, PartNumber: aws.Int32(number)}) + r.mu.Unlock() +} + +// uploadParts reads the body part by part and uploads them through a bounded pool. A free-list of +// buffers caps memory at one buffer per worker. +func (m *multipart) uploadParts(ctx context.Context, first, carry []byte, body io.Reader) ([]types.CompletedPart, int64, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + group, gctx := errgroup.WithContext(ctx) + + free := make(chan []byte, m.store.cfg.concurrency) + for range m.store.cfg.concurrency { + free <- nil + } + result := &partResult{} + submit := func(number int32, data []byte) { + group.Go(func() error { + defer func() { free <- data }() + etag, err := m.uploadPart(gctx, number, data) + if err != nil { + return err + } + result.add(number, etag) + return nil + }) + } + + feed := &partFeed{free: free, submit: submit} + total, produceErr := m.produce(gctx, feed, first, carry, body) + if produceErr != nil { + cancel() + } + waitErr := group.Wait() + if produceErr != nil { + return nil, 0, produceErr + } + if waitErr != nil { + return nil, 0, waitErr + } + sort.Slice(result.parts, func(i, j int) bool { + return aws.ToInt32(result.parts[i].PartNumber) < aws.ToInt32(result.parts[j].PartNumber) + }) + return result.parts, total, nil +} + +// partFeed hands parts to the worker pool. Its free list holds one buffer slot per worker. +type partFeed struct { + free chan []byte + submit func(number int32, data []byte) +} + +// produce feeds parts to the workers, waiting for a free buffer before reading each next part so +// the reader never runs ahead of the uploaders. +func (m *multipart) produce(ctx context.Context, feed *partFeed, first, carry []byte, body io.Reader) (int64, error) { + partSize := m.plan.partSize + free, submit := feed.free, feed.submit + if err := acquire(ctx, free); err != nil { + return 0, err + } + number := int32(1) + data := first + total := int64(0) + for { + submit(number, data) + total += int64(len(data)) + if int64(len(data)) < partSize { + return total, nil + } + buf, err := acquireBuffer(ctx, free, partSize) + if err != nil { + return 0, err + } + n, readErr := readPart(body, buf, carry) + carry = nil + if readErr != nil { + return 0, blob.NewError(blob.KindOther, blob.OpPut, m.plan.key, fmt.Errorf("read body: %w", readErr)) + } + if n == 0 { + free <- buf + return total, nil + } + number++ + if number > MaxParts { + return 0, blob.NewError(blob.KindInvalid, blob.OpPut, m.plan.key, + fmt.Errorf("%w: upload exceeds %d parts; use a larger part size", errUtils.ErrBlobInvalid, MaxParts)) + } + data = buf[:n] + } +} + +// acquire waits for a worker slot and discards its buffer. +func acquire(ctx context.Context, free chan []byte) error { + select { + case <-free: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// acquireBuffer waits for a worker slot and returns its buffer, sized to a part. +func acquireBuffer(ctx context.Context, free chan []byte, partSize int64) ([]byte, error) { + select { + case buf := <-free: + if int64(cap(buf)) < partSize { + buf = make([]byte, partSize) + } + return buf[:partSize], nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// readPart fills buf from body, starting with a carried byte when one was already consumed. It +// returns the bytes in buf; zero means the body ended exactly on the previous boundary. +func readPart(body io.Reader, buf, carry []byte) (int, error) { + offset := copy(buf, carry) + n, err := io.ReadFull(body, buf[offset:]) + switch { + case err == nil: + return offset + n, nil + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + return offset + n, nil + default: + return 0, err + } +} + +// uploadPart sends one part and returns its ETag. +func (m *multipart) uploadPart(ctx context.Context, number int32, data []byte) (*string, error) { + out, err := m.plan.api.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + PartNumber: aws.Int32(number), + Body: bytes.NewReader(data), + ContentLength: aws.Int64(int64(len(data))), + }, skipOptionalChecksums) + if err != nil { + return nil, classify(m.plan.request(false), err) + } + return out.ETag, nil +} + +// complete assembles the parts into the object under the write condition. +func (m *multipart) complete(ctx context.Context, parts []types.CompletedPart, total int64) (*blob.ObjectInfo, error) { + cond := m.plan.opts.Condition + in := &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + MultipartUpload: &types.CompletedMultipartUpload{Parts: parts}, + } + in.IfNoneMatch, in.IfMatch = conditionHeaders(cond) + out, err := m.plan.api.CompleteMultipartUpload(ctx, in, append(retryOptions(cond.IsConditional()), skipOptionalChecksums)...) + if err != nil { + classified := classify(m.plan.request(true), err) + if blob.IsAmbiguous(classified) { + return nil, classified + } + return nil, m.abort(ctx, classified) + } + return &blob.ObjectInfo{Key: m.plan.key, Size: total, ETag: aws.ToString(out.ETag)}, nil +} + +// abort removes the upload's stored parts. It uses a context detached from cancellation so a +// canceled operation still cleans up, and reports a failed cleanup alongside the original error. +func (m *multipart) abort(ctx context.Context, cause error) error { + abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortTimeout) + defer cancel() + _, err := m.plan.api.AbortMultipartUpload(abortCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + }) + if err != nil { + return errors.Join(cause, blob.NewError(blob.KindOther, "abort", m.plan.key, err)) + } + return cause +} + +// asBlobError classifies err as a plain failure unless it already is a blob error. +func asBlobError(op, key string, err error) error { + var classified *blob.Error + if errors.As(err, &classified) { + return err + } + return blob.NewError(blob.KindOther, op, key, err) +} diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go new file mode 100644 index 00000000000..7540ffcf237 --- /dev/null +++ b/pkg/artifact/blob/s3/options.go @@ -0,0 +1,176 @@ +package s3 + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Defaults for streaming uploads. +const ( + // DefaultPartSize is the single-request threshold and multipart part size. S3 requires every + // part except the last to be at least 5 MiB. + DefaultPartSize int64 = 8 << 20 + + // DefaultConcurrency is the number of multipart parts uploaded at once. + DefaultConcurrency = 4 + + // MaxParts is the S3 limit on parts in one multipart upload. + MaxParts = 10000 +) + +// ConditionalMode selects how the store treats conditional writes. +type ConditionalMode int + +// Conditional modes. +const ( + // ConditionalAuto sends If-None-Match/If-Match and trusts the endpoint. An endpoint that rejects + // the headers fails the operation with ErrBlobConditionalUnsupported. + ConditionalAuto ConditionalMode = iota + + // ConditionalProbe is like Auto, but first proves once per store that the endpoint enforces + // If-None-Match (see [Store.ProbeConditionalWrites]). Use it for third-party endpoints that + // might silently ignore the headers. + ConditionalProbe + + // ConditionalDisabled never sends conditions: any conditional operation fails clearly. + ConditionalDisabled +) + +// settings holds the resolved configuration of a Store. +type settings struct { + bucket string + prefix string + tags map[string]string + partSize int64 + concurrency int + conditional ConditionalMode + opaqueKeys bool + client ClientConfig + api API +} + +// Option configures a [Store]. +type Option func(*settings) + +// WithPrefix prefixes every key with prefix and a slash. Leading and trailing slashes are trimmed. +func WithPrefix(prefix string) Option { + defer perf.Track(nil, "blob.s3.WithPrefix")() + + return func(s *settings) { s.prefix = trimSlashes(prefix) } +} + +// WithRegion sets the AWS region. +func WithRegion(region string) Option { + defer perf.Track(nil, "blob.s3.WithRegion")() + + return func(s *settings) { s.client.Region = region } +} + +// WithEndpoint overrides the S3 endpoint, for example an emulator. Path-style addressing is used +// unless [WithPathStyle] says otherwise. +func WithEndpoint(url string) Option { + defer perf.Track(nil, "blob.s3.WithEndpoint")() + + return func(s *settings) { s.client.EndpointURL = url } +} + +// WithPathStyle forces path-style (true) or virtual-hosted (false) addressing. +func WithPathStyle(pathStyle bool) Option { + defer perf.Track(nil, "blob.s3.WithPathStyle")() + + return func(s *settings) { s.client.PathStyle = aws.Bool(pathStyle) } +} + +// WithIdentity authenticates through an Atmos identity. Credentials resolve lazily on first use and +// an unresolvable identity fails closed. +func WithIdentity(name string, resolver AuthResolver) Option { + defer perf.Track(nil, "blob.s3.WithIdentity")() + + return func(s *settings) { + s.client.Identity = name + s.client.Resolver = resolver + } +} + +// WithHTTPClient replaces the SDK HTTP client. +func WithHTTPClient(client aws.HTTPClient) Option { + defer perf.Track(nil, "blob.s3.WithHTTPClient")() + + return func(s *settings) { s.client.HTTPClient = client } +} + +// WithTags sets default object tags copied onto every object the store writes. Tags passed to an +// individual Put override these. +func WithTags(tags map[string]string) Option { + defer perf.Track(nil, "blob.s3.WithTags")() + + return func(s *settings) { s.tags = cloneTags(tags) } +} + +// WithPartSize sets the multipart part size and the single-request threshold. Real S3 requires at +// least 5 MiB for every part but the last; smaller sizes are for emulators and tests. +func WithPartSize(size int64) Option { + defer perf.Track(nil, "blob.s3.WithPartSize")() + + return func(s *settings) { + if size > 0 { + s.partSize = size + } + } +} + +// WithConcurrency bounds the multipart parts uploaded at once. Memory use is at most +// (concurrency + 1) part buffers. +func WithConcurrency(workers int) Option { + defer perf.Track(nil, "blob.s3.WithConcurrency")() + + return func(s *settings) { + if workers > 0 { + s.concurrency = workers + } + } +} + +// WithConditionalMode selects how conditional writes are enforced; see [ConditionalMode]. +func WithConditionalMode(mode ConditionalMode) Option { + defer perf.Track(nil, "blob.s3.WithConditionalMode")() + + return func(s *settings) { s.conditional = mode } +} + +// WithOpaqueKeys accepts any non-empty key up to the S3 limit instead of requiring portable +// relative keys. The legacy CI adapter uses it to keep its historical key patterns. +func WithOpaqueKeys() Option { + defer perf.Track(nil, "blob.s3.WithOpaqueKeys")() + + return func(s *settings) { s.opaqueKeys = true } +} + +// WithAPI injects a ready S3 client (or a test double), skipping lazy client construction. +func WithAPI(api API) Option { + defer perf.Track(nil, "blob.s3.WithAPI")() + + return func(s *settings) { s.api = api } +} + +func cloneTags(tags map[string]string) map[string]string { + if len(tags) == 0 { + return nil + } + out := make(map[string]string, len(tags)) + for k, v := range tags { + out[k] = v + } + return out +} + +func trimSlashes(s string) string { + for len(s) > 0 && s[0] == '/' { + s = s[1:] + } + for len(s) > 0 && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + return s +} diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go new file mode 100644 index 00000000000..b138e7d278b --- /dev/null +++ b/pkg/artifact/blob/s3/put.go @@ -0,0 +1,181 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "sort" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// putPlan is a validated write ready to send. +type putPlan struct { + api API + key string + objectKey string + opts blob.PutOptions + tags string + partSize int64 +} + +func (p *putPlan) request(mutating bool) request { + return request{op: blob.OpPut, key: p.key, mutating: mutating, conditional: p.opts.Condition.IsConditional()} +} + +// Put implements [blob.Store]. A body that fits one part is sent with a single PutObject; anything +// larger, or of unknown length, streams as a bounded multipart upload. +func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.s3.Put")() + + plan, err := s.planPut(ctx, key, opts) + if err != nil { + return nil, err + } + first, carry, err := readFirstPart(body, plan.partSize, opts.Size) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, key, fmt.Errorf("read body: %w", err)) + } + if len(carry) == 0 { + return s.putSingle(ctx, plan, first) + } + return s.putMultipart(ctx, plan, first, carry, body) +} + +// planPut validates the request and prepares the client, tags, and part size. +func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) (*putPlan, error) { + if err := s.validateKey(key); err != nil { + return nil, err + } + tags := blob.MergeTags(s.cfg.tags, opts.Tags) + if err := blob.ValidateTags(tags); err != nil { + return nil, err + } + if opts.Condition.Kind == blob.ConditionIfMatch && opts.Condition.ETag == "" { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + } + if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { + return nil, err + } + if err := s.ensureConditional(ctx, key, opts.Condition); err != nil { + return nil, err + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + return &putPlan{ + api: api, + key: key, + objectKey: s.objectKey(key), + opts: opts, + tags: encodeTags(tags), + partSize: effectivePartSize(opts.Size, s.cfg.partSize), + }, nil +} + +// effectivePartSize grows the part size so a known-size body fits within the S3 part limit. +func effectivePartSize(size, partSize int64) int64 { + if size <= 0 { + return partSize + } + needed := (size + MaxParts - 1) / MaxParts + return max(partSize, needed) +} + +// readFirstPart reads up to one part. When the body continues past the part, carry holds the one +// byte already consumed from the next part; an empty carry means the body ended. +func readFirstPart(body io.Reader, partSize, size int64) (first, carry []byte, err error) { + var buf bytes.Buffer + if size >= 0 && size < partSize { + buf.Grow(int(size)) + } + if _, err := io.Copy(&buf, io.LimitReader(body, partSize)); err != nil { + return nil, nil, err + } + if int64(buf.Len()) < partSize { + return buf.Bytes(), nil, nil + } + probe := make([]byte, 1) + n, err := io.ReadFull(body, probe) + switch { + case n == 1: + return buf.Bytes(), probe, nil + case errors.Is(err, io.EOF): + return buf.Bytes(), nil, nil + default: + return nil, nil, err + } +} + +// putSingle sends a body that fits one part with one PutObject. +func (s *Store) putSingle(ctx context.Context, plan *putPlan, data []byte) (*blob.ObjectInfo, error) { + size := int64(len(data)) + if plan.opts.Size >= 0 && size != plan.opts.Size { + return nil, sizeMismatch(plan.key, size, plan.opts.Size) + } + cond := plan.opts.Condition + in := &s3.PutObjectInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(plan.objectKey), + Body: bytes.NewReader(data), + ContentLength: aws.Int64(size), + ContentType: optionalString(plan.opts.ContentType), + Metadata: plan.opts.Metadata, + Tagging: optionalString(plan.tags), + } + in.IfNoneMatch, in.IfMatch = conditionHeaders(cond) + out, err := plan.api.PutObject(ctx, in, retryOptions(cond.IsConditional())...) + if err != nil { + return nil, classify(plan.request(true), err) + } + return &blob.ObjectInfo{Key: plan.key, Size: size, ETag: aws.ToString(out.ETag)}, nil +} + +// conditionHeaders maps a condition to the If-None-Match and If-Match request headers. +func conditionHeaders(cond blob.Condition) (ifNoneMatch, ifMatch *string) { + switch cond.Kind { + case blob.ConditionIfAbsent: + return aws.String("*"), nil + case blob.ConditionIfMatch: + return nil, aws.String(cond.ETag) + default: + return nil, nil + } +} + +func optionalString(value string) *string { + if value == "" { + return nil + } + return aws.String(value) +} + +// encodeTags formats tags as the URL query string S3 expects, with stable ordering. +func encodeTags(tags map[string]string) string { + if len(tags) == 0 { + return "" + } + keys := make([]string, 0, len(tags)) + for key := range tags { + keys = append(keys, key) + } + sort.Strings(keys) + values := url.Values{} + for _, key := range keys { + values.Set(key, tags[key]) + } + return values.Encode() +} + +func sizeMismatch(key string, got, declared int64) error { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: body is %d bytes, declared %d", errUtils.ErrBlobInvalid, got, declared)) +} diff --git a/pkg/artifact/blob/s3/put_test.go b/pkg/artifact/blob/s3/put_test.go new file mode 100644 index 00000000000..0abef912bb9 --- /dev/null +++ b/pkg/artifact/blob/s3/put_test.go @@ -0,0 +1,262 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func TestSmallBodyUsesOneSignedRequest(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("tiny"), blob.PutOptions{ContentType: "text/plain", Metadata: map[string]string{"Owner": "atmos"}}) + + requests := fake.Requests() + require.Len(t, requests, 1, "no multipart machinery for a small body") + assert.Equal(t, s3test.OpPutObject, requests[0].Op) + assert.Equal(t, "text/plain", requests[0].Header.Get("Content-Type")) + assert.Equal(t, "atmos", requests[0].Header.Get("X-Amz-Meta-Owner")) + assert.Empty(t, requests[0].Header.Get("If-None-Match"), "unconditional puts carry no condition") + assert.Empty(t, fake.OpenUploads()) +} + +func TestZeroLengthObjectIsOnePutWithEmptyBody(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "empty", nil, blob.PutOptions{}) + requests := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, requests, 1) + assert.Equal(t, int64(0), requests[0].Bytes) + object, ok := fake.Object("empty") + require.True(t, ok) + assert.Zero(t, object.Size) +} + +func TestBodyExactlyOnePartIsNotMultipart(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "exact", patterned(smallPart), blob.PutOptions{}) + assert.Empty(t, fake.RequestsFor(s3test.OpCreateMultipartUpload)) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestMultipartRequestOrderAndContent(t *testing.T) { + fake := newFake(t, s3test.WithMinPartSize(smallPart)) + s := newTestStore(t, fake, WithConcurrency(1)) + data := patterned(smallPart*3 + 17) + info := putBytes(t, s, "big", data, blob.PutOptions{ + ContentType: "application/octet-stream", + Tags: map[string]string{"team": "platform"}, + Condition: blob.IfAbsent(), + }) + + var ops []string + for _, req := range fake.Requests() { + ops = append(ops, req.Op) + } + assert.Equal(t, []string{ + s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, s3test.OpUploadPart, s3test.OpUploadPart, s3test.OpUploadPart, + s3test.OpCompleteMultipartUpload, + }, ops, "create, then every part, then complete last") + + create := fake.RequestsFor(s3test.OpCreateMultipartUpload)[0] + assert.Equal(t, "team=platform", create.Header.Get("X-Amz-Tagging"), "tags are set when the upload is created") + assert.Empty(t, create.Header.Get("If-None-Match"), "the condition belongs to the completing request") + complete := fake.RequestsFor(s3test.OpCompleteMultipartUpload)[0] + assert.Equal(t, "*", complete.Header.Get("If-None-Match")) + for i, part := range fake.RequestsFor(s3test.OpUploadPart) { + assert.Equal(t, []string{string(rune('1' + i))}, part.Query["partNumber"], "parts are numbered in order") + } + + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) + assert.Equal(t, info.ETag, object.ETag) + assert.Contains(t, object.ETag, "-4", "a multipart ETag records its part count and is not a payload checksum") + assert.Equal(t, map[string]string{"team": "platform"}, object.Tags) + assert.Equal(t, "application/octet-stream", object.ContentType) + assert.Equal(t, 1, fake.CompletedUploads()) + assert.Empty(t, fake.OpenUploads()) +} + +func TestUnknownSizeStreamsAsMultipart(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 5) + info, err := s.Put(context.Background(), "stream", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), info.Size) + object, _ := fake.Object("stream") + assert.Equal(t, data, object.Data) + assert.Len(t, fake.RequestsFor(s3test.OpUploadPart), 3) +} + +func TestKnownSizeGrowsPartsToStayUnderPartLimit(t *testing.T) { + assert.Equal(t, int64(10), effectivePartSize(10*MaxParts, 1)) + assert.Equal(t, int64(smallPart), effectivePartSize(100, smallPart)) + assert.Equal(t, int64(smallPart), effectivePartSize(blob.UnknownSize, smallPart)) +} + +func TestTagsAreCopiedToEveryObject(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithTags(map[string]string{"managed-by": "atmos", "env": "dev"})) + + putBytes(t, s, "single", []byte("x"), blob.PutOptions{}) + putBytes(t, s, "multi", patterned(smallPart*2+1), blob.PutOptions{}) + putBytes(t, s, "override", []byte("x"), blob.PutOptions{Tags: map[string]string{"env": "prod", "extra": "1"}}) + putBytes(t, s, "empty", nil, blob.PutOptions{}) + + want := map[string]map[string]string{ + "single": {"managed-by": "atmos", "env": "dev"}, + "multi": {"managed-by": "atmos", "env": "dev"}, + "override": {"managed-by": "atmos", "env": "prod", "extra": "1"}, + "empty": {"managed-by": "atmos", "env": "dev"}, + } + for key, tags := range want { + object, ok := fake.Object(key) + require.True(t, ok, key) + assert.Equal(t, tags, object.Tags, "tags on %s", key) + } +} + +func TestTooManyTagsRejectedBeforeAnyRequest(t *testing.T) { + fake := newFake(t) + tags := map[string]string{} + for _, key := range []string{"a", "b", "c", "d", "e", "f"} { + tags[key] = "v" + } + s := newTestStore(t, fake, WithTags(tags)) + _, err := s.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Tags: map[string]string{"g": "v", "h": "v", "i": "v", "j": "v", "k": "v"}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) +} + +func TestSizeMismatchIsRejectedAndMultipartAborted(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "short", bytes.NewReader([]byte("ab")), blob.PutOptions{Size: 9}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests(), "a single-part size mismatch is caught before sending") + + data := patterned(smallPart*2 + 3) + _, err = s.Put(context.Background(), "long", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data) - 1)}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.OpenUploads(), "the half-built upload was aborted") + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestCanceledOperationAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Action: s3test.Delay, Delay: 200 * time.Millisecond, Times: -1}) + s := newTestStore(t, fake, WithConcurrency(2)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + source := &countingReader{size: 64 * smallPart, onRead: func(read int64) { + if read > 3*smallPart { + cancel() + } + }} + _, err := s.Put(ctx, "canceled", source, blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + assert.False(t, blob.IsAmbiguous(err), "a canceled part upload cannot have created the object") + + assert.Empty(t, fake.OpenUploads(), "no multipart upload may outlive its canceled operation") + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestFailedSourceAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + boom := errors.New("disk vanished") + body := &errAfterReader{data: patterned(smallPart*2 + 10), err: boom} + _, err := s.Put(context.Background(), "k", body, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, boom) + assert.Empty(t, fake.OpenUploads()) + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestFailedPartAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Skip: 1, Times: -1, Status: 400, Code: "InvalidPart"}) + s := newTestStore(t, fake, WithConcurrency(1)) + _, err := s.Put(context.Background(), "k", bytes.NewReader(patterned(smallPart*4)), blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.Empty(t, fake.OpenUploads()) + assert.Equal(t, 1, fake.AbortedUploads()) +} + +func TestLostConditionAtCompletionAbortsTheUpload(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Seed("taken", []byte("existing"), nil) + + _, err := s.Put(context.Background(), "taken", bytes.NewReader(patterned(smallPart*2+1)), blob.PutOptions{Size: blob.UnknownSize, Condition: blob.IfAbsent()}) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + assert.Empty(t, fake.OpenUploads(), "the losing writer cleans up its parts") + object, _ := fake.Object("taken") + assert.Equal(t, []byte("existing"), object.Data) +} + +func TestMemoryStaysBoundedWhileStreaming(t *testing.T) { + const ( + part = 1 << 20 + total = 48 << 20 + concurrency = 3 + ) + fake := newFake(t, s3test.WithDiscardBodies()) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Action: s3test.Delay, Delay: 2 * time.Millisecond, Times: -1}) + s := newTestStore(t, fake, WithPartSize(part), WithConcurrency(concurrency)) + + var maxInFlight atomic.Int64 + source := &countingReader{size: total} + source.onRead = func(read int64) { + inFlight := read - fake.PartBytesReceived() + for { + current := maxInFlight.Load() + if inFlight <= current || maxInFlight.CompareAndSwap(current, inFlight) { + return + } + } + } + info, err := s.Put(context.Background(), "huge", source, blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Equal(t, int64(total), info.Size) + + bound := int64(concurrency+1) * part + assert.LessOrEqual(t, maxInFlight.Load(), bound, "bytes read but not yet uploaded must stay within the buffer pool, not the object size") + assert.LessOrEqual(t, fake.MaxConcurrentParts(), concurrency, "worker concurrency is bounded") + assert.GreaterOrEqual(t, fake.MaxConcurrentParts(), 2, "parts do upload in parallel") + assert.Equal(t, int64(total), fake.PartBytesReceived()) +} + +// errAfterReader yields data and then fails. +type errAfterReader struct { + data []byte + err error +} + +func (r *errAfterReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} diff --git a/pkg/artifact/blob/s3/read.go b/pkg/artifact/blob/s3/read.go new file mode 100644 index 00000000000..517958f6d4c --- /dev/null +++ b/pkg/artifact/blob/s3/read.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Number parsing parameters for Content-Range totals. +const ( + decimalBase = 10 + bitSize64 = 64 +) + +// Get implements [blob.Store]. +func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blob.Object, error) { + defer perf.Track(nil, "blob.s3.Get")() + + if err := s.validateKey(key); err != nil { + return nil, err + } + if opts.Offset < 0 { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: negative offset", errUtils.ErrBlobInvalid)) + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + in := &s3.GetObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), Range: rangeHeader(opts)} + out, err := api.GetObject(ctx, in) + if err != nil { + return nil, classify(request{op: blob.OpGet, key: key}, err) + } + info := blob.ObjectInfo{ + Key: key, + Size: objectSize(out), + ETag: aws.ToString(out.ETag), + ContentType: aws.ToString(out.ContentType), + LastModified: aws.ToTime(out.LastModified), + Metadata: out.Metadata, + } + return &blob.Object{ObjectInfo: info, Body: out.Body}, nil +} + +// rangeHeader renders a partial read as an HTTP Range header. +func rangeHeader(opts blob.GetOptions) *string { + if !opts.IsRange() { + return nil + } + if opts.Length > 0 { + return aws.String(fmt.Sprintf("bytes=%d-%d", opts.Offset, opts.Offset+opts.Length-1)) + } + return aws.String(fmt.Sprintf("bytes=%d-", opts.Offset)) +} + +// objectSize returns the full object size, using Content-Range for partial responses. +func objectSize(out *s3.GetObjectOutput) int64 { + if out.ContentRange != nil { + if _, total, ok := strings.Cut(aws.ToString(out.ContentRange), "/"); ok { + if size, err := strconv.ParseInt(total, decimalBase, bitSize64); err == nil { + return size + } + } + } + return aws.ToInt64(out.ContentLength) +} + +// Stat implements [blob.Store]. +func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.s3.Stat")() + + if err := s.validateKey(key); err != nil { + return nil, err + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + out, err := api.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))}) + if err != nil { + return nil, classify(request{op: blob.OpStat, key: key}, err) + } + info := &blob.ObjectInfo{ + Key: key, + Size: aws.ToInt64(out.ContentLength), + ETag: aws.ToString(out.ETag), + ContentType: aws.ToString(out.ContentType), + LastModified: aws.ToTime(out.LastModified), + Metadata: out.Metadata, + } + if opts.IncludeTags { + if info.Tags, err = s.tags(ctx, api, key); err != nil { + return nil, err + } + } + return info, nil +} + +// tags loads the object tags. +func (s *Store) tags(ctx context.Context, api API, key string) (map[string]string, error) { + out, err := api.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))}) + if err != nil { + return nil, classify(request{op: "tags", key: key}, err) + } + tags := make(map[string]string, len(out.TagSet)) + for _, tag := range out.TagSet { + tags[aws.ToString(tag.Key)] = aws.ToString(tag.Value) + } + return tags, nil +} diff --git a/pkg/artifact/blob/s3/s3test/chunked.go b/pkg/artifact/blob/s3/s3test/chunked.go new file mode 100644 index 00000000000..a0142da0d47 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/chunked.go @@ -0,0 +1,57 @@ +package s3test + +import ( + "bufio" + "bytes" + "io" + "net/http" + "strconv" + "strings" +) + +// hexBase is the radix of aws-chunked chunk sizes. +const hexBase = 16 + +// readBody returns the request payload, decoding aws-chunked framing when the SDK used it. +func readBody(r *http.Request) ([]byte, error) { + raw, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + if !isChunked(r) { + return raw, nil + } + return decodeAWSChunked(raw) +} + +func isChunked(r *http.Request) bool { + return strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || + strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") +} + +// decodeAWSChunked strips the aws-chunked framing: "[;ext]\r\n\r\n" repeated, ended +// by a zero-size chunk and optional trailers. +func decodeAWSChunked(raw []byte) ([]byte, error) { + reader := bufio.NewReader(bytes.NewReader(raw)) + var out bytes.Buffer + for { + line, err := reader.ReadString('\n') + if err != nil { + return nil, err + } + sizeText, _, _ := strings.Cut(strings.TrimSpace(line), ";") + size, err := strconv.ParseInt(sizeText, hexBase, bitSize64) + if err != nil { + return nil, err + } + if size == 0 { + return out.Bytes(), nil + } + if _, err := io.CopyN(&out, reader, size); err != nil { + return nil, err + } + if _, err := reader.Discard(2); err != nil { + return nil, err + } + } +} diff --git a/pkg/artifact/blob/s3/s3test/cond.go b/pkg/artifact/blob/s3/s3test/cond.go new file mode 100644 index 00000000000..c618a8497d7 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/cond.go @@ -0,0 +1,75 @@ +package s3test + +import ( + "net/http" + "strings" +) + +// conditionResult is the outcome of evaluating If-None-Match and If-Match. +type conditionResult int + +const ( + conditionOK conditionResult = iota + conditionFailed + conditionMissing + conditionNotImplemented +) + +// evaluate checks the conditional headers of r against the current object (nil when absent). +func (s *Server) evaluate(r *http.Request, current *object) conditionResult { + ifNoneMatch := r.Header.Get("If-None-Match") + ifMatch := r.Header.Get("If-Match") + if ifNoneMatch == "" && ifMatch == "" { + return conditionOK + } + switch s.cfg.conditional { + case ConditionalIgnore: + return conditionOK + case ConditionalNotImplemented: + return conditionNotImplemented + case ConditionalEnforce: + } + return enforce(ifNoneMatch, ifMatch, current) +} + +// enforce applies If-None-Match and If-Match to the current object (nil when absent). +func enforce(ifNoneMatch, ifMatch string, current *object) conditionResult { + if ifNoneMatch != "" && current != nil && (ifNoneMatch == "*" || etagMatches(ifNoneMatch, current.etag)) { + return conditionFailed + } + if ifMatch == "" { + return conditionOK + } + switch { + case current == nil: + return conditionMissing + case ifMatch != "*" && !etagMatches(ifMatch, current.etag): + return conditionFailed + default: + return conditionOK + } +} + +// etagMatches compares ETags ignoring quotes and the weak prefix. +func etagMatches(header, etag string) bool { + return normalizeETag(header) == normalizeETag(etag) +} + +func normalizeETag(etag string) string { + return strings.Trim(strings.TrimPrefix(strings.TrimSpace(etag), "W/"), `"`) +} + +// reject writes the response for a failed condition and reports whether it did. +func reject(w http.ResponseWriter, result conditionResult) bool { + switch result { + case conditionFailed: + writeError(w, http.StatusPreconditionFailed, "PreconditionFailed", "At least one of the pre-conditions you specified did not hold") + case conditionMissing: + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + case conditionNotImplemented: + writeError(w, http.StatusNotImplemented, "NotImplemented", "A header you provided implies functionality that is not implemented") + case conditionOK: + return false + } + return true +} diff --git a/pkg/artifact/blob/s3/s3test/doc.go b/pkg/artifact/blob/s3/s3test/doc.go new file mode 100644 index 00000000000..d2c359c8d3d --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/doc.go @@ -0,0 +1,18 @@ +// Package s3test is an in-process, S3-compatible HTTP server for tests. It speaks the S3 REST +// protocol over a real listener, so the AWS SDK exercises its real request and response handling +// against it, and it models the behavior publication code depends on rather than always succeeding: +// +// - ListObjectsV2 pagination (MaxKeys, ContinuationToken, delimiter roll-up, a configurable page cap). +// - Conditional PUT, multipart completion, and DELETE with If-None-Match and If-Match, answering +// 412 on a lost race and 404 for If-Match on a missing key, plus endpoints that ignore the +// headers or answer 501. +// - ETags (MD5 for single puts, MD5-of-MD5s with a part count for multipart uploads). +// - Multipart create, upload-part, complete, and abort, with tracking of open and aborted uploads +// and of concurrent part uploads. +// - Object tags (the x-amz-tagging header and GetObjectTagging) and user metadata. +// - Byte ranges, HEAD, and zero-length objects. +// - Fault injection: fail after N requests, drop the response after the mutation committed +// (response ambiguity), drop it before the commit, and delay. +// +// Every request is recorded so tests can assert ordering and headers. Signatures are not verified. +package s3test diff --git a/pkg/artifact/blob/s3/s3test/emulator.go b/pkg/artifact/blob/s3/s3test/emulator.go new file mode 100644 index 00000000000..eccd62b9ace --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/emulator.go @@ -0,0 +1,49 @@ +package s3test + +import ( + "os" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Environment variables that opt tests in to a real S3-compatible endpoint (Floci, MinIO, AWS). +// Credentials come from the usual AWS_* variables and the bucket must already exist. +const ( + EmulatorEndpointEnv = "ATMOS_BLOB_S3_ENDPOINT" + EmulatorBucketEnv = "ATMOS_BLOB_S3_BUCKET" + EmulatorRegionEnv = "ATMOS_BLOB_S3_REGION" +) + +// Emulator describes a real S3-compatible endpoint chosen through the environment. +type Emulator struct { + Endpoint string + Bucket string + Region string +} + +// lookup returns the value of an opt-in test variable, or empty when it is unset. +func lookup(name string) string { + value, _ := os.LookupEnv(name) + return value +} + +// EmulatorFromEnv returns the opt-in emulator, or skips the test when the environment does not +// name one. Emulator-backed tests supplement the fake and must never be required by ordinary runs. +func EmulatorFromEnv(t testing.TB) Emulator { + defer perf.Track(nil, "s3test.EmulatorFromEnv")() + + t.Helper() + emulator := Emulator{ + Endpoint: lookup(EmulatorEndpointEnv), + Bucket: lookup(EmulatorBucketEnv), + Region: lookup(EmulatorRegionEnv), + } + if emulator.Endpoint == "" || emulator.Bucket == "" { + t.Skipf("set %s and %s to run against an S3-compatible endpoint", EmulatorEndpointEnv, EmulatorBucketEnv) + } + if emulator.Region == "" { + emulator.Region = "us-east-1" + } + return emulator +} diff --git a/pkg/artifact/blob/s3/s3test/fault.go b/pkg/artifact/blob/s3/s3test/fault.go new file mode 100644 index 00000000000..5a6e00bd892 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/fault.go @@ -0,0 +1,138 @@ +package s3test + +import ( + "net/http" + "strings" + "sync" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// FaultAction is what an injected fault does to a matching request. +type FaultAction int8 + +// Fault actions. +const ( + // FailStatus answers with an S3 error and does not apply the request. + FailStatus FaultAction = iota + + // DropAfterCommit applies the request, then closes the connection without answering. The client + // sees a transport error although the mutation happened: response ambiguity. + DropAfterCommit + + // DropBeforeCommit closes the connection without applying the request. + DropBeforeCommit + + // Delay sleeps before handling the request normally. + Delay +) + +// Fault describes a failure to inject. A request matches when Op and KeyPrefix match (empty +// matches anything). The first Skip matches pass through untouched ("fail after N requests"), then +// the fault applies Times times; Times zero means once and a negative Times means forever. +type Fault struct { + // Op is the S3 operation name, for example "PutObject" or "CompleteMultipartUpload". + Op string + + // KeyPrefix restricts the fault to keys beginning with it. + KeyPrefix string + + // Skip is the number of matching requests that pass before the fault applies. + Skip int32 + + // Times is how many requests the fault applies to. + Times int32 + + // Action is the fault behavior. + Action FaultAction + + // Status and Code describe the S3 error for [FailStatus]; they default to 500 InternalError. + Status int32 + Code string + + // Delay is the pause for [Delay]. + Delay time.Duration +} + +// faultRule is a registered fault with its remaining counters. +type faultRule struct { + Fault + skip int32 + times int32 +} + +// faults holds the registered rules. +type faults struct { + mu sync.Mutex + rules []*faultRule +} + +func (f *faults) add(fault *Fault) { + f.mu.Lock() + defer f.mu.Unlock() + times := fault.Times + if times == 0 { + times = 1 + } + f.rules = append(f.rules, &faultRule{Fault: *fault, skip: fault.Skip, times: times}) +} + +func (f *faults) clear() { + f.mu.Lock() + defer f.mu.Unlock() + f.rules = nil +} + +// match returns the fault to apply to req, consuming its counters. +func (f *faults) match(req *Request) *Fault { + f.mu.Lock() + defer f.mu.Unlock() + for _, rule := range f.rules { + if rule.Op != "" && rule.Op != req.Op { + continue + } + if rule.KeyPrefix != "" && !strings.HasPrefix(req.Key, rule.KeyPrefix) { + continue + } + if rule.skip > 0 { + rule.skip-- + continue + } + if rule.times == 0 { + continue + } + if rule.times > 0 { + rule.times-- + } + fault := rule.Fault + return &fault + } + return nil +} + +// status returns the HTTP status and S3 code of a FailStatus fault. +func (f *Fault) status() (int, string) { + status, code := int(f.Status), f.Code + if status == 0 { + status = http.StatusInternalServerError + } + if code == "" { + code = "InternalError" + } + return status, code +} + +// Inject registers a fault. +func (s *Server) Inject(fault Fault) { + defer perf.Track(nil, "s3test.Server.Inject")() + + s.faults.add(&fault) +} + +// ClearFaults removes every registered fault. +func (s *Server) ClearFaults() { + defer perf.Track(nil, "s3test.Server.ClearFaults")() + + s.faults.clear() +} diff --git a/pkg/artifact/blob/s3/s3test/list.go b/pkg/artifact/blob/s3/s3test/list.go new file mode 100644 index 00000000000..882a6e0fcdf --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/list.go @@ -0,0 +1,163 @@ +package s3test + +import ( + "encoding/base64" + "encoding/xml" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +type listResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Delimiter string `xml:"Delimiter,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + Contents []listContent `xml:"Contents"` + CommonPrefixes []commonPrefixXM `xml:"CommonPrefixes"` +} + +type listContent struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type commonPrefixXM struct { + Prefix string `xml:"Prefix"` +} + +// listEntry is one item of a sorted listing: an object or a rolled-up prefix. +type listEntry struct { + token string + object *object +} + +func (s *Server) listObjects(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + prefix, delimiter := query.Get("prefix"), query.Get("delimiter") + limit := min(s.cfg.pageCap, maxKeysCap) + if raw := query.Get("max-keys"); raw != "" { + requested, err := strconv.Atoi(raw) + if err != nil || requested < 0 { + writeError(w, http.StatusBadRequest, "InvalidArgument", "max-keys must be a non-negative integer") + return + } + limit = min(limit, requested) + } + cursor, err := decodeToken(query.Get("continuation-token")) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidArgument", "The continuation token provided is incorrect") + return + } + if start := query.Get("start-after"); start > cursor { + cursor = start + } + page := s.page(prefix, delimiter, cursor, limit) + + encode := func(v string) string { return v } + if query.Get("encoding-type") == "url" { + encode = func(v string) string { return strings.ReplaceAll(url.QueryEscape(v), "+", "%20") } + } + result := listResult{ + Name: s.cfg.bucket, Prefix: encode(prefix), Delimiter: delimiter, EncodingType: query.Get("encoding-type"), + MaxKeys: limit, ContinuationToken: query.Get("continuation-token"), + } + for _, entry := range page.entries { + if entry.object == nil { + result.CommonPrefixes = append(result.CommonPrefixes, commonPrefixXM{Prefix: encode(entry.token)}) + continue + } + result.Contents = append(result.Contents, listContent{ + Key: encode(entry.object.key), LastModified: entry.object.modified.UTC().Format("2006-01-02T15:04:05.000Z"), + ETag: entry.object.etag, Size: entry.object.size, StorageClass: "STANDARD", + }) + } + result.KeyCount = len(page.entries) + if page.next != "" { + result.IsTruncated = true + result.NextContinuationToken = base64.RawURLEncoding.EncodeToString([]byte(page.next)) + } + writeXML(w, result) +} + +func decodeToken(token string) (string, error) { + if token == "" { + return "", nil + } + raw, err := base64.RawURLEncoding.DecodeString(token) + return string(raw), err +} + +type listPage struct { + entries []listEntry + next string +} + +// snapshotUnder returns the sorted keys under prefix and a copy of their objects. +func (s *Server) snapshotUnder(prefix string) ([]string, map[string]object) { + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.objects)) + for key := range s.objects { + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + snapshot := make(map[string]object, len(keys)) + for _, key := range keys { + snapshot[key] = *s.objects[key] + } + return keys, snapshot +} + +// rollUp returns the ordering token of a key: the key itself, or its common prefix when the +// delimiter occurs after the listing prefix. +func rollUp(key, prefix, delimiter string) (string, bool) { + if delimiter == "" { + return key, false + } + rest := key[len(prefix):] + idx := strings.Index(rest, delimiter) + if idx < 0 { + return key, false + } + return prefix + rest[:idx+len(delimiter)], true +} + +// page builds one page: keys under prefix, delimiter roll-up, resumed after the cursor token. +func (s *Server) page(prefix, delimiter, cursor string, limit int) listPage { + keys, snapshot := s.snapshotUnder(prefix) + + var page listPage + last := "" + for _, key := range keys { + token, rolled := rollUp(key, prefix, delimiter) + if (cursor != "" && token <= cursor) || (rolled && token == last) { + continue + } + if len(page.entries) == limit { + page.next = last + return page + } + entry := listEntry{token: token} + if !rolled { + obj := snapshot[key] + entry.object = &obj + } + page.entries = append(page.entries, entry) + last = token + } + return page +} diff --git a/pkg/artifact/blob/s3/s3test/multipart.go b/pkg/artifact/blob/s3/s3test/multipart.go new file mode 100644 index 00000000000..0263d624fa3 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/multipart.go @@ -0,0 +1,196 @@ +package s3test + +import ( + "encoding/xml" + "fmt" + "net/http" + "strconv" + "strings" +) + +// upload is an open multipart upload. +type upload struct { + id string + key string + contentType string + metadata map[string]string + tags map[string]string + parts map[int]part +} + +type part struct { + etag string + digest string + size int64 + data []byte +} + +type initiateResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadID string `xml:"UploadId"` +} + +type completeRequest struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + } `xml:"Part"` +} + +type completeResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +func (s *Server) createUpload(w http.ResponseWriter, r *http.Request, key string) { + attrs, err := attributes(r) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidTag", err.Error()) + return + } + s.mu.Lock() + id := s.newID() + s.uploads[id] = &upload{id: id, key: key, contentType: attrs.contentType, metadata: attrs.metadata, tags: attrs.tags, parts: map[int]part{}} + s.mu.Unlock() + writeXML(w, initiateResult{Bucket: s.cfg.bucket, Key: key, UploadID: id}) +} + +// maxPartNumber is the highest multipart part number S3 accepts. +const maxPartNumber = 10000 + +// trackConcurrency records the peak number of UploadPart requests in flight. The returned function +// ends the request's contribution. +func (s *Server) trackConcurrency() func() { + inFlight := s.partsInFlight.Add(1) + for { + current := s.maxPartsInFlow.Load() + if inFlight <= current || s.maxPartsInFlow.CompareAndSwap(current, inFlight) { + break + } + } + return func() { s.partsInFlight.Add(-1) } +} + +func (s *Server) uploadPart(w http.ResponseWriter, r *http.Request, key string) { + defer s.trackConcurrency()() + + data, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + number, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil || number < 1 || number > maxPartNumber { + writeError(w, http.StatusBadRequest, "InvalidArgument", "Part number must be an integer between 1 and 10000") + return + } + s.partBytes.Add(int64(len(data))) + p := newPart(data, s.cfg.discard) + + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.uploads[r.URL.Query().Get("uploadId")] + if !ok || up.key != key { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + up.parts[number] = p + w.Header().Set("ETag", p.etag) + w.WriteHeader(http.StatusOK) +} + +// newPart builds a stored part, dropping the bytes when bodies are discarded. +func newPart(data []byte, discard bool) part { + p := part{etag: quotedDigest(data), digest: digestHex(data), size: int64(len(data))} + if !discard { + p.data = data + } + return p +} + +func (s *Server) completeUpload(w http.ResponseWriter, r *http.Request, key string) { + body, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + var req completeRequest + if err := xml.Unmarshal(body, &req); err != nil { + writeError(w, http.StatusBadRequest, "MalformedXML", err.Error()) + return + } + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.uploads[r.URL.Query().Get("uploadId")] + if !ok || up.key != key { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + obj, problem := s.assemble(up, req) + if problem != "" { + writeError(w, http.StatusBadRequest, "InvalidPart", problem) + return + } + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + s.objects[key] = obj + delete(s.uploads, up.id) + s.completed.Add(1) + writeXML(w, completeResult{Bucket: s.cfg.bucket, Key: key, ETag: obj.etag}) +} + +// assemble validates the requested parts and builds the final object. A non-empty string is the +// reason the parts were rejected. +func (s *Server) assemble(up *upload, req completeRequest) (*object, string) { + if len(req.Parts) == 0 { + return nil, "You must specify at least one part" + } + var ( + data []byte + sums strings.Builder + size int64 + lastNo = 0 + ) + for i, requested := range req.Parts { + stored, ok := up.parts[requested.PartNumber] + switch { + case !ok || !etagMatches(requested.ETag, stored.etag): + return nil, fmt.Sprintf("part %d is missing or its ETag does not match", requested.PartNumber) + case requested.PartNumber <= lastNo: + return nil, "parts must be in ascending order" + case i < len(req.Parts)-1 && stored.size < s.cfg.minPartSize: + return nil, "Your proposed upload is smaller than the minimum allowed size" + } + lastNo = requested.PartNumber + sums.WriteString(stored.digest) + size += stored.size + data = append(data, stored.data...) + } + obj := &object{ + key: up.key, size: size, modified: timeNow(), contentType: up.contentType, metadata: up.metadata, tags: up.tags, + etag: fmt.Sprintf(`"%s-%d"`, digestHex([]byte(sums.String())), len(req.Parts)), + } + if !s.cfg.discard { + obj.data = data + } + return obj, "" +} + +func (s *Server) abortUpload(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + id := r.URL.Query().Get("uploadId") + if _, ok := s.uploads[id]; !ok { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + delete(s.uploads, id) + s.aborted.Add(1) + w.WriteHeader(http.StatusNoContent) +} diff --git a/pkg/artifact/blob/s3/s3test/objects.go b/pkg/artifact/blob/s3/s3test/objects.go new file mode 100644 index 00000000000..bfb93c71c73 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/objects.go @@ -0,0 +1,217 @@ +package s3test + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// Number parsing parameters and the S3 tag limit. +const ( + maxTagCount = 10 + decimalBase = 10 + bitSize64 = 64 +) + +// objectAttrs are the attributes a write request carries. +type objectAttrs struct { + contentType string + metadata map[string]string + tags map[string]string +} + +// errTooManyTags reports a tag set above the S3 limit. +var errTooManyTags = errors.New("object tags cannot be greater than 10") + +// attributes extracts content type, user metadata, and tags from request headers. +func attributes(r *http.Request) (objectAttrs, error) { + attrs := objectAttrs{contentType: r.Header.Get("Content-Type")} + for name, values := range r.Header { + lower := strings.ToLower(name) + if strings.HasPrefix(lower, "x-amz-meta-") && len(values) > 0 { + if attrs.metadata == nil { + attrs.metadata = map[string]string{} + } + attrs.metadata[strings.TrimPrefix(lower, "x-amz-meta-")] = values[0] + } + } + raw := r.Header.Get("X-Amz-Tagging") + if raw == "" { + return attrs, nil + } + parsed, err := url.ParseQuery(raw) + if err != nil { + return objectAttrs{}, err + } + if len(parsed) > maxTagCount { + return objectAttrs{}, errTooManyTags + } + attrs.tags = make(map[string]string, len(parsed)) + for key, values := range parsed { + attrs.tags[key] = values[0] + } + return attrs, nil +} + +func (s *Server) putObject(w http.ResponseWriter, r *http.Request, key string) { + data, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + attrs, err := attributes(r) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidTag", err.Error()) + return + } + s.mu.Lock() + defer s.mu.Unlock() + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + obj := &object{ + key: key, size: int64(len(data)), etag: quotedDigest(data), modified: timeNow(), + contentType: attrs.contentType, metadata: attrs.metadata, tags: attrs.tags, + } + if !s.cfg.discard { + obj.data = data + } + s.objects[key] = obj + w.Header().Set("ETag", obj.etag) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) getObject(w http.ResponseWriter, r *http.Request, key string, head bool) { + s.mu.Lock() + obj, ok := s.objects[key] + var snapshot object + if ok { + snapshot = *obj + } + s.mu.Unlock() + if !ok { + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + return + } + window, valid := parseRange(r.Header.Get("Range"), snapshot.size) + if !valid { + writeError(w, http.StatusRequestedRangeNotSatisfiable, "InvalidRange", "The requested range is not satisfiable") + return + } + setObjectHeaders(w, &snapshot) + w.Header().Set("Content-Length", strconv.FormatInt(window.end-window.start, decimalBase)) + if window.partial { + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", window.start, window.end-1, snapshot.size)) + w.WriteHeader(http.StatusPartialContent) + } else { + w.WriteHeader(http.StatusOK) + } + if head { + return + } + if s.cfg.discard { + s.t.Errorf("s3test: cannot read object %q: bodies are discarded", key) + return + } + _, _ = w.Write(snapshot.data[window.start:window.end]) +} + +// setObjectHeaders writes the response headers shared by GET and HEAD. +func setObjectHeaders(w http.ResponseWriter, obj *object) { + header := w.Header() + header.Set("ETag", obj.etag) + header.Set("Last-Modified", obj.modified.UTC().Format(http.TimeFormat)) + if obj.contentType != "" { + header.Set("Content-Type", obj.contentType) + } + for key, value := range obj.metadata { + header.Set("X-Amz-Meta-"+key, value) + } + if len(obj.tags) > 0 { + header.Set("X-Amz-Tagging-Count", strconv.Itoa(len(obj.tags))) + } +} + +// byteRange is a half-open window into an object. +type byteRange struct { + start, end int64 + partial bool +} + +// parseRange resolves a Range header against size. It reports false for an unsatisfiable range. +func parseRange(header string, size int64) (byteRange, bool) { + if header == "" { + return byteRange{end: size}, true + } + spec, found := strings.CutPrefix(header, "bytes=") + first, last, hasDash := strings.Cut(spec, "-") + if !found || !hasDash { + return byteRange{}, false + } + start, err := strconv.ParseInt(first, decimalBase, bitSize64) + if err != nil || start > size || (start == size && size > 0) { + return byteRange{}, false + } + end, ok := rangeEnd(last, start, size) + return byteRange{start: start, end: end, partial: true}, ok +} + +// rangeEnd resolves the exclusive end of a byte range from its last-byte text. +func rangeEnd(last string, start, size int64) (int64, bool) { + if last == "" { + return size, true + } + through, err := strconv.ParseInt(last, decimalBase, bitSize64) + if err != nil || through < start { + return 0, false + } + return min(through+1, size), true +} + +func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, key string) { + s.mu.Lock() + defer s.mu.Unlock() + if r.Header.Get("If-Match") != "" || r.Header.Get("If-None-Match") != "" { + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + } + delete(s.objects, key) + w.WriteHeader(http.StatusNoContent) +} + +// tagging is the GetObjectTagging response. +type tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet struct { + Tags []tag `xml:"Tag"` + } `xml:"TagSet"` +} + +type tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +func (s *Server) getTagging(w http.ResponseWriter, key string) { + s.mu.Lock() + obj, ok := s.objects[key] + var tags map[string]string + if ok { + tags = cloneMap(obj.tags) + } + s.mu.Unlock() + if !ok { + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + return + } + var doc tagging + for key, value := range tags { + doc.TagSet.Tags = append(doc.TagSet.Tags, tag{Key: key, Value: value}) + } + writeXML(w, doc) +} diff --git a/pkg/artifact/blob/s3/s3test/options.go b/pkg/artifact/blob/s3/s3test/options.go new file mode 100644 index 00000000000..4ae1657a1cb --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/options.go @@ -0,0 +1,74 @@ +package s3test + +import "github.com/cloudposse/atmos/pkg/perf" + +// ConditionalBehavior selects how the server treats If-None-Match and If-Match. +type ConditionalBehavior int + +// Conditional behaviors. +const ( + // ConditionalEnforce honors the conditions like S3. + ConditionalEnforce ConditionalBehavior = iota + + // ConditionalIgnore silently ignores the headers, like a non-conforming emulator. + ConditionalIgnore + + // ConditionalNotImplemented answers 501 NotImplemented to any conditional request. + ConditionalNotImplemented +) + +// defaultBucket is the bucket the server serves unless overridden. +const defaultBucket = "test-bucket" + +// maxKeysCap is the page cap S3 applies to ListObjectsV2. +const maxKeysCap = 1000 + +// settings holds server configuration. +type settings struct { + bucket string + discard bool + conditional ConditionalBehavior + pageCap int + minPartSize int64 +} + +// Option configures a [Server]. +type Option func(*settings) + +// WithBucket sets the bucket name the server serves. +func WithBucket(name string) Option { + defer perf.Track(nil, "s3test.WithBucket")() + + return func(s *settings) { s.bucket = name } +} + +// WithDiscardBodies keeps only sizes and ETags, so streaming tests can push large bodies without +// the server holding them. Reading such an object fails. +func WithDiscardBodies() Option { + defer perf.Track(nil, "s3test.WithDiscardBodies")() + + return func(s *settings) { s.discard = true } +} + +// WithConditionalBehavior selects how conditional headers are treated. +func WithConditionalBehavior(behavior ConditionalBehavior) Option { + defer perf.Track(nil, "s3test.WithConditionalBehavior")() + + return func(s *settings) { s.conditional = behavior } +} + +// WithPageCap caps the entries per ListObjectsV2 page below the client's max-keys, to force +// pagination with few objects. +func WithPageCap(n int) Option { + defer perf.Track(nil, "s3test.WithPageCap")() + + return func(s *settings) { s.pageCap = n } +} + +// WithMinPartSize rejects completion when a non-final part is smaller than n bytes, as S3 does +// with 5 MiB. +func WithMinPartSize(n int64) Option { + defer perf.Track(nil, "s3test.WithMinPartSize")() + + return func(s *settings) { s.minPartSize = n } +} diff --git a/pkg/artifact/blob/s3/s3test/record.go b/pkg/artifact/blob/s3/s3test/record.go new file mode 100644 index 00000000000..b7e21b95bf9 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/record.go @@ -0,0 +1,145 @@ +package s3test + +import ( + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// S3 operation names as reported in [Request.Op]. +const ( + OpPutObject = "PutObject" + OpGetObject = "GetObject" + OpHeadObject = "HeadObject" + OpDeleteObject = "DeleteObject" + OpListObjectsV2 = "ListObjectsV2" + OpGetObjectTagging = "GetObjectTagging" + OpCreateMultipartUpload = "CreateMultipartUpload" + OpUploadPart = "UploadPart" + OpCompleteMultipartUpload = "CompleteMultipartUpload" + OpAbortMultipartUpload = "AbortMultipartUpload" + opUnknown = "Unknown" +) + +// Request is one recorded request. +type Request struct { + // Seq is the 1-based arrival order. + Seq int + + // Op is the S3 operation name. + Op string + + // Method is the HTTP method. + Method string + + // Key is the object key, empty for bucket-level requests. + Key string + + // Query holds the query parameters. + Query url.Values + + // Header holds the request headers. + Header http.Header + + // Bytes is the number of body bytes received. + Bytes int64 + + // Status is the status the server answered, or 0 when the connection was dropped. + Status int + + // Faulted reports that an injected fault affected the request. + Faulted bool + + // At is the arrival time. + At time.Time +} + +// recorder collects requests. +type recorder struct { + mu sync.Mutex + requests []Request +} + +// begin records an arriving request and returns its index. +func (r *recorder) begin(req *Request) int { + r.mu.Lock() + defer r.mu.Unlock() + req.Seq = len(r.requests) + 1 + r.requests = append(r.requests, *req) + return len(r.requests) - 1 +} + +func (r *recorder) update(index int, fn func(*Request)) { + r.mu.Lock() + defer r.mu.Unlock() + fn(&r.requests[index]) +} + +// Requests returns a copy of every recorded request in arrival order. +func (s *Server) Requests() []Request { + defer perf.Track(nil, "s3test.Server.Requests")() + + s.rec.mu.Lock() + defer s.rec.mu.Unlock() + return append([]Request(nil), s.rec.requests...) +} + +// RequestsFor returns the recorded requests of one operation. +func (s *Server) RequestsFor(op string) []Request { + defer perf.Track(nil, "s3test.Server.RequestsFor")() + + var matched []Request + for _, req := range s.Requests() { + if req.Op == op { + matched = append(matched, req) + } + } + return matched +} + +// ResetRequests clears the request log. +func (s *Server) ResetRequests() { + defer perf.Track(nil, "s3test.Server.ResetRequests")() + + s.rec.mu.Lock() + defer s.rec.mu.Unlock() + s.rec.requests = nil +} + +// operations maps an HTTP method and the query marker that distinguishes the operation to its +// S3 name. An empty marker is the method's default operation. +var operations = map[string]map[string]string{ + http.MethodHead: {"": OpHeadObject}, + http.MethodGet: {"tagging": OpGetObjectTagging, "": OpGetObject}, + http.MethodPut: {"uploadId": OpUploadPart, "": OpPutObject}, + http.MethodPost: {"uploads": OpCreateMultipartUpload, "uploadId": OpCompleteMultipartUpload}, + http.MethodDelete: {"uploadId": OpAbortMultipartUpload, "": OpDeleteObject}, +} + +// classifyRequest names the S3 operation for a request. +func classifyRequest(method string, query url.Values, key string) string { + if key == "" && method == http.MethodGet { + return OpListObjectsV2 + } + byMarker := operations[method] + for marker, op := range byMarker { + if marker != "" && query.Has(marker) { + return op + } + } + if op, ok := byMarker[""]; ok { + return op + } + return opUnknown +} + +// splitPath separates the bucket and key from a path-style request path. +func splitPath(path string) (bucket, key string) { + trimmed := strings.TrimPrefix(path, "/") + bucket, key, _ = strings.Cut(trimmed, "/") + return bucket, key +} diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go new file mode 100644 index 00000000000..d45d3202cca --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -0,0 +1,239 @@ +package s3test + +import ( + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Server is an in-process S3-compatible endpoint serving one bucket in path-style addressing. +type Server struct { + t testing.TB + cfg settings + http *httptest.Server + rec recorder + faults faults + + mu sync.Mutex + objects map[string]*object + uploads map[string]*upload + nextID int + + aborted atomic.Int64 + completed atomic.Int64 + partBytes atomic.Int64 + partsInFlight atomic.Int64 + maxPartsInFlow atomic.Int64 +} + +// NewServer starts a server and registers its shutdown with t.Cleanup. +func NewServer(t testing.TB, opts ...Option) *Server { + defer perf.Track(nil, "s3test.NewServer")() + + t.Helper() + cfg := settings{bucket: defaultBucket, pageCap: maxKeysCap} + for _, opt := range opts { + opt(&cfg) + } + server := &Server{t: t, cfg: cfg, objects: map[string]*object{}, uploads: map[string]*upload{}} + server.http = httptest.NewServer(http.HandlerFunc(server.serve)) + t.Cleanup(server.http.Close) + return server +} + +// Endpoint returns the base URL to configure as the S3 endpoint. +func (s *Server) Endpoint() string { + defer perf.Track(nil, "s3test.Server.Endpoint")() + + return s.http.URL +} + +// Close stops the server early, for tests of connection failures. It is safe to call twice. +func (s *Server) Close() { + defer perf.Track(nil, "s3test.Server.Close")() + + s.http.Close() +} + +// Bucket returns the served bucket name. +func (s *Server) Bucket() string { + defer perf.Track(nil, "s3test.Server.Bucket")() + + return s.cfg.bucket +} + +// Client returns an SDK client wired to the server with path-style addressing, static credentials, +// and optional request checksums disabled (the settings the generic store uses). +func (s *Server) Client() *s3.Client { + defer perf.Track(nil, "s3test.Server.Client")() + + return s.client(func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) +} + +// LegacyClient returns a client that keeps the SDK's default checksum behavior, as the legacy CI +// adapter does. +func (s *Server) LegacyClient() *s3.Client { + defer perf.Track(nil, "s3test.Server.LegacyClient")() + + return s.client() +} + +func (s *Server) client(extra ...func(*s3.Options)) *s3.Client { + cfg := aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("test", "test", ""), + } + optFns := append([]func(*s3.Options){func(o *s3.Options) { + o.BaseEndpoint = aws.String(s.http.URL) + o.UsePathStyle = true + }}, extra...) + return s3.NewFromConfig(cfg, optFns...) +} + +// serve records, applies faults to, and dispatches one request. +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + bucket, key := splitPath(r.URL.Path) + query := r.URL.Query() + req := Request{Op: classifyRequest(r.Method, query, key), Method: r.Method, Key: key, Query: query, Header: r.Header.Clone(), At: timeNow()} + index := s.rec.begin(&req) + + fault := s.faults.match(&req) + if fault != nil { + s.rec.update(index, func(rr *Request) { rr.Faulted = true }) + } + rw := &statusWriter{ResponseWriter: w} + if fault != nil && s.applyEarlyFault(rw, r, fault) { + s.finish(index, rw) + return + } + if bucket != s.cfg.bucket { + writeError(rw, http.StatusNotFound, "NoSuchBucket", "The specified bucket does not exist") + s.finish(index, rw) + return + } + if fault != nil && fault.Action == DropAfterCommit { + s.handle(&mockDiscardWriter{header: http.Header{}}, r, key, req.Op) + s.drop(w) + s.rec.update(index, func(rr *Request) { rr.Status = 0 }) + return + } + s.handle(rw, r, key, req.Op) + s.finish(index, rw) +} + +// applyEarlyFault applies faults that decide the request before it is handled. It reports whether +// the request is finished. +func (s *Server) applyEarlyFault(w *statusWriter, r *http.Request, fault *Fault) bool { + switch fault.Action { + case FailStatus: + _, _ = io.Copy(io.Discard, r.Body) + status, code := fault.status() + writeError(w, status, code, "injected fault") + return true + case DropBeforeCommit: + _, _ = io.Copy(io.Discard, r.Body) + s.drop(w.ResponseWriter) + w.dropped = true + return true + case Delay: + sleep(r.Context(), fault.Delay) + case DropAfterCommit: + } + return false +} + +// finish records the response status and body size of a handled request. +func (s *Server) finish(index int, w *statusWriter) { + s.rec.update(index, func(rr *Request) { + if !w.dropped { + rr.Status = w.status() + } + }) +} + +// drop closes the client connection without writing a response. +func (s *Server) drop(w http.ResponseWriter) { + hijacker, ok := w.(http.Hijacker) + if !ok { + s.t.Errorf("s3test: response writer cannot be hijacked") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + s.t.Errorf("s3test: hijack failed: %v", err) + return + } + _ = conn.Close() +} + +// handle dispatches a request to its operation. +func (s *Server) handle(w http.ResponseWriter, r *http.Request, key, op string) { + handlers := map[string]func(){ + OpPutObject: func() { s.putObject(w, r, key) }, + OpGetObject: func() { s.getObject(w, r, key, false) }, + OpHeadObject: func() { s.getObject(w, r, key, true) }, + OpDeleteObject: func() { s.deleteObject(w, r, key) }, + OpListObjectsV2: func() { s.listObjects(w, r) }, + OpGetObjectTagging: func() { s.getTagging(w, key) }, + OpCreateMultipartUpload: func() { s.createUpload(w, r, key) }, + OpUploadPart: func() { s.uploadPart(w, r, key) }, + OpCompleteMultipartUpload: func() { s.completeUpload(w, r, key) }, + OpAbortMultipartUpload: func() { s.abortUpload(w, r) }, + } + if handler, ok := handlers[op]; ok { + handler() + return + } + writeError(w, http.StatusNotImplemented, "NotImplemented", "operation is not supported by the fake") +} + +// statusWriter remembers the status code written. +type statusWriter struct { + http.ResponseWriter + code int + dropped bool +} + +func (w *statusWriter) WriteHeader(code int) { + defer perf.Track(nil, "s3test.statusWriter.WriteHeader")() + + if w.code == 0 { + w.code = code + } + w.ResponseWriter.WriteHeader(code) +} + +func (w *statusWriter) Write(p []byte) (int, error) { + defer perf.Track(nil, "s3test.statusWriter.Write")() + + if w.code == 0 { + w.code = http.StatusOK + } + return w.ResponseWriter.Write(p) +} + +func (w *statusWriter) status() int { + if w.code == 0 { + return http.StatusOK + } + return w.code +} + +// mockDiscardWriter swallows a response, used when the response is dropped after the commit. +type mockDiscardWriter struct{ header http.Header } + +func (d *mockDiscardWriter) Header() http.Header { return d.header } +func (d *mockDiscardWriter) Write(p []byte) (int, error) { return len(p), nil } +func (d *mockDiscardWriter) WriteHeader(int) {} diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go new file mode 100644 index 00000000000..4d5e4b8bdca --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -0,0 +1,368 @@ +package s3test + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func apiCode(t *testing.T, err error) string { + t.Helper() + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + return apiErr.ErrorCode() +} + +func putObject(t *testing.T, client *s3.Client, srv *Server, key, body string, mutate ...func(*s3.PutObjectInput)) (*s3.PutObjectOutput, error) { + t.Helper() + in := &s3.PutObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String(key), Body: strings.NewReader(body)} + for _, fn := range mutate { + fn(in) + } + return client.PutObject(context.Background(), in) +} + +func TestConditionalPutEnforced(t *testing.T) { + srv := NewServer(t) + client := srv.Client() + first, err := putObject(t, client, srv, "k", "one", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err) + + _, err = putObject(t, client, srv, "k", "two", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + + _, err = putObject(t, client, srv, "k", "three", func(in *s3.PutObjectInput) { in.IfMatch = aws.String(`"bogus"`) }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + + _, err = putObject(t, client, srv, "k", "four", func(in *s3.PutObjectInput) { in.IfMatch = first.ETag }) + require.NoError(t, err) + + _, err = putObject(t, client, srv, "ghost", "x", func(in *s3.PutObjectInput) { in.IfMatch = aws.String(`"x"`) }) + assert.Equal(t, "NoSuchKey", apiCode(t, err), "If-Match on a missing key is a 404") + object, _ := srv.Object("k") + assert.Equal(t, []byte("four"), object.Data) +} + +func TestConditionalBehaviorsOfNonConformingEndpoints(t *testing.T) { + ignore := NewServer(t, WithConditionalBehavior(ConditionalIgnore)) + for range 2 { + _, err := putObject(t, ignore.Client(), ignore, "k", "x", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err, "an ignoring endpoint overwrites silently") + } + notImplemented := NewServer(t, WithConditionalBehavior(ConditionalNotImplemented)) + _, err := putObject(t, notImplemented.Client(), notImplemented, "k", "x", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + assert.Equal(t, "NotImplemented", apiCode(t, err)) + _, err = putObject(t, notImplemented.Client(), notImplemented, "plain", "x") + require.NoError(t, err, "unconditional requests still work") +} + +func TestListPagination(t *testing.T) { + srv := NewServer(t, WithPageCap(4)) + for i := range 10 { + srv.Seed(fmt.Sprintf("p/%02d", i), nil, nil) + } + srv.Seed("q/x", nil, nil) + client := srv.Client() + + var keys []string + var token *string + pages := 0 + for { + out, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(srv.Bucket()), Prefix: aws.String("p/"), MaxKeys: aws.Int32(100), ContinuationToken: token, + }) + require.NoError(t, err) + pages++ + assert.LessOrEqual(t, len(out.Contents), 4, "the server's page cap wins over max-keys") + for _, obj := range out.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + if !aws.ToBool(out.IsTruncated) { + break + } + require.NotEmpty(t, aws.ToString(out.NextContinuationToken)) + token = out.NextContinuationToken + } + assert.Len(t, keys, 10) + assert.Equal(t, 3, pages) + + _, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(srv.Bucket()), ContinuationToken: aws.String("!!!")}) + assert.Equal(t, "InvalidArgument", apiCode(t, err)) +} + +func TestListKeysWithSpecialCharactersRoundTrip(t *testing.T) { + srv := NewServer(t) + keys := []string{"a b/c+d", "ünï/ç", "x&y=z/1", "plain"} + for _, key := range keys { + srv.Seed(key, nil, nil) + } + out, err := srv.Client().ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(srv.Bucket())}) + require.NoError(t, err) + var got []string + for _, obj := range out.Contents { + got = append(got, aws.ToString(obj.Key)) + } + assert.ElementsMatch(t, keys, got) +} + +func TestMultipartLifecycle(t *testing.T) { + srv := NewServer(t, WithMinPartSize(4)) + client := srv.Client() + ctx := context.Background() + created, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), Tagging: aws.String("a=b"), ContentType: aws.String("text/x"), + }) + require.NoError(t, err) + part := func(n int32, body string) types.CompletedPart { + out, partErr := client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, PartNumber: aws.Int32(n), Body: strings.NewReader(body), + }) + require.NoError(t, partErr) + return types.CompletedPart{ETag: out.ETag, PartNumber: aws.Int32(n)} + } + p1, p2 := part(1, "aaaa"), part(2, "bb") + assert.Len(t, srv.OpenUploads(), 1) + + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{p2, p1}}, + }) + assert.Equal(t, "InvalidPart", apiCode(t, err), "parts must be in ascending order") + + done, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{p1, p2}}, + }) + require.NoError(t, err) + assert.True(t, strings.HasSuffix(strings.Trim(aws.ToString(done.ETag), `"`), "-2")) + object, _ := srv.Object("m") + assert.Equal(t, []byte("aaaabb"), object.Data) + assert.Equal(t, map[string]string{"a": "b"}, object.Tags) + assert.Equal(t, "text/x", object.ContentType) + assert.Empty(t, srv.OpenUploads()) + + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId}) + assert.Equal(t, "NoSuchUpload", apiCode(t, err)) +} + +func TestMultipartMinimumPartSizeAndAbort(t *testing.T) { + srv := NewServer(t, WithMinPartSize(10)) + client := srv.Client() + ctx := context.Background() + created, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m")}) + require.NoError(t, err) + var parts []types.CompletedPart + for n := int32(1); n <= 2; n++ { + out, partErr := client.UploadPart(ctx, &s3.UploadPartInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, PartNumber: aws.Int32(n), Body: strings.NewReader("tiny")}) + require.NoError(t, partErr) + parts = append(parts, types.CompletedPart{ETag: out.ETag, PartNumber: aws.Int32(n)}) + } + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, MultipartUpload: &types.CompletedMultipartUpload{Parts: parts}, + }) + assert.Equal(t, "InvalidPart", apiCode(t, err), "a non-final part below the minimum is rejected") + + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId}) + require.NoError(t, err) + assert.Equal(t, 1, srv.AbortedUploads()) + assert.Empty(t, srv.OpenUploads()) + assert.Empty(t, srv.Keys()) +} + +func TestTaggingMetadataAndHead(t *testing.T) { + srv := NewServer(t) + client := srv.Client() + _, err := putObject(t, client, srv, "k", "body", func(in *s3.PutObjectInput) { + in.Tagging = aws.String("a=1&b=2") + in.Metadata = map[string]string{"Owner": "me"} + in.ContentType = aws.String("text/plain") + }) + require.NoError(t, err) + head, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k")}) + require.NoError(t, err) + assert.Equal(t, int64(4), aws.ToInt64(head.ContentLength)) + assert.Equal(t, "text/plain", aws.ToString(head.ContentType)) + assert.Equal(t, "me", head.Metadata["owner"]) + tags, err := client.GetObjectTagging(context.Background(), &s3.GetObjectTaggingInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k")}) + require.NoError(t, err) + assert.Len(t, tags.TagSet, 2) + + _, err = putObject(t, client, srv, "toomany", "x", func(in *s3.PutObjectInput) { + in.Tagging = aws.String("a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11") + }) + assert.Equal(t, "InvalidTag", apiCode(t, err)) + _, err = client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("nope")}) + var notFound *types.NotFound + assert.ErrorAs(t, err, ¬Found) +} + +func TestRanges(t *testing.T) { + srv := NewServer(t) + srv.Seed("k", []byte("0123456789"), nil) + get := func(r string) (string, error) { + out, err := srv.Client().GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k"), Range: aws.String(r)}) + if err != nil { + return "", err + } + defer out.Body.Close() + data, _ := io.ReadAll(out.Body) + return string(data), nil + } + got, err := get("bytes=2-4") + require.NoError(t, err) + assert.Equal(t, "234", got) + got, err = get("bytes=8-") + require.NoError(t, err) + assert.Equal(t, "89", got) + _, err = get("bytes=20-30") + assert.Equal(t, "InvalidRange", apiCode(t, err)) +} + +func TestUnknownBucketAndZeroLength(t *testing.T) { + srv := NewServer(t, WithBucket("mine")) + _, err := srv.Client().PutObject(context.Background(), &s3.PutObjectInput{Bucket: aws.String("other"), Key: aws.String("k"), Body: strings.NewReader("x")}) + assert.Equal(t, "NoSuchBucket", apiCode(t, err)) + + _, err = putObject(t, srv.Client(), srv, "empty", "") + require.NoError(t, err) + object, ok := srv.Object("empty") + require.True(t, ok) + assert.Zero(t, object.Size) + assert.NotEmpty(t, object.ETag) +} + +func TestFaultInjection(t *testing.T) { + t.Run("skip then fail forever", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Skip: 2, Times: -1, Status: http.StatusInternalServerError}) + client := srv.LegacyClient() + client = s3.NewFromConfig(aws.Config{Region: "us-east-1", Credentials: client.Options().Credentials, RetryMaxAttempts: 1}, func(o *s3.Options) { + o.BaseEndpoint = aws.String(srv.Endpoint()) + o.UsePathStyle = true + o.Retryer = aws.NopRetryer{} + }) + for i := range 2 { + _, err := putObject(t, client, srv, fmt.Sprint("ok", i), "x") + require.NoError(t, err) + } + _, err := putObject(t, client, srv, "boom", "x") + assert.Equal(t, "InternalError", apiCode(t, err)) + _, ok := srv.Object("boom") + assert.False(t, ok, "a failed request is not applied") + }) + t.Run("drop after commit applies the write", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: DropAfterCommit}) + client := noRetryClient(srv) + _, err := putObject(t, client, srv, "k", "x") + require.Error(t, err) + _, ok := srv.Object("k") + assert.True(t, ok) + assert.Equal(t, 0, srv.Requests()[0].Status) + assert.True(t, srv.Requests()[0].Faulted) + }) + t.Run("drop before commit does not", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: DropBeforeCommit}) + _, err := putObject(t, noRetryClient(srv), srv, "k", "x") + require.Error(t, err) + _, ok := srv.Object("k") + assert.False(t, ok) + }) + t.Run("delay", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: Delay, Delay: 60 * time.Millisecond}) + start := time.Now() + _, err := putObject(t, srv.Client(), srv, "k", "x") + require.NoError(t, err) + assert.GreaterOrEqual(t, time.Since(start), 60*time.Millisecond) + }) + t.Run("key prefix scoping and clearing", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{KeyPrefix: "bad/", Times: -1, Status: http.StatusBadRequest, Code: "Nope"}) + _, err := putObject(t, srv.Client(), srv, "good/k", "x") + require.NoError(t, err) + _, err = putObject(t, srv.Client(), srv, "bad/k", "x") + assert.Equal(t, "Nope", apiCode(t, err)) + srv.ClearFaults() + _, err = putObject(t, srv.Client(), srv, "bad/k", "x") + require.NoError(t, err) + }) +} + +func noRetryClient(srv *Server) *s3.Client { + return s3.NewFromConfig(aws.Config{Region: "us-east-1"}, func(o *s3.Options) { + o.BaseEndpoint = aws.String(srv.Endpoint()) + o.UsePathStyle = true + o.Retryer = aws.NopRetryer{} + o.Credentials = srv.Client().Options().Credentials + }) +} + +func TestRequestLogRecordsOrderAndHeaders(t *testing.T) { + srv := NewServer(t) + _, err := putObject(t, srv.Client(), srv, "a/b", "hello", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err) + _, err = srv.Client().HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("a/b")}) + require.NoError(t, err) + + requests := srv.Requests() + require.Len(t, requests, 2) + assert.Equal(t, []int{1, 2}, []int{requests[0].Seq, requests[1].Seq}) + assert.Equal(t, OpPutObject, requests[0].Op) + assert.Equal(t, "a/b", requests[0].Key) + assert.Equal(t, "*", requests[0].Header.Get("If-None-Match")) + assert.Equal(t, http.StatusOK, requests[0].Status) + assert.Equal(t, OpHeadObject, requests[1].Op) + assert.Len(t, srv.RequestsFor(OpHeadObject), 1) + srv.ResetRequests() + assert.Empty(t, srv.Requests()) +} + +func TestDecodeAWSChunked(t *testing.T) { + raw := []byte("5;chunk-signature=abc\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:AAAA\r\n\r\n") + got, err := decodeAWSChunked(raw) + require.NoError(t, err) + assert.Equal(t, "hello world", string(got)) + _, err = decodeAWSChunked([]byte("zz\r\n")) + require.Error(t, err) + _, err = decodeAWSChunked([]byte("5\r\nhel")) + require.Error(t, err) +} + +func TestDiscardBodiesKeepsSizesAndETags(t *testing.T) { + srv := NewServer(t, WithDiscardBodies()) + _, err := putObject(t, srv.Client(), srv, "k", "payload") + require.NoError(t, err) + object, ok := srv.Object("k") + require.True(t, ok) + assert.Equal(t, int64(7), object.Size) + assert.Empty(t, object.Data) + assert.NotEmpty(t, object.ETag) +} + +func TestEmulatorFromEnvSkipsWhenUnset(t *testing.T) { + t.Setenv(EmulatorEndpointEnv, "") + t.Setenv(EmulatorBucketEnv, "") + result := t.Run("inner", func(t *testing.T) { + EmulatorFromEnv(t) + t.Error("EmulatorFromEnv must skip when the environment is unset") + }) + assert.True(t, result, "a skipped subtest still counts as passed") + + t.Setenv(EmulatorEndpointEnv, "http://localhost:4566") + t.Setenv(EmulatorBucketEnv, "b") + emulator := EmulatorFromEnv(t) + assert.Equal(t, Emulator{Endpoint: "http://localhost:4566", Bucket: "b", Region: "us-east-1"}, emulator) +} diff --git a/pkg/artifact/blob/s3/s3test/state.go b/pkg/artifact/blob/s3/s3test/state.go new file mode 100644 index 00000000000..2b8e3ca10f7 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/state.go @@ -0,0 +1,157 @@ +package s3test + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// object is a stored object. +type object struct { + key string + data []byte + size int64 + etag string + modified time.Time + contentType string + metadata map[string]string + tags map[string]string +} + +// Object is a read-only snapshot of a stored object. +type Object struct { + Key string + Data []byte + Size int64 + ETag string + ContentType string + Metadata map[string]string + Tags map[string]string +} + +func (o *object) snapshot() Object { + return Object{ + Key: o.key, Data: append([]byte(nil), o.data...), Size: o.size, ETag: o.etag, + ContentType: o.contentType, Metadata: cloneMap(o.metadata), Tags: cloneMap(o.tags), + } +} + +func cloneMap(m map[string]string) map[string]string { + if m == nil { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// etagLength is the number of hex characters in a fake ETag, the length of an MD5 digest. +const etagLength = 32 + +// digestHex returns a short opaque digest. Real S3 ETags are MD5-based for unencrypted single +// requests, but they are concurrency tokens and callers must never rely on that, so the fake +// uses a different hash to catch code that does. +func digestHex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:])[:etagLength] +} + +// quotedDigest returns the ETag of a single-request object. +func quotedDigest(data []byte) string { + return `"` + digestHex(data) + `"` +} + +// Object returns a snapshot of the object stored under key. +func (s *Server) Object(key string) (Object, bool) { + defer perf.Track(nil, "s3test.Server.Object")() + + s.mu.Lock() + defer s.mu.Unlock() + obj, ok := s.objects[key] + if !ok { + return Object{}, false + } + return obj.snapshot(), true +} + +// Keys returns every stored key in order. +func (s *Server) Keys() []string { + defer perf.Track(nil, "s3test.Server.Keys")() + + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.objects)) + for key := range s.objects { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// Seed stores an object directly, bypassing the HTTP surface and the request log. +func (s *Server) Seed(key string, data []byte, tags map[string]string) Object { + defer perf.Track(nil, "s3test.Server.Seed")() + + s.mu.Lock() + defer s.mu.Unlock() + obj := &object{ + key: key, data: append([]byte(nil), data...), size: int64(len(data)), etag: quotedDigest(data), + modified: timeNow(), tags: cloneMap(tags), + } + s.objects[key] = obj + return obj.snapshot() +} + +// OpenUploads returns the IDs of multipart uploads that were neither completed nor aborted. +func (s *Server) OpenUploads() []string { + defer perf.Track(nil, "s3test.Server.OpenUploads")() + + s.mu.Lock() + defer s.mu.Unlock() + ids := make([]string, 0, len(s.uploads)) + for id := range s.uploads { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// AbortedUploads returns how many multipart uploads were aborted. +func (s *Server) AbortedUploads() int { + defer perf.Track(nil, "s3test.Server.AbortedUploads")() + + return int(s.aborted.Load()) +} + +// CompletedUploads returns how many multipart uploads were completed. +func (s *Server) CompletedUploads() int { + defer perf.Track(nil, "s3test.Server.CompletedUploads")() + + return int(s.completed.Load()) +} + +// PartBytesReceived returns the bytes of every UploadPart body received so far. +func (s *Server) PartBytesReceived() int64 { + defer perf.Track(nil, "s3test.Server.PartBytesReceived")() + + return s.partBytes.Load() +} + +// MaxConcurrentParts returns the most UploadPart requests that were in flight at once. +func (s *Server) MaxConcurrentParts() int { + defer perf.Track(nil, "s3test.Server.MaxConcurrentParts")() + + return int(s.maxPartsInFlow.Load()) +} + +// newID returns a unique upload ID. +func (s *Server) newID() string { + s.nextID++ + return fmt.Sprintf("upload-%d", s.nextID) +} diff --git a/pkg/artifact/blob/s3/s3test/util.go b/pkg/artifact/blob/s3/s3test/util.go new file mode 100644 index 00000000000..c3e4fb60f1e --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/util.go @@ -0,0 +1,50 @@ +package s3test + +import ( + "context" + "encoding/xml" + "net/http" + "time" +) + +func timeNow() time.Time { + return time.Now() +} + +// sleep pauses for d or until ctx ends. +func sleep(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } +} + +// errorBody is the S3 XML error document. +type errorBody struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + RequestID string `xml:"RequestId"` +} + +// writeError answers with an S3 XML error. HEAD responses carry no body. +func writeError(w http.ResponseWriter, status int, code, message string) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + body, _ := xml.Marshal(errorBody{Code: code, Message: message, RequestID: "s3test"}) + _, _ = w.Write(append([]byte(xml.Header), body...)) +} + +// writeXML answers 200 with an XML document. +func writeXML(w http.ResponseWriter, value any) { + body, err := xml.Marshal(value) + if err != nil { + writeError(w, http.StatusInternalServerError, "InternalError", err.Error()) + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(append([]byte(xml.Header), body...)) +} diff --git a/pkg/artifact/blob/s3/store.go b/pkg/artifact/blob/s3/store.go new file mode 100644 index 00000000000..543d82f8765 --- /dev/null +++ b/pkg/artifact/blob/s3/store.go @@ -0,0 +1,129 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +const storeName = "s3" + +// Store is an S3 [blob.Store]. +type Store struct { + cfg settings + + mu sync.Mutex + api API + + probeMu sync.Mutex + probed bool + probeError error +} + +// New returns a Store for bucket. No network or credential work happens until the first operation. +func New(bucket string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.s3.New")() + + if bucket == "" { + return nil, blob.NewError(blob.KindInvalid, blob.OpNew, "", fmt.Errorf("%w: bucket is required", errUtils.ErrBlobInvalid)) + } + cfg := settings{bucket: bucket, partSize: DefaultPartSize, concurrency: DefaultConcurrency} + for _, opt := range opts { + opt(&cfg) + } + return &Store{cfg: cfg, api: cfg.api}, nil +} + +// NewFromClient returns a Store over an existing client. +func NewFromClient(api API, bucket string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.s3.NewFromClient")() + + return New(bucket, append(opts, WithAPI(api))...) +} + +// Name implements [blob.Store]. +func (s *Store) Name() string { + defer perf.Track(nil, "s3.Store.Name")() + + return storeName +} + +// Bucket returns the bucket name. +func (s *Store) Bucket() string { + defer perf.Track(nil, "s3.Store.Bucket")() + + return s.cfg.bucket +} + +// Capabilities implements [blob.Store]. +func (s *Store) Capabilities() blob.Capabilities { + defer perf.Track(nil, "s3.Store.Capabilities")() + + conditional := s.cfg.conditional != ConditionalDisabled + return blob.Capabilities{ + ConditionalPut: conditional, + ConditionalDelete: conditional, + Multipart: true, + Tags: true, + Metadata: true, + Range: true, + Listing: true, + ListDelimiter: true, + } +} + +// client returns the S3 client, building it on first use. A failed build is not cached, so a +// transient identity failure can be retried by the next operation. +func (s *Store) client(ctx context.Context) (API, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.api != nil { + return s.api, nil + } + client, err := NewClient(ctx, &s.cfg.client) + if err != nil { + return nil, blob.NewError(blob.KindIdentity, "client", "", err) + } + s.api = client + return s.api, nil +} + +// validateKey applies the configured key rules. +func (s *Store) validateKey(key string) error { + if s.cfg.opaqueKeys { + return blob.ValidateOpaqueKey(key) + } + return blob.ValidateKey(key) +} + +// objectKey maps a store key to the bucket key. +func (s *Store) objectKey(key string) string { + if s.cfg.prefix == "" { + return key + } + return s.cfg.prefix + "/" + key +} + +// storeKey maps a bucket key back to a store key. +func (s *Store) storeKey(objectKey string) string { + if s.cfg.prefix == "" { + return objectKey + } + return strings.TrimPrefix(objectKey, s.cfg.prefix+"/") +} + +// retryOptions disables SDK retries for conditional mutations. A retried conditional request whose +// first attempt already committed would otherwise answer 412 and look like a lost race. +func retryOptions(conditional bool) []func(*sdkOptions) { + if !conditional { + return nil + } + return []func(*sdkOptions){noRetry} +} + +var _ blob.Store = (*Store)(nil) diff --git a/pkg/artifact/blob/s3/store_test.go b/pkg/artifact/blob/s3/store_test.go new file mode 100644 index 00000000000..0ee1556657d --- /dev/null +++ b/pkg/artifact/blob/s3/store_test.go @@ -0,0 +1,147 @@ +package s3 + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func TestConformance(t *testing.T) { + variants := map[string]struct { + fake []s3test.Option + store []Option + }{ + "single-request": {store: []Option{WithPartSize(DefaultPartSize)}}, + "multipart": {store: []Option{WithPartSize(smallPart)}}, + "prefixed": {store: []Option{WithPrefix("/team/artifacts/")}}, + "capped-pages": {fake: []s3test.Option{s3test.WithPageCap(7)}}, + } + for name, variant := range variants { + t.Run(name, func(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { + fake := newFake(t, variant.fake...) + return newTestStore(t, fake, variant.store...) + }) + }) + } +} + +func TestNewValidates(t *testing.T) { + _, err := New("") + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + s, err := New("bucket") + require.NoError(t, err) + assert.Equal(t, "s3", s.Name()) + assert.Equal(t, "bucket", s.Bucket()) + assert.Equal(t, DefaultPartSize, s.cfg.partSize) + assert.Equal(t, DefaultConcurrency, s.cfg.concurrency) +} + +func TestCapabilities(t *testing.T) { + enabled, err := New("b") + require.NoError(t, err) + caps := enabled.Capabilities() + assert.True(t, caps.ConditionalPut && caps.ConditionalDelete && caps.Multipart && caps.Tags && caps.Listing) + + disabled, err := New("b", WithConditionalMode(ConditionalDisabled)) + require.NoError(t, err) + assert.False(t, disabled.Capabilities().ConditionalPut) + assert.False(t, disabled.Capabilities().ConditionalDelete) +} + +func TestKeyMappingWithPrefix(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithPrefix("/p/q/")) + putBytes(t, s, "a/b", []byte("x"), blob.PutOptions{}) + assert.Equal(t, []string{"p/q/a/b"}, fake.Keys()) + + objects, _, err := blob.ListAll(context.Background(), s, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, objects, 1) + assert.Equal(t, "a/b", objects[0].Key, "listed keys are relative to the configured prefix") +} + +func TestKeyValidationPortableAndOpaque(t *testing.T) { + fake := newFake(t) + strict := newTestStore(t, fake) + _, err := strict.Put(context.Background(), "a/../b", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + + opaque := newTestStore(t, fake, WithOpaqueKeys()) + putBytes(t, opaque, "a//odd/../key", []byte("x"), blob.PutOptions{}) + assert.Contains(t, fake.Keys(), "a//odd/../key", "legacy key patterns are passed through untouched") + _, err = opaque.Put(context.Background(), "", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestRangedGetSendsRangeHeader(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("0123456789"), blob.PutOptions{}) + fake.ResetRequests() + + object, err := s.Get(context.Background(), "k", blob.GetOptions{Offset: 2, Length: 3}) + require.NoError(t, err) + require.NoError(t, object.Body.Close()) + assert.Equal(t, int64(10), object.Size) + require.Len(t, fake.RequestsFor(s3test.OpGetObject), 1) + assert.Equal(t, "bytes=2-4", fake.RequestsFor(s3test.OpGetObject)[0].Header.Get("Range")) + + _, err = s.Get(context.Background(), "k", blob.GetOptions{Offset: 50}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "416 maps to an invalid request") +} + +func TestStatLoadsTagsOnlyWhenAsked(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) + fake.ResetRequests() + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{}) + require.NoError(t, err) + assert.Nil(t, info.Tags) + assert.Empty(t, fake.RequestsFor(s3test.OpGetObjectTagging)) + + info, err = s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"a": "b"}, info.Tags) + assert.Len(t, fake.RequestsFor(s3test.OpGetObjectTagging), 1) +} + +func TestListPaginatesAcrossMoreThanAThousandKeys(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + const total = 2503 + for i := range total { + fake.Seed(fmt.Sprintf("many/%05d", i), nil, nil) + } + fake.ResetRequests() + + objects, _, err := blob.ListAll(context.Background(), s, blob.ListOptions{Prefix: "many/"}) + require.NoError(t, err) + require.Len(t, objects, total) + assert.Equal(t, "many/00000", objects[0].Key) + assert.Equal(t, fmt.Sprintf("many/%05d", total-1), objects[total-1].Key) + assert.Len(t, fake.RequestsFor(s3test.OpListObjectsV2), 3, "1000 + 1000 + 503 entries") + for _, object := range objects[:3] { + assert.NotEmpty(t, object.ETag, "S3 listings carry ETags") + } +} + +func TestListDefaultsToServerPageSizeWhenNoLimit(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Seed("a", nil, nil) + page, err := s.List(context.Background(), blob.ListOptions{}) + require.NoError(t, err) + assert.Empty(t, page.NextCursor) + assert.NotContains(t, fake.RequestsFor(s3test.OpListObjectsV2)[0].Query, "max-keys") +} diff --git a/pkg/artifact/blob/store.go b/pkg/artifact/blob/store.go new file mode 100644 index 00000000000..317f067ba61 --- /dev/null +++ b/pkg/artifact/blob/store.go @@ -0,0 +1,197 @@ +package blob + +import ( + "context" + "io" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate go run go.uber.org/mock/mockgen -package blob -destination mock_store.go github.com/cloudposse/atmos/pkg/artifact/blob Store + +// UnknownSize is the Size value of a [PutOptions] whose body length is not known in advance. +const UnknownSize int64 = -1 + +// Store is transport-only object I/O. Keys are slash-separated and already fully formed by the +// caller; a Store never interprets them beyond validation. Every method honors context +// cancellation. Implementations are safe for concurrent use. +type Store interface { + // Name returns the implementation name, for example "local" or "s3". + Name() string + + // Capabilities describes what the store can enforce without network access. + Capabilities() Capabilities + + // Put streams body to key. An unmet or unenforceable [PutOptions.Condition] fails the call + // without changing the object. Returned info describes the stored object, including its ETag. + Put(ctx context.Context, key string, body io.Reader, opts PutOptions) (*ObjectInfo, error) + + // Get opens key for streaming. Callers must close the returned body. A missing object fails + // with ErrBlobNotFound. + Get(ctx context.Context, key string, opts GetOptions) (*Object, error) + + // Stat returns object attributes without reading the body. A missing object fails with + // ErrBlobNotFound. + Stat(ctx context.Context, key string, opts StatOptions) (*ObjectInfo, error) + + // Delete removes key. Deleting a missing object without a condition succeeds. With + // [DeleteOptions.IfMatch] set, a missing object fails with ErrBlobNotFound and a different ETag + // fails with ErrBlobPreconditionFailed. + Delete(ctx context.Context, key string, opts DeleteOptions) error + + // List returns one lexicographically ordered page of objects under a prefix. Pass the returned + // NextCursor back to continue; an empty NextCursor means the listing is complete. + List(ctx context.Context, opts ListOptions) (*ListPage, error) +} + +// ConditionKind selects how a write is guarded. +type ConditionKind int + +// Condition kinds. +const ( + // ConditionNone writes unconditionally. + ConditionNone ConditionKind = iota + + // ConditionIfAbsent writes only when the key does not exist (S3 If-None-Match: *). + ConditionIfAbsent + + // ConditionIfMatch writes only when the current ETag equals Condition.ETag (S3 If-Match). + ConditionIfMatch +) + +// Condition guards a write. The zero value is unconditional. +type Condition struct { + // Kind selects the guard. + Kind ConditionKind + + // ETag is the expected current ETag for [ConditionIfMatch]. It is an opaque token. + ETag string +} + +// IfAbsent returns a condition that succeeds only when the key does not exist. +func IfAbsent() Condition { + defer perf.Track(nil, "blob.IfAbsent")() + + return Condition{Kind: ConditionIfAbsent} +} + +// IfMatch returns a condition that succeeds only when the current ETag equals etag. +func IfMatch(etag string) Condition { + defer perf.Track(nil, "blob.IfMatch")() + + return Condition{Kind: ConditionIfMatch, ETag: etag} +} + +// IsConditional reports whether the condition constrains the write. +func (c Condition) IsConditional() bool { + defer perf.Track(nil, "blob.Condition.IsConditional")() + + return c.Kind != ConditionNone +} + +// PutOptions describe one write. +type PutOptions struct { + // Size is the body length in bytes, or [UnknownSize]. A known size is verified: a body that + // is shorter or longer fails the call and leaves no object behind. + Size int64 + + // ContentType is stored with the object when non-empty. + ContentType string + + // Metadata is stored as user metadata. + Metadata map[string]string + + // Tags are explicit object tags. Stores that merge default tags apply them first; these win. + Tags map[string]string + + // Condition guards the write. The zero value is unconditional. + Condition Condition +} + +// GetOptions describe one read. +type GetOptions struct { + // Offset is the first byte to read. + Offset int64 + + // Length is the number of bytes to read from Offset. Zero or negative reads to the end. + Length int64 +} + +// IsRange reports whether the read is a partial read. +func (o GetOptions) IsRange() bool { + defer perf.Track(nil, "blob.GetOptions.IsRange")() + + return o.Offset > 0 || o.Length > 0 +} + +// StatOptions describe one head request. +type StatOptions struct { + // IncludeTags also loads the object tags, which may cost an extra request. + IncludeTags bool +} + +// DeleteOptions describe one delete. +type DeleteOptions struct { + // IfMatch deletes only when the current ETag equals this opaque token. + IfMatch string +} + +// ListOptions describe one listing page. +type ListOptions struct { + // Prefix restricts the listing to keys beginning with it. + Prefix string + + // Delimiter groups keys sharing a prefix up to the delimiter into CommonPrefixes. + Delimiter string + + // Cursor continues a previous listing. + Cursor string + + // Limit caps the entries (objects plus common prefixes) in one page; zero uses the store default. + Limit int +} + +// ObjectInfo describes a stored object. +type ObjectInfo struct { + // Key is the object key. + Key string + + // Size is the full object size in bytes, even for a ranged read. + Size int64 + + // ETag is an opaque concurrency token, never a payload checksum. + ETag string + + // ContentType is the stored content type, when known. + ContentType string + + // LastModified is the last modification time. + LastModified time.Time + + // Metadata is user metadata, when loaded. + Metadata map[string]string + + // Tags are object tags, loaded only when requested. + Tags map[string]string +} + +// Object is an open object. Callers must close Body. +type Object struct { + ObjectInfo + + // Body streams the (possibly ranged) content. + Body io.ReadCloser +} + +// ListPage is one page of a listing. +type ListPage struct { + // Objects are the keys found, lexicographically ordered. + Objects []ObjectInfo + + // CommonPrefixes are rolled-up prefixes when a delimiter was requested. + CommonPrefixes []string + + // NextCursor continues the listing; empty means complete. + NextCursor string +} diff --git a/pkg/artifact/blob/tags.go b/pkg/artifact/blob/tags.go new file mode 100644 index 00000000000..655796a045c --- /dev/null +++ b/pkg/artifact/blob/tags.go @@ -0,0 +1,64 @@ +package blob + +import ( + "fmt" + "maps" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// S3 object tag limits, enforced for every store so tags are portable. +const ( + MaxTags = 10 + MaxTagKeyChars = 128 + MaxTagValueChars = 256 +) + +// MergeTags returns defaults overlaid with overrides. It returns nil when both are empty and never +// aliases either input. +func MergeTags(defaults, overrides map[string]string) map[string]string { + defer perf.Track(nil, "blob.MergeTags")() + + if len(defaults) == 0 && len(overrides) == 0 { + return nil + } + merged := make(map[string]string, len(defaults)+len(overrides)) + maps.Copy(merged, defaults) + maps.Copy(merged, overrides) + return merged +} + +// ValidateTags checks the S3 object tag limits: at most [MaxTags] tags, non-empty keys of at most +// [MaxTagKeyChars] characters, and values of at most [MaxTagValueChars] characters. +func ValidateTags(tags map[string]string) error { + defer perf.Track(nil, "blob.ValidateTags")() + + if len(tags) > MaxTags { + return tagError(fmt.Sprintf("%d tags exceed the limit of %d", len(tags), MaxTags)) + } + for key, value := range tags { + if key == "" || utf8.RuneCountInString(key) > MaxTagKeyChars { + return tagError(fmt.Sprintf("tag key %q must be 1-%d characters", key, MaxTagKeyChars)) + } + if utf8.RuneCountInString(value) > MaxTagValueChars { + return tagError(fmt.Sprintf("tag %q value exceeds %d characters", key, MaxTagValueChars)) + } + } + return nil +} + +func tagError(reason string) error { + return NewError(KindInvalid, OpValidate, "", fmt.Errorf("%w: %s", errUtils.ErrBlobInvalid, reason)) +} + +// CloneMap returns a copy of m, or nil when m is empty. +func CloneMap(m map[string]string) map[string]string { + defer perf.Track(nil, "blob.CloneMap")() + + if len(m) == 0 { + return nil + } + return maps.Clone(m) +} diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index 521056a0edf..52ec2f9f8f2 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -1,6 +1,7 @@ package local import ( + "bytes" "context" "encoding/json" "fmt" @@ -13,12 +14,17 @@ import ( "time" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobLocal "github.com/cloudposse/atmos/pkg/artifact/blob/local" "github.com/cloudposse/atmos/pkg/ci/artifact" "github.com/cloudposse/atmos/pkg/config/homedir" "github.com/cloudposse/atmos/pkg/perf" ) const ( + // Format that wraps a failure to open the artifact directory with the operation's sentinel. + openDirFormat = "%w: failed to open artifact directory for %s: %w" + storeName = "local/dir" metadataSuffix = ".metadata.json" defaultDirPerms = 0o755 @@ -26,7 +32,9 @@ const ( ) // Store implements the artifact.Backend interface using the local filesystem. -// It stores each artifact as a single file with a metadata JSON sidecar. +// It stores each artifact as a single file with a metadata JSON sidecar. Object I/O is delegated +// to the shared transport in pkg/artifact/blob/local; this type only owns the legacy CI +// concerns: artifact naming, the metadata sidecar format, and query matching. type Store struct { basePath string } @@ -62,6 +70,26 @@ func (s *Store) Name() string { return storeName } +// transport returns the shared filesystem blob store rooted at the base path. The store is +// stateless apart from a per-root lock registry, so it is built on demand. +func (s *Store) transport() (*blobLocal.Store, error) { + root := s.basePath + if root == "" { + root = "." + } + return blobLocal.New(root) +} + +// key maps a validated artifact name to the transport key. Names are cleaned the same way the +// filesystem would clean them, so "a/./b" and "a//b" address the same object as before. +func (s *Store) key(name string) string { + rel, err := filepath.Rel(s.basePath, filepath.Join(s.basePath, name)) + if err != nil { + return filepath.ToSlash(name) + } + return filepath.ToSlash(rel) +} + // Upload uploads a single data stream to the local filesystem. // Writes the data as a single file and the metadata as a JSON sidecar. func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size int64, metadata *Metadata) error { @@ -70,25 +98,16 @@ func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size in if err := s.validateName(name); err != nil { return err } - - filePath := filepath.Join(s.basePath, name) - - // Create parent directories for the artifact file. - if err := os.MkdirAll(filepath.Dir(filePath), defaultDirPerms); err != nil { - return fmt.Errorf("%w: failed to create directory for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) - } - - // Write the data stream to a single file. - f, err := os.Create(filePath) + transport, err := s.transport() if err != nil { - return fmt.Errorf("%w: failed to create file %s: %w", errUtils.ErrArtifactUploadFailed, name, err) + return fmt.Errorf(openDirFormat, errUtils.ErrArtifactUploadFailed, name, err) } + key := s.key(name) - if _, err := io.Copy(f, data); err != nil { - f.Close() + // The declared size is advisory for this backend, as it always was. + if _, err := transport.Put(ctx, key, data, blob.PutOptions{Size: blob.UnknownSize}); err != nil { return fmt.Errorf("%w: failed to write file %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } - f.Close() // Write metadata sidecar. if metadata == nil { @@ -97,18 +116,12 @@ func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size in } } - metadataPath := filepath.Join(s.basePath, name+metadataSuffix) - - // Create parent directories for metadata file. - if err := os.MkdirAll(filepath.Dir(metadataPath), defaultDirPerms); err != nil { - return fmt.Errorf("%w: failed to create directory for metadata: %w", errUtils.ErrArtifactUploadFailed, err) - } - metadataJSON, err := json.MarshalIndent(metadata, "", " ") if err != nil { return fmt.Errorf("%w: failed to marshal metadata for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } - if err := os.WriteFile(metadataPath, metadataJSON, defaultFilePerms); err != nil { + _, err = transport.Put(ctx, key+metadataSuffix, bytes.NewReader(metadataJSON), blob.PutOptions{Size: int64(len(metadataJSON))}) + if err != nil { return fmt.Errorf("%w: failed to write metadata for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } @@ -127,30 +140,33 @@ func (s *Store) Download(ctx context.Context, name string) (io.ReadCloser, *arti if err := s.validateName(name); err != nil { return nil, nil, err } - - filePath := filepath.Join(s.basePath, name) - - // Check if artifact file exists and is not a directory. - info, err := os.Stat(filePath) + transport, err := s.transport() if err != nil { - if os.IsNotExist(err) { - return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) - } - return nil, nil, fmt.Errorf("%w: failed to stat artifact %s: %w", errUtils.ErrArtifactDownloadFailed, name, err) - } - if info.IsDir() { - return nil, nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactDownloadFailed, name) + return nil, nil, fmt.Errorf(openDirFormat, errUtils.ErrArtifactDownloadFailed, name, err) } + key := s.key(name) - f, err := os.Open(filePath) + object, err := transport.Get(ctx, key, blob.GetOptions{}) if err != nil { + if blob.IsNotFound(err) { + if s.isDirectory(name) { + return nil, nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactDownloadFailed, name) + } + return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) + } return nil, nil, fmt.Errorf("%w: failed to open artifact %s: %w", errUtils.ErrArtifactDownloadFailed, name, err) } // Load metadata. - metadata, _ := s.loadMetadata(filepath.Join(s.basePath, name+metadataSuffix)) + metadata, _ := s.loadMetadata(ctx, transport, key+metadataSuffix) - return f, metadata, nil + return object.Body, metadata, nil +} + +// isDirectory reports whether name addresses a directory, which is not an artifact. +func (s *Store) isDirectory(name string) bool { + info, err := os.Stat(filepath.Join(s.basePath, name)) + return err == nil && info.IsDir() } // Delete deletes an artifact from the local filesystem. @@ -162,78 +178,72 @@ func (s *Store) Delete(ctx context.Context, name string) error { if err := s.validateName(name); err != nil { return err } + transport, err := s.transport() + if err != nil { + return fmt.Errorf(openDirFormat, errUtils.ErrArtifactDeleteFailed, name, err) + } + key := s.key(name) - filePath := filepath.Join(s.basePath, name) - - // Delete the artifact file. - if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + // Delete the artifact file; the transport also removes emptied parent directories. + if err := transport.Delete(ctx, key, blob.DeleteOptions{}); err != nil { return fmt.Errorf("%w: failed to delete artifact %s: %w", errUtils.ErrArtifactDeleteFailed, name, err) } // Delete metadata sidecar (ignore errors if it doesn't exist). - _ = os.Remove(filepath.Join(s.basePath, name+metadataSuffix)) - - // Try to clean up empty parent directories. - s.cleanupEmptyDirs(filepath.Dir(filePath)) + _ = transport.Delete(ctx, key+metadataSuffix, blob.DeleteOptions{}) return nil } // List lists artifacts matching the given query. -// Walks basePath looking for metadata sidecar files with a corresponding artifact file (not dir), +// Finds metadata sidecar files with a corresponding artifact file (not dir), // applies Query filters, and sorts newest first. func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { defer perf.Track(nil, "artifact.local.List")() - var artifacts []artifact.ArtifactInfo - - // Walk basePath looking for metadata sidecar files. - err := filepath.WalkDir(s.basePath, func(path string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return nil // Skip entries with errors. - } + transport, err := s.transport() + if err != nil { + return nil, fmt.Errorf("%w: failed to open artifact directory: %w", errUtils.ErrArtifactListFailed, err) + } + objects, _, err := blob.ListAll(ctx, transport, blob.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("%w: failed to list artifacts: %w", errUtils.ErrArtifactListFailed, err) + } - // We only care about .metadata.json files. - if d.IsDir() || !strings.HasSuffix(path, metadataSuffix) { - return nil - } + byKey := make(map[string]blob.ObjectInfo, len(objects)) + for _, object := range objects { + byKey[object.Key] = object + } - // Derive artifact name from metadata path. - relPath, err := filepath.Rel(s.basePath, path) - if err != nil { - return nil + var artifacts []artifact.ArtifactInfo + for _, object := range objects { + name, ok := strings.CutSuffix(object.Key, metadataSuffix) + if !ok { + continue } - artifactName := strings.TrimSuffix(relPath, metadataSuffix) - // Verify the artifact file exists and is not a directory. - artifactFile := filepath.Join(s.basePath, artifactName) - fileInfo, err := os.Stat(artifactFile) - if err != nil || fileInfo.IsDir() { - return nil + // Only metadata sidecars with a corresponding artifact file are artifacts. + file, ok := byKey[name] + if !ok { + continue } - // Load metadata. - metadata, err := s.loadMetadata(path) + metadata, err := s.loadMetadata(ctx, transport, object.Key) if err != nil { - return nil + continue } // Apply query filters. if !query.All && !s.matchesQuery(metadata, query) { - return nil + continue } artifacts = append(artifacts, artifact.ArtifactInfo{ - Name: artifactName, - Size: fileInfo.Size(), - LastModified: fileInfo.ModTime(), + Name: filepath.FromSlash(name), + Size: file.Size, + LastModified: file.LastModified, Metadata: metadata, }) - - return nil - }) - if err != nil { - return nil, fmt.Errorf("%w: failed to list artifacts: %w", errUtils.ErrArtifactListFailed, err) } // Sort by last modified (newest first). @@ -251,16 +261,18 @@ func (s *Store) Exists(ctx context.Context, name string) (bool, error) { if err := s.validateName(name); err != nil { return false, err } - - filePath := filepath.Join(s.basePath, name) - info, err := os.Stat(filePath) + transport, err := s.transport() if err != nil { - if os.IsNotExist(err) { + return false, fmt.Errorf("failed to check if artifact %s exists: %w", name, err) + } + + if _, err := transport.Stat(ctx, s.key(name), blob.StatOptions{}); err != nil { + if blob.IsNotFound(err) { return false, nil } return false, fmt.Errorf("failed to check if artifact %s exists: %w", name, err) } - return !info.IsDir(), nil + return true, nil } // GetMetadata retrieves metadata for an artifact without downloading the content. @@ -270,29 +282,32 @@ func (s *Store) GetMetadata(ctx context.Context, name string) (*artifact.Metadat if err := s.validateName(name); err != nil { return nil, err } + transport, err := s.transport() + if err != nil { + return nil, fmt.Errorf(openDirFormat, errUtils.ErrArtifactMetadataFailed, name, err) + } + key := s.key(name) // Check if the artifact file exists. - filePath := filepath.Join(s.basePath, name) - fileInfo, err := os.Stat(filePath) + info, err := transport.Stat(ctx, key, blob.StatOptions{}) if err != nil { - if os.IsNotExist(err) { + if blob.IsNotFound(err) { + if s.isDirectory(name) { + return nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactNotFound, name) + } return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) } return nil, fmt.Errorf("%w: failed to stat artifact %s: %w", errUtils.ErrArtifactMetadataFailed, name, err) } - if fileInfo.IsDir() { - return nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactNotFound, name) - } - metadataPath := filepath.Join(s.basePath, name+metadataSuffix) - metadata, err := s.loadMetadata(metadataPath) + metadata, err := s.loadMetadata(ctx, transport, key+metadataSuffix) if err != nil { return nil, fmt.Errorf("%w: failed to load metadata for %s: %w", errUtils.ErrArtifactMetadataFailed, name, err) } if metadata == nil { // Return minimal metadata from file info. metadata = &artifact.Metadata{ - CreatedAt: fileInfo.ModTime(), + CreatedAt: info.LastModified, } } @@ -311,15 +326,21 @@ func (s *Store) validateName(name string) error { return nil } -// loadMetadata loads metadata from a JSON sidecar file. -func (s *Store) loadMetadata(metadataPath string) (*artifact.Metadata, error) { - data, err := os.ReadFile(metadataPath) +// loadMetadata loads metadata from a JSON sidecar object. A missing sidecar yields nil metadata. +func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, metadataKey string) (*artifact.Metadata, error) { + object, err := transport.Get(ctx, metadataKey, blob.GetOptions{}) if err != nil { - if os.IsNotExist(err) { + if blob.IsNotFound(err) { return nil, nil } return nil, err } + defer object.Body.Close() + + data, err := io.ReadAll(object.Body) + if err != nil { + return nil, err + } var metadata artifact.Metadata if err := json.Unmarshal(data, &metadata); err != nil { @@ -358,18 +379,6 @@ func (s *Store) matchesQuery(metadata *artifact.Metadata, query artifact.Query) return true } -// cleanupEmptyDirs removes empty parent directories up to the base path. -func (s *Store) cleanupEmptyDirs(dir string) { - for dir != s.basePath && dir != "" { - entries, err := os.ReadDir(dir) - if err != nil || len(entries) > 0 { - return - } - _ = os.Remove(dir) - dir = filepath.Dir(dir) - } -} - func init() { artifact.Register(storeName, NewStore) } diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go new file mode 100644 index 00000000000..192ae4583e9 --- /dev/null +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -0,0 +1,178 @@ +package local + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +// dirEntries lists every regular file under root as slash-separated relative paths. +func dirEntries(t *testing.T, root string) []string { + t.Helper() + var entries []string + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + require.NoError(t, err) + if !d.IsDir() { + rel, relErr := filepath.Rel(root, path) + require.NoError(t, relErr) + entries = append(entries, filepath.ToSlash(rel)) + } + return nil + })) + sort.Strings(entries) + return entries +} + +// TestTransportKeepsTheLegacyOnDiskLayout pins the wire format: one file per artifact plus a +// ".metadata.json" sidecar holding indented JSON, and nothing else in the directory. +func TestTransportKeepsTheLegacyOnDiskLayout(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + created := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc", CreatedAt: created, Custom: map[string]string{"k": "v"}} + body := []byte("tar-bytes\x00\x01") + + require.NoError(t, store.Upload(context.Background(), "dev/vpc/abc.tar", bytes.NewReader(body), int64(len(body)), metadata)) + + assert.Equal(t, []string{"dev/vpc/abc.tar", "dev/vpc/abc.tar.metadata.json"}, dirEntries(t, root)) + onDisk, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "abc.tar")) + require.NoError(t, err) + assert.Equal(t, body, onDisk) + sidecar, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "abc.tar.metadata.json")) + require.NoError(t, err) + want, err := json.MarshalIndent(metadata, "", " ") + require.NoError(t, err) + assert.Equal(t, want, sidecar, "the sidecar is byte-identical to the legacy encoding") + if info, statErr := os.Stat(filepath.Join(root, "dev", "vpc", "abc.tar")); statErr == nil && filepath.Separator == '/' { + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()&0o644) + } +} + +// TestBundledStoreTarBytesAndChecksumSurviveTheTransport verifies the tar archive round trip and +// the SHA-256 recorded in the sidecar still describe the stored bytes. +func TestBundledStoreTarBytesAndChecksumSurviveTheTransport(t *testing.T) { + root := t.TempDir() + bundled := artifact.NewBundledStore(&Store{basePath: root}) + files := []artifact.FileEntry{ + {Name: "plan.tfplan", Data: strings.NewReader("plan-bytes"), Size: 10}, + {Name: "lock.hcl", Data: strings.NewReader("lock"), Size: 4}, + } + require.NoError(t, bundled.Upload(context.Background(), "dev/vpc/sha.tfplan.tar", files, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "sha"})) + + stored, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "sha.tfplan.tar")) + require.NoError(t, err) + sum := sha256.Sum256(stored) + meta, err := bundled.GetMetadata(context.Background(), "dev/vpc/sha.tfplan.tar") + require.NoError(t, err) + assert.Equal(t, hex.EncodeToString(sum[:]), meta.SHA256) + + got, _, err := bundled.Download(context.Background(), "dev/vpc/sha.tfplan.tar") + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "plan.tfplan", got[0].Name) + assert.Equal(t, "lock.hcl", got[1].Name) +} + +// TestNamesAreCleanedLikeTheFilesystemDid pins that equivalent spellings address one artifact. +func TestNamesAreCleanedLikeTheFilesystemDid(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a/./b//c.tar", strings.NewReader("x"), 1, nil)) + assert.Equal(t, []string{"a/b/c.tar", "a/b/c.tar.metadata.json"}, dirEntries(t, root)) + + exists, err := store.Exists(ctx, "a/b/c.tar") + require.NoError(t, err) + assert.True(t, exists) + exists, err = store.Exists(ctx, "a/x/../b/c.tar") + require.NoError(t, err) + assert.True(t, exists) +} + +// TestDirectoryNamesKeepTheirLegacyErrors pins error behavior for a name that is a directory. +func TestDirectoryNamesKeepTheirLegacyErrors(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "dev/vpc/a.tar", strings.NewReader("x"), 1, nil)) + + _, _, err := store.Download(ctx, "dev/vpc") + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) + _, err = store.GetMetadata(ctx, "dev/vpc") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + exists, err := store.Exists(ctx, "dev/vpc") + require.NoError(t, err) + assert.False(t, exists) +} + +// TestUploadReplacesAtomically checks a failed overwrite leaves the previous artifact intact. +func TestUploadReplacesAtomically(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a.tar", strings.NewReader("original"), 8, nil)) + + err := store.Upload(ctx, "a.tar", &brokenReader{}, -1, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + + reader, _, err := store.Download(ctx, "a.tar") + require.NoError(t, err) + defer reader.Close() + var buf bytes.Buffer + _, err = buf.ReadFrom(reader) + require.NoError(t, err) + assert.Equal(t, "original", buf.String(), "a failed upload no longer truncates the existing artifact") + for _, entry := range dirEntries(t, root) { + assert.NotContains(t, entry, ".atmos-blob-tmp-", "no temporary files are left behind") + } +} + +type brokenReader struct{ done bool } + +func (b *brokenReader) Read(p []byte) (int, error) { + if !b.done { + b.done = true + return copy(p, "partial"), nil + } + return 0, os.ErrClosed +} + +// TestListSkipsOrphanSidecarsAndFiltersByQuery covers the sidecar-driven listing semantics. +func TestListSkipsOrphanSidecarsAndFiltersByQuery(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "dev/vpc/1.tar", strings.NewReader("a"), 1, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "1"})) + require.NoError(t, store.Upload(ctx, "prod/vpc/2.tar", strings.NewReader("bb"), 2, &artifact.Metadata{Stack: "prod", Component: "vpc", SHA: "2"})) + require.NoError(t, os.WriteFile(filepath.Join(root, "orphan.tar.metadata.json"), []byte(`{"stack":"x"}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "no-sidecar.tar"), []byte("data"), 0o644)) + + all, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + names := make([]string, 0, len(all)) + for _, info := range all { + names = append(names, filepath.ToSlash(info.Name)) + } + sort.Strings(names) + assert.Equal(t, []string{"dev/vpc/1.tar", "prod/vpc/2.tar"}, names, "artifacts need both the file and its sidecar") + + prod, err := store.List(ctx, artifact.Query{Stacks: []string{"prod"}}) + require.NoError(t, err) + require.Len(t, prod, 1) + assert.Equal(t, int64(2), prod[0].Size) + assert.Equal(t, "prod", prod[0].Metadata.Stack) +} diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index 44e68567145..716eb8451c1 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -8,15 +8,16 @@ import ( "fmt" "io" "path" + "strings" "sync" - "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" "github.com/cloudposse/atmos/pkg/ci/artifact" "github.com/cloudposse/atmos/pkg/perf" ) @@ -29,6 +30,12 @@ const ( ) // Store implements the artifact.Backend interface using AWS S3. +// +// It is a CI adapter over the shared transport in pkg/artifact/blob/s3: this type owns the legacy +// key pattern (prefix joined with path.Join), the ".metadata.json" sidecar, and the legacy error +// sentinels, while client construction and object I/O live in the transport. Unlike the generic +// transport, it keeps the historical fallback to the default credential chain when an identity is +// configured but no resolver was injected. type Store struct { client *s3.Client bucket string @@ -41,6 +48,11 @@ type Store struct { authResolver artifact.AuthContextResolver initOnce sync.Once initErr error + + // Shared transport over client, built on first use. + transportOnce sync.Once + transportStore *blobs3.Store + transportErr error } // NewStore creates a new S3 backend. @@ -92,41 +104,33 @@ func (s *Store) SetAuthContext(resolver artifact.AuthContextResolver, identityNa // initDefaultClient initializes the AWS client using the default credential chain. func (s *Store) initDefaultClient(ctx context.Context) error { - var awsOpts []func(*config.LoadOptions) error - if s.region != "" { - awsOpts = append(awsOpts, config.WithRegion(s.region)) - } - - cfg, err := config.LoadDefaultConfig(ctx, awsOpts...) + client, err := blobs3.NewClient(ctx, &blobs3.ClientConfig{Region: s.region, Legacy: true}) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + return err } - - s.client = s3.NewFromConfig(cfg) + s.client = client return nil } // initIdentityClient initializes the AWS client using identity-based credentials // resolved through the injected AuthContextResolver. func (s *Store) initIdentityClient(ctx context.Context) error { - if s.authResolver == nil { - // No resolver was injected; fall back to the default chain. - return s.initDefaultClient(ctx) - } - - authContext, err := s.authResolver.ResolveAWSAuthContext(ctx, s.identityName) - if err != nil { - return fmt.Errorf("%w: failed to resolve AWS auth context for identity %q: %w", - errUtils.ErrAWSConfigLoadFailed, s.identityName, err) - } - - cfgOpts := s.buildAuthConfigOpts(authContext) - cfg, err := config.LoadDefaultConfig(ctx, cfgOpts...) + // A missing resolver falls back to the default chain: legacy CI behavior that the generic + // transport deliberately does not share. + var resolver blobs3.AuthResolver + if s.authResolver != nil { + resolver = s.authResolver + } + client, err := blobs3.NewClient(ctx, &blobs3.ClientConfig{ + Region: s.region, + Identity: s.identityName, + Resolver: resolver, + Legacy: true, + }) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + return err } - - s.client = s3.NewFromConfig(cfg, buildClientOptFns(authContext)...) + s.client = client return nil } @@ -135,42 +139,12 @@ func (s *Store) initIdentityClient(ctx context.Context) error { // Without this, an identity scoped to an emulator (e.g. Floci) still routes // S3 calls to the real AWS endpoint. func buildClientOptFns(authContext *artifact.AWSAuthConfig) []func(*s3.Options) { - var optFns []func(*s3.Options) - if authContext != nil && authContext.EndpointURL != "" { - optFns = append(optFns, func(o *s3.Options) { - o.BaseEndpoint = aws.String(authContext.EndpointURL) - o.UsePathStyle = true - }) - } - return optFns + return blobs3.BuildClientOptFns(authContext) } // buildAuthConfigOpts constructs AWS SDK config options from the auth context. func (s *Store) buildAuthConfigOpts(authContext *artifact.AWSAuthConfig) []func(*config.LoadOptions) error { - var cfgOpts []func(*config.LoadOptions) error - if authContext == nil { - return cfgOpts - } - if authContext.CredentialsFile != "" { - cfgOpts = append(cfgOpts, config.WithSharedCredentialsFiles([]string{authContext.CredentialsFile})) - } - if authContext.ConfigFile != "" { - cfgOpts = append(cfgOpts, config.WithSharedConfigFiles([]string{authContext.ConfigFile})) - } - if authContext.Profile != "" { - cfgOpts = append(cfgOpts, config.WithSharedConfigProfile(authContext.Profile)) - } - - // Use region from auth context if the store doesn't specify one. - region := s.region - if region == "" && authContext.Region != "" { - region = authContext.Region - } - if region != "" { - cfgOpts = append(cfgOpts, config.WithRegion(region)) - } - - return cfgOpts + return blobs3.BuildAuthConfigOpts(s.region, authContext) } // ensureClient lazily initializes the AWS client on first operation. @@ -207,30 +181,33 @@ func (s *Store) fullKey(key string) string { return path.Join(s.prefix, key) } +// transport returns the shared S3 transport over the lazily initialized client. Keys are passed +// through untouched (fullKey already applied the legacy prefix), so the transport is created +// without a prefix and with opaque-key validation to keep historical key patterns valid. +func (s *Store) transport(ctx context.Context) (*blobs3.Store, error) { + if err := s.ensureClient(ctx); err != nil { + return nil, err + } + s.transportOnce.Do(func() { + s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys()) + }) + return s.transportStore, s.transportErr +} + // Upload uploads a single data stream to S3 with a metadata sidecar. func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int64, metadata *artifact.Metadata) error { defer perf.Track(nil, "s3.Upload")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return fmt.Errorf("%w: %w", errUtils.ErrArtifactUploadFailed, err) } fullKey := s.fullKey(key) - // Read all data into memory for S3 PutObject (needs content length). - dataBytes, err := io.ReadAll(data) - if err != nil { - return fmt.Errorf("%w: failed to read data: %w", errUtils.ErrArtifactUploadFailed, err) - } - - // Upload the data. - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - Body: bytes.NewReader(dataBytes), - ContentLength: aws.Int64(int64(len(dataBytes))), - }) - if err != nil { + // Upload the data. The declared size is advisory, as it always was; the transport streams the + // body and uses a multipart upload only for bodies larger than one part. + if _, err := transport.Put(ctx, fullKey, data, blob.PutOptions{Size: blob.UnknownSize}); err != nil { return fmt.Errorf("%w: failed to upload artifact to S3: %w", errUtils.ErrArtifactUploadFailed, err) } @@ -242,12 +219,9 @@ func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int return fmt.Errorf("%w: failed to marshal metadata: %w", errUtils.ErrArtifactUploadFailed, err) } - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(metadataKey), - Body: bytes.NewReader(metadataJSON), - ContentLength: aws.Int64(int64(len(metadataJSON))), - ContentType: aws.String("application/json"), + _, err = transport.Put(ctx, metadataKey, bytes.NewReader(metadataJSON), blob.PutOptions{ + Size: int64(len(metadataJSON)), + ContentType: "application/json", }) if err != nil { return fmt.Errorf("%w: failed to upload metadata to S3: %w", errUtils.ErrArtifactUploadFailed, err) @@ -262,54 +236,46 @@ func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int func (s *Store) Download(ctx context.Context, key string) (io.ReadCloser, *artifact.Metadata, error) { defer perf.Track(nil, "s3.Download")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return nil, nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactDownloadFailed, err) } fullKey := s.fullKey(key) // Download the data. - result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) + object, err := transport.Get(ctx, fullKey, blob.GetOptions{}) if err != nil { - if isNoSuchKeyError(err) { + if isMissing(err) { return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, key) } return nil, nil, fmt.Errorf("%w: failed to download artifact from S3: %w", errUtils.ErrArtifactDownloadFailed, err) } // Try to load metadata. - metadata, _ := s.loadMetadata(ctx, fullKey) + metadata, _ := s.loadMetadata(ctx, transport, fullKey) - return result.Body, metadata, nil + return object.Body, metadata, nil } // Delete deletes an artifact from S3. func (s *Store) Delete(ctx context.Context, key string) error { defer perf.Track(nil, "s3.Delete")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) } fullKey := s.fullKey(key) // Delete the artifact. - _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) - if err != nil { + if err := transport.Delete(ctx, fullKey, blob.DeleteOptions{}); err != nil { return fmt.Errorf("%w: failed to delete artifact from S3: %w", errUtils.ErrArtifactDeleteFailed, err) } // Try to delete metadata (ignore errors). - _, _ = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey + metadataSuffix), - }) + _ = transport.Delete(ctx, fullKey+metadataSuffix, blob.DeleteOptions{}) return nil } @@ -318,7 +284,8 @@ func (s *Store) Delete(ctx context.Context, key string) error { func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { defer perf.Track(nil, "s3.List")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) } @@ -326,47 +293,35 @@ func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.Arti prefix := s.queryToPrefix(query) fullPrefix := s.fullKey(prefix) + objects, _, err := blob.ListAll(ctx, transport, blob.ListOptions{Prefix: fullPrefix}) + if err != nil { + return nil, fmt.Errorf("%w: failed to list artifacts in S3: %w", errUtils.ErrArtifactListFailed, err) + } + var files []artifact.ArtifactInfo - paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(fullPrefix), - }) + for _, obj := range objects { + key := obj.Key - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("%w: failed to list artifacts in S3: %w", errUtils.ErrArtifactListFailed, err) + // Skip metadata files. + if strings.HasSuffix(key, metadataSuffix) && len(key) > len(metadataSuffix) { + continue } - for _, obj := range page.Contents { - key := aws.ToString(obj.Key) - - // Skip metadata files. - if len(key) > len(metadataSuffix) && key[len(key)-len(metadataSuffix):] == metadataSuffix { - continue - } - - // Remove prefix to get relative key. - relKey := key - if s.prefix != "" && len(key) > len(s.prefix)+1 { - relKey = key[len(s.prefix)+1:] - } - - // Try to load metadata. - metadata, _ := s.loadMetadata(ctx, key) - - var lastModified time.Time - if obj.LastModified != nil { - lastModified = *obj.LastModified - } - - files = append(files, artifact.ArtifactInfo{ - Name: relKey, - Size: aws.ToInt64(obj.Size), - LastModified: lastModified, - Metadata: metadata, - }) + // Remove prefix to get relative key. + relKey := key + if s.prefix != "" && len(key) > len(s.prefix)+1 { + relKey = key[len(s.prefix)+1:] } + + // Try to load metadata. + metadata, _ := s.loadMetadata(ctx, transport, key) + + files = append(files, artifact.ArtifactInfo{ + Name: relKey, + Size: obj.Size, + LastModified: obj.LastModified, + Metadata: metadata, + }) } return files, nil @@ -376,18 +331,15 @@ func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.Arti func (s *Store) Exists(ctx context.Context, key string) (bool, error) { defer perf.Track(nil, "s3.Exists")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return false, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) } fullKey := s.fullKey(key) - _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) - if err != nil { - if isNotFoundError(err) { + if _, err := transport.Stat(ctx, fullKey, blob.StatOptions{}); err != nil { + if isMissing(err) { return false, nil } return false, fmt.Errorf("%w: failed to check if %s exists in S3: %w", errUtils.ErrArtifactListFailed, key, err) @@ -400,45 +352,37 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) { func (s *Store) GetMetadata(ctx context.Context, key string) (*artifact.Metadata, error) { defer perf.Track(nil, "s3.GetMetadata")() - if err := s.ensureClient(ctx); err != nil { + transport, err := s.transport(ctx) + if err != nil { return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactMetadataFailed, err) } fullKey := s.fullKey(key) // Check if the artifact exists. - headResult, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) + info, err := transport.Stat(ctx, fullKey, blob.StatOptions{}) if err != nil { - if isNotFoundError(err) { + if isMissing(err) { return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, key) } return nil, fmt.Errorf("%w: failed to get metadata for %s from S3: %w", errUtils.ErrArtifactMetadataFailed, key, err) } // Try to load metadata from separate file. - metadata, err := s.loadMetadata(ctx, fullKey) + metadata, err := s.loadMetadata(ctx, transport, fullKey) if err != nil || metadata == nil { // Return minimal metadata from S3 object. - metadata = &artifact.Metadata{} - if headResult.LastModified != nil { - metadata.CreatedAt = *headResult.LastModified - } + metadata = &artifact.Metadata{CreatedAt: info.LastModified} } return metadata, nil } // loadMetadata loads metadata from the metadata file in S3. -func (s *Store) loadMetadata(ctx context.Context, artifactKey string) (*artifact.Metadata, error) { +func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, artifactKey string) (*artifact.Metadata, error) { metadataKey := artifactKey + metadataSuffix - result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(metadataKey), - }) + result, err := transport.Get(ctx, metadataKey, blob.GetOptions{}) if err != nil { return nil, fmt.Errorf("%w: failed to get metadata from S3: %w", errUtils.ErrArtifactMetadataFailed, err) } @@ -469,6 +413,12 @@ func (s *Store) queryToPrefix(query artifact.Query) string { return prefix } +// isMissing reports whether err means the object does not exist, whether it was classified by the +// shared transport or is a raw SDK error. +func isMissing(err error) bool { + return blob.IsNotFound(err) || isNoSuchKeyError(err) || isNotFoundError(err) +} + // isNoSuchKeyError checks if the error is a NoSuchKey error. func isNoSuchKeyError(err error) bool { var noSuchKey *types.NoSuchKey diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go new file mode 100644 index 00000000000..7f6a0d3d78f --- /dev/null +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -0,0 +1,203 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +// newFakeStore returns the legacy adapter over a fake S3 endpoint, using the SDK's default +// checksum behavior like production. +func newFakeStore(t *testing.T, prefix string) (*Store, *s3test.Server) { + t.Helper() + fake := s3test.NewServer(t) + return &Store{client: fake.LegacyClient(), bucket: fake.Bucket(), prefix: prefix}, fake +} + +func TestAdapterKeepsKeyPatternAndSidecarFormat(t *testing.T) { + store, fake := newFakeStore(t, "planfiles") + created := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc", CreatedAt: created, Custom: map[string]string{"k": "v"}} + body := []byte("tar-bytes\x00\x01") + + require.NoError(t, store.Upload(context.Background(), "dev/vpc/abc.tar", bytes.NewReader(body), int64(len(body)), metadata)) + + assert.Equal(t, []string{"planfiles/dev/vpc/abc.tar", "planfiles/dev/vpc/abc.tar.metadata.json"}, fake.Keys()) + data, ok := fake.Object("planfiles/dev/vpc/abc.tar") + require.True(t, ok) + assert.Equal(t, body, data.Data) + assert.Empty(t, data.Tags, "the legacy adapter adds no object tags") + + sidecar, ok := fake.Object("planfiles/dev/vpc/abc.tar.metadata.json") + require.True(t, ok) + want, err := json.MarshalIndent(metadata, "", " ") + require.NoError(t, err) + assert.Equal(t, want, sidecar.Data, "the sidecar is byte-identical to the legacy encoding") + assert.Equal(t, "application/json", sidecar.ContentType) + + puts := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, puts, 2, "one request per object, data first and sidecar second") + assert.Equal(t, "planfiles/dev/vpc/abc.tar", puts[0].Key) + assert.Equal(t, "planfiles/dev/vpc/abc.tar.metadata.json", puts[1].Key) + for _, put := range puts { + assert.Empty(t, put.Header.Get("If-None-Match"), "legacy writes stay unconditional") + assert.Empty(t, put.Header.Get("X-Amz-Tagging")) + } +} + +func TestAdapterLegacyKeyQuirksArePreserved(t *testing.T) { + store, fake := newFakeStore(t, "pre/fix/") + require.NoError(t, store.Upload(context.Background(), "a//b/../c.tar", strings.NewReader("x"), 1, nil)) + assert.Equal(t, []string{"pre/fix/a/c.tar"}, fake.Keys(), "path.Join cleaning of prefix and key is unchanged; nil metadata writes no sidecar") +} + +func TestAdapterRoundTripWithoutPrefix(t *testing.T) { + store, fake := newFakeStore(t, "") + ctx := context.Background() + metadata := &artifact.Metadata{Stack: "s", Component: "c", SHA: "1"} + require.NoError(t, store.Upload(ctx, "k.tar", strings.NewReader("payload"), 7, metadata)) + assert.Equal(t, []string{"k.tar", "k.tar.metadata.json"}, fake.Keys()) + + reader, got, err := store.Download(ctx, "k.tar") + require.NoError(t, err) + var buf bytes.Buffer + _, err = buf.ReadFrom(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "payload", buf.String()) + require.NotNil(t, got) + assert.Equal(t, "s", got.Stack) + + exists, err := store.Exists(ctx, "k.tar") + require.NoError(t, err) + assert.True(t, exists) + exists, err = store.Exists(ctx, "missing.tar") + require.NoError(t, err) + assert.False(t, exists) + + _, _, err = store.Download(ctx, "missing.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = store.GetMetadata(ctx, "missing.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestAdapterDownloadWithoutSidecarReturnsNilMetadata(t *testing.T) { + store, fake := newFakeStore(t, "") + fake.Seed("bare.tar", []byte("data"), nil) + reader, meta, err := store.Download(context.Background(), "bare.tar") + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Nil(t, meta) + + minimal, err := store.GetMetadata(context.Background(), "bare.tar") + require.NoError(t, err) + assert.False(t, minimal.CreatedAt.IsZero(), "falls back to the object's modification time") +} + +func TestAdapterDeleteRemovesObjectAndSidecarAndIsIdempotent(t *testing.T) { + store, fake := newFakeStore(t, "p") + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a.tar", strings.NewReader("x"), 1, &artifact.Metadata{})) + require.NoError(t, store.Delete(ctx, "a.tar")) + assert.Empty(t, fake.Keys()) + require.NoError(t, store.Delete(ctx, "a.tar")) +} + +func TestAdapterListHandlesPrefixSidecarsAndPagination(t *testing.T) { + store, fake := newFakeStore(t, "plans") + const total = 1105 + for i := range total { + key := fmt.Sprintf("plans/dev/c%04d.tar", i) + fake.Seed(key, []byte("x"), nil) + fake.Seed(key+metadataSuffix, []byte(fmt.Sprintf(`{"stack":"dev","component":"c%04d","sha":"s"}`, i)), nil) + } + fake.Seed("plans/prod/other.tar", []byte("x"), nil) + fake.Seed("elsewhere/dev/x.tar", []byte("x"), nil) + + infos, err := store.List(context.Background(), artifact.Query{Stacks: []string{"dev"}}) + require.NoError(t, err) + require.Len(t, infos, total, "sidecars are skipped and every page is consumed") + assert.Equal(t, "dev/c0000.tar", infos[0].Name, "names are relative to the prefix") + assert.Equal(t, "dev/c1104.tar", infos[total-1].Name) + require.NotNil(t, infos[0].Metadata) + assert.Equal(t, "c0000", infos[0].Metadata.Component) + assert.Equal(t, int64(1), infos[0].Size) + assert.False(t, infos[0].LastModified.IsZero()) +} + +func TestAdapterBundledStoreChecksumAndTarSurviveTheTransport(t *testing.T) { + store, fake := newFakeStore(t, "ci") + bundled := artifact.NewBundledStore(store) + files := []artifact.FileEntry{{Name: "plan.tfplan", Data: strings.NewReader("plan-bytes"), Size: 10}} + require.NoError(t, bundled.Upload(context.Background(), "dev/vpc/sha.tar", files, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "sha"})) + + stored, ok := fake.Object("ci/dev/vpc/sha.tar") + require.True(t, ok) + sum := sha256.Sum256(stored.Data) + meta, err := bundled.GetMetadata(context.Background(), "dev/vpc/sha.tar") + require.NoError(t, err) + assert.Equal(t, hex.EncodeToString(sum[:]), meta.SHA256) + + got, _, err := bundled.Download(context.Background(), "dev/vpc/sha.tar") + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "plan.tfplan", got[0].Name) +} + +func TestAdapterLargeUploadStreamsAsMultipartWithLegacyClient(t *testing.T) { + store, fake := newFakeStore(t, "") + body := bytes.Repeat([]byte("0123456789abcdef"), (9<<20)/16) + require.NoError(t, store.Upload(context.Background(), "big.tar", bytes.NewReader(body), int64(len(body)), nil)) + + stored, ok := fake.Object("big.tar") + require.True(t, ok) + assert.Equal(t, len(body), len(stored.Data)) + assert.True(t, bytes.Equal(body, stored.Data)) + assert.Equal(t, 1, fake.CompletedUploads()) + assert.Empty(t, fake.OpenUploads()) + for _, req := range fake.RequestsFor(s3test.OpUploadPart) { + assert.Empty(t, req.Header.Get("X-Amz-Checksum-Crc32"), "multipart requests carry no optional checksums") + } +} + +func TestAdapterWrapsTransportFailuresInLegacySentinels(t *testing.T) { + store, fake := newFakeStore(t, "") + ctx := context.Background() + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: 403, Code: "AccessDenied", Times: -1}) + err := store.Upload(ctx, "k.tar", strings.NewReader("x"), 1, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpGetObject, Status: 403, Code: "AccessDenied", Times: -1}) + _, _, err = store.Download(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpListObjectsV2, Status: 403, Code: "AccessDenied", Times: -1}) + _, err = store.List(ctx, artifact.Query{All: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactListFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpHeadObject, Status: 403, Code: "Forbidden", Times: -1}) + _, err = store.Exists(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactListFailed) + _, err = store.GetMetadata(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactMetadataFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: 403, Code: "AccessDenied", Times: -1}) + require.ErrorIs(t, store.Delete(ctx, "k.tar"), errUtils.ErrArtifactDeleteFailed) +} From 006b6c28e5eb2cb38477697f2d49f0773106d264 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 10:40:23 -0500 Subject: [PATCH 02/43] feat(artifacts): add local directory and archive repositories (A04) Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/artifacttest/verifier.go | 75 ++++ pkg/artifact/layout/admit.go | 128 +++++++ pkg/artifact/layout/claims.go | 86 +++++ pkg/artifact/layout/codec.go | 83 +++++ pkg/artifact/layout/doc.go | 29 ++ pkg/artifact/layout/evidence.go | 120 +++++++ pkg/artifact/layout/layout_test.go | 459 +++++++++++++++++++++++++ pkg/artifact/layout/location.go | 220 ++++++++++++ pkg/artifact/layout/paths.go | 205 +++++++++++ pkg/artifact/layout/records.go | 428 +++++++++++++++++++++++ pkg/artifact/local/conformance_test.go | 54 +++ pkg/artifact/local/delete.go | 82 +++++ pkg/artifact/local/delete_test.go | 227 ++++++++++++ pkg/artifact/local/doc.go | 48 +++ pkg/artifact/local/edge_test.go | 350 +++++++++++++++++++ pkg/artifact/local/evidence.go | 302 ++++++++++++++++ pkg/artifact/local/faults.go | 62 ++++ pkg/artifact/local/fsys.go | 255 ++++++++++++++ pkg/artifact/local/helpers_test.go | 262 ++++++++++++++ pkg/artifact/local/integrity_test.go | 258 ++++++++++++++ pkg/artifact/local/job.go | 247 +++++++++++++ pkg/artifact/local/kind_archive.go | 210 +++++++++++ pkg/artifact/local/kind_dir.go | 300 ++++++++++++++++ pkg/artifact/local/location_test.go | 241 +++++++++++++ pkg/artifact/local/lock.go | 177 ++++++++++ pkg/artifact/local/lock_test.go | 380 ++++++++++++++++++++ pkg/artifact/local/options.go | 57 +++ pkg/artifact/local/permissions_test.go | 166 +++++++++ pkg/artifact/local/publish.go | 375 ++++++++++++++++++++ pkg/artifact/local/read.go | 231 +++++++++++++ pkg/artifact/local/recovery_test.go | 292 ++++++++++++++++ pkg/artifact/local/register.go | 72 ++++ pkg/artifact/local/register_test.go | 126 +++++++ pkg/artifact/local/repository.go | 288 ++++++++++++++++ pkg/artifact/local/reserve.go | 91 +++++ pkg/artifact/local/resolve.go | 232 +++++++++++++ pkg/artifact/local/roundtrip_test.go | 287 ++++++++++++++++ pkg/artifact/local/service_test.go | 146 ++++++++ pkg/artifact/local/signature_test.go | 302 ++++++++++++++++ pkg/artifact/local/state.go | 167 +++++++++ pkg/artifact/local/verifier_test.go | 144 ++++++++ 41 files changed, 8264 insertions(+) create mode 100644 pkg/artifact/artifacttest/verifier.go create mode 100644 pkg/artifact/layout/admit.go create mode 100644 pkg/artifact/layout/claims.go create mode 100644 pkg/artifact/layout/codec.go create mode 100644 pkg/artifact/layout/doc.go create mode 100644 pkg/artifact/layout/evidence.go create mode 100644 pkg/artifact/layout/layout_test.go create mode 100644 pkg/artifact/layout/location.go create mode 100644 pkg/artifact/layout/paths.go create mode 100644 pkg/artifact/layout/records.go create mode 100644 pkg/artifact/local/conformance_test.go create mode 100644 pkg/artifact/local/delete.go create mode 100644 pkg/artifact/local/delete_test.go create mode 100644 pkg/artifact/local/doc.go create mode 100644 pkg/artifact/local/edge_test.go create mode 100644 pkg/artifact/local/evidence.go create mode 100644 pkg/artifact/local/faults.go create mode 100644 pkg/artifact/local/fsys.go create mode 100644 pkg/artifact/local/helpers_test.go create mode 100644 pkg/artifact/local/integrity_test.go create mode 100644 pkg/artifact/local/job.go create mode 100644 pkg/artifact/local/kind_archive.go create mode 100644 pkg/artifact/local/kind_dir.go create mode 100644 pkg/artifact/local/location_test.go create mode 100644 pkg/artifact/local/lock.go create mode 100644 pkg/artifact/local/lock_test.go create mode 100644 pkg/artifact/local/options.go create mode 100644 pkg/artifact/local/permissions_test.go create mode 100644 pkg/artifact/local/publish.go create mode 100644 pkg/artifact/local/read.go create mode 100644 pkg/artifact/local/recovery_test.go create mode 100644 pkg/artifact/local/register.go create mode 100644 pkg/artifact/local/register_test.go create mode 100644 pkg/artifact/local/repository.go create mode 100644 pkg/artifact/local/reserve.go create mode 100644 pkg/artifact/local/resolve.go create mode 100644 pkg/artifact/local/roundtrip_test.go create mode 100644 pkg/artifact/local/service_test.go create mode 100644 pkg/artifact/local/signature_test.go create mode 100644 pkg/artifact/local/state.go create mode 100644 pkg/artifact/local/verifier_test.go diff --git a/pkg/artifact/artifacttest/verifier.go b/pkg/artifact/artifacttest/verifier.go new file mode 100644 index 00000000000..ddaec659442 --- /dev/null +++ b/pkg/artifact/artifacttest/verifier.go @@ -0,0 +1,75 @@ +package artifacttest + +import ( + "context" + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// FakeVerifier is an [artifact.Verifier] for the evidence [FakeSigner] produces. Repositories that +// delegate cryptography to an injected verifier (the local repositories) use it in conformance +// tests, so those tests exercise the real storage and policy flow without Sigstore. It is a +// harness: it checks subject binding and the trust set, nothing cryptographic. +type FakeVerifier struct{} + +var _ artifact.Verifier = (*FakeVerifier)(nil) + +// Formats implements [artifact.Verifier]. +func (*FakeVerifier) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "artifacttest.FakeVerifier.Formats")() + + return []artifact.EvidenceFormat{FakeEvidenceFormat} +} + +// Verify implements [artifact.Verifier]. Every trust set must be satisfied by at least one +// evidence item bound to the subject. Malformed, unsupported, and unrelated evidence is reported +// and ignored; it can never override matching evidence. +func (*FakeVerifier) Verify(_ context.Context, subject artifact.SignatureSubject, evidence []artifact.EvidenceBlob, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "artifacttest.FakeVerifier.Verify")() + + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, err := policy.Fingerprint(); err == nil { + result.PolicyFingerprint = fingerprint + } + if !policy.HasTrustConstraints() { + result.Status = artifact.VerificationUnchecked + return result, nil + } + var claims []artifact.SignerClaim + for index := range evidence { + claim, ok := authenticateFake(subject, &evidence[index], result) + if ok { + claims = append(claims, claim) + result.EvidenceDigests = append(result.EvidenceDigests, evidence[index].Evidence.Descriptor.Digest) + } + } + if !satisfies(policy, claims) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-permitted-signature", Message: "no valid permitted signature satisfies the trust set"}) + return result, fmt.Errorf("%w: no valid permitted signature for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + result.Claims = claims + result.Status = artifact.VerificationVerified + return result, nil +} + +func authenticateFake(subject artifact.SignatureSubject, held *artifact.EvidenceBlob, result *artifact.VerificationResult) (artifact.SignerClaim, bool) { + skip := func(code, message string) (artifact.SignerClaim, bool) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) + return artifact.SignerClaim{}, false + } + if held.Evidence.Format != FakeEvidenceFormat { + return skip("unsupported-evidence", "evidence in an unsupported format was ignored") + } + var parsed fakeEvidence + if err := json.Unmarshal(held.Data, &parsed); err != nil { + return skip("malformed-evidence", "malformed evidence was ignored") + } + if parsed.Subject != subject.Digest || held.Evidence.Subject.Digest != subject.Digest { + return skip("subject-mismatch", "evidence bound to a different subject was ignored") + } + return artifact.SignerClaim{Identity: parsed.Identity, Issuer: parsed.Issuer}, true +} diff --git a/pkg/artifact/layout/admit.go b/pkg/artifact/layout/admit.go new file mode 100644 index 00000000000..e1c21179439 --- /dev/null +++ b/pkg/artifact/layout/admit.go @@ -0,0 +1,128 @@ +package layout + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Admission is the decision a repository takes for one publication after reading the current +// reference under the writer guard. +type Admission string + +// Admission decisions. +const ( + // AdmitCreate writes a new reference. + AdmitCreate Admission = "create" + + // AdmitReplace replaces a completed reference under force. + AdmitReplace Admission = "replace" + + // AdmitUnchanged leaves the completed reference untouched: the same intent already exists. + AdmitUnchanged Admission = "unchanged" + + // AdmitResumeCreate resumes a pending first publication of the same intent. + AdmitResumeCreate Admission = "resume-create" + + // AdmitResumeReplace resumes a pending replacement of the same intent. + AdmitResumeReplace Admission = "resume-replace" +) + +// Outcome maps an admission to the operation outcome it produces. +func (a Admission) Outcome() artifact.Outcome { + defer perf.Track(nil, "layout.Admission.Outcome")() + + switch a { + case AdmitReplace, AdmitResumeReplace: + return artifact.OutcomeReplaced + case AdmitUnchanged: + return artifact.OutcomeUnchanged + default: + return artifact.OutcomeCreated + } +} + +// Resuming reports whether the admission continues an interrupted publication. +func (a Admission) Resuming() bool { + defer perf.Track(nil, "layout.Admission.Resuming")() + + return a == AdmitResumeCreate || a == AdmitResumeReplace +} + +// AdmitInput is everything the admission rules need. +type AdmitInput struct { + // Current is the reference as stored, or nil when it does not exist. + Current *Reference + + // CurrentIntent is the intent digest of the completed Record. It is only read when Current is + // completed; empty means it could not be computed, which never counts as unchanged. + CurrentIntent digest.Digest + + // IncomingRecord and IncomingIntent identify the artifact being published. + IncomingRecord digest.Digest + IncomingIntent digest.Digest + + // ExpectedPrior is the reference the service observed; nil means the reference must not exist. + ExpectedPrior *artifact.PinnedReference + + // Force permits replacing a conflicting completed reference. + Force bool +} + +// Admit applies the compare-and-swap, intent, force, and recovery rules of the publication +// lifecycle: +// +// - A stale expected-prior reference is a conflict, never a silent overwrite. +// - The same record or intent as the completed reference is unchanged. +// - A different completed intent conflicts unless force is set. +// - A pending reference of the same intent resumes; a pending reference of a different intent +// is unavailable and names the owning operation, even under force. A pending operation is +// never stolen automatically. +func Admit(in *AdmitInput) (Admission, error) { + defer perf.Track(nil, "layout.Admit")() + + if in.Current == nil { + if in.ExpectedPrior != nil { + return "", fmt.Errorf("%w: the reference no longer exists but the writer expected %s", errUtils.ErrArtifactConflict, in.ExpectedPrior.ResolvedDigest) + } + return AdmitCreate, nil + } + if in.Current.State == StatePending { + return admitPending(in) + } + if err := checkExpectedPrior(in.Current, in.ExpectedPrior); err != nil { + return "", err + } + if in.Current.RecordDigest == in.IncomingRecord || (in.CurrentIntent != "" && in.CurrentIntent == in.IncomingIntent) { + return AdmitUnchanged, nil + } + if !in.Force { + return "", fmt.Errorf("%w: %s:%s already exists with a different intent; use force to replace it", errUtils.ErrArtifactConflict, in.Current.Name, in.Current.Version) + } + return AdmitReplace, nil +} + +func admitPending(in *AdmitInput) (Admission, error) { + if in.Current.IntentDigest != in.IncomingIntent { + return "", fmt.Errorf("%w: %s:%s is pending for operation %s with a different intent; recover or repair it with the same intent before publishing", + errUtils.ErrArtifactPublicationUnavailable, in.Current.Name, in.Current.Version, in.Current.OperationID) + } + if in.Current.PriorRecordDigest != "" { + return AdmitResumeReplace, nil + } + return AdmitResumeCreate, nil +} + +func checkExpectedPrior(current *Reference, expected *artifact.PinnedReference) error { + switch { + case expected == nil: + return fmt.Errorf("%w: %s:%s now exists (%s) but the writer expected it to be absent", errUtils.ErrArtifactConflict, current.Name, current.Version, current.RecordDigest) + case expected.ResolvedDigest != current.RecordDigest: + return fmt.Errorf("%w: %s:%s is %s but the writer expected %s", errUtils.ErrArtifactConflict, current.Name, current.Version, current.RecordDigest, expected.ResolvedDigest) + } + return nil +} diff --git a/pkg/artifact/layout/claims.go b/pkg/artifact/layout/claims.go new file mode 100644 index 00000000000..c42d28f65a1 --- /dev/null +++ b/pkg/artifact/layout/claims.go @@ -0,0 +1,86 @@ +package layout + +import ( + "path" + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Claim is the set of storage paths one reference owns. Two references may never own overlapping +// paths, which is how two references with a colliding key_pattern are detected: the second fails +// with a conflict and is never bound to the first reference's payload. +type Claim struct { + // ReferenceKey identifies the owning reference. + ReferenceKey string + + // Name and Version identify the owner for diagnostics. + Name string + Version string + + // OperationID is the pending operation holding the claim, empty for a completed reference. + OperationID string + + // Paths are the owned slash-separated paths relative to the repository root. + Paths []string +} + +// PayloadPaths returns the paths a payload at location owns for the repository kind: a copied +// directory owns the directory; an archive owns the file, its metadata sidecar, and its +// documents directory. +func PayloadPaths(repositoryKind, location string) []string { + defer perf.Track(nil, "layout.PayloadPaths")() + + if repositoryKind == artifact.RepositoryKindLocalArchive { + return []string{location, MetadataSidecarPath(location), DocsSidecarDir(location)} + } + return []string{location} +} + +// FoldPath returns the comparison form of a slash-separated path: cleaned and case folded, because +// common filesystems alias names that differ only by case. +func FoldPath(p string) string { + defer perf.Track(nil, "layout.FoldPath")() + + return strings.ToLower(path.Clean(p)) +} + +// PathsOverlap reports whether two paths are equal or one contains the other. +func PathsOverlap(a, b string) bool { + defer perf.Track(nil, "layout.PathsOverlap")() + + a, b = FoldPath(a), FoldPath(b) + return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/") +} + +// Overlaps reports whether any path of c overlaps any path of other. +func (c *Claim) Overlaps(other *Claim) bool { + defer perf.Track(nil, "layout.Claim.Overlaps")() + + for _, mine := range c.Paths { + for _, theirs := range other.Paths { + if PathsOverlap(mine, theirs) { + return true + } + } + } + return false +} + +// FindConflict returns the first existing claim, owned by a different reference, that overlaps the +// candidate, or nil. A reference never conflicts with itself, so replacement and same-intent +// retry keep working. +func FindConflict(candidate *Claim, existing []Claim) *Claim { + defer perf.Track(nil, "layout.FindConflict")() + + for index := range existing { + if existing[index].ReferenceKey == candidate.ReferenceKey { + continue + } + if candidate.Overlaps(&existing[index]) { + return &existing[index] + } + } + return nil +} diff --git a/pkg/artifact/layout/codec.go b/pkg/artifact/layout/codec.go new file mode 100644 index 00000000000..6db36b993db --- /dev/null +++ b/pkg/artifact/layout/codec.go @@ -0,0 +1,83 @@ +package layout + +import ( + "bytes" + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// MaxDocumentBytes bounds every layout document (reference, representation, operation journal). +const MaxDocumentBytes int64 = 1 << 20 + +// MaxEvidenceFileBytes bounds one stored evidence file: the base64 form of the largest evidence +// blob plus its envelope. +const MaxEvidenceFileBytes = 2*artifact.MaxEvidenceSize + 4096 + +// Document kinds, used in error messages. +const ( + kindReference = "reference" + kindRepresentation = "representation" + kindOperation = "operation" + kindEvidence = "evidence" +) + +// invalid wraps a malformed or inconsistent managed document. It matches both the integrity and +// the record-invalid sentinels, because a stored managed document that does not parse is, to the +// reader, corrupted state. +func invalid(kind, format string, args ...any) error { + return fmt.Errorf("%w: %w: %s: %s", errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactRecordInvalid, kind, fmt.Sprintf(format, args...)) +} + +// validator is implemented by every layout document. +type validator interface { + Validate() error +} + +// encode canonicalizes v: sorted keys, no insignificant whitespace, no trailing newline. Stored +// bytes are therefore a function of the document alone. +func encode(v validator, limit int64) ([]byte, error) { + if err := v.Validate(); err != nil { + return nil, err + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + canonical, err := artifact.CanonicalizeJSON(buf.Bytes()) + if err != nil { + return nil, err + } + if int64(len(canonical)) > limit { + return nil, fmt.Errorf("%w: layout document is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(canonical), limit) + } + return canonical, nil +} + +// decode strictly parses raw into v: it rejects oversize input, unknown fields, invalid +// documents, and any bytes that are not exactly the canonical serialization of the result. +func decode(kind string, raw []byte, v validator, limit int64) error { + if int64(len(raw)) > limit { + return invalid(kind, "%d bytes exceeds the %d byte limit", len(raw), limit) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(v); err != nil { + return invalid(kind, "%v", err) + } + if err := v.Validate(); err != nil { + return invalid(kind, "%v", err) + } + canonical, err := encode(v, limit) + if err != nil { + return invalid(kind, "%v", err) + } + if !bytes.Equal(canonical, raw) { + return invalid(kind, "bytes are not the canonical serialization") + } + return nil +} diff --git a/pkg/artifact/layout/doc.go b/pkg/artifact/layout/doc.go new file mode 100644 index 00000000000..0a49063a385 --- /dev/null +++ b/pkg/artifact/layout/doc.go @@ -0,0 +1,29 @@ +// Package layout defines the backend-neutral record layout shared by the local directory, local +// archive, and S3 artifact repositories. It is pure data and logic: it performs no filesystem, +// network, or AWS work, so every backend stores the same documents and agrees on names, paths, +// state transitions, and collision rules. +// +// Below a repository root (a directory, or a bucket prefix) the managed records live in the +// reserved ".atmos-artifacts/v1/" namespace: +// +// .atmos-artifacts/v1/refs/.json stable-root reference record +// .atmos-artifacts/v1/records/.json immutable canonical Record +// .atmos-artifacts/v1/representations/.json mutable storage description +// .atmos-artifacts/v1/operations/.json publication journal (recovery only) +// .atmos-artifacts/v1/signatures///.json detached evidence +// payload (outside the namespace) +// +// The reference key is the SHA-256 of "name:version", so names containing slashes can never +// collide with control paths. A reference record binds a name and version to the digest of its +// immutable Record and carries the pending or completed publication state. The Record never holds +// a mutable storage locator: the resolved payload location is rendered from the repository +// key_pattern exactly once, at publication, from the captured creation time, and is persisted in +// the representation record. Reads, deletes, and retries use the persisted location and never +// render the pattern again, so a date rollover cannot move an artifact. +// +// Publication order: write the pending operation journal and mark the reference pending, stage +// and install the payload, write the Record, the representation, and the adjacent documents, and +// only then flip the reference to completed. Pending references are unavailable to readers and +// absent from search results; readers recheck the reference generation after reading from a +// mutable payload location. +package layout diff --git a/pkg/artifact/layout/evidence.go b/pkg/artifact/layout/evidence.go new file mode 100644 index 00000000000..731175b5e55 --- /dev/null +++ b/pkg/artifact/layout/evidence.go @@ -0,0 +1,120 @@ +package layout + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Evidence is the stored form of one detached signature evidence blob. The file name is the +// SHA-256 of Data, so the evidence digest the verifier reports identifies exactly the signed +// bytes, and a stored file is rejected when its name and content disagree. +// +// Evidence is stored beside, never inside, the signed Record, so adding a signature cannot change +// the Record, its digest, or the reference. +type Evidence struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // Subject is the immutable subject the evidence claims to cover. + Subject artifact.SignatureSubject `json:"subject"` + + // Format is the evidence encoding. + Format artifact.EvidenceFormat `json:"format"` + + // MediaType is the evidence media type. + MediaType string `json:"media_type"` + + // Data is the evidence bytes, at most [artifact.MaxEvidenceSize]. + Data []byte `json:"data"` +} + +// NewEvidence builds the stored form of a freshly produced evidence payload. +func NewEvidence(subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (*Evidence, error) { + defer perf.Track(nil, "layout.NewEvidence")() + + evidence := &Evidence{SchemaVersion: SchemaVersion, Subject: subject, Format: payload.Format, MediaType: payload.MediaType, Data: payload.Data} + if err := evidence.Validate(); err != nil { + return nil, err + } + return evidence, nil +} + +// Digest returns the evidence digest: the SHA-256 of the evidence bytes. +func (e *Evidence) Digest() digest.Digest { + defer perf.Track(nil, "layout.Evidence.Digest")() + + return digest.FromBytes(e.Data) +} + +// Validate checks the evidence envelope. +func (e *Evidence) Validate() error { + defer perf.Track(nil, "layout.Evidence.Validate")() + + if e.SchemaVersion != SchemaVersion { + return invalid(kindEvidence, "unsupported schema_version %d", e.SchemaVersion) + } + if err := e.Subject.Validate(); err != nil { + return err + } + if e.Format == "" || e.MediaType == "" { + return invalid(kindEvidence, "format and media_type are required") + } + if len(e.Data) == 0 { + return invalid(kindEvidence, "data is empty") + } + if int64(len(e.Data)) > artifact.MaxEvidenceSize { + return fmt.Errorf("%w: evidence is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(e.Data), artifact.MaxEvidenceSize) + } + return nil +} + +// Marshal returns the canonical bytes of the envelope. +func (e *Evidence) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Evidence.Marshal")() + + return encode(e, MaxEvidenceFileBytes) +} + +// DecodeEvidence strictly parses a stored evidence file. The wantDigest argument is the evidence +// digest taken from the file name; it must equal the digest of the stored bytes. +func DecodeEvidence(raw []byte, wantDigest digest.Digest) (*Evidence, error) { + defer perf.Track(nil, "layout.DecodeEvidence")() + + var evidence Evidence + if err := decode(kindEvidence, raw, &evidence, MaxEvidenceFileBytes); err != nil { + return nil, err + } + if evidence.Digest() != wantDigest { + return nil, invalid(kindEvidence, "content digest %s does not match its file name %s", evidence.Digest(), wantDigest) + } + return &evidence, nil +} + +// Blob converts the stored evidence into an [artifact.EvidenceBlob]. The locator argument is the +// display location of the evidence file. +func (e *Evidence) Blob(locator string) artifact.EvidenceBlob { + defer perf.Track(nil, "layout.Evidence.Blob")() + + return artifact.EvidenceBlob{Evidence: e.Reference(locator), Data: append([]byte(nil), e.Data...)} +} + +// Reference returns the descriptor of the stored evidence. The locator argument is the display +// location. +func (e *Evidence) Reference(locator string) artifact.SignatureEvidence { + defer perf.Track(nil, "layout.Evidence.Reference")() + + return artifact.SignatureEvidence{ + Subject: e.Subject, + Format: e.Format, + Descriptor: ocispec.Descriptor{ + MediaType: e.MediaType, Digest: e.Digest(), Size: int64(len(e.Data)), + }, + Locator: locator, + } +} diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go new file mode 100644 index 00000000000..3f27c2e7592 --- /dev/null +++ b/pkg/artifact/layout/layout_test.go @@ -0,0 +1,459 @@ +package layout + +import ( + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +var ( + digestA = digest.FromString("a") + digestB = digest.FromString("b") + digestC = digest.FromString("c") +) + +func TestReferenceKey(t *testing.T) { + key := ReferenceKey("components/terraform/vpc", "1.2.0") + assert.Len(t, key, 64, "lowercase hex SHA-256") + assert.Equal(t, digest.FromString("components/terraform/vpc:1.2.0").Encoded(), key, "the key is the SHA-256 of name:version") + assert.NotEqual(t, key, ReferenceKey("components/terraform", "vpc:1.2.0"), "names with slashes cannot collide with other name/version splits") + assert.NotContains(t, key, "/", "no slash can reach a control path") +} + +func TestPaths(t *testing.T) { + key := ReferenceKey("a", "1") + assert.Equal(t, ".atmos-artifacts/v1/refs/"+key+".json", ReferencePath(key)) + assert.Equal(t, ".atmos-artifacts/v1/representations/"+key+".json", RepresentationPath(key)) + assert.Equal(t, ".atmos-artifacts/v1/records/"+digestA.Encoded()+".json", RecordPath(digestA)) + assert.Equal(t, ".atmos-artifacts/v1/refs", ReferencesDir()) + assert.Equal(t, + ".atmos-artifacts/v1/signatures/record/"+digestA.Encoded()+"/"+digestB.Encoded()+".json", + SignaturePath(artifact.SubjectRecord, digestA, digestB)) + assert.Equal(t, "x/y.tgz.metadata.json", MetadataSidecarPath("x/y.tgz")) + assert.Equal(t, "x/y.tgz.docs/README.md", DocumentSidecarPath("x/y.tgz", artifact.DocumentReadme)) + assert.Equal(t, "x/y.tgz.docs/CHANGELOG.md", DocumentSidecarPath("x/y.tgz", artifact.DocumentChangelog)) + assert.Equal(t, "x/y/.atmos/artifact/manifest.json", DistributionPath("x/y", ManifestName)) + assert.True(t, strings.HasPrefix(LockPath("any resource/with:odd chars"), ManagedRoot+"/locks/")) + assert.NotEqual(t, LockPath("a"), LockPath("b")) +} + +func TestOperationFileName(t *testing.T) { + tests := []struct { + name string + id string + safe bool + }{ + {"uuid", "9f1c6d52-48b6-4c3c-9f0e-3a5b7f1d1c11", true}, + {"dotted", "op.1_x-2", true}, + {"slash", "op-team/app-1.0.0", false}, + {"space", "op one", false}, + {"traversal", "../x", false}, + {"overlong", strings.Repeat("a", 200), false}, + {"empty", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := OperationFileName(tt.id) + if tt.safe { + assert.Equal(t, tt.id, got) + } else { + assert.True(t, strings.HasPrefix(got, "sha256-"), got) + } + assert.NotContains(t, got, "/") + assert.Equal(t, got, OperationFileName(tt.id), "stable") + assert.Equal(t, ManagedRoot+"/operations/"+got+".json", OperationPath(tt.id)) + assert.Equal(t, ManagedRoot+"/staging/"+got, StagingPath(tt.id)) + }) + } +} + +func TestInManagedNamespace(t *testing.T) { + assert.True(t, InManagedNamespace(".atmos-artifacts/v1/refs/x.json")) + assert.True(t, InManagedNamespace("./.ATMOS-Artifacts")) + assert.False(t, InManagedNamespace("team/.atmos-artifacts"), "only the first segment is the root namespace") + assert.False(t, InManagedNamespace("atmos-artifacts/x")) +} + +func TestRenderLocation(t *testing.T) { + captured := time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + in := &PatternInput{Name: "components/terraform/vpc", Version: "1.2.0", Kind: "component/terraform", CreatedAt: captured} + tests := []struct { + name string + pattern string + packaging artifact.Packaging + want string + wantErr error + }{ + {"default copied files", "", artifact.PackagingNone, "components/terraform/vpc/1.2.0", nil}, + {"default tgz", "", artifact.PackagingTgz, "components/terraform/vpc/1.2.0.tgz", nil}, + {"default tar", "", artifact.PackagingTar, "components/terraform/vpc/1.2.0.tar", nil}, + {"default zip", "", artifact.PackagingZip, "components/terraform/vpc/1.2.0.zip", nil}, + {"dated root from the PRD", `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, artifact.PackagingTgz, "2026/10/30/components/terraform/vpc/1.2.0.tgz", nil}, + {"kind field", "{{ .Kind }}/{{ .Version }}", artifact.PackagingNone, "component/terraform/1.2.0", nil}, + {"trailing slash is trimmed", "x/{{ .Version }}/", artifact.PackagingNone, "x/1.2.0", nil}, + {"sprig function", "{{ .Name | upper }}/{{ .Version }}", artifact.PackagingNone, "COMPONENTS/TERRAFORM/VPC/1.2.0", nil}, + {"mismatched archive extension", "{{ .Name }}.zip", artifact.PackagingTgz, "", errUtils.ErrArtifactPackagingInvalid}, + {"tar.gz agrees with tgz", "{{ .Name }}.tar.gz", artifact.PackagingTgz, "components/terraform/vpc.tar.gz", nil}, + {"no extension is not renamed", "{{ .Name }}", artifact.PackagingTgz, "components/terraform/vpc", nil}, + {"root escape", "../{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"rendered root escape", "{{ \"..\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"absolute", "/abs/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"empty segment", "a//b", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"reserved namespace", ".atmos-artifacts/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"reserved namespace any case", "x/.ATMOS-ARTIFACTS/y", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"windows reserved character", "a:b/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"unknown field", "{{ .Stack }}", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"syntax error", "{{ .Name", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"clock is not available", "{{ now | date \"2006\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"environment is not available", "{{ env \"HOME\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"empty result", "{{ \"\" }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderLocation(tt.pattern, in, tt.packaging) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRenderLocation_DateIsUTCAndStable(t *testing.T) { + // 23:30 at UTC-5 is already the next day in UTC; the captured instant decides, not the zone. + zone := time.FixedZone("UTC-5", -5*60*60) + local := time.Date(2026, 10, 30, 23, 30, 0, 0, zone) + in := &PatternInput{Name: "a", Version: "1", Kind: "files", CreatedAt: local} + pattern := `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}` + first, err := RenderLocation(pattern, in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, "2026/10/31/a", first) + + // Rendering again with the same captured time is stable even if the wall clock moved on. + second, err := RenderLocation(pattern, in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, first, second) + + rolled := *in + rolled.CreatedAt = local.Add(48 * time.Hour) + third, err := RenderLocation(pattern, &rolled, artifact.PackagingNone) + require.NoError(t, err) + assert.NotEqual(t, first, third, "a different captured time renders a different dated root; callers persist the first") +} + +func TestCleanLocation(t *testing.T) { + for _, valid := range []string{"a", "a/b", "a/b/", "a/b.tgz", "team/app/1.0.0", "ünï/çode"} { + _, err := CleanLocation(valid) + require.NoError(t, err, valid) + } + for _, bad := range []string{"", "/", "/a", "a/../b", "./a", "a/./b", `a\b`, "a\x00b", "a\nb", ".atmos-artifacts", "a/.Atmos-Artifacts/b", "a|b", "a\xff", strings.Repeat("a", 2000)} { + _, err := CleanLocation(bad) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, "%q", bad) + } +} + +func TestClaims(t *testing.T) { + archive := func(key, location string) Claim { + return Claim{ReferenceKey: key, Paths: PayloadPaths(artifact.RepositoryKindLocalArchive, location)} + } + dir := func(key, location string) Claim { + return Claim{ReferenceKey: key, Paths: PayloadPaths(artifact.RepositoryKindLocalDir, location)} + } + assert.Equal(t, []string{"x/1.tgz", "x/1.tgz.metadata.json", "x/1.tgz.docs"}, archive("k", "x/1.tgz").Paths) + assert.Equal(t, []string{"x/1"}, dir("k", "x/1").Paths) + + tests := []struct { + name string + candidate Claim + existing []Claim + conflict bool + }{ + {"identical location", dir("b", "x/1"), []Claim{dir("a", "x/1")}, true}, + {"case-folded location", dir("b", "X/1"), []Claim{dir("a", "x/1")}, true}, + {"candidate inside existing directory", dir("b", "x/1/y"), []Claim{dir("a", "x/1")}, true}, + {"existing inside candidate directory", dir("b", "x"), []Claim{dir("a", "x/1")}, true}, + {"archive sidecar collides with another archive", archive("b", "x/1.tgz.metadata.json"), []Claim{archive("a", "x/1.tgz")}, true}, + {"archive docs directory collides", archive("b", "x/1.tgz.docs/README.md"), []Claim{archive("a", "x/1.tgz")}, true}, + {"siblings do not collide", dir("b", "x/2"), []Claim{dir("a", "x/1")}, false}, + {"prefix without a separator is not an overlap", dir("b", "x/10"), []Claim{dir("a", "x/1")}, false}, + {"the same reference never conflicts with itself", dir("a", "x/1"), []Claim{dir("a", "x/1")}, false}, + {"no claims", dir("a", "x/1"), nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + found := FindConflict(&tt.candidate, tt.existing) + assert.Equal(t, tt.conflict, found != nil) + if tt.conflict { + assert.Equal(t, "a", found.ReferenceKey) + } + }) + } +} + +func TestReference_StateModel(t *testing.T) { + first := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + require.NoError(t, first.Validate()) + assert.False(t, first.Available(), "a pending reference is unavailable to readers") + assert.EqualValues(t, 1, first.Generation) + assert.Empty(t, first.PriorRecordDigest) + assert.Equal(t, ReferenceKey("team/app", "1.0.0"), first.Key()) + + done := first.Completed(digestB) + require.NoError(t, done.Validate()) + assert.True(t, done.Available()) + assert.EqualValues(t, 2, done.Generation, "every state change bumps the generation") + assert.Equal(t, digestB, done.RecordDigest) + assert.Equal(t, StatePending, first.State, "Completed does not mutate its receiver") + + replacing := NewPendingReference("team/app", "1.0.0", "op-2", digestC, &done) + require.NoError(t, replacing.Validate()) + assert.False(t, replacing.Available()) + assert.EqualValues(t, 3, replacing.Generation) + assert.Equal(t, digestB, replacing.PriorRecordDigest, "a replacement remembers what it replaces") + assert.Empty(t, replacing.RecordDigest, "the pending reference must not advertise either record") + + again := NewPendingReference("team/app", "1.0.0", "op-3", digestC, &replacing) + assert.Equal(t, digestB, again.PriorRecordDigest, "a re-pending interrupted replacement keeps the original prior") + assert.EqualValues(t, 4, again.Generation) + + finished := replacing.Completed(digestC) + assert.Empty(t, finished.PriorRecordDigest) + assert.EqualValues(t, 4, finished.Generation) +} + +func TestReference_RoundTripAndStrictness(t *testing.T) { + ref := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + ref = ref.Completed(digestB) + raw, err := ref.Marshal() + require.NoError(t, err) + assert.NotContains(t, string(raw), "\n") + + decoded, err := DecodeReference(raw) + require.NoError(t, err) + assert.Equal(t, ref, *decoded) + + again, err := decoded.Marshal() + require.NoError(t, err) + assert.Equal(t, raw, again, "serialization is deterministic") + + tests := []struct { + name string + mutate func(string) string + }{ + {"unknown field", func(s string) string { return strings.Replace(s, `{"`, `{"extra":1,"`, 1) }}, + {"not canonical (whitespace)", func(s string) string { return strings.Replace(s, `,"`, `, "`, 1) }}, + {"truncated", func(s string) string { return s[:len(s)-2] }}, + {"wrong schema", func(s string) string { return strings.Replace(s, `"schema_version":1`, `"schema_version":2`, 1) }}, + {"garbage", func(string) string { return "not json" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DecodeReference([]byte(tt.mutate(string(raw)))) + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } +} + +func TestReference_ValidateRejectsInconsistentStates(t *testing.T) { + base := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + completed := base.Completed(digestB) + tests := []struct { + name string + ref func() Reference + wantOK bool + }{ + {"pending without record", func() Reference { return base }, true}, + {"completed", func() Reference { return completed }, true}, + {"pending naming a record", func() Reference { r := base; r.RecordDigest = digestB; return r }, false}, + {"completed without a record", func() Reference { r := completed; r.RecordDigest = ""; return r }, false}, + {"completed with a prior", func() Reference { r := completed; r.PriorRecordDigest = digestA; return r }, false}, + {"zero generation", func() Reference { r := base; r.Generation = 0; return r }, false}, + {"missing operation", func() Reference { r := base; r.OperationID = ""; return r }, false}, + {"unknown state", func() Reference { r := base; r.State = "weird"; return r }, false}, + {"bad intent digest", func() Reference { r := base; r.IntentDigest = "sha256:zz"; return r }, false}, + {"invalid name", func() Reference { r := base; r.Name = "../x"; return r }, false}, + {"invalid version", func() Reference { r := base; r.Version = ""; return r }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref := tt.ref() + err := ref.Validate() + if tt.wantOK { + require.NoError(t, err) + return + } + require.Error(t, err) + }) + } +} + +func sampleRepresentation() Representation { + return Representation{ + SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalArchive, Name: "team/app", Version: "1.0.0", + RecordDigest: digestA, OperationID: "op-1", Packaging: artifact.PackagingTgz, Location: "2026/10/30/team/app/1.0.0.tgz", + Payload: []ocispec.Descriptor{{MediaType: artifact.MediaTypeTarGzip, Digest: digestB, Size: 10}}, + Documents: []ocispec.Descriptor{}, + RepresentationDigest: digestB, + } +} + +func TestRepresentation_PersistsResolvedLocation(t *testing.T) { + rep := sampleRepresentation() + raw, err := rep.Marshal() + require.NoError(t, err) + decoded, err := DecodeRepresentation(raw) + require.NoError(t, err) + assert.Equal(t, rep.Location, decoded.Location, "the dated location survives serialization verbatim") + assert.Equal(t, rep.Payload, decoded.Payload) + assert.Equal(t, []string{rep.Location, rep.Location + ".metadata.json", rep.Location + ".docs"}, decoded.Claim().Paths) + + converted := decoded.Artifact("/abs/root/" + rep.Location) + assert.Equal(t, artifact.PackagingTgz, converted.Packaging) + assert.Equal(t, "/abs/root/"+rep.Location, converted.Locator) + assert.Equal(t, digestA, converted.RecordDigest) + converted.Payload[0].Size = 99 + assert.EqualValues(t, 10, decoded.Payload[0].Size, "the conversion copies the descriptors") +} + +func TestRepresentation_RejectsUnsafeStoredLocation(t *testing.T) { + for _, location := range []string{"", "../escape", "/abs", "a/../b", ".atmos-artifacts/x", "a//b", "a/"} { + rep := sampleRepresentation() + rep.Location = location + _, err := rep.Marshal() + require.Error(t, err, "%q", location) + } + missing := sampleRepresentation() + missing.Payload = nil + _, err := missing.Marshal() + require.Error(t, err) +} + +func sampleOperation() Operation { + return Operation{ + SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op/1", Name: "team/app", Version: "1.0.0", + IntentDigest: digestA, RecordDigest: digestB, CapturedAt: artifact.NewTimestamp(time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC)), + Packaging: artifact.PackagingNone, Location: "2026/10/30/team/app/1.0.0", ExpectedPrior: digestC, + Prior: &Prior{RecordDigest: digestC, Location: "2026/10/29/team/app/1.0.0"}, + Payload: []ocispec.Descriptor{}, Documents: []ocispec.Descriptor{}, Owner: "host:1", + } +} + +func TestOperation_JournalRoundTrip(t *testing.T) { + op := sampleOperation() + raw, err := op.Marshal() + require.NoError(t, err) + decoded, err := DecodeOperation(raw) + require.NoError(t, err) + assert.Equal(t, op.CapturedAt.Time(), decoded.CapturedAt.Time(), "the captured time is journaled") + assert.Equal(t, op.Location, decoded.Location) + assert.Equal(t, op.Prior, decoded.Prior) + assert.Equal(t, op.ExpectedPrior, decoded.ExpectedPrior) + + claim := decoded.Claim() + assert.Equal(t, ReferenceKey("team/app", "1.0.0"), claim.ReferenceKey) + assert.Equal(t, "op/1", claim.OperationID) + assert.Equal(t, []string{"2026/10/30/team/app/1.0.0"}, claim.Paths) + + broken := sampleOperation() + broken.CapturedAt = artifact.Timestamp{} + _, err = broken.Marshal() + require.Error(t, err, "the captured time is required") + broken = sampleOperation() + broken.Prior = &Prior{} + _, err = broken.Marshal() + require.Error(t, err) +} + +func TestEvidence_EnvelopeBindsFileNameToContent(t *testing.T) { + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digestA, MediaType: artifact.MediaTypeConfig, Size: 12} + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte(`{"bundle":true}`)} + evidence, err := NewEvidence(subject, payload) + require.NoError(t, err) + assert.Equal(t, digest.FromBytes(payload.Data), evidence.Digest()) + + raw, err := evidence.Marshal() + require.NoError(t, err) + decoded, err := DecodeEvidence(raw, evidence.Digest()) + require.NoError(t, err) + assert.Equal(t, payload.Data, decoded.Data) + + blob := decoded.Blob("loc") + assert.Equal(t, subject, blob.Evidence.Subject) + assert.Equal(t, evidence.Digest(), blob.Evidence.Descriptor.Digest) + assert.EqualValues(t, len(payload.Data), blob.Evidence.Descriptor.Size) + assert.Equal(t, "loc", blob.Evidence.Locator) + blob.Data[0] = 'X' + assert.Equal(t, byte('{'), decoded.Data[0], "Blob copies the bytes") + + _, err = DecodeEvidence(raw, digestB) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a file whose name disagrees with its content is rejected") + + for _, bad := range []*artifact.EvidencePayload{ + {Format: "", MediaType: "x", Data: []byte("x")}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "", Data: []byte("x")}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: nil}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: make([]byte, artifact.MaxEvidenceSize+1)}, + } { + _, err := NewEvidence(subject, bad) + require.Error(t, err) + } + _, err = NewEvidence(artifact.SignatureSubject{Kind: "bogus", Digest: digestA}, payload) + require.Error(t, err) +} + +func TestAdmit(t *testing.T) { + completed := NewPendingReference("team/app", "1.0.0", "op-0", digestA, nil) + completed = completed.Completed(digestB) + pendingFirst := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + pendingReplace := NewPendingReference("team/app", "1.0.0", "op-2", digestA, &completed) + expectCurrent := &artifact.PinnedReference{ResolvedDigest: digestB} + + tests := []struct { + name string + in AdmitInput + want Admission + outcome artifact.Outcome + wantErr error + }{ + {"create when absent", AdmitInput{IncomingRecord: digestC, IncomingIntent: digestA}, AdmitCreate, artifact.OutcomeCreated, nil}, + {"absent but a prior was expected", AdmitInput{IncomingRecord: digestC, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"same record is unchanged", AdmitInput{Current: &completed, IncomingRecord: digestB, ExpectedPrior: expectCurrent}, AdmitUnchanged, artifact.OutcomeUnchanged, nil}, + {"same intent is unchanged", AdmitInput{Current: &completed, CurrentIntent: digestA, IncomingRecord: digestC, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, AdmitUnchanged, artifact.OutcomeUnchanged, nil}, + {"unknown current intent is never unchanged", AdmitInput{Current: &completed, IncomingRecord: digestC, IncomingIntent: "", ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"different intent conflicts without force", AdmitInput{Current: &completed, CurrentIntent: digestC, IncomingRecord: digestA, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"different intent replaces under force", AdmitInput{Current: &completed, CurrentIntent: digestC, IncomingRecord: digestA, IncomingIntent: digestA, ExpectedPrior: expectCurrent, Force: true}, AdmitReplace, artifact.OutcomeReplaced, nil}, + {"stale expected prior", AdmitInput{Current: &completed, IncomingRecord: digestC, ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digestC}, Force: true}, "", "", errUtils.ErrArtifactConflict}, + {"exists but writer expected none", AdmitInput{Current: &completed, IncomingRecord: digestC, Force: true}, "", "", errUtils.ErrArtifactConflict}, + {"same-intent pending first publication resumes", AdmitInput{Current: &pendingFirst, IncomingIntent: digestA}, AdmitResumeCreate, artifact.OutcomeCreated, nil}, + {"same-intent pending replacement resumes", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA}, AdmitResumeReplace, artifact.OutcomeReplaced, nil}, + {"different-intent pending names the operation", AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}, "", "", errUtils.ErrArtifactPublicationUnavailable}, + {"force never steals a different pending intent", AdmitInput{Current: &pendingReplace, IncomingIntent: digestC, Force: true}, "", "", errUtils.ErrArtifactPublicationUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Admit(&tt.in) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.outcome, got.Outcome()) + assert.Equal(t, tt.want == AdmitResumeCreate || tt.want == AdmitResumeReplace, got.Resuming()) + }) + } + + _, err := Admit(&AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}) + require.ErrorContains(t, err, "op-1", "the recovery diagnostic includes the owning operation ID") +} diff --git a/pkg/artifact/layout/location.go b/pkg/artifact/layout/location.go new file mode 100644 index 00000000000..120929d73f8 --- /dev/null +++ b/pkg/artifact/layout/location.go @@ -0,0 +1,220 @@ +package layout + +import ( + "bytes" + "fmt" + "strings" + "text/template" + "time" + "unicode/utf8" + + "github.com/Masterminds/sprig/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/templatefuncs" +) + +// maxLocationLength bounds one resolved location, in bytes. +const maxLocationLength = 1024 + +// windowsReserved lists characters that are invalid in Windows file names. Locations avoid them on +// every platform so a repository root can be moved between hosts. +const windowsReserved = `<>:"|?*` + +// deleteRune is the ASCII DEL control character. +const deleteRune = 0x7f + +// nonHermeticFunctions are Sprig functions removed from key_pattern rendering. A location must +// depend only on the captured publication inputs: not on the clock, the process environment, the +// network, or randomness. +var nonHermeticFunctions = []string{ + "now", "env", "expandenv", "getHostByName", "uuidv4", "randAlphaNum", "randAlpha", "randNumeric", + "randAscii", "randBytes", "shuffle", "genPrivateKey", "genCA", "genCAWithKey", "genSelfSignedCert", + "genSelfSignedCertWithKey", "genSignedCert", "genSignedCertWithKey", "buildCustomCert", "derivePassword", + "htpasswd", "bcrypt", "encryptAES", "decryptAES", +} + +// PatternInput holds the values a key_pattern may reference: .Name, .Version, .Kind, and +// .CreatedAt. CreatedAt is the creation time captured once for a publication. +type PatternInput struct { + // Name is the artifact name. + Name string + + // Version is the artifact version. + Version string + + // Kind is the artifact kind. + Kind string + + // CreatedAt is the captured creation time; it is rendered in UTC. + CreatedAt time.Time +} + +// patternData is the template data. CreatedAt is a time.Time so date formatting applies. +type patternData struct { + Name string + Version string + Kind string + CreatedAt time.Time +} + +// DefaultLocation returns the default payload location below the repository root for the +// packaging: "/" for copied files, "/." for an archive. +func DefaultLocation(name, version string, packaging artifact.Packaging) string { + defer perf.Track(nil, "layout.DefaultLocation")() + + return name + "/" + version + ArchiveExtension(packaging) +} + +// ArchiveExtension returns the file extension (with the dot) of an archive packaging, or the empty +// string for copied files and unknown packagings. +func ArchiveExtension(packaging artifact.Packaging) string { + defer perf.Track(nil, "layout.ArchiveExtension")() + + switch packaging { + case artifact.PackagingTar: + return ".tar" + case artifact.PackagingTgz: + return ".tgz" + case artifact.PackagingZip: + return ".zip" + default: + return "" + } +} + +// RenderLocation resolves the payload location for one publication. An empty pattern selects +// [DefaultLocation]. A non-empty pattern is rendered as a Go template with the Sprig functions +// (minus the non-hermetic ones) and a UTC "date". The result is validated: it must stay below the +// repository root, avoid the reserved namespace, and for archives agree with the packaging. +// +// Callers render once, at publication, persist the result in the representation record, and use +// the persisted value for every later read, delete, and retry; they never render again. +func RenderLocation(pattern string, in *PatternInput, packaging artifact.Packaging) (string, error) { + defer perf.Track(nil, "layout.RenderLocation")() + + raw := DefaultLocation(in.Name, in.Version, packaging) + if pattern != "" { + rendered, err := renderPattern(pattern, in) + if err != nil { + return "", err + } + raw = rendered + } + location, err := CleanLocation(raw) + if err != nil { + return "", fmt.Errorf("key_pattern %q rendered %q: %w", pattern, raw, err) + } + if err := checkExtension(location, packaging); err != nil { + return "", fmt.Errorf("key_pattern %q rendered %q: %w", pattern, raw, err) + } + return location, nil +} + +func renderPattern(pattern string, in *PatternInput) (string, error) { + funcs := sprig.TxtFuncMap() + for _, name := range nonHermeticFunctions { + delete(funcs, name) + } + funcs["date"] = utcDate + for name, fn := range templatefuncs.FuncMap() { + funcs[name] = fn + } + tmpl, err := template.New("key_pattern").Funcs(funcs).Option("missingkey=error").Parse(pattern) + if err != nil { + return "", fmt.Errorf("%w: key_pattern %q: %w", errUtils.ErrArtifactConfigInvalid, pattern, err) + } + data := patternData{Name: in.Name, Version: in.Version, Kind: in.Kind, CreatedAt: in.CreatedAt.UTC()} + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("%w: key_pattern %q: %w", errUtils.ErrArtifactConfigInvalid, pattern, err) + } + return out.String(), nil +} + +// utcDate formats a time in UTC. Sprig's date uses the local zone, which would make the persisted +// location depend on the host that published it. +func utcDate(format string, value any) string { + switch typed := value.(type) { + case time.Time: + return typed.UTC().Format(format) + case *time.Time: + if typed != nil { + return typed.UTC().Format(format) + } + } + return "" +} + +// CleanLocation validates a resolved or persisted payload location and returns it in canonical +// form (a single trailing slash is removed). A location is relative, slash separated, free of +// empty, "." and ".." segments, control and Windows-reserved characters, and never enters the +// reserved namespace, however the case is written. Locations are never rewritten otherwise. +func CleanLocation(location string) (string, error) { + defer perf.Track(nil, "layout.CleanLocation")() + + location = strings.TrimSuffix(location, "/") + if err := checkLocationShape(location); err != nil { + return "", err + } + for _, segment := range strings.Split(location, "/") { + switch { + case segment == "" || segment == "." || segment == "..": + return "", fmt.Errorf("%w: location %q has an empty, '.', or '..' segment", errUtils.ErrArtifactUnsafePath, location) + case strings.EqualFold(segment, artifact.ReservedNamespace): + return "", fmt.Errorf("%w: location %q enters the reserved %s namespace", errUtils.ErrArtifactUnsafePath, location, artifact.ReservedNamespace) + } + } + return location, nil +} + +// checkLocationShape checks length, encoding, rootedness, separators, and character classes. +func checkLocationShape(location string) error { + switch { + case location == "" || len(location) > maxLocationLength: + return fmt.Errorf("%w: location must be 1-%d bytes", errUtils.ErrArtifactUnsafePath, maxLocationLength) + case !utf8.ValidString(location): + return fmt.Errorf("%w: location %q is not valid UTF-8", errUtils.ErrArtifactUnsafePath, location) + case strings.HasPrefix(location, "/") || strings.Contains(location, `\`): + return fmt.Errorf("%w: location %q must be relative and slash-separated", errUtils.ErrArtifactUnsafePath, location) + case strings.ContainsAny(location, windowsReserved) || strings.IndexFunc(location, isControl) >= 0: + return fmt.Errorf("%w: location %q contains control or reserved characters", errUtils.ErrArtifactUnsafePath, location) + } + return nil +} + +func isControl(r rune) bool { + return r < ' ' || r == deleteRune +} + +// archiveSuffixes maps archive file extensions to their packaging, longest first. +var archiveSuffixes = []struct { + suffix string + packaging artifact.Packaging +}{ + {".tar.gz", artifact.PackagingTgz}, + {".tgz", artifact.PackagingTgz}, + {".tar", artifact.PackagingTar}, + {".zip", artifact.PackagingZip}, +} + +// checkExtension rejects an archive extension that disagrees with the packaging. A location +// without an archive extension is accepted; it is never silently renamed. +func checkExtension(location string, packaging artifact.Packaging) error { + if ArchiveExtension(packaging) == "" { + return nil + } + lower := strings.ToLower(location) + for _, candidate := range archiveSuffixes { + if !strings.HasSuffix(lower, candidate.suffix) { + continue + } + if candidate.packaging != packaging { + return fmt.Errorf("%w: %w: location ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, candidate.suffix, packaging) + } + return nil + } + return nil +} diff --git a/pkg/artifact/layout/paths.go b/pkg/artifact/layout/paths.go new file mode 100644 index 00000000000..d9fcef6918f --- /dev/null +++ b/pkg/artifact/layout/paths.go @@ -0,0 +1,205 @@ +package layout + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Names of the managed layout. +const ( + // SchemaVersion is the schema version of every layout document. + SchemaVersion = 1 + + // Version is the layout version directory below the reserved namespace. + Version = "v1" + + // ManagedRoot is the slash-separated managed directory below a repository root. + ManagedRoot = artifact.ReservedNamespace + "/" + Version + + // MetadataSuffix is appended to an archive payload path for its adjacent metadata sidecar. + MetadataSuffix = ".metadata.json" + + // DocsSuffix is appended to an archive payload path for its adjacent documents directory. + DocsSuffix = ".docs" + + // DistributionDir is the distribution directory below a copied-files payload, and inside an + // archive, that holds README, CHANGELOG, and manifest.json. Managed readers omit it when + // materializing user content. + DistributionDir = ".atmos/artifact" + + // ManifestName is the generated manifest file name inside the distribution directory. + ManifestName = "manifest.json" +) + +// Directory names inside the managed root. +const ( + refsDir = "refs" + recordsDir = "records" + representationsDir = "representations" + operationsDir = "operations" + signaturesDir = "signatures" + locksDir = "locks" + stagingDirName = "staging" + jsonSuffix = ".json" + lockSuffix = ".lock" + + // The longest operation identifier used verbatim as a file name. + maxSafeOperationID = 128 + + // The prefix of a file name derived from an unsafe operation identifier. + hashedOperationPrefix = "sha256-" +) + +// safeOperationID matches identifiers that are safe path segments on every platform. +var safeOperationID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// ReferenceKey returns the private reference key for a name and version: the lowercase hex +// SHA-256 of "name:version". The colon is unambiguous because names never contain one. +func ReferenceKey(name, version string) string { + defer perf.Track(nil, "layout.ReferenceKey")() + + sum := sha256.Sum256([]byte(name + ":" + version)) + return hex.EncodeToString(sum[:]) +} + +// ReferencePath returns the slash-separated path of the reference record for key. +func ReferencePath(key string) string { + defer perf.Track(nil, "layout.ReferencePath")() + + return ManagedRoot + "/" + refsDir + "/" + key + jsonSuffix +} + +// ReferencesDir returns the directory that holds every reference record. +func ReferencesDir() string { + defer perf.Track(nil, "layout.ReferencesDir")() + + return ManagedRoot + "/" + refsDir +} + +// RecordPath returns the slash-separated path of the immutable Record with the given digest. +func RecordPath(recordDigest digest.Digest) string { + defer perf.Track(nil, "layout.RecordPath")() + + return ManagedRoot + "/" + recordsDir + "/" + recordDigest.Encoded() + jsonSuffix +} + +// RepresentationPath returns the slash-separated path of the representation record for key. +func RepresentationPath(key string) string { + defer perf.Track(nil, "layout.RepresentationPath")() + + return ManagedRoot + "/" + representationsDir + "/" + key + jsonSuffix +} + +// OperationFileName returns the file stem for an operation identifier. Safe identifiers are used +// verbatim; anything else (slashes, spaces, overlong values) maps to a hash so that no identifier +// can address a path outside the operations directory. +func OperationFileName(operationID string) string { + defer perf.Track(nil, "layout.OperationFileName")() + + if len(operationID) <= maxSafeOperationID && safeOperationID.MatchString(operationID) { + return operationID + } + sum := sha256.Sum256([]byte(operationID)) + return hashedOperationPrefix + hex.EncodeToString(sum[:]) +} + +// OperationPath returns the slash-separated path of the journal for an operation. +func OperationPath(operationID string) string { + defer perf.Track(nil, "layout.OperationPath")() + + return ManagedRoot + "/" + operationsDir + "/" + OperationFileName(operationID) + jsonSuffix +} + +// StagingPath returns the slash-separated staging directory owned by an operation. Staging lives +// below the managed root so it shares a filesystem with the payload for atomic renames. +func StagingPath(operationID string) string { + defer perf.Track(nil, "layout.StagingPath")() + + return ManagedRoot + "/" + stagingDirName + "/" + OperationFileName(operationID) +} + +// TempDir returns the directory for short-lived temporary files used by atomic replacement. +func TempDir() string { + defer perf.Track(nil, "layout.TempDir")() + + return ManagedRoot + "/tmp" +} + +// LockPath returns the slash-separated path of the lock file for a resource. The file name is the +// hash of the resource so any resource string is safe. +func LockPath(resource string) string { + defer perf.Track(nil, "layout.LockPath")() + + sum := sha256.Sum256([]byte(resource)) + return ManagedRoot + "/" + locksDir + "/" + hex.EncodeToString(sum[:]) + lockSuffix +} + +// SignatureDir returns the directory that holds the detached evidence for a subject. +func SignatureDir(kind artifact.SubjectKind, subject digest.Digest) string { + defer perf.Track(nil, "layout.SignatureDir")() + + return ManagedRoot + "/" + signaturesDir + "/" + string(kind) + "/" + subject.Encoded() +} + +// SignaturePath returns the slash-separated path of one detached evidence file: +// .atmos-artifacts/v1/signatures///.json, with +// the digests written as lowercase hex. +func SignaturePath(kind artifact.SubjectKind, subject, evidence digest.Digest) string { + defer perf.Track(nil, "layout.SignaturePath")() + + return SignatureDir(kind, subject) + "/" + evidence.Encoded() + jsonSuffix +} + +// MetadataSidecarPath returns the adjacent metadata sidecar of an archive payload location. +func MetadataSidecarPath(location string) string { + defer perf.Track(nil, "layout.MetadataSidecarPath")() + + return location + MetadataSuffix +} + +// DocsSidecarDir returns the adjacent documents directory of an archive payload location. +func DocsSidecarDir(location string) string { + defer perf.Track(nil, "layout.DocsSidecarDir")() + + return location + DocsSuffix +} + +// DocumentFileName returns the file name of a distribution document. +func DocumentFileName(role artifact.DocumentRole) string { + defer perf.Track(nil, "layout.DocumentFileName")() + + if role == artifact.DocumentChangelog { + return "CHANGELOG.md" + } + return "README.md" +} + +// DocumentSidecarPath returns the adjacent document file of an archive payload location. +func DocumentSidecarPath(location string, role artifact.DocumentRole) string { + defer perf.Track(nil, "layout.DocumentSidecarPath")() + + return DocsSidecarDir(location) + "/" + DocumentFileName(role) +} + +// DistributionPath returns the distribution file of a copied-files payload location. +func DistributionPath(location, name string) string { + defer perf.Track(nil, "layout.DistributionPath")() + + return location + "/" + DistributionDir + "/" + name +} + +// InManagedNamespace reports whether a slash-separated path enters the reserved namespace. The +// comparison folds case because common filesystems alias names that differ only by case. +func InManagedNamespace(p string) bool { + defer perf.Track(nil, "layout.InManagedNamespace")() + + first, _, _ := strings.Cut(strings.TrimPrefix(p, "./"), "/") + return strings.EqualFold(first, artifact.ReservedNamespace) +} diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go new file mode 100644 index 00000000000..5365a3fd1bc --- /dev/null +++ b/pkg/artifact/layout/records.go @@ -0,0 +1,428 @@ +package layout + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// State is the publication state of a reference. +type State string + +// Reference states. A pending reference is unavailable to managed readers and absent from +// completed search results. The completed state is written last in the publication sequence. +const ( + // StatePending marks a reference whose publication started and has not completed. It stays + // pending after an interruption until the same intent retries and repairs it. + StatePending State = "pending" + + // StateCompleted marks a reference whose Record, payload, and documents are all in place. + StateCompleted State = "completed" +) + +// Reference is the stable-root reference record: it binds a name and version to the digest of the +// immutable Record. It is the only mutable document readers consult, and it is replaced +// atomically. Generation increases on every state change so a reader can detect that the +// reference moved while it was reading from a mutable payload location. +type Reference struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // State is pending or completed. + State State `json:"state"` + + // Generation increases by one on every state change; the first record has generation 1. + Generation int64 `json:"generation"` + + // OperationID is the operation that wrote this state: the pending operation, or the operation + // that completed the publication. + OperationID string `json:"operation_id"` + + // IntentDigest is the intent digest of the publication that wrote this state. + IntentDigest digest.Digest `json:"intent_digest"` + + // RecordDigest is the digest of the canonical Record. It is required when completed and empty + // while a first publication is pending. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // PriorRecordDigest is the Record this pending publication replaces, kept for recovery + // diagnostics. It is empty for a first publication and once completed. + PriorRecordDigest digest.Digest `json:"prior_record_digest,omitempty"` +} + +// Available reports whether managed readers may use the reference. +func (r *Reference) Available() bool { + defer perf.Track(nil, "layout.Reference.Available")() + + return r.State == StateCompleted +} + +// Key returns the reference key. +func (r *Reference) Key() string { + defer perf.Track(nil, "layout.Reference.Key")() + + return ReferenceKey(r.Name, r.Version) +} + +// Validate checks the reference's shape and state invariants. +func (r *Reference) Validate() error { + defer perf.Track(nil, "layout.Reference.Validate")() + + if err := r.validateIdentity(); err != nil { + return err + } + if err := r.validateState(); err != nil { + return err + } + return validateOptionalDigests(r.RecordDigest, r.PriorRecordDigest) +} + +func (r *Reference) validateIdentity() error { + if r.SchemaVersion != SchemaVersion { + return invalid(kindReference, "unsupported schema_version %d", r.SchemaVersion) + } + if err := artifact.ValidateName(r.Name); err != nil { + return err + } + if err := artifact.ValidateVersion(r.Version); err != nil { + return err + } + if r.Generation < 1 || r.OperationID == "" { + return invalid(kindReference, "generation and operation_id are required") + } + return artifact.ValidateDigest(r.IntentDigest) +} + +func (r *Reference) validateState() error { + switch r.State { + case StateCompleted: + if r.RecordDigest == "" || r.PriorRecordDigest != "" { + return invalid(kindReference, "a completed reference needs record_digest and no prior_record_digest") + } + case StatePending: + if r.RecordDigest != "" { + return invalid(kindReference, "a pending reference must not name a record_digest") + } + default: + return invalid(kindReference, "unsupported state %q", r.State) + } + return nil +} + +func validateOptionalDigests(digests ...digest.Digest) error { + for _, d := range digests { + if d == "" { + continue + } + if err := artifact.ValidateDigest(d); err != nil { + return err + } + } + return nil +} + +// NewPendingReference returns the pending state that replaces prior (nil for a first publication). +func NewPendingReference(name, version, operationID string, intent digest.Digest, prior *Reference) Reference { + defer perf.Track(nil, "layout.NewPendingReference")() + + pending := Reference{ + SchemaVersion: SchemaVersion, Name: name, Version: version, State: StatePending, + Generation: 1, OperationID: operationID, IntentDigest: intent, + } + if prior != nil { + pending.Generation = prior.Generation + 1 + if prior.State == StateCompleted { + pending.PriorRecordDigest = prior.RecordDigest + } else { + pending.PriorRecordDigest = prior.PriorRecordDigest + } + } + return pending +} + +// Completed returns the completed state that follows a pending reference once the Record, +// payload, and documents are in place. +func (r *Reference) Completed(recordDigest digest.Digest) Reference { + defer perf.Track(nil, "layout.Reference.Completed")() + + done := *r + done.State = StateCompleted + done.Generation = r.Generation + 1 + done.RecordDigest = recordDigest + done.PriorRecordDigest = "" + return done +} + +// Marshal returns the canonical bytes of the reference. +func (r *Reference) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Reference.Marshal")() + + return encode(r, MaxDocumentBytes) +} + +// DecodeReference strictly parses a stored reference record. +func DecodeReference(raw []byte) (*Reference, error) { + defer perf.Track(nil, "layout.DecodeReference")() + + var ref Reference + if err := decode(kindReference, raw, &ref, MaxDocumentBytes); err != nil { + return nil, err + } + return &ref, nil +} + +// Representation is the mutable storage description of one reference's current Record. It holds +// the resolved payload location rendered once at publication; later reads and deletes use it and +// never render the key_pattern again. +type Representation struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // RepositoryKind is the repository kind that wrote it, for example "local/archive". + RepositoryKind string `json:"repository_kind"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // RecordDigest is the digest of the Record this representation belongs to. + RecordDigest digest.Digest `json:"record_digest"` + + // OperationID is the operation that wrote the representation. + OperationID string `json:"operation_id"` + + // Packaging is the payload packaging. + Packaging artifact.Packaging `json:"packaging"` + + // Location is the resolved payload location, relative to the repository root. + Location string `json:"location"` + + // Payload lists the payload blob descriptors: one archive, or one per file. + Payload []ocispec.Descriptor `json:"payload"` + + // Documents lists the document blob descriptors stored alongside the payload. + Documents []ocispec.Descriptor `json:"documents"` + + // RepresentationDigest is the checksum of the archive, or of the per-file payload digests. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` +} + +// Validate checks the representation's shape. +func (r *Representation) Validate() error { + defer perf.Track(nil, "layout.Representation.Validate")() + + if r.SchemaVersion != SchemaVersion { + return invalid(kindRepresentation, "unsupported schema_version %d", r.SchemaVersion) + } + if err := validateNameVersion(r.Name, r.Version); err != nil { + return err + } + if err := artifact.ValidateDigest(r.RecordDigest); err != nil { + return err + } + if r.RepositoryKind == "" || r.OperationID == "" { + return invalid(kindRepresentation, "repository_kind and operation_id are required") + } + if _, err := CleanLocation(r.Location); err != nil || r.Location != cleanedOrSelf(r.Location) { + return invalid(kindRepresentation, "location %q is not a canonical relative location", r.Location) + } + if r.Payload == nil || r.Documents == nil { + return invalid(kindRepresentation, "payload and documents lists are required (use empty lists)") + } + return validateOptionalDigests(r.RepresentationDigest) +} + +// validateNameVersion checks the name and version of a document. +func validateNameVersion(name, version string) error { + if err := artifact.ValidateName(name); err != nil { + return err + } + return artifact.ValidateVersion(version) +} + +// cleanedOrSelf returns the canonical form of location, or location itself when it is invalid. +func cleanedOrSelf(location string) string { + if cleaned, err := CleanLocation(location); err == nil { + return cleaned + } + return location +} + +// Marshal returns the canonical bytes of the representation. +func (r *Representation) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Representation.Marshal")() + + return encode(r, MaxDocumentBytes) +} + +// DecodeRepresentation strictly parses a stored representation record. +func DecodeRepresentation(raw []byte) (*Representation, error) { + defer perf.Track(nil, "layout.DecodeRepresentation")() + + var rep Representation + if err := decode(kindRepresentation, raw, &rep, MaxDocumentBytes); err != nil { + return nil, err + } + return &rep, nil +} + +// Artifact converts the stored description into the repository-neutral [artifact.Representation]. +// The locator argument is the display locator of the payload, for example an absolute path or an +// s3:// URI. +func (r *Representation) Artifact(locator string) artifact.Representation { + defer perf.Track(nil, "layout.Representation.Artifact")() + + return artifact.Representation{ + Packaging: r.Packaging, + Payload: append([]ocispec.Descriptor(nil), r.Payload...), + Documents: append([]ocispec.Descriptor(nil), r.Documents...), + Locator: locator, + RecordDigest: r.RecordDigest, + RepresentationDigest: r.RepresentationDigest, + } +} + +// Prior describes the state an operation replaces. +type Prior struct { + // RecordDigest is the replaced Record. + RecordDigest digest.Digest `json:"record_digest"` + + // Location is where the replaced payload lives, so a recovery can retire it. + Location string `json:"location,omitempty"` + + // Packaging is the packaging of the replaced payload, for backends such as S3 whose object + // keys depend on it. Local backends leave it empty. + Packaging artifact.Packaging `json:"packaging,omitempty"` +} + +// Operation is the publication journal: an implementation recovery record, not a public catalog. +// It is written before any payload path changes and records everything a same-intent retry needs +// to resume or repair the publication: the captured time, the intent, the prior reference, the +// resolved location, and the prepared representation checksums. +type Operation struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // RepositoryKind is the repository kind performing the publication. + RepositoryKind string `json:"repository_kind"` + + // OperationID is the logical operation identifier, stable across retries. + OperationID string `json:"operation_id"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // IntentDigest is the intent digest of the artifact being published. + IntentDigest digest.Digest `json:"intent_digest"` + + // RecordDigest is the digest of the Record being published. + RecordDigest digest.Digest `json:"record_digest"` + + // CapturedAt is the creation time captured once for the publication. + CapturedAt artifact.Timestamp `json:"captured_at"` + + // Packaging is the payload packaging being written. + Packaging artifact.Packaging `json:"packaging"` + + // Location is the resolved payload location reserved for this operation. + Location string `json:"location"` + + // ExpectedPrior is the record digest the writer observed under the guard; empty means none. + ExpectedPrior digest.Digest `json:"expected_prior,omitempty"` + + // Prior describes the state being replaced, if any. + Prior *Prior `json:"prior,omitempty"` + + // Payload and Documents are the prepared representation checksums. + Payload []ocispec.Descriptor `json:"payload"` + Documents []ocispec.Descriptor `json:"documents"` + + // Owner identifies the writer for recovery diagnostics, for example host and process. + Owner string `json:"owner,omitempty"` +} + +// Validate checks the journal's shape. +func (o *Operation) Validate() error { + defer perf.Track(nil, "layout.Operation.Validate")() + + if o.SchemaVersion != SchemaVersion { + return invalid(kindOperation, "unsupported schema_version %d", o.SchemaVersion) + } + if o.OperationID == "" || o.RepositoryKind == "" { + return invalid(kindOperation, "operation_id and repository_kind are required") + } + if err := validateNameVersion(o.Name, o.Version); err != nil { + return err + } + if err := o.validateState(); err != nil { + return err + } + return validateOptionalDigests(o.IntentDigest, o.RecordDigest, o.ExpectedPrior) +} + +func (o *Operation) validateState() error { + if o.CapturedAt.IsZero() { + return invalid(kindOperation, "captured_at is required") + } + if _, err := CleanLocation(o.Location); err != nil { + return invalid(kindOperation, "location %q is not a valid relative location", o.Location) + } + if o.Payload == nil || o.Documents == nil { + return invalid(kindOperation, "payload and documents lists are required (use empty lists)") + } + if o.Prior != nil && o.Prior.RecordDigest == "" { + return invalid(kindOperation, "prior needs a record_digest") + } + return nil +} + +// Claim returns the paths the operation reserves. +func (o *Operation) Claim() Claim { + defer perf.Track(nil, "layout.Operation.Claim")() + + return Claim{ + ReferenceKey: ReferenceKey(o.Name, o.Version), Name: o.Name, Version: o.Version, + OperationID: o.OperationID, Paths: PayloadPaths(o.RepositoryKind, o.Location), + } +} + +// Marshal returns the canonical bytes of the journal. +func (o *Operation) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Operation.Marshal")() + + return encode(o, MaxDocumentBytes) +} + +// DecodeOperation strictly parses a stored operation journal. +func DecodeOperation(raw []byte) (*Operation, error) { + defer perf.Track(nil, "layout.DecodeOperation")() + + var op Operation + if err := decode(kindOperation, raw, &op, MaxDocumentBytes); err != nil { + return nil, err + } + return &op, nil +} + +// Claim returns the paths a completed representation owns. +func (r *Representation) Claim() Claim { + defer perf.Track(nil, "layout.Representation.Claim")() + + return Claim{ + ReferenceKey: ReferenceKey(r.Name, r.Version), Name: r.Name, Version: r.Version, + Paths: PayloadPaths(r.RepositoryKind, r.Location), + } +} diff --git a/pkg/artifact/local/conformance_test.go b/pkg/artifact/local/conformance_test.go new file mode 100644 index 00000000000..8db2036a99b --- /dev/null +++ b/pkg/artifact/local/conformance_test.go @@ -0,0 +1,54 @@ +package local + +import ( + "errors" + "testing" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +// errCrash is the error the fault hooks simulate a crash with. +var errCrash = errors.New("simulated crash") + +func TestDirRepositoryConformance(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRepo(t, &repoSpec{kind: artifact.RepositoryKindLocalDir, opts: []Option{WithVerifier(&artifacttest.FakeVerifier{})}}) + }, + Packaging: artifact.PackagingNone, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) +} + +func TestArchiveRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRepo(t, &repoSpec{kind: artifact.RepositoryKindLocalArchive, packaging: packaging, opts: []Option{WithVerifier(&artifacttest.FakeVerifier{})}}) + }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} + +// Without a verifier the repositories advertise no evidence formats and Sign returns a capability +// error instead of a successful no-op. +func TestRepositoryConformance_WithoutVerifier(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { return newRepo(t, spec) }, + }) + }) + } +} diff --git a/pkg/artifact/local/delete.go b/pkg/artifact/local/delete.go new file mode 100644 index 00000000000..5c3b7255cf1 --- /dev/null +++ b/pkg/artifact/local/delete.go @@ -0,0 +1,82 @@ +package local + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Repository]. Deletion is scoped to the selected artifact: its +// reference, its Record, its representation, its detached evidence, and the payload and sidecars +// it exclusively owns. Nothing outside the repository root is touched, and files the repository +// does not manage (a user's extra file inside a payload directory) stay in place. +// +// The reference record is removed first, so the artifact disappears atomically; an interruption +// afterwards leaves garbage that a later publication of the same reference clears, never a +// reference that points at missing bytes. A pending publication is never deleted: its owner +// must repair it with the same intent. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + defer perf.Track(nil, "local.Repository.Delete")() + + if options.Scope != "" && options.Scope != artifact.DeleteObject { + return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/"+string(options.Scope), + "local repositories delete the reference and the payload it owns together (object scope)") + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + record, _, err := t.loadRecord(pinned.RecordDigest) + if err != nil { + return nil, err + } + var held *fileLock + if options.Lock == nil { + if held, err = r.acquire(ctx, record.Name+":"+record.Version, r.settings.owner, ""); err != nil { + return nil, err + } + } + result, err := r.deleteLocked(ctx, t, &pinned, &options) + if held != nil { + if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { + err = releaseErr + } + } + return result, err +} + +func (r *Repository) deleteLocked(ctx context.Context, t *tree, pinned *artifact.PinnedReference, options *artifact.DeleteOptions) (*artifact.DeleteResult, error) { + v, err := r.openView(ctx, t, pinned) + if err != nil { + return nil, err + } + if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != v.ref.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is %s but the delete expected %s", errUtils.ErrArtifactConflict, v.ref.Name, v.ref.Version, v.ref.RecordDigest, options.ExpectedCurrent.ResolvedDigest) + } + if err := t.Remove(native(layout.ReferencePath(v.ref.Key()))); err != nil { + return nil, fmt.Errorf("%w: remove reference: %w", errUtils.ErrArtifactDeleteFailed, err) + } + removed, err := r.store.remove(t, v) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) + } + for _, owned := range []string{ + layout.SignatureDir(artifact.SubjectRecord, v.ref.RecordDigest), + layout.RecordPath(v.ref.RecordDigest), + layout.RepresentationPath(v.ref.Key()), + } { + if err := t.removeAll(owned); err != nil { + return nil, fmt.Errorf("%w: remove %s: %w", errUtils.ErrArtifactDeleteFailed, owned, err) + } + } + r.tidyOperation(t, v.ref.OperationID) + t.removeEmptyParents(v.rep.Location) + return &artifact.DeleteResult{Deleted: removed, Digest: v.ref.RecordDigest}, nil +} diff --git a/pkg/artifact/local/delete_test.go b/pkg/artifact/local/delete_test.go new file mode 100644 index 00000000000..250ff908e65 --- /dev/null +++ b/pkg/artifact/local/delete_test.go @@ -0,0 +1,227 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +func TestDelete_IsScopedToTheSelectedArtifact(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{name: "team/keep", version: "1.0.0"}, artifact.PublishOptions{}) + publish(t, repo, &prepareSpec{name: "team/keep", version: "2.0.0", files: map[string]string{"data.txt": "two"}}, artifact.PublishOptions{}) + target := publish(t, repo, &prepareSpec{name: "team/drop", version: "1.0.0", files: map[string]string{"data.txt": "drop"}}, artifact.PublishOptions{}) + + userFile := filepath.Join(repo.root, "notes", "user.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(userFile), 0o755)) + require.NoError(t, os.WriteFile(userFile, []byte("mine"), 0o644)) + outside := filepath.Join(filepath.Dir(repo.root), "outside.txt") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + + pinned := resolve(t, repo, ref("team/drop", "1.0.0")) + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.Equal(t, pinned.RecordDigest, result.Digest) + require.NotEmpty(t, result.Deleted) + assert.Equal(t, target.Location, result.Deleted[0]) + + _, err = repo.Resolve(context.Background(), ref("team/drop", "1.0.0")) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assertIntact(t, repo, ref("team/keep", "1.0.0")) + assertIntact(t, repo, ref("team/keep", "2.0.0")) + assert.FileExists(t, userFile, "files the repository does not manage are never touched") + assert.FileExists(t, outside, "nothing outside the root is touched") + assert.DirExists(t, repo.root) + assert.NoDirExists(t, filepath.Join(repo.root, "team", "drop"), "empty parents of the deleted artifact are pruned") + assert.DirExists(t, filepath.Join(repo.root, "team", "keep")) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.RecordPath(pinned.RecordDigest)))) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(layout.ReferenceKey("team/drop", "1.0.0"))))) + assert.Equal(t, []string{"team/keep:1.0.0", "team/keep:2.0.0"}, listNames(t, repo)) + + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "deleting twice reports not found") + }) + } +} + +func TestDelete_LeavesUnmanagedFilesInsideADirectoryPayload(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + extra := filepath.Join(result.Location, "mine.txt") + require.NoError(t, os.WriteFile(extra, []byte("user data"), 0o644)) + + pinned := resolve(t, repo, defaultRef()) + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.FileExists(t, extra, "delete removes what the artifact owns, not a user's extra file") + assert.NoFileExists(t, filepath.Join(result.Location, "README.md")) + assert.NoDirExists(t, filepath.Join(result.Location, ".atmos")) + assert.Equal(t, []string{"team/app/1.0.0/mine.txt"}, payloadFiles(t, repo.root)) +} + +func TestDelete_RejectsStaleTargetsAndScopes(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + + stale := pinned + stale.ResolvedDigest = digest.FromString("other") + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &stale, Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assertIntact(t, repo, defaultRef()) + + for _, scope := range []artifact.DeleteSemantics{artifact.DeleteNone, artifact.DeleteManifest, artifact.DeleteReference} { + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: scope}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, scope) + } + assertIntact(t, repo, defaultRef()) +} + +func TestDelete_DoesNotDeletePendingPublications(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Delete(context.Background(), old, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.Error(t, err, "a pending operation is never deleted out from under its owner") + require.ErrorContains(t, err, "op-2") +} + +// A payload location replaced by a symlink to a directory outside the root must not let a delete +// reach the outside files: every operation is confined to the repository root. +func TestDelete_NeverEscapesTheRootThroughASymlink(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + outside := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(outside, "README.md"), []byte("outside readme"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "empty.txt"), nil, 0o644)) + require.NoError(t, os.RemoveAll(result.Location)) + if err := os.Symlink(outside, result.Location); err != nil { + t.Skipf("symlinks are unavailable: %v", err) + } + + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.Error(t, err, "the swapped location cannot be verified or removed") + assert.FileExists(t, filepath.Join(outside, "README.md")) + assert.FileExists(t, filepath.Join(outside, "empty.txt")) +} + +// A stored representation that names a location outside the root is rejected on load. +func TestStoredLocationOutsideTheRootIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + key := layout.ReferenceKey(testName, testVersion) + + tr, err := repo.openTree(false) + require.NoError(t, err) + rep, err := tr.loadRepresentation(key) + require.NoError(t, err) + require.NoError(t, tr.Close()) + + rep.Location = "../escape" + _, marshalErr := rep.Marshal() + require.Error(t, marshalErr, "the layout refuses to serialize an escaping location") + + // Write the document by hand to simulate tampering. + raw := `{"documents":[],"location":"../escape","name":"team/app","operation_id":"x","packaging":"none","payload":[],"record_digest":"` + pinned.RecordDigest.String() + `","repository_kind":"local/dir","schema_version":1,"version":"1.0.0"}` + require.NoError(t, os.WriteFile(filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(key))), []byte(raw), 0o644)) + _, err = repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestList_EnumeratesManagedReferencesOnly(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + var fault armedFault + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}} { + publish(t, repo, &prepareSpec{name: item.name, version: item.version, labels: map[string]string{"team": item.name}}, artifact.PublishOptions{}) + } + // Unmanaged files, a stray directory, and a pending publication are all ignored. + require.NoError(t, os.WriteFile(filepath.Join(repo.root, "user.txt"), []byte("x"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(repo.root, "a", "stray"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(repo.root, "a", "stray", "file.json"), []byte("{}"), 0o644)) + pending := prepare(t, repo, &prepareSpec{name: "c/pending", opID: "op-pending"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), pending, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + assert.Equal(t, []string{"a/app:1", "a/app:2", "b/app:1"}, listNames(t, repo)) + + named, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) + require.NoError(t, err) + require.Len(t, named.Entries, 2) + assert.Equal(t, "1", named.Entries[0].Version) + assert.Equal(t, "2", named.Entries[1].Version) + + labeled, err := repo.List(context.Background(), artifact.ListQuery{Labels: []artifact.Label{{Key: "team", Value: "b/app"}}}) + require.NoError(t, err) + require.Len(t, labeled.Entries, 1) + assert.Equal(t, "b/app", labeled.Entries[0].Name) + assert.Equal(t, artifact.KindFiles, labeled.Entries[0].Kind) + assert.Equal(t, repo.root, labeled.Entries[0].Pinned.Repository) + }) + } +} + +func TestList_PaginatesInLexicalOrder(t *testing.T) { + repo := newRepo(t, dirSpec()) + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "10"}, {"a/app", "2"}, {"a/app-x", "1"}, {"c/app", "1"}} { + publish(t, repo, &prepareSpec{name: item.name, version: item.version}, artifact.PublishOptions{}) + } + var seen []string + cursor := "" + for pages := 0; pages < 10; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{PageSize: 2, Cursor: cursor}) + require.NoError(t, err) + for index := range page.Entries { + seen = append(seen, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + require.NotEqual(t, cursor, page.NextCursor) + cursor = page.NextCursor + } + assert.Equal(t, []string{"a/app:10", "a/app:2", "a/app-x:1", "b/app:1", "c/app:1"}, seen, "name then version, every entry exactly once") +} + +func TestList_EmptyOrMissingRoot(t *testing.T) { + repo := newRepo(t, dirSpec()) + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Empty(t, page.Entries) + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "listing never creates the root") + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, statErr = os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "reads never create the root") +} diff --git a/pkg/artifact/local/doc.go b/pkg/artifact/local/doc.go new file mode 100644 index 00000000000..87ac02a6fff --- /dev/null +++ b/pkg/artifact/local/doc.go @@ -0,0 +1,48 @@ +// Package local implements the local directory (local/dir) and local archive (local/archive) +// artifact repositories behind [artifact.Repository]. They share one record layout (package +// layout), one publication lifecycle, one lock implementation, and one detached-evidence store; +// they differ only in how a payload is stored. +// +// # Directory repositories +// +// A local/dir payload is a tree of ordinary copied files below its resolved location, with the +// distribution documents and manifest.json adjacent at /.atmos/artifact/. Managed +// readers omit that distribution directory when they materialize user content. +// +// # Archive repositories +// +// A local/archive payload is one tar, tgz, or zip file. The repository stores adjacent +// .metadata.json and .docs/README.md|CHANGELOG.md sidecars, so metadata and +// documents are readable without opening the archive, while the archive itself embeds portable +// copies for transport. The configured quoted-octal permissions (default 0600) apply to the +// archive file and its sidecars only; they are distinct from member modes inside the archive. An +// explicit request the host cannot enforce fails with an unsupported-capability error instead of +// silently weakening the mode. +// +// # Publication and recovery +// +// A publication follows the contracts sequence: reserve the resolved location and record a pending +// operation journal, mark the reference pending, stage the payload on the same filesystem, write +// the immutable Record, install the payload and sidecars, write the representation, and only then +// flip the reference to completed. Pending references are unavailable to readers and invisible to +// listing. Readers recheck the reference generation after reading from the mutable payload +// location, so replaced bytes are never served under an old digest. +// +// An interrupted or failed force replacement leaves the reference unavailable until a retry with +// the same intent repairs it; there is no rollback. A pending operation of a different intent is +// never stolen: publication fails with a recovery diagnostic naming the operation ID. +// +// # Locks +// +// Lock returns an OS-backed file lock (host scope only) with an ownership token. It is held for +// the reference and, during publication, for the resolved location. The OS releases the lock when +// its owner exits. Acquisition is cancel-safe: a canceled wait leaks no ownership. +// +// # Signatures +// +// Signature evidence is stored as detached files below the reserved namespace and is bound to the +// canonical Record digest. Adding evidence never changes the Record, the reference, or the +// archive: evidence added after publication does not claim that the original archive bytes were +// signed, only the Record that describes them. Cryptographic verification is delegated to an +// injected [artifact.Verifier]; without one, the repository advertises no evidence formats. +package local diff --git a/pkg/artifact/local/edge_test.go b/pkg/artifact/local/edge_test.go new file mode 100644 index 00000000000..80da4df2942 --- /dev/null +++ b/pkg/artifact/local/edge_test.go @@ -0,0 +1,350 @@ +package local + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// cloneWithBlobs returns a prepared artifact sharing the Record but with different blobs, without +// copying the original's once-guard. +func cloneWithBlobs(p *artifact.PreparedArtifact, blobs []artifact.StagedBlob) *artifact.PreparedArtifact { + return &artifact.PreparedArtifact{ + Record: p.Record, IntentDigest: p.IntentDigest, OperationID: p.OperationID, PreparedAt: p.PreparedAt, + Packaging: p.Packaging, Blobs: blobs, + } +} + +func fileBlob(path string, data []byte) artifact.StagedBlob { + return artifact.StagedBlob{ + Descriptor: artifact.StagedBlob{}.Descriptor, Role: artifact.BlobPayload, Path: path, + Open: func(context.Context) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(data)), nil }, + } +} + +func TestPublish_RejectsInconsistentPreparedBlobs(t *testing.T) { + files := map[string]string{"a.txt": "alpha", "b/c.txt": "gamma"} + docFiles := map[string]string{"a.txt": "alpha", ".atmos/artifact/README.md": "docs\n"} + tests := []struct { + name string + spec *repoSpec + files map[string]string + edit func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact + want error + }{ + {"unopenable blob", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[0].Open = nil + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactConfigInvalid}, + {"inventory file without a blob", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, p.Blobs[:1]) + }, errUtils.ErrArtifactIntegrityFailed}, + {"blob that is not in the inventory", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + extra := p.Blobs[0] + extra.Path = "unlisted.txt" + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), extra)) + }, errUtils.ErrArtifactIntegrityFailed}, + {"blob without a path", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[0].Path = "" + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate path", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[0])) + }, errUtils.ErrArtifactIntegrityFailed}, + {"document blob missing", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + var blobs []artifact.StagedBlob + for _, blob := range p.Blobs { + if blob.Role == artifact.BlobPayload { + blobs = append(blobs, blob) + } + } + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate document blob", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[len(p.Blobs)-1])) + }, errUtils.ErrArtifactRecordInvalid}, + {"document blob with the wrong digest", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[len(blobs)-1].Descriptor.Digest = digest.FromString("other") + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"archive without a payload blob", archiveSpec(artifact.PackagingTgz), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, nil) + }, errUtils.ErrArtifactPackagingInvalid}, + {"archive with two payload blobs", archiveSpec(artifact.PackagingTgz), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[0])) + }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newRepo(t, tt.spec) + prepared := tt.edit(prepare(t, repo, &prepareSpec{files: tt.files})) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, tt.want) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a rejected preparation leaves no pending reference behind") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "and creates nothing") + }) + } +} + +// A blob whose bytes disagree with its descriptor fails the copy; the reference stays unavailable +// until a correct retry repairs it, and the bad bytes are never visible. +func TestPublish_CorruptStagedBlobIsNeverCommitted(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + good := prepare(t, repo, &prepareSpec{}) + for _, lie := range []string{"digest", "short", "long"} { + blobs := append([]artifact.StagedBlob(nil), good.Blobs...) + original := blobs[0].Open + switch lie { + case "digest": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(reader) + _ = reader.Close() + if len(data) > 0 { + data[0] ^= 0xff + } + return io.NopCloser(bytes.NewReader(data)), readErr + } + case "short": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, _ := io.ReadAll(reader) + _ = reader.Close() + return io.NopCloser(bytes.NewReader(data[:len(data)-1])), nil + } + case "long": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, _ := io.ReadAll(reader) + _ = reader.Close() + return io.NopCloser(bytes.NewReader(append(data, 'x'))), nil + } + } + _, err := repo.Publish(context.Background(), cloneWithBlobs(good, blobs), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, lie) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "%s: no partial success is advertised", lie) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(good.OperationID))), "%s: a failed attempt cleans its staging", lie) + } + _, err := repo.Publish(context.Background(), good, artifact.PublishOptions{}) + require.NoError(t, err, "the correct retry repairs the reference") + assertIntact(t, repo, defaultRef()) + }) + } +} + +// Cancellation mid-publication surfaces as a context error and leaves the same repairable state. +func TestPublish_CancellationLeavesARepairableState(t *testing.T) { + repo := newRepo(t, dirSpec()) + good := prepare(t, repo, &prepareSpec{files: map[string]string{"a.txt": "a", "b.txt": "b"}}) + ctx, cancel := context.WithCancel(context.Background()) + blobs := append([]artifact.StagedBlob(nil), good.Blobs...) + original := blobs[0].Open + blobs[0].Open = func(openCtx context.Context) (io.ReadCloser, error) { + cancel() + return original(openCtx) + } + _, err := repo.Publish(ctx, cloneWithBlobs(good, blobs), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + + _, err = repo.Publish(context.Background(), good, artifact.PublishOptions{}) + require.NoError(t, err) + assertIntact(t, repo, defaultRef()) +} + +// If the journal was lost, a same-intent retry still repairs the pending reference. +func TestPendingReference_RepairsWithoutItsJournal(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + require.NoError(t, os.RemoveAll(filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations"))) + retry, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, retry.Outcome) + assertIntact(t, repo, defaultRef()) + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestAbsentRootReportsNotFoundEverywhere(t *testing.T) { + repo := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + ctx := context.Background() + pinned := artifact.PinnedReference{Ref: defaultRef(), RecordDigest: digest.FromString("x"), ResolvedDigest: digest.FromString("x")} + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("x")} + + _, err := repo.Resolve(ctx, defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Pin(ctx, defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Inspect(ctx, pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Read(ctx, pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.OpenDocument(ctx, pinned, artifact.DocumentReadme) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Sign(ctx, subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Verify(ctx, subject, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.ReadEvidence(ctx, &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "no read or failed mutation created the root") +} + +func TestCanceledContextStopsReads(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := repo.Resolve(ctx, defaultRef()) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Read(ctx, pinned) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.ReadEvidence(ctx, &pinned) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, context.Canceled) +} + +func TestRead_OpeningFailsCleanlyWhenAPayloadFileIsMissing(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location, "nested", "deep", "file.txt"))) + content, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.Nil(t, content) + + require.NoError(t, os.RemoveAll(result.Location)) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a missing payload directory is damage, not absence") +} + +func TestRead_ReaderReportsTheReferenceMovingOrDisappearing(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + // Remove only the reference record: payload handles stay valid on every platform. + require.NoError(t, os.Remove(filepath.Join(repo.root, filepath.FromSlash(layout.ReferencePath(layout.ReferenceKey(testName, testVersion)))))) + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactNotFound, "a deleted reference is reported, not served") + }) + } +} + +func TestRead_PayloadThatGrowsIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + result := publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "short"}}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, filepath.Join(result.Location, "data.txt"))), appendJunk) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed, "bytes beyond the declared size are never served") +} + +func TestUnchangedRepublishDetectsAMissingRepresentation(t *testing.T) { + repo := newRepo(t, dirSpec()) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + current := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)))))) + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{ExpectedPrior: ¤t}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestRootThatIsAFileFailsClearly(t *testing.T) { + file := filepath.Join(t.TempDir(), "root") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + repo := newRepo(t, &repoSpec{root: file}) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repo.Lock(context.Background(), lockRequest("r")) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repo.List(context.Background(), artifact.ListQuery{}) + require.Error(t, err) +} + +func TestInventoryInsideTheGeneratedDistributionDirIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + prepared := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "x"}}) + record := prepared.Record + record.Inventory = append(append([]artifact.Entry(nil), record.Inventory...), artifact.Entry{ + Path: ".atmos/artifact/extra.txt", Type: artifact.EntryFile, Size: 1, Mode: 0o644, Digest: digest.FromString("x"), + }) + require.NoError(t, record.Seal()) + conflicting := &artifact.PreparedArtifact{ + Record: record, IntentDigest: prepared.IntentDigest, OperationID: "op-conflict", PreparedAt: prepared.PreparedAt, + Packaging: artifact.PackagingNone, + Blobs: append(append([]artifact.StagedBlob(nil), prepared.Blobs...), fileBlob(".atmos/artifact/extra.txt", []byte("x"))), + } + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor = prepared.Blobs[0].Descriptor + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor.Digest = digest.FromString("x") + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor.Size = 1 + _, err := repo.Publish(context.Background(), conflicting, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestFormattedErrorsNameTheOperationAndReference(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + fault.hook = crashAt(faultBeforePayload) + prepared := prepare(t, repo, &prepareSpec{opID: "op-visible"}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorContains(t, err, fmt.Sprintf("%s:%s", testName, testVersion)) + require.ErrorContains(t, err, "op-visible") + require.ErrorContains(t, err, "same intent") +} diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go new file mode 100644 index 00000000000..97f09c86d1a --- /dev/null +++ b/pkg/artifact/local/evidence.go @@ -0,0 +1,302 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io/fs" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxEvidenceFiles bounds how many evidence files one verification or export reads. More than +// this is reported, not silently dropped. +const maxEvidenceFiles = 64 + +// loadedEvidence is one stored evidence file that parsed and matched its file name. +type loadedEvidence struct { + evidence *layout.Evidence + locator string +} + +// loadEvidence reads the evidence stored for a Record subject. Malformed or oversize files are +// reported as diagnostics and skipped; they can never override valid matching evidence. +func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic) { + dir := layout.SignatureDir(artifact.SubjectRecord, subject) + entries, err := fs.ReadDir(t.FS(), dir) + if err != nil { + return nil, nil + } + var ( + loaded []loadedEvidence + diags []artifact.Diagnostic + ) + for _, entry := range entries { + if len(loaded) >= maxEvidenceFiles { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: fmt.Sprintf("more than %d evidence files; the rest were not read", maxEvidenceFiles)}) + break + } + stem, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + continue + } + locator := dir + "/" + entry.Name() + item, problem := t.readEvidenceFile(locator, stem) + if problem != nil { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}) + continue + } + loaded = append(loaded, loadedEvidence{evidence: item, locator: locator}) + } + return loaded, diags +} + +func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, error) { + want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) + if err := want.Validate(); err != nil { + return nil, err + } + raw, err := t.readBounded(locator, layout.MaxEvidenceFileBytes) + if err != nil { + return nil, err + } + return layout.DecodeEvidence(raw, want) +} + +// listEvidence returns the stored evidence descriptors for a subject, skipping malformed files. +func (t *tree) listEvidence(subject digest.Digest) []artifact.SignatureEvidence { + loaded, _ := t.loadEvidence(subject) + out := make([]artifact.SignatureEvidence, 0, len(loaded)) + for index := range loaded { + out = append(out, loaded[index].evidence.Reference(loaded[index].locator)) + } + return out +} + +// requireCurrentRecord checks that digest names the current completed Record of its reference and +// returns the canonical bytes. Evidence only ever binds to a live subject. +func (r *Repository) requireCurrentRecord(t *tree, recordDigest digest.Digest) ([]byte, error) { + record, raw, err := t.loadRecord(recordDigest) + if err != nil { + return nil, err + } + if _, err := r.resolveRef(t, artifact.Ref{Name: record.Name, Digest: recordDigest}); err != nil { + return nil, err + } + return raw, nil +} + +// checkSubject validates a signature subject for this repository. +func (r *Repository) checkSubject(operation string, subject artifact.SignatureSubject) error { + if err := subject.Validate(); err != nil { + return err + } + if caps := r.Capabilities(); !caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.alias, operation, "subject/"+string(subject.Kind), + "local repositories bind detached evidence to the canonical Record only") + } + return nil +} + +// Sign implements [artifact.Repository]: it asks the signer for evidence over the Record subject +// and stores it as a detached file named by the evidence digest. Evidence lives beside the Record, +// so signing never changes the Record, the reference, or the payload. A signature added after +// publication covers the Record that describes the archive, not the original archive bytes. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "local.Repository.Sign")() + + if err := r.checkSubject("sign", subject); err != nil { + return nil, err + } + if signer == nil { + return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, artifact.Ref{Name: "(subject)", Digest: subject.Digest}) + } + defer t.Close() + raw, err := r.requireCurrentRecord(t, subject.Digest) + if err != nil { + return nil, err + } + if subject.Size != 0 && subject.Size != int64(len(raw)) { + return nil, fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, len(raw)) + } + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, err + } + if payload == nil { + return nil, fmt.Errorf("%w: the signer returned no evidence", errUtils.ErrArtifactVerificationFailed) + } + stored, err := r.storeEvidence(t, subject, payload) + if err != nil { + return nil, err + } + return []artifact.SignatureEvidence{stored}, nil +} + +// storeEvidence validates and writes one evidence payload. +func (r *Repository) storeEvidence(t *tree, subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (artifact.SignatureEvidence, error) { + if caps := r.Capabilities(); !caps.HasEvidenceFormat(payload.Format) { + return artifact.SignatureEvidence{}, artifact.NewUnsupportedCapability(r.alias, "sign", "evidence/"+string(payload.Format), + fmt.Sprintf("the repository stores and verifies %v evidence; configure a verifier for this format", caps.EvidenceFormats)) + } + evidence, err := layout.NewEvidence(subject, payload) + if err != nil { + return artifact.SignatureEvidence{}, err + } + raw, err := evidence.Marshal() + if err != nil { + return artifact.SignatureEvidence{}, err + } + locator := layout.SignaturePath(subject.Kind, subject.Digest, evidence.Digest()) + if err := t.writeAtomic(locator, raw); err != nil { + return artifact.SignatureEvidence{}, fmt.Errorf("%w: store evidence: %w", errUtils.ErrArtifactUploadFailed, err) + } + return evidence.Reference(locator), nil +} + +// Verify implements [artifact.Repository]. It authenticates the stored evidence with the injected +// verifier; it never trusts a cached verdict. No evidence, an unverifiable format, or an +// unsatisfied policy is a failure. +func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "local.Repository.Verify")() + + if err := r.checkSubject("verify", subject); err != nil { + return nil, err + } + verifier := r.settings.verifier + if verifier == nil { + return nil, artifact.NewUnsupportedCapability(r.alias, "verify", "evidence/verify", "no signature verifier is configured for this repository") + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, artifact.Ref{Name: "(subject)", Digest: subject.Digest}) + } + defer t.Close() + if _, err := r.requireCurrentRecord(t, subject.Digest); err != nil { + return nil, err + } + loaded, diags := t.loadEvidence(subject.Digest) + blobs := evidenceBlobs(subject, loaded, &diags) + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed, Diagnostics: diags} + if fingerprint, fingerprintErr := policy.Fingerprint(); fingerprintErr == nil { + result.PolicyFingerprint = fingerprint + } + if len(blobs) == 0 { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-evidence", Message: "no signature evidence is stored for the subject"}) + return result, fmt.Errorf("%w: no signature evidence is stored for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + verified, verifyErr := verifier.Verify(ctx, subject, blobs, policy) + return r.finishVerification(result, verified, verifyErr) +} + +// evidenceBlobs converts the evidence bound to the subject, reporting any bound to another subject. +func evidenceBlobs(subject artifact.SignatureSubject, loaded []loadedEvidence, diags *[]artifact.Diagnostic) []artifact.EvidenceBlob { + blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) + for index := range loaded { + stored := loaded[index].evidence + if stored.Subject.Kind != subject.Kind || stored.Subject.Digest != subject.Digest { + *diags = append(*diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "subject-mismatch", Message: loaded[index].locator + " is bound to a different subject and was ignored"}) + continue + } + blobs = append(blobs, stored.Blob(loaded[index].locator)) + } + return blobs +} + +// finishVerification merges the verifier's result with the storage diagnostics and turns anything +// but a verified status into an error. +func (r *Repository) finishVerification(base, verified *artifact.VerificationResult, verifyErr error) (*artifact.VerificationResult, error) { + result := base + if verified != nil { + merged := *verified + merged.Subject = base.Subject + merged.Diagnostics = append(append([]artifact.Diagnostic(nil), base.Diagnostics...), verified.Diagnostics...) + if merged.PolicyFingerprint == "" { + merged.PolicyFingerprint = base.PolicyFingerprint + } + result = &merged + } + switch { + case verifyErr != nil && errors.Is(verifyErr, errUtils.ErrArtifactVerificationFailed): + return result, verifyErr + case verifyErr != nil: + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) + case result.Status == artifact.VerificationFailed: + return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) + } + return result, nil +} + +// ReadEvidence implements [artifact.EvidenceReader]: it exports the stored evidence blobs of a +// pinned artifact so a copy through the shared staging path can carry them along. The evidence is +// untrusted until verified. +func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "local.Repository.ReadEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { + return nil, err + } + loaded, _ := t.loadEvidence(pinned.RecordDigest) + blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) + for index := range loaded { + blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) + } + return blobs, nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this +// repository already holds. The evidence must bind to the pinned Record and carry matching bytes; +// importing never changes the Record or the reference. +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "local.Repository.WriteEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { + return nil, err + } + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + blob := &evidence[index] + subject := blob.Evidence.Subject + if err := r.checkSubject("write evidence", subject); err != nil { + return stored, err + } + if subject.Digest != pinned.RecordDigest { + return stored, fmt.Errorf("%w: evidence is bound to %s, not to the pinned Record %s", errUtils.ErrArtifactRefInvalid, subject.Digest, pinned.RecordDigest) + } + if blob.Evidence.Descriptor.Digest != digest.FromBytes(blob.Data) { + return stored, fmt.Errorf("%w: evidence bytes do not match their descriptor %s", errUtils.ErrArtifactIntegrityFailed, blob.Evidence.Descriptor.Digest) + } + item, err := r.storeEvidence(t, subject, &artifact.EvidencePayload{Format: blob.Evidence.Format, MediaType: blob.Evidence.Descriptor.MediaType, Data: blob.Data}) + if err != nil { + return stored, err + } + stored = append(stored, item) + } + return stored, nil +} diff --git a/pkg/artifact/local/faults.go b/pkg/artifact/local/faults.go new file mode 100644 index 00000000000..cd07592fccc --- /dev/null +++ b/pkg/artifact/local/faults.go @@ -0,0 +1,62 @@ +package local + +import ( + "errors" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// faultPoint names a point in the publication lifecycle at which tests simulate a crash. +type faultPoint string + +// Fault points, in lifecycle order. +const ( + // The before-payload point fires after the pending state was recorded and before any payload + // byte is staged. + faultBeforePayload faultPoint = "before-payload" + + // The during-payload point fires while the first payload blob is being staged. + faultDuringPayload faultPoint = "during-payload" + + // The after-metadata point fires after the Record, payload, and representation are in place + // and before the reference is completed. + faultAfterMetadata faultPoint = "after-metadata" + + // The after-commit point fires after the reference was completed, before cleanup and before + // the caller sees the response. + faultAfterCommit faultPoint = "after-commit" +) + +// faultHook returns a non-nil error to simulate a crash at the point. +type faultHook func(point faultPoint) error + +// crashError marks an injected crash. A crashed process cannot clean up after itself, so the +// publication leaves its staging and journal exactly as a real interruption would. +type crashError struct{ err error } + +func (c *crashError) Error() string { + defer perf.Track(nil, "local.crashError.Error")() + + return c.err.Error() +} + +func (c *crashError) Unwrap() error { + defer perf.Track(nil, "local.crashError.Unwrap")() + + return c.err +} + +func (s *settings) fault(point faultPoint) error { + if s.faults == nil { + return nil + } + if err := s.faults(point); err != nil { + return &crashError{err: err} + } + return nil +} + +func isCrash(err error) bool { + var crash *crashError + return errors.As(err, &crash) +} diff --git a/pkg/artifact/local/fsys.go b/pkg/artifact/local/fsys.go new file mode 100644 index 00000000000..ffd8b9dad96 --- /dev/null +++ b/pkg/artifact/local/fsys.go @@ -0,0 +1,255 @@ +package local + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Modes of directories and files this package creates itself. +const ( + dirMode fs.FileMode = 0o755 + privateDir fs.FileMode = 0o700 + privateFile fs.FileMode = 0o600 + publicFile fs.FileMode = 0o644 + randomBytes = 8 + copyBufBytes = 32 * 1024 +) + +// errRootMissing reports that the repository root directory does not exist yet. Read paths treat +// it as an empty repository; only writes create the root. +var errRootMissing = errors.New("repository root does not exist") + +// tree is the repository root opened as an os.Root, so no operation can leave the root even +// through a symlink planted inside it. Slash-separated relative paths are converted for the host. +type tree struct { + *os.Root + + abs string +} + +// openTree opens the repository root. With create set, a missing root is created first. +func (r *Repository) openTree(create bool) (*tree, error) { + if create { + if err := os.MkdirAll(r.root, dirMode); err != nil { + return nil, fmt.Errorf("%w: create repository root %q: %w", errUtils.ErrArtifactConfigInvalid, r.root, err) + } + } + root, err := os.OpenRoot(r.root) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, errRootMissing + case err != nil: + return nil, fmt.Errorf("%w: open repository root %q: %w", errUtils.ErrArtifactConfigInvalid, r.root, err) + } + return &tree{Root: root, abs: r.root}, nil +} + +// native converts a slash-separated relative path to the host form. +func native(rel string) string { + return filepath.FromSlash(rel) +} + +// absolute returns the absolute host path of a slash-separated relative path. +func (t *tree) absolute(rel string) string { + return filepath.Join(t.abs, native(rel)) +} + +// exists reports whether a path exists without following a final symlink. +func (t *tree) exists(rel string) (bool, error) { + _, err := t.Lstat(native(rel)) + switch { + case err == nil: + return true, nil + case errors.Is(err, fs.ErrNotExist): + return false, nil + default: + return false, err + } +} + +// readBounded reads a whole regular file of at most limit bytes. A missing file returns an error +// that wraps fs.ErrNotExist. +func (t *tree) readBounded(rel string, limit int64) ([]byte, error) { + file, err := t.Open(native(rel)) + if err != nil { + return nil, err + } + defer file.Close() + return artifact.ReadBounded(file, limit) +} + +// ensureParent creates the parent directories of rel. +func (t *tree) ensureParent(rel string) error { + parent := path.Dir(rel) + if parent == "." || parent == "" { + return nil + } + return t.MkdirAll(native(parent), dirMode) +} + +// writeAtomic writes data at rel through a temporary file in the managed temp directory, syncs +// it, applies the public file mode, and renames it over the destination. Readers see the old or the new file, +// never a partial one. +func (t *tree) writeAtomic(rel string, data []byte) error { + if err := t.MkdirAll(native(layout.TempDir()), privateDir); err != nil { + return err + } + if err := t.ensureParent(rel); err != nil { + return err + } + suffix, err := randomSuffix() + if err != nil { + return err + } + tmp := layout.TempDir() + "/" + path.Base(rel) + "." + suffix + ".tmp" + file, err := t.OpenFile(native(tmp), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + _, writeErr := file.Write(data) + if writeErr == nil { + writeErr = file.Sync() + } + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr == nil { + writeErr = t.Chmod(native(tmp), publicFile) + } + if writeErr == nil { + writeErr = t.Rename(native(tmp), native(rel)) + } + if writeErr != nil { + _ = t.Remove(native(tmp)) + return writeErr + } + return nil +} + +func randomSuffix() (string, error) { + buf := make([]byte, randomBytes) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} + +// removeAll deletes a path and everything below it inside the root. Directories that lack owner +// permissions (a payload may legitimately contain mode 0555 directories) are made writable first +// so removal cannot be blocked by the artifact's own modes. +func (t *tree) removeAll(rel string) error { + if ok, err := t.exists(rel); err != nil || !ok { + return err + } + t.makeWritable(rel) + return t.RemoveAll(native(rel)) +} + +// makeWritable gives every directory below rel owner access. Failures are ignored: removal reports +// the real error. +func (t *tree) makeWritable(rel string) { + info, err := t.Lstat(native(rel)) + if err != nil || !info.IsDir() { + return + } + _ = t.Chmod(native(rel), info.Mode().Perm()|privateDir) + children, err := fs.ReadDir(t.FS(), filepath.ToSlash(native(rel))) + if err != nil { + return + } + for _, child := range children { + if child.IsDir() { + t.makeWritable(rel + "/" + child.Name()) + } + } +} + +// removeEmptyParents removes the now-empty parent directories of rel up to, never including, the +// root. A non-empty directory stops the walk without error. +func (t *tree) removeEmptyParents(rel string) { + for parent := path.Dir(rel); parent != "." && parent != "" && !layout.InManagedNamespace(parent); parent = path.Dir(parent) { + if err := t.Remove(native(parent)); err != nil { + return + } + } +} + +// copyVerified copies src to dst, enforcing the declared size and SHA-256 digest. It never writes +// more than size bytes plus the one extra byte needed to detect an oversize source. +func copyVerified(dst io.Writer, src io.Reader, want digest.Digest, size int64) error { + hasher := want.Algorithm().Hash() + written, err := io.CopyBuffer(io.MultiWriter(dst, hasher), io.LimitReader(src, size+1), make([]byte, copyBufBytes)) + switch { + case err != nil: + return err + case written != size: + return fmt.Errorf("%w: blob is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, written, size) + } + if got := digest.NewDigest(want.Algorithm(), hasher); got != want { + return fmt.Errorf("%w: blob digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, want) + } + return nil +} + +// stageFile creates rel exclusively in the tree and copies the verified blob into it. +func (t *tree) stageFile(rel string, src io.Reader, want digest.Digest, size int64) error { + if err := t.ensureParent(rel); err != nil { + return err + } + file, err := t.OpenFile(native(rel), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + copyErr := copyVerified(file, src, want, size) + if copyErr == nil { + copyErr = file.Sync() + } + if closeErr := file.Close(); copyErr == nil { + copyErr = closeErr + } + return copyErr +} + +// writeNew creates rel exclusively with data and mode. +func (t *tree) writeNew(rel string, data []byte, mode fs.FileMode) error { + if err := t.ensureParent(rel); err != nil { + return err + } + file, err := t.OpenFile(native(rel), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + _, writeErr := file.Write(data) + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr == nil { + writeErr = t.Chmod(native(rel), mode) + } + return writeErr +} + +// moveAside renames rel to dest when rel exists, creating the parent of dest. +func (t *tree) moveAside(rel, dest string) error { + ok, err := t.exists(rel) + if err != nil || !ok { + return err + } + if err := t.ensureParent(dest); err != nil { + return err + } + return t.Rename(native(rel), native(dest)) +} diff --git a/pkg/artifact/local/helpers_test.go b/pkg/artifact/local/helpers_test.go new file mode 100644 index 00000000000..40987ca3d49 --- /dev/null +++ b/pkg/artifact/local/helpers_test.go @@ -0,0 +1,262 @@ +package local + +import ( + "context" + "io" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guards for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Packaging: "none", Permissions: "0600"} + +const ( + testName = "team/app" + testVersion = "1.0.0" +) + +// capturedAt is the fixed preparation time of the test artifacts. +var capturedAt = time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + +// repoSpec configures a repository under test. +type repoSpec struct { + kind string + packaging artifact.Packaging + permissions string + keyPattern string + root string + opts []Option +} + +func (s *repoSpec) config(t *testing.T) *schema.ArtifactRepositoryConfig { + t.Helper() + options := map[string]any{"path": s.root} + if s.keyPattern != "" { + options["key_pattern"] = s.keyPattern + } + cfg := &schema.ArtifactRepositoryConfig{Kind: s.kind, Options: options, Permissions: s.permissions} + if s.packaging != "" { + cfg.Packaging = string(s.packaging) + } + return cfg +} + +// newRepo builds a repository under a fresh temporary root unless the spec names one. +func newRepo(t *testing.T, spec *repoSpec) *Repository { + t.Helper() + spec = spec.clone() + if spec.root == "" { + spec.root = filepath.Join(t.TempDir(), "repo") + } + if spec.kind == "" { + spec.kind = artifact.RepositoryKindLocalDir + } + repo, err := newRepository("test", spec.kind, spec.config(t), spec.opts) + require.NoError(t, err) + return repo +} + +func dirSpec() *repoSpec { return &repoSpec{kind: artifact.RepositoryKindLocalDir} } + +func archiveSpec(packaging artifact.Packaging) *repoSpec { + return &repoSpec{kind: artifact.RepositoryKindLocalArchive, packaging: packaging} +} + +// clone returns an independent copy, so a case can adjust its spec without leaking into others. +func (s *repoSpec) clone() *repoSpec { + c := *s + c.opts = append([]Option(nil), s.opts...) + return &c +} + +func (s *repoSpec) withOpts(opts ...Option) *repoSpec { + c := s.clone() + c.opts = append(c.opts, opts...) + return c +} + +func (s *repoSpec) withRoot(root string) *repoSpec { + c := s.clone() + c.root = root + return c +} + +func (s *repoSpec) withPattern(pattern string) *repoSpec { + c := s.clone() + c.keyPattern = pattern + return c +} + +func (s *repoSpec) withPermissions(permissions string) *repoSpec { + c := s.clone() + c.permissions = permissions + return c +} + +// prepareSpec describes the artifact a test prepares with the production files producer. +type prepareSpec struct { + name string + version string + packaging artifact.Packaging + files map[string]string + at time.Time + opID string + labels map[string]string +} + +func (p *prepareSpec) clone() *prepareSpec { + c := *p + return &c +} + +func (p *prepareSpec) withDefaults(repo *Repository) *prepareSpec { + p = p.clone() + if p.name == "" { + p.name = testName + } + if p.version == "" { + p.version = testVersion + } + if p.packaging == "" { + p.packaging = repo.packaging + } + if p.at.IsZero() { + p.at = capturedAt + } + if p.opID == "" { + p.opID = "op-" + p.name + "-" + p.version + } + if p.files == nil { + p.files = map[string]string{"data.txt": "hello\n"} + } + return p +} + +func prepare(t *testing.T, repo *Repository, spec *prepareSpec) *artifact.PreparedArtifact { + t.Helper() + spec = spec.withDefaults(repo) + src := t.TempDir() + for name, content := range spec.files { + fixtures.WriteFile(t, src, name, content, 0o644) + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: spec.name, Version: spec.version, Packaging: spec.packaging, + Metadata: artifact.Metadata{Labels: spec.labels}, + OperationID: spec.opID, PreparedAt: spec.at, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +// publish prepares and publishes, failing the test on error. +func publish(t *testing.T, repo *Repository, spec *prepareSpec, options artifact.PublishOptions) *artifact.PublishResult { + t.Helper() + prepared := prepare(t, repo, spec) + result, err := repo.Publish(context.Background(), prepared, options) + require.NoError(t, err) + return result +} + +func ref(name, version string) artifact.Ref { return artifact.Ref{Name: name, Version: version} } + +func defaultRef() artifact.Ref { return ref(testName, testVersion) } + +func resolve(t *testing.T, repo *Repository, r artifact.Ref) artifact.PinnedReference { + t.Helper() + pinned, err := repo.Resolve(context.Background(), r) + require.NoError(t, err) + return pinned +} + +// readAll drains every stream of the content, returning the first read error. +func readAll(t *testing.T, content *artifact.Content) error { + t.Helper() + for index := range content.Streams { + if _, err := io.Copy(io.Discard, content.Streams[index].Reader); err != nil { + return err + } + } + return nil +} + +// listFiles returns the sorted slash-separated relative paths of every file below dir. +func listFiles(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.WalkDir(dir, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() { + rel, relErr := filepath.Rel(dir, p) + require.NoError(t, relErr) + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + require.NoError(t, err) + sort.Strings(out) + return out +} + +// payloadFiles lists the files outside the managed namespace. +func payloadFiles(t *testing.T, root string) []string { + t.Helper() + var out []string + for _, p := range listFiles(t, root) { + if !layout.InManagedNamespace(p) { + out = append(out, p) + } + } + return out +} + +func newSigner() *artifacttest.FakeSigner { + return &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} +} + +func withFaults(hook faultHook) Option { + return func(s *settings) { s.faults = hook } +} + +// crashAt returns a fault hook that fails once at the point. +func crashAt(point faultPoint) faultHook { + fired := false + return func(p faultPoint) error { + if p == point && !fired { + fired = true + return errCrash + } + return nil + } +} + +// armedFault is a fault hook that does nothing until a test arms it. +type armedFault struct{ hook faultHook } + +func (a *armedFault) option() Option { + return withFaults(func(point faultPoint) error { + if a.hook == nil { + return nil + } + return a.hook(point) + }) +} + +// helperConfig is the configuration the lock helper process builds its repository from. +func helperConfig(root string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": root}} +} diff --git a/pkg/artifact/local/integrity_test.go b/pkg/artifact/local/integrity_test.go new file mode 100644 index 00000000000..b7f9da4aaca --- /dev/null +++ b/pkg/artifact/local/integrity_test.go @@ -0,0 +1,258 @@ +package local + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// tamper rewrites one file below the repository root after applying edit to its bytes. +func tamper(t *testing.T, root, rel string, edit func([]byte) []byte) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + original, err := os.ReadFile(path) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.Chmod(path, 0o644)) + require.NoError(t, os.WriteFile(path, edit(original), info.Mode().Perm())) +} + +func flipFirstByte(data []byte) []byte { + out := append([]byte(nil), data...) + out[0] ^= 0xff + return out +} + +func appendJunk(data []byte) []byte { + return append(append([]byte(nil), data...), []byte("junk")...) +} + +func truncate(data []byte) []byte { return data[:len(data)/2] } + +// integrityCase names one corruption and where it surfaces. +type integrityCase struct { + name string + // target returns the file to corrupt, relative to the root, given the stored location. + target func(location string, record *artifact.Record) string + edit func([]byte) []byte + // surfaces lists the operations that must fail with an integrity error. + surfaces []string +} + +func TestTamperedState_FailsIntegrity(t *testing.T) { + ref := defaultRef() + managed := func(path func(key string, recordDigest string) string) func(string, *artifact.Record) string { + return func(_ string, record *artifact.Record) string { + digestValue, err := record.RecordDigest() + require.NoError(t, err) + return path(layout.ReferenceKey(record.Name, record.Version), digestValue.Encoded()) + } + } + cases := []integrityCase{ + { + name: "record file bytes", + target: managed(func(_, recordHex string) string { + return ".atmos-artifacts/v1/records/" + recordHex + ".json" + }), + edit: appendJunk, surfaces: []string{"resolve", "list"}, + }, + { + name: "reference file garbage", + target: managed(func(key, _ string) string { return layout.ReferencePath(key) }), + edit: truncate, + surfaces: []string{"resolve", "list"}, + }, + { + name: "representation file", + target: managed(func(key, _ string) string { return layout.RepresentationPath(key) }), + edit: flipFirstByte, + surfaces: []string{"inspect", "read", "document"}, + }, + } + for _, spec := range allSpecs { + for _, tc := range cases { + t.Run(specName(spec)+"/"+tc.name, func(t *testing.T) { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, ref) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + tamper(t, repo.root, tc.target(result.Location, inspection.Record), tc.edit) + assertIntegrityFailure(t, repo, ref, &pinned, tc.surfaces) + }) + } + } +} + +func assertIntegrityFailure(t *testing.T, repo *Repository, r artifact.Ref, pinned *artifact.PinnedReference, surfaces []string) { + t.Helper() + ctx := context.Background() + for _, surface := range surfaces { + var err error + switch surface { + case "resolve": + _, err = repo.Resolve(ctx, r) + case "list": + _, err = repo.List(ctx, artifact.ListQuery{}) + case "inspect": + _, err = repo.Inspect(ctx, *pinned) + case "read": + _, err = repo.Read(ctx, *pinned) + case "document": + _, err = repo.OpenDocument(ctx, *pinned, artifact.DocumentReadme) + } + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, surface) + } +} + +func TestTamperedMetadataSidecar_FailsIntegrity(t *testing.T) { + for _, tc := range []struct { + spec *repoSpec + sidecar func(location string) string + describe string + }{ + {archiveSpec(artifact.PackagingTgz), func(l string) string { return l + ".metadata.json" }, "archive metadata sidecar"}, + {dirSpec(), func(l string) string { return l + "/.atmos/artifact/manifest.json" }, "directory manifest"}, + } { + t.Run(tc.describe, func(t *testing.T) { + repo := newRepo(t, tc.spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + rel := filepath.ToSlash(mustRel(t, repo.root, result.Location)) + tamper(t, repo.root, tc.sidecar(rel), appendJunk) + + _, err := repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } +} + +func mustRel(t *testing.T, base, target string) string { + t.Helper() + rel, err := filepath.Rel(base, target) + require.NoError(t, err) + return rel +} + +func TestMissingSidecars_FailExplicitly(t *testing.T) { + t.Run("archive metadata sidecar", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(result.Location+".metadata.json")) + _, err := repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a completed reference promised the sidecar exists") + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("archive document sidecar", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location+".docs", "README.md"))) + _, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.NotErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a lost sidecar is damage, not an absent optional document") + _, err = repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.NoError(t, err, "the other sidecar is unaffected") + }) + t.Run("directory distribution", func(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location, ".atmos", "artifact", "CHANGELOG.md"))) + _, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +func TestTamperedPayloadAndDocuments_FailAtTheDeclaredDigest(t *testing.T) { + t.Run("archive bytes", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, result.Location)), flipFirstByteInPlace) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err, "opening is cheap; verification happens as the bytes are consumed") + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("archive truncated", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingZip)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, result.Location)), truncate) + _, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a size mismatch is caught before any byte is read") + }) + t.Run("directory file", func(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, filepath.Join(result.Location, "nested", "deep", "file.txt"))), sameLengthEdit) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("document", func(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + docPath := filepath.Join(result.Location, ".atmos", "artifact", "README.md") + if spec.kind == artifact.RepositoryKindLocalArchive { + docPath = filepath.Join(result.Location+".docs", "README.md") + } + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, docPath)), sameLengthEdit) + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + _, err = io.Copy(io.Discard, stream.Reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, spec.kind) + } + }) +} + +func flipFirstByteInPlace(data []byte) []byte { + out := append([]byte(nil), data...) + out[len(out)/2] ^= 0xff + return out +} + +// sameLengthEdit changes content without changing its size, so only the digest can catch it. +func sameLengthEdit(data []byte) []byte { + out := append([]byte(nil), data...) + out[0] = 'X' + if len(data) > 0 && data[0] == 'X' { + out[0] = 'Y' + } + return out +} + +func TestExtraFilesInsideAPayloadAreNotAdopted(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.WriteFile(filepath.Join(result.Location, "injected.txt"), []byte("not in the inventory"), 0o644)) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + for index := range content.Streams { + assert.NotEqual(t, "injected.txt", content.Streams[index].Path, "only inventory files are ever served") + } +} diff --git a/pkg/artifact/local/job.go b/pkg/artifact/local/job.go new file mode 100644 index 00000000000..13687cd5ab2 --- /dev/null +++ b/pkg/artifact/local/job.go @@ -0,0 +1,247 @@ +package local + +import ( + "fmt" + "io/fs" + "runtime" + "sort" + + "github.com/google/uuid" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// publishJob is everything one publication attempt derives from the prepared artifact, computed +// once before any state changes. +type publishJob struct { + prepared *artifact.PreparedArtifact + record *artifact.Record + recordBytes []byte + recordDigest digest.Digest + inventory []artifact.Entry + key string + opID string + location string + staging string + payload []ocispec.Descriptor + documents []ocispec.Descriptor + staged int + installing bool +} + +// stagedPayload is the staging path that becomes the payload directory. +func (j *publishJob) stagedPayload() string { + return j.staging + "/payload" +} + +// retiredDir is where replaced content is moved aside until the new reference is completed. +func (j *publishJob) retiredDir() string { + return j.staging + "/retired" +} + +// blobStaged records one staged payload blob and fires the during-payload fault after the first. +func (j *publishJob) blobStaged(r *Repository) error { + j.staged++ + if j.staged == 1 { + return r.settings.fault(faultDuringPayload) + } + return nil +} + +// newJob validates a prepared artifact against the repository and derives the publication inputs. +func (r *Repository) newJob(prepared *artifact.PreparedArtifact) (*publishJob, error) { + if prepared == nil { + return nil, fmt.Errorf("%w: no prepared artifact to publish", errUtils.ErrArtifactConfigInvalid) + } + if !r.store.supports(prepared.Packaging) { + return nil, artifact.NewUnsupportedCapability(r.alias, "publish", "packaging/"+string(prepared.Packaging), + fmt.Sprintf("%s repositories store %v payloads", r.kind, r.Capabilities().Packagings)) + } + if err := r.checkArchivePermissions(); err != nil { + return nil, err + } + canonical, err := prepared.Record.Canonical() + if err != nil { + return nil, err + } + record := prepared.Record + opID := prepared.OperationID + if opID == "" { + opID = uuid.NewString() + } + job := &publishJob{ + prepared: prepared, record: &record, recordBytes: canonical, recordDigest: digest.FromBytes(canonical), + inventory: sortedEntries(record.Inventory), key: layout.ReferenceKey(record.Name, record.Version), + opID: opID, staging: layout.StagingPath(opID), + } + if err := job.describeBlobs(); err != nil { + return nil, err + } + if err := r.store.validate(job); err != nil { + return nil, err + } + return job, nil +} + +func sortedEntries(entries []artifact.Entry) []artifact.Entry { + out := append([]artifact.Entry(nil), entries...) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// describeBlobs derives the representation descriptors from the staged blobs and checks that the +// blobs agree with the Record before anything is written. +func (j *publishJob) describeBlobs() error { + j.payload, j.documents = []ocispec.Descriptor{}, []ocispec.Descriptor{} + docs := map[artifact.DocumentRole]bool{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Open == nil { + return fmt.Errorf("%w: staged blob %q cannot be opened", errUtils.ErrArtifactConfigInvalid, blob.Path) + } + descriptor := blob.Descriptor + if blob.Role == artifact.BlobDocument { + if docs[blob.DocumentRole] { + return fmt.Errorf("%w: duplicate %s document blob", errUtils.ErrArtifactRecordInvalid, blob.DocumentRole) + } + docs[blob.DocumentRole] = true + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: layout.DocumentFileName(blob.DocumentRole)} + j.documents = append(j.documents, descriptor) + continue + } + if blob.Path != "" { + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: blob.Path} + } + j.payload = append(j.payload, descriptor) + } + for index := range j.record.Documents { + if !docs[j.record.Documents[index].Role] { + return fmt.Errorf("%w: the Record lists a %s document with no staged blob", errUtils.ErrArtifactIntegrityFailed, j.record.Documents[index].Role) + } + } + if len(docs) != len(j.record.Documents) { + return fmt.Errorf("%w: %d document blobs but %d documents in the Record", errUtils.ErrArtifactIntegrityFailed, len(docs), len(j.record.Documents)) + } + return nil +} + +// indexBlobs maps file payload blobs by inventory path and document blobs by role. +func (j *publishJob) indexBlobs() (map[string]*artifact.StagedBlob, map[artifact.DocumentRole]*artifact.StagedBlob, error) { + files := map[string]*artifact.StagedBlob{} + docs := map[artifact.DocumentRole]*artifact.StagedBlob{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Role == artifact.BlobDocument { + docs[blob.DocumentRole] = blob + continue + } + if _, duplicate := files[blob.Path]; duplicate || blob.Path == "" { + return nil, nil, fmt.Errorf("%w: payload blob path %q is missing or duplicated", errUtils.ErrArtifactIntegrityFailed, blob.Path) + } + files[blob.Path] = blob + } + if err := j.matchDocuments(docs); err != nil { + return nil, nil, err + } + return files, docs, nil +} + +// archiveBlobs returns the single archive payload blob and the document blobs. +func (j *publishJob) archiveBlobs() (*artifact.StagedBlob, map[artifact.DocumentRole]*artifact.StagedBlob, error) { + var payload *artifact.StagedBlob + docs := map[artifact.DocumentRole]*artifact.StagedBlob{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Role == artifact.BlobDocument { + docs[blob.DocumentRole] = blob + continue + } + if payload != nil { + return nil, nil, fmt.Errorf("%w: an archive publication needs exactly one payload blob", errUtils.ErrArtifactPackagingInvalid) + } + payload = blob + } + if payload == nil { + return nil, nil, fmt.Errorf("%w: an archive publication needs exactly one payload blob", errUtils.ErrArtifactPackagingInvalid) + } + if err := j.matchDocuments(docs); err != nil { + return nil, nil, err + } + return payload, docs, nil +} + +// matchDocuments checks every document blob against the Record's descriptor for its role. +func (j *publishJob) matchDocuments(docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + for role, blob := range docs { + doc, ok := documentDescriptor(j.record, role) + if !ok || doc.Digest != blob.Descriptor.Digest || doc.Size != blob.Descriptor.Size { + return fmt.Errorf("%w: the staged %s document does not match the Record", errUtils.ErrArtifactIntegrityFailed, role) + } + } + return nil +} + +// representationDigest is the checksum of the transferred payload representation. +func (j *publishJob) representationDigest(packaging artifact.Packaging) digest.Digest { + if packaging == artifact.PackagingNone { + return dirRepresentationDigest(j.record) + } + if len(j.payload) == 1 { + return j.payload[0].Digest + } + return "" +} + +// permissionBits holds the ordinary permission bits of a mode. +const permissionBits fs.FileMode = 0o777 + +// fsMode converts a Record mode to a file mode. +func fsMode(mode uint32) fs.FileMode { + return fs.FileMode(mode) & permissionBits +} + +// checkArchivePermissions fails before any state changes when an explicitly requested archive +// mode cannot be enforced on this host. +func (r *Repository) checkArchivePermissions() error { + if r.kind != artifact.RepositoryKindLocalArchive || !r.modeExplicit || r.hostModes { + return nil + } + return artifact.NewUnsupportedCapability(r.alias, "publish", "permissions/archive-file-mode", + fmt.Sprintf("the host (%s) cannot enforce permissions %04o on the archive file; remove the permissions setting to accept the host default, or publish from a POSIX host", runtime.GOOS, r.archiveMode)) +} + +// chmodIfSupported applies a mode when the host can enforce one. +func (r *Repository) chmodIfSupported(t *tree, rel string, mode fs.FileMode) error { + if !r.hostModes { + return nil + } + return t.Chmod(native(rel), mode) +} + +// setArchiveMode applies the archive permissions to a staged archive file, sidecar, or sidecar +// directory. For an explicit request it also reads the mode back, because a filesystem that +// silently ignores chmod must not be reported as enforcing the configured permissions. +func (r *Repository) setArchiveMode(t *tree, rel string, mode fs.FileMode) error { + if !r.hostModes { + return nil + } + if err := t.Chmod(native(rel), mode); err != nil { + return err + } + if !r.modeExplicit { + return nil + } + info, err := t.Lstat(native(rel)) + if err != nil { + return err + } + if info.Mode().Perm() != mode { + return artifact.NewUnsupportedCapability(r.alias, "publish", "permissions/archive-file-mode", + fmt.Sprintf("the filesystem under %q did not keep mode %04o (got %04o)", r.root, mode, info.Mode().Perm())) + } + return nil +} diff --git a/pkg/artifact/local/kind_archive.go b/pkg/artifact/local/kind_archive.go new file mode 100644 index 00000000000..d841aeec9c1 --- /dev/null +++ b/pkg/artifact/local/kind_archive.go @@ -0,0 +1,210 @@ +package local + +import ( + "context" + "fmt" + "io" + "io/fs" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Staging names inside an operation's staging directory. +const ( + stagedArchive = "archive" + stagedMetadata = "metadata.json" + stagedDocs = "docs" +) + +// archiveStore stores a payload as one tar, tgz, or zip file with adjacent metadata and document +// sidecars. The configured archive permissions apply to the file and its sidecars, never to the +// members inside the archive. +type archiveStore struct { + repo *Repository +} + +var _ payloadStore = archiveStore{} + +func (archiveStore) supports(packaging artifact.Packaging) bool { + switch packaging { + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return true + default: + return false + } +} + +// validate checks that the publication carries exactly one archive blob matching the Record's +// documents, before any state changes. +func (archiveStore) validate(job *publishJob) error { + _, _, err := job.archiveBlobs() + return err +} + +// stage copies the archive and the document sidecars into the staging directory and applies the +// archive permissions before anything becomes visible at the final location. +func (s archiveStore) stage(ctx context.Context, t *tree, job *publishJob) error { + payload, docs, err := job.archiveBlobs() + if err != nil { + return err + } + archivePath := job.staging + "/" + stagedArchive + if err := s.copyBlob(ctx, t, payload, archivePath); err != nil { + return err + } + if err := job.blobStaged(s.repo); err != nil { + return err + } + if err := s.repo.setArchiveMode(t, archivePath, s.repo.archiveMode); err != nil { + return err + } + if err := s.stageDocuments(ctx, t, job, docs); err != nil { + return err + } + metadata := job.staging + "/" + stagedMetadata + if err := t.writeNew(metadata, job.recordBytes, privateFile); err != nil { + return err + } + return s.repo.setArchiveMode(t, metadata, s.repo.archiveMode) +} + +// stageDocuments copies the document sidecars and gives their directory the matching mode. +func (s archiveStore) stageDocuments(ctx context.Context, t *tree, job *publishJob, docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + for role, blob := range docs { + docPath := job.staging + "/" + stagedDocs + "/" + layout.DocumentFileName(role) + if err := s.copyBlob(ctx, t, blob, docPath); err != nil { + return err + } + if err := s.repo.setArchiveMode(t, docPath, s.repo.archiveMode); err != nil { + return err + } + } + if len(docs) == 0 { + return nil + } + return s.repo.setArchiveMode(t, job.staging+"/"+stagedDocs, directoryMode(s.repo.archiveMode)) +} + +func (archiveStore) copyBlob(ctx context.Context, t *tree, blob *artifact.StagedBlob, dest string) error { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(dest, reader, blob.Descriptor.Digest, blob.Descriptor.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %s: %w", dest, copyErr) + } + return nil +} + +// install retires the previous archive and sidecars, then renames the staged ones into place. +func (archiveStore) install(t *tree, job *publishJob) error { + retired := job.retiredDir() + for _, move := range []struct{ from, to string }{ + {job.location, retired + "/archive"}, + {layout.DocsSidecarDir(job.location), retired + "/docs"}, + {layout.MetadataSidecarPath(job.location), retired + "/metadata.json"}, + } { + if err := t.moveAside(move.from, move.to); err != nil { + return err + } + } + if err := t.ensureParent(job.location); err != nil { + return err + } + installs := []struct{ from, to string }{{job.staging + "/" + stagedArchive, job.location}} + if ok, err := t.exists(job.staging + "/" + stagedDocs); err != nil { + return err + } else if ok { + installs = append(installs, struct{ from, to string }{job.staging + "/" + stagedDocs, layout.DocsSidecarDir(job.location)}) + } + installs = append(installs, struct{ from, to string }{job.staging + "/" + stagedMetadata, layout.MetadataSidecarPath(job.location)}) + for _, install := range installs { + if err := t.Rename(native(install.from), native(install.to)); err != nil { + return err + } + } + return nil +} + +// streams returns the archive as the single payload stream plus the document sidecars. +func (s archiveStore) streams(_ context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) { + if len(v.rep.Payload) != 1 { + return nil, fmt.Errorf("%w: an archive representation must have exactly one payload, found %d", errUtils.ErrArtifactIntegrityFailed, len(v.rep.Payload)) + } + descriptor := v.rep.Payload[0] + file, info, err := t.openFile(v.rep.Location, "archive") + if err != nil { + return nil, err + } + if info.Size() != descriptor.Size { + _ = file.Close() + return nil, fmt.Errorf("%w: archive %q is %d bytes; the representation declares %d", errUtils.ErrArtifactIntegrityFailed, v.rep.Location, info.Size(), descriptor.Size) + } + streams := []artifact.ContentStream{{ + Descriptor: descriptor, Role: artifact.BlobPayload, + Reader: newVerifyingReader(file, descriptor.Digest, descriptor.Size, guard), + }} + for index := range v.record.Documents { + doc := &v.record.Documents[index] + reader, err := s.document(t, v, doc, guard) + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{Descriptor: blobDescriptor(doc), Role: artifact.BlobDocument, Reader: reader}) + } + return streams, nil +} + +// document opens a document from its adjacent sidecar. A missing sidecar is an integrity failure: +// reading it must not fall back to opening the archive, and the completed reference promised the +// sidecar exists. +func (archiveStore) document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) { + file, _, err := t.openFile(layout.DocumentSidecarPath(v.rep.Location, doc.Role), string(doc.Role)+" sidecar") + if err != nil { + return nil, err + } + return newVerifyingReader(file, doc.Digest, doc.Size, guard), nil +} + +func (archiveStore) verifySidecars(t *tree, v *view) error { + return t.verifyBytes(layout.MetadataSidecarPath(v.rep.Location), "metadata sidecar", v.recordBytes) +} + +// remove deletes the archive, its metadata sidecar, and its document sidecars. +func (archiveStore) remove(t *tree, v *view) ([]string, error) { + location := v.rep.Location + removed := []string{} + targets := []string{location, layout.MetadataSidecarPath(location)} + for index := range v.record.Documents { + targets = append(targets, layout.DocumentSidecarPath(location, v.record.Documents[index].Role)) + } + for _, target := range targets { + ok, err := t.exists(target) + if err != nil { + return removed, err + } + if !ok { + continue + } + if err := t.Remove(native(target)); err != nil { + return removed, err + } + removed = append(removed, t.absolute(target)) + } + _ = t.Remove(native(layout.DocsSidecarDir(location))) + return removed, nil +} + +// readBits holds the read permission bits; execute follows read for a directory. +const readBits fs.FileMode = 0o444 + +// directoryMode derives the directory mode that goes with a file mode: execute follows read. +func directoryMode(file fs.FileMode) fs.FileMode { + return file | (file&readBits)>>2 +} diff --git a/pkg/artifact/local/kind_dir.go b/pkg/artifact/local/kind_dir.go new file mode 100644 index 00000000000..ad4b444aadc --- /dev/null +++ b/pkg/artifact/local/kind_dir.go @@ -0,0 +1,300 @@ +package local + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// filePayloadMediaType is the media type of one copied file. +const filePayloadMediaType = "application/octet-stream" + +// dirStore stores a payload as ordinary copied files below its resolved location, with the +// distribution documents and manifest adjacent at /.atmos/artifact/. +type dirStore struct { + repo *Repository +} + +var _ payloadStore = dirStore{} + +func (dirStore) supports(packaging artifact.Packaging) bool { + return packaging == artifact.PackagingNone +} + +// validate checks the staged blobs against the inventory before the publication starts, so a +// malformed preparation never leaves a pending reference behind. +func (dirStore) validate(job *publishJob) error { + blobs, _, err := job.indexBlobs() + if err != nil { + return err + } + files := 0 + for index := range job.inventory { + entry := &job.inventory[index] + if entry.Path == layout.DistributionDir || strings.HasPrefix(entry.Path, layout.DistributionDir+"/") { + return fmt.Errorf("%w: inventory path %q is inside the generated %s directory", errUtils.ErrArtifactUnsafePath, entry.Path, layout.DistributionDir) + } + if entry.Type != artifact.EntryFile { + continue + } + blob, ok := blobs[entry.Path] + if !ok || blob.Descriptor.Digest != entry.Digest || blob.Descriptor.Size != entry.Size { + return fmt.Errorf("%w: no staged blob matches inventory file %q", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + files++ + } + if files != len(blobs) { + return fmt.Errorf("%w: %d staged payload blobs but %d inventory files", errUtils.ErrArtifactIntegrityFailed, len(blobs), files) + } + return nil +} + +// stage copies every inventory entry and the distribution directory into the staging tree. +func (s dirStore) stage(ctx context.Context, t *tree, job *publishJob) error { + root := job.stagedPayload() + if err := t.MkdirAll(native(root), privateDir); err != nil { + return err + } + blobs, docs, err := job.indexBlobs() + if err != nil { + return err + } + if err := s.writeInventory(ctx, t, job, blobs); err != nil { + return err + } + if err := s.writeDistribution(ctx, t, job, docs); err != nil { + return err + } + return s.applyModes(t, job) +} + +func (s dirStore) writeInventory(ctx context.Context, t *tree, job *publishJob, blobs map[string]*artifact.StagedBlob) error { + root := job.stagedPayload() + for index := range job.inventory { + entry := &job.inventory[index] + target := root + "/" + entry.Path + switch entry.Type { + case artifact.EntryDirectory: + if err := t.MkdirAll(native(target), privateDir); err != nil { + return err + } + case artifact.EntrySymlink: + if err := t.ensureParent(target); err != nil { + return err + } + if err := t.Symlink(native(entry.Target), native(target)); err != nil { + return fmt.Errorf("%w: create link %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + default: + if err := s.stageEntry(ctx, t, blobs[entry.Path], target, entry); err != nil { + return err + } + if err := job.blobStaged(s.repo); err != nil { + return err + } + } + } + return nil +} + +func (s dirStore) stageEntry(ctx context.Context, t *tree, blob *artifact.StagedBlob, target string, entry *artifact.Entry) error { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(target, reader, entry.Digest, entry.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %q: %w", entry.Path, copyErr) + } + return s.repo.chmodIfSupported(t, target, fsMode(entry.Mode)) +} + +// writeDistribution writes the documents and the generated manifest.json that the payload +// carries so it stays self-describing. The manifest is the canonical Record. +func (s dirStore) writeDistribution(ctx context.Context, t *tree, job *publishJob, docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + dist := job.stagedPayload() + "/" + layout.DistributionDir + if err := t.MkdirAll(native(dist), dirMode); err != nil { + return err + } + for role, blob := range docs { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(dist+"/"+layout.DocumentFileName(role), reader, blob.Descriptor.Digest, blob.Descriptor.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %s: %w", role, copyErr) + } + if err := s.repo.chmodIfSupported(t, dist+"/"+layout.DocumentFileName(role), publicFile); err != nil { + return err + } + } + return t.writeNew(dist+"/"+layout.ManifestName, job.recordBytes, publicFile) +} + +// applyModes sets directory modes last, deepest first, so a read-only directory never blocks the +// writes below it. +func (s dirStore) applyModes(t *tree, job *publishJob) error { + if !s.repo.hostModes { + return nil + } + root := job.stagedPayload() + for _, generated := range []string{root, root + "/.atmos", root + "/" + layout.DistributionDir} { + if err := t.Chmod(native(generated), dirMode); err != nil { + return err + } + } + dirs := make([]artifact.Entry, 0) + for index := range job.inventory { + if job.inventory[index].Type == artifact.EntryDirectory { + dirs = append(dirs, job.inventory[index]) + } + } + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i].Path) > len(dirs[j].Path) }) + for index := range dirs { + if err := t.Chmod(native(root+"/"+dirs[index].Path), fsMode(dirs[index].Mode)); err != nil { + return err + } + } + return nil +} + +// install retires the previous payload directory and renames the staged tree into place. +func (dirStore) install(t *tree, job *publishJob) error { + if err := t.moveAside(job.location, job.retiredDir()+"/payload"); err != nil { + return err + } + if err := t.ensureParent(job.location); err != nil { + return err + } + return t.Rename(native(job.stagedPayload()), native(job.location)) +} + +// streams returns one verified stream per inventory file plus the documents. +func (s dirStore) streams(_ context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) { + if ok, err := t.exists(v.rep.Location); err != nil || !ok { + return nil, fmt.Errorf("%w: payload directory %q is missing", errUtils.ErrArtifactIntegrityFailed, v.rep.Location) + } + streams := make([]artifact.ContentStream, 0, len(v.record.Inventory)+len(v.record.Documents)) + for index := range v.record.Inventory { + entry := &v.record.Inventory[index] + if entry.Type != artifact.EntryFile { + continue + } + file, _, err := t.openFile(v.rep.Location+"/"+entry.Path, "payload file") + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{ + Descriptor: ocispec.Descriptor{MediaType: filePayloadMediaType, Digest: entry.Digest, Size: entry.Size}, + Role: artifact.BlobPayload, + Path: entry.Path, + Reader: newVerifyingReader(file, entry.Digest, entry.Size, guard), + }) + } + for index := range v.record.Documents { + doc := &v.record.Documents[index] + reader, err := s.document(t, v, doc, guard) + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{Descriptor: blobDescriptor(doc), Role: artifact.BlobDocument, Reader: reader}) + } + return streams, nil +} + +func (dirStore) document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) { + file, _, err := t.openFile(layout.DistributionPath(v.rep.Location, layout.DocumentFileName(doc.Role)), string(doc.Role)+" document") + if err != nil { + return nil, err + } + return newVerifyingReader(file, doc.Digest, doc.Size, guard), nil +} + +func (dirStore) verifySidecars(t *tree, v *view) error { + return t.verifyBytes(layout.DistributionPath(v.rep.Location, layout.ManifestName), "manifest", v.recordBytes) +} + +// remove deletes the inventory files, the distribution directory, and then the directories that +// became empty. Files the artifact does not own stay in place and keep their directory alive. +func (dirStore) remove(t *tree, v *view) ([]string, error) { + location := v.rep.Location + if ok, err := t.exists(location); err != nil || !ok { + return nil, err + } + t.makeWritable(location) + if err := removeFiles(t, location, v.record); err != nil { + return nil, err + } + removeDirectories(t, location, v.record) + return []string{t.absolute(location)}, nil +} + +// removeFiles deletes every non-directory inventory entry and the generated distribution files. +func removeFiles(t *tree, location string, record *artifact.Record) error { + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type != artifact.EntryDirectory { + if err := removeIfPresent(t, location+"/"+entry.Path); err != nil { + return err + } + } + } + for _, name := range []string{layout.ManifestName, "README.md", "CHANGELOG.md"} { + if err := removeIfPresent(t, layout.DistributionPath(location, name)); err != nil { + return err + } + } + return nil +} + +// removeDirectories removes the artifact's directories deepest first. A directory that still holds +// unmanaged files is left in place. +func removeDirectories(t *tree, location string, record *artifact.Record) { + var dirs []string + for index := range record.Inventory { + if record.Inventory[index].Type == artifact.EntryDirectory { + dirs = append(dirs, location+"/"+record.Inventory[index].Path) + } + } + dirs = append(dirs, location+"/"+layout.DistributionDir, location+"/.atmos", location) + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + for _, dir := range dirs { + _ = t.Remove(native(dir)) + } +} + +// removeIfPresent removes a path, ignoring a path that is already gone. +func removeIfPresent(t *tree, rel string) error { + if ok, err := t.exists(rel); err != nil || !ok { + return err + } + return t.Remove(native(rel)) +} + +// dirRepresentationDigest is the checksum of the per-file payload: the SHA-256 of the sorted +// "path NUL digest LF" lines. +func dirRepresentationDigest(record *artifact.Record) digest.Digest { + hasher := sha256.New() + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type == artifact.EntryFile { + _, _ = io.WriteString(hasher, entry.Path+"\x00"+entry.Digest.String()+"\n") + } + } + return digest.NewDigest(digest.SHA256, hasher) +} diff --git a/pkg/artifact/local/location_test.go b/pkg/artifact/local/location_test.go new file mode 100644 index 00000000000..f61897e9e01 --- /dev/null +++ b/pkg/artifact/local/location_test.go @@ -0,0 +1,241 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +const datedPattern = `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}` + +func datedSpec(kind string, packaging artifact.Packaging) *repoSpec { + suffix := "" + if kind == artifact.RepositoryKindLocalArchive { + suffix = layout.ArchiveExtension(packaging) + } + return &repoSpec{kind: kind, packaging: packaging, keyPattern: datedPattern + suffix} +} + +var datedSpecs = []*repoSpec{ + datedSpec(artifact.RepositoryKindLocalDir, ""), + datedSpec(artifact.RepositoryKindLocalArchive, artifact.PackagingTgz), +} + +func storedLocation(t *testing.T, repo *Repository, name, version string) string { + t.Helper() + tr, err := repo.openTree(false) + require.NoError(t, err) + defer tr.Close() + rep, err := tr.loadRepresentation(layout.ReferenceKey(name, version)) + require.NoError(t, err) + require.NotNil(t, rep) + return rep.Location +} + +func TestDatedRoot_LocationIsRenderedOnceAndPersisted(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{at: time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC)}, artifact.PublishOptions{}) + want := layout.DefaultLocation(testName, testVersion, repo.packaging) + want = "2026/10/30/" + want + assert.Equal(t, filepath.Join(repo.root, filepath.FromSlash(want)), result.Location) + assert.Equal(t, want, storedLocation(t, repo, testName, testVersion)) + + // A new repository object at the same root with a different pattern (and the wall + // clock long past the capture date) still reads and deletes the persisted location. + reader := newRepo(t, spec.withRoot(repo.root).withPattern("somewhere/else/{{ .Version }}")) + assertIntact(t, reader, defaultRef()) + pinned := resolve(t, reader, defaultRef()) + deleted, err := reader.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + require.NotEmpty(t, deleted.Deleted) + assert.Equal(t, result.Location, deleted.Deleted[0], "delete uses the persisted dated location, never a re-rendered one") + assert.NoDirExists(t, filepath.Join(repo.root, "2026"), "empty dated parents are pruned") + assert.NoDirExists(t, filepath.Join(repo.root, "somewhere")) + }) + } +} + +func TestDatedRoot_RolloverReusesTheCapturedLocation(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + dayOne := time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + first := prepare(t, repo, &prepareSpec{at: dayOne, opID: "op-day-one"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + // The same prepared artifact, retried after midnight, lands in the day-one location. + result, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Contains(t, filepath.ToSlash(result.Location), "2026/10/30/") + + // A later invocation (new preparation time) of the same intent after an interruption + // also resumes at the journaled location instead of creating a second dated tree. + repo2 := newRepo(t, spec.withRoot(repo.root).withOpts(withFaults(crashAt(faultAfterMetadata)))) + second := prepare(t, repo2, &prepareSpec{name: "team/other", at: dayOne, opID: "op-other-1"}) + _, err = repo2.Publish(context.Background(), second, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + dayTwo := prepare(t, repo2, &prepareSpec{name: "team/other", at: dayOne.Add(26 * time.Hour), opID: "op-other-2"}) + resumed, err := repo2.Publish(context.Background(), dayTwo, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Contains(t, filepath.ToSlash(resumed.Location), "2026/10/30/", "the journaled location is reused, not re-rendered for the new day") + assert.NoDirExists(t, filepath.Join(repo.root, "2026", "10", "31")) + assertIntact(t, repo2, ref("team/other", testVersion)) + }) + } +} + +func TestDatedRoot_ForceReplacementAcrossDaysRetiresTheOldLocation(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + dayOne := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + first := publish(t, repo, &prepareSpec{at: dayOne, files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + second := publish(t, repo, &prepareSpec{at: dayOne.Add(48 * time.Hour), opID: "op-2", files: map[string]string{"data.txt": "v2\n"}}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + assert.Equal(t, artifact.OutcomeReplaced, second.Outcome) + assert.Contains(t, filepath.ToSlash(second.Location), "2026/11/01/") + assert.NoFileExists(t, first.Location, "the replaced dated payload is retired") + assert.NoDirExists(t, filepath.Join(repo.root, "2026", "10"), "its empty dated parents are pruned") + assertIntact(t, repo, defaultRef()) + }) + } +} + +func collidingSpec(kind string) *repoSpec { + spec := &repoSpec{kind: kind, keyPattern: "shared/{{ .Version }}"} + if kind == artifact.RepositoryKindLocalArchive { + spec.packaging = artifact.PackagingTgz + spec.keyPattern += ".tgz" + } + return spec +} + +// Two references whose key_pattern resolves to one path: the second fails with a conflict and is +// never bound to the first reference's payload. +func TestCollidingKeyPattern_SecondReferenceConflicts(t *testing.T) { + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + t.Run(kind, func(t *testing.T) { + repo := newRepo(t, collidingSpec(kind)) + first := publish(t, repo, &prepareSpec{name: "a/one", files: map[string]string{"data.txt": "one\n"}}, artifact.PublishOptions{}) + before := resolve(t, repo, ref("a/one", testVersion)) + + second := prepare(t, repo, &prepareSpec{name: "b/two", files: map[string]string{"data.txt": "two\n"}}) + _, err := repo.Publish(context.Background(), second, artifact.PublishOptions{Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "a/one:1.0.0", "the conflict names the owning reference") + + _, err = repo.Resolve(context.Background(), ref("b/two", testVersion)) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the loser has no reference at all") + assert.Equal(t, before.RecordDigest, resolve(t, repo, ref("a/one", testVersion)).RecordDigest) + content, err := repo.Read(context.Background(), before) + require.NoError(t, err) + defer content.Close() + require.NoError(t, readAll(t, content)) + assert.Equal(t, []string{"a/one:1.0.0"}, listNames(t, repo)) + assert.True(t, pathExists(first.Location), "the first payload is untouched") + assert.Empty(t, listFiles(t, filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations")), "a rejected reservation leaves no journal behind") + }) + } +} + +func TestCollidingKeyPattern_AncestorAndCaseAliases(t *testing.T) { + spec := &repoSpec{kind: artifact.RepositoryKindLocalDir, keyPattern: "{{ .Kind }}/{{ .Name }}"} + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{name: "tools"}, artifact.PublishOptions{}) + nested := prepare(t, repo, &prepareSpec{name: "tools/cli"}) + _, err := repo.Publish(context.Background(), nested, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a payload directory nested in another artifact's directory is a collision") + + caseSpec := &repoSpec{kind: artifact.RepositoryKindLocalDir, keyPattern: "same/{{ .Version }}"} + caseRepo := newRepo(t, caseSpec) + publish(t, caseRepo, &prepareSpec{name: "a/x", version: "RC1"}, artifact.PublishOptions{}) + aliased := prepare(t, caseRepo, &prepareSpec{name: "b/y", version: "rc1"}) + _, err = caseRepo.Publish(context.Background(), aliased, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "locations that differ only by case alias on common filesystems") +} + +// Content that exists at the location but is not managed by this repository is never adopted or +// overwritten. +func TestUnmanagedContentAtTheLocationIsNeverOverwritten(t *testing.T) { + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + t.Run(kind, func(t *testing.T) { + spec := &repoSpec{kind: kind} + if kind == artifact.RepositoryKindLocalArchive { + spec.packaging = artifact.PackagingTgz + } + repo := newRepo(t, spec) + location := layout.DefaultLocation(testName, testVersion, repo.packaging) + userFile := filepath.Join(repo.root, filepath.FromSlash(location)) + if kind == artifact.RepositoryKindLocalDir { + userFile = filepath.Join(userFile, "mine.txt") + } + require.NoError(t, os.MkdirAll(filepath.Dir(userFile), 0o755)) + require.NoError(t, os.WriteFile(userFile, []byte("precious"), 0o644)) + + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "not managed") + got, readErr := os.ReadFile(userFile) + require.NoError(t, readErr) + assert.Equal(t, "precious", string(got)) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// Concurrent publications of colliding references: exactly one wins. +func TestCollidingKeyPattern_ConcurrentWritersExactlyOneWins(t *testing.T) { + repo := newRepo(t, collidingSpec(artifact.RepositoryKindLocalDir)) + names := []string{"a/one", "b/two", "c/three", "d/four"} + prepared := make([]*artifact.PreparedArtifact, len(names)) + for index, name := range names { + prepared[index] = prepare(t, repo, &prepareSpec{name: name, files: map[string]string{"data.txt": name}}) + } + errs := make([]error, len(names)) + var wg sync.WaitGroup + for index := range names { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[index] = repo.Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wg.Wait() + winners := 0 + for _, err := range errs { + if err == nil { + winners++ + } else { + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + } + assert.Equal(t, 1, winners, "ownership of the shared location is exclusive") + assert.Len(t, listNames(t, repo), 1) + for _, name := range listNames(t, repo) { + parsed, err := artifact.ParseRef(name) + require.NoError(t, err) + assertIntact(t, repo, parsed) + } +} + +func pathExists(p string) bool { + _, err := os.Stat(p) + return err == nil +} diff --git a/pkg/artifact/local/lock.go b/pkg/artifact/local/lock.go new file mode 100644 index 00000000000..953743d752d --- /dev/null +++ b/pkg/artifact/local/lock.go @@ -0,0 +1,177 @@ +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/filelock" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Internal lock resources. They start with a NUL byte, which no artifact reference can contain, +// so they can never collide with a caller's reference lock. +const ( + claimsResource = "\x00claims" + locationResource = "\x00location\x00" +) + +// heldLocks tracks the locks this process holds, keyed by lock file path. The OS lock excludes +// other processes; this table supplies the ownership-token check on release and diagnostics. +var ( + heldMu sync.Mutex + heldLocks = map[string]*fileLock{} +) + +// fileLock is an OS file lock held until Release. It implements [artifact.LockHandle]. +type fileLock struct { + path string + resource string + owner string + operationID string + token string + release chan struct{} + done chan error + released bool +} + +var _ artifact.LockHandle = (*fileLock)(nil) + +// Lock implements [artifact.LockProvider] with an OS-backed file lock of host scope. It waits for +// a competing owner until ctx ends; a canceled wait leaks no ownership. Leases are rejected +// because a paused writer could not be fenced, and repository scope is rejected because a local +// file lock cannot exclude writers on other hosts. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "local.Repository.Lock")() + + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.alias, "lock", "lease", "local file locks do not expire, so a lease cannot be fenced") + } + if request.Required != "" && !artifact.LockScopeHost.Satisfies(request.Required) { + return nil, artifact.NewUnsupportedCapability(r.alias, "lock", "lock/"+string(request.Required), + "local repositories provide host-scope locks only; use an S3 repository or a lock delegate for repository scope") + } + if request.Resource == "" { + return nil, fmt.Errorf("%w: a lock needs a resource", errUtils.ErrArtifactRefInvalid) + } + owner := request.Owner + if owner == "" { + owner = r.settings.owner + } + return r.acquire(ctx, request.Resource, owner, request.OperationID) +} + +// acquire takes the OS lock for resource. +func (r *Repository) acquire(ctx context.Context, resource, owner, operationID string) (*fileLock, error) { + lockPath := filepath.Join(r.root, native(layout.LockPath(resource))) + if err := os.MkdirAll(filepath.Dir(lockPath), dirMode); err != nil { + return nil, fmt.Errorf("%w: create lock directory for %q: %w", errUtils.ErrArtifactConfigInvalid, resource, err) + } + held := make(chan struct{}) + lock := &fileLock{ + path: lockPath, resource: resource, owner: owner, operationID: operationID, + token: uuid.NewString(), release: make(chan struct{}), done: make(chan error, 1), + } + go func() { + lock.done <- filelock.New(lockPath).WithExclusive(ctx, func() error { + close(held) + <-lock.release + return nil + }) + }() + select { + case <-held: + heldMu.Lock() + heldLocks[lockPath] = lock + heldMu.Unlock() + return lock, nil + case err := <-lock.done: + return nil, fmt.Errorf("waiting for lock %q in repository %q: %w", resource, r.alias, err) + } +} + +// Resource implements [artifact.LockHandle]. +func (l *fileLock) Resource() string { + defer perf.Track(nil, "local.fileLock.Resource")() + + return l.resource +} + +// Owner implements [artifact.LockHandle]. +func (l *fileLock) Owner() string { + defer perf.Track(nil, "local.fileLock.Owner")() + + return l.owner +} + +// Scope implements [artifact.LockHandle]; local locks are host scope. +func (l *fileLock) Scope() artifact.LockScope { + defer perf.Track(nil, "local.fileLock.Scope")() + + return artifact.LockScopeHost +} + +// Token implements [artifact.LockHandle]. +func (l *fileLock) Token() string { + defer perf.Track(nil, "local.fileLock.Token")() + + return l.token +} + +// ExpiresAt implements [artifact.LockHandle]; local locks never expire. +func (l *fileLock) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "local.fileLock.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; local locks have no leases. +func (l *fileLock) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "local.fileLock.Renew")() + + return artifact.NewUnsupportedCapability("", "lock", "lease", "local file locks do not expire") +} + +// Release implements [artifact.LockHandle]. It checks the ownership token against the lock this +// process holds, so a stale or foreign handle can never release another operation's lock. The OS +// also releases the lock if its owner exits. +func (l *fileLock) Release(context.Context) error { + defer perf.Track(nil, "local.fileLock.Release")() + + heldMu.Lock() + current, owned := heldLocks[l.path] + if !owned || current != l || l.released || current.token != l.token { + heldMu.Unlock() + return fmt.Errorf("%w: lock %q is not held by token %s", errUtils.ErrArtifactConflict, l.resource, l.token) + } + delete(heldLocks, l.path) + l.released = true + heldMu.Unlock() + close(l.release) + if err := <-l.done; err != nil { + return fmt.Errorf("release lock %q: %w", l.resource, err) + } + return nil +} + +// releaseQuietly releases a lock acquired internally and returns the first error alongside err. +func releaseAll(ctx context.Context, locks ...*fileLock) error { + var first error + for index := len(locks) - 1; index >= 0; index-- { + if locks[index] == nil { + continue + } + if err := locks[index].Release(context.WithoutCancel(ctx)); err != nil && first == nil { + first = err + } + } + return first +} diff --git a/pkg/artifact/local/lock_test.go b/pkg/artifact/local/lock_test.go new file mode 100644 index 00000000000..ff4c5e6cc8f --- /dev/null +++ b/pkg/artifact/local/lock_test.go @@ -0,0 +1,380 @@ +package local + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Environment variables that turn the test binary into a lock helper process. The test binary +// doubles as a cross-platform subprocess: no Unix-only helper programs are involved. +const ( + helperModeEnv = "_ATMOS_LOCAL_LOCK_HELPER" + helperRootEnv = "_ATMOS_LOCAL_LOCK_ROOT" + helperResourceEnv = "_ATMOS_LOCAL_LOCK_RESOURCE" + + helperExitAcquired = 0 + helperExitBusy = 3 + helperExitError = 4 +) + +func TestMain(m *testing.M) { + if mode := os.Getenv(helperModeEnv); mode != "" { + os.Exit(runLockHelper(mode)) + } + os.Exit(m.Run()) +} + +// runLockHelper is the body of the helper process. +// +// try: attempt the lock for a short time; exit 0 when acquired, 3 when busy. +// hold: acquire the lock, announce it on stdout, and hold it until stdin closes or the process is killed. +func runLockHelper(mode string) int { + repo, err := newRepository("helper", artifact.RepositoryKindLocalDir, helperConfig(os.Getenv(helperRootEnv)), nil) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + timeout := 400 * time.Millisecond + if mode == "hold" { + timeout = 20 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: os.Getenv(helperResourceEnv), Owner: "helper-process"}) + switch { + case errors.Is(err, context.DeadlineExceeded): + return helperExitBusy + case err != nil: + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + if mode == "hold" { + fmt.Println("locked") + _, _ = bufio.NewReader(os.Stdin).ReadString('\n') + } + if err := handle.Release(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + return helperExitAcquired +} + +func helperCommand(t *testing.T, mode, root, resource string) *exec.Cmd { + t.Helper() + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command(exe, "-test.run=^$") + cmd.Env = append(os.Environ(), helperModeEnv+"="+mode, helperRootEnv+"="+root, helperResourceEnv+"="+resource) + return cmd +} + +func exitCode(t *testing.T, cmd *exec.Cmd) int { + t.Helper() + err := cmd.Run() + if err == nil { + return 0 + } + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + return exitErr.ExitCode() +} + +func lockRequest(resource string) artifact.LockRequest { + return artifact.LockRequest{Resource: resource, Owner: "test-owner", OperationID: "op-lock", Required: artifact.LockScopeHost} +} + +func TestLock_HandleContract(t *testing.T) { + repo := newRepo(t, dirSpec()) + handle, err := repo.Lock(context.Background(), lockRequest("team/app:1.0.0")) + require.NoError(t, err) + assert.Equal(t, "team/app:1.0.0", handle.Resource()) + assert.Equal(t, "test-owner", handle.Owner()) + assert.Equal(t, artifact.LockScopeHost, handle.Scope()) + assert.NotEmpty(t, handle.Token()) + _, expires := handle.ExpiresAt() + assert.False(t, expires, "local locks never expire") + require.ErrorIs(t, handle.Renew(context.Background(), time.Minute), errUtils.ErrArtifactUnsupportedCapability) + assert.FileExists(t, lockFile(repo, "team/app:1.0.0"), "the lock file lives in the managed namespace and is never removed") + + require.NoError(t, handle.Release(context.Background())) + require.ErrorIs(t, handle.Release(context.Background()), errUtils.ErrArtifactConflict, "a handle cannot be released twice") + assert.FileExists(t, lockFile(repo, "team/app:1.0.0")) + + other, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "x"}) + require.NoError(t, err) + assert.NotEmpty(t, other.Owner(), "an empty owner defaults to the repository owner") + require.NoError(t, other.Release(context.Background())) +} + +func TestLock_RejectsUnsupportedGuarantees(t *testing.T) { + repo := newRepo(t, dirSpec()) + request := lockRequest("r") + request.Lease = time.Minute + _, err := repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a lease cannot be fenced") + + request = lockRequest("r") + request.Required = artifact.LockScopeRepository + _, err = repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a host file lock is never reported as repository scope") + capErr, ok := artifact.AsCapabilityError(err) + require.True(t, ok) + assert.Equal(t, "lock/repository", capErr.Capability) + + _, err = repo.Lock(context.Background(), artifact.LockRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + caps := repo.Capabilities() + assert.True(t, caps.HasLockScope(artifact.LockScopeHost)) + assert.False(t, caps.HasLockScope(artifact.LockScopeRepository)) +} + +func TestLock_ReleaseChecksTheOwnershipToken(t *testing.T) { + repo := newRepo(t, dirSpec()) + handle, err := repo.Lock(context.Background(), lockRequest("r")) + require.NoError(t, err) + real, ok := handle.(*fileLock) + require.True(t, ok) + + forged := &fileLock{path: real.path, resource: real.resource, token: "not-the-token", release: make(chan struct{}), done: make(chan error, 1)} + require.ErrorIs(t, forged.Release(context.Background()), errUtils.ErrArtifactConflict, "a wrong token never releases another operation's lock") + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, err = repo.Lock(ctx, lockRequest("r")) + require.ErrorIs(t, err, context.DeadlineExceeded, "the real owner still holds the lock") + require.NoError(t, handle.Release(context.Background())) +} + +func TestLock_CompetingHandlesAreMutuallyExclusive(t *testing.T) { + // Separate repository objects over the same root model separate writers in one process. + root := dirSpec().withRoot(filepath.Join(t.TempDir(), "shared")) + writers := make([]*Repository, 4) + for index := range writers { + writers[index] = newRepo(t, root) + } + var ( + inside, maxInside, total atomic.Int32 + wg sync.WaitGroup + ) + for _, writer := range writers { + wg.Add(1) + go func() { + defer wg.Done() + for range 5 { + handle, err := writer.Lock(context.Background(), lockRequest("team/app:1.0.0")) + if !assert.NoError(t, err) { + return + } + now := inside.Add(1) + for { + seen := maxInside.Load() + if now <= seen || maxInside.CompareAndSwap(seen, now) { + break + } + } + time.Sleep(2 * time.Millisecond) + total.Add(1) + inside.Add(-1) + assert.NoError(t, handle.Release(context.Background())) + } + }() + } + wg.Wait() + assert.EqualValues(t, 1, maxInside.Load(), "never two owners inside the critical section") + assert.EqualValues(t, 20, total.Load()) +} + +func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder, err := repo.Lock(context.Background(), lockRequest("r")) + require.NoError(t, err) + + var wg sync.WaitGroup + for range 5 { + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + handle, err := repo.Lock(ctx, lockRequest("r")) + assert.Nil(t, handle) + assert.ErrorIs(t, err, context.DeadlineExceeded) + }() + } + wg.Wait() + require.NoError(t, holder.Release(context.Background())) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + next, err := repo.Lock(ctx, lockRequest("r")) + require.NoError(t, err, "the canceled waiters took no ownership") + require.NoError(t, next.Release(context.Background())) + + canceled, cancelNow := context.WithCancel(context.Background()) + cancelNow() + _, err = repo.Lock(canceled, lockRequest("r2")) + if err != nil { + require.ErrorIs(t, err, context.Canceled) + } + again, err := repo.Lock(context.Background(), lockRequest("r2")) + require.NoError(t, err, "an already-canceled attempt leaves the resource free") + require.NoError(t, again.Release(context.Background())) +} + +// Two writers to one reference: the OS lock excludes them across the comparison and commit, so +// exactly one creates the reference and every other sees the conflict. +func TestTwoWritersToOneReference(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + const writers = 6 + prepared := make([]*artifact.PreparedArtifact, writers) + for index := range writers { + prepared[index] = prepare(t, repo, &prepareSpec{opID: fmt.Sprintf("op-%d", index)}) + } + outcomes := make([]error, writers) + var wg sync.WaitGroup + for index := range writers { + wg.Add(1) + go func() { + defer wg.Done() + _, outcomes[index] = repo.Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wg.Wait() + created := 0 + for _, err := range outcomes { + if err == nil { + created++ + } else { + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + } + assert.Equal(t, 1, created) + assertIntact(t, repo, defaultRef()) + assert.Empty(t, listFiles(t, filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations")), "no journal outlives its publication") + }) + } +} + +// A publication without a caller-held lock takes the reference lock itself, so it waits for an +// existing holder; with a caller-held lock it proceeds without deadlocking on itself. +func TestPublishHonorsTheReferenceLock(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) + require.NoError(t, err) + + prepared := prepare(t, repo, &prepareSpec{}) + done := make(chan error, 1) + go func() { + _, publishErr := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + done <- publishErr + }() + select { + case err := <-done: + t.Fatalf("the publication did not wait for the held lock (returned %v)", err) + case <-time.After(200 * time.Millisecond): + } + require.NoError(t, holder.Release(context.Background())) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("the publication never resumed after the lock was released") + } + + second := newRepo(t, dirSpec()) + held, err := second.Lock(context.Background(), lockRequest("team/other:1.0.0")) + require.NoError(t, err) + t.Cleanup(func() { _ = held.Release(context.Background()) }) + _, err = second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "team/other"}), artifact.PublishOptions{Lock: held}) + require.NoError(t, err, "a service-held lock is honored, not re-acquired") +} + +func TestPublishCancellationWhileWaitingLeaksNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err = repo.Publish(ctx, prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, holder.Release(context.Background())) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "nothing was written") + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) +} + +func TestLock_ExcludesOtherProcesses(t *testing.T) { + repo := newRepo(t, dirSpec()) + resource := "team/app:1.0.0" + handle, err := repo.Lock(context.Background(), lockRequest(resource)) + require.NoError(t, err) + + assert.Equal(t, helperExitBusy, exitCode(t, helperCommand(t, "try", repo.root, resource)), "another process cannot take a held lock") + other := resource + "-other" + assert.Equal(t, helperExitAcquired, exitCode(t, helperCommand(t, "try", repo.root, other)), "locks on other resources are independent") + + require.NoError(t, handle.Release(context.Background())) + assert.Equal(t, helperExitAcquired, exitCode(t, helperCommand(t, "try", repo.root, resource)), "the lock is free after release") +} + +func TestLock_ReleasesWhenItsOwnerProcessExits(t *testing.T) { + repo := newRepo(t, dirSpec()) + resource := "team/app:1.0.0" + cmd := helperCommand(t, "hold", repo.root, resource) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + exited := false + defer func() { + if !exited { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + + line, err := bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + require.Equal(t, "locked", strings.TrimSpace(line)) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err = repo.Lock(ctx, lockRequest(resource)) + require.ErrorIs(t, err, context.DeadlineExceeded, "the helper process holds the lock") + + // Kill the owner without letting it release: the OS must free the lock. + _ = stdin.Close() + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + exited = true + + acquire, cancelAcquire := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelAcquire() + handle, err := repo.Lock(acquire, lockRequest(resource)) + require.NoError(t, err, "the lock is released when its owner exits") + require.NoError(t, handle.Release(context.Background())) +} + +func lockFile(repo *Repository, resource string) string { + return filepath.Join(repo.root, filepath.FromSlash(layout.LockPath(resource))) +} diff --git a/pkg/artifact/local/options.go b/pkg/artifact/local/options.go new file mode 100644 index 00000000000..be3b0f6cd4f --- /dev/null +++ b/pkg/artifact/local/options.go @@ -0,0 +1,57 @@ +package local + +import ( + "fmt" + "os" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// settings holds the construction options of a local repository. +type settings struct { + verifier artifact.Verifier + owner string + faults faultHook +} + +// Option configures a local repository or its registration. +type Option func(*settings) + +// WithVerifier injects the cryptographic verifier used by Verify. The formats it understands are +// the evidence formats the repository advertises. Without a verifier the repository stores no +// evidence and Sign and Verify return unsupported-capability errors. +func WithVerifier(verifier artifact.Verifier) Option { + defer perf.Track(nil, "local.WithVerifier")() + + return func(s *settings) { s.verifier = verifier } +} + +// WithOwner sets the owner recorded in lock requests and operation journals when the caller does +// not supply one. The default is ":". +func WithOwner(owner string) Option { + defer perf.Track(nil, "local.WithOwner")() + + return func(s *settings) { s.owner = owner } +} + +func applySettings(opts []Option) settings { + var applied settings + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.owner == "" { + applied.owner = defaultOwner() + } + return applied +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s:%d", host, os.Getpid()) +} diff --git a/pkg/artifact/local/permissions_test.go b/pkg/artifact/local/permissions_test.go new file mode 100644 index 00000000000..42be34f0a53 --- /dev/null +++ b/pkg/artifact/local/permissions_test.go @@ -0,0 +1,166 @@ +package local + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" +) + +func modeOf(t *testing.T, path string) fs.FileMode { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err) + return info.Mode().Perm() +} + +// tgzMemberMode returns the mode recorded in the archive for one member. +func tgzMemberMode(t *testing.T, archivePath, member string) int64 { + t.Helper() + file, err := os.Open(archivePath) + require.NoError(t, err) + defer file.Close() + gz, err := gzip.NewReader(file) + require.NoError(t, err) + reader := tar.NewReader(gz) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + t.Fatalf("member %q not found in %s", member, archivePath) + } + require.NoError(t, err) + if header.Name == member { + return header.Mode & 0o777 + } + } +} + +func TestArchivePermissions_DefaultIs0600OnTheArchiveAndSidecars(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + + assert.Equal(t, fs.FileMode(0o600), modeOf(t, result.Location), "default archive permissions") + assert.Equal(t, fs.FileMode(0o600), modeOf(t, result.Location+".metadata.json")) + assert.Equal(t, fs.FileMode(0o600), modeOf(t, filepath.Join(result.Location+".docs", "README.md"))) + assert.Equal(t, fs.FileMode(0o700), modeOf(t, result.Location+".docs"), "the sidecar directory keeps owner traversal") + assert.False(t, repo.modeExplicit, "a default is not an explicit request") +} + +func TestArchivePermissions_ExplicitModeIsDistinctFromMemberModes(t *testing.T) { + fixtures.RequirePOSIXModes(t) + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0640") + repo := newRepo(t, spec) + require.True(t, repo.modeExplicit) + result, _ := publishFilePackage(t, repo) + + assert.Equal(t, fs.FileMode(0o640), modeOf(t, result.Location), "the configured mode applies to the archive file") + assert.Equal(t, fs.FileMode(0o640), modeOf(t, result.Location+".metadata.json")) + assert.Equal(t, fs.FileMode(0o750), modeOf(t, result.Location+".docs")) + assert.EqualValues(t, 0o755, tgzMemberMode(t, result.Location, "bin/helper.sh"), "member modes are independent of the archive file mode") + assert.EqualValues(t, 0o644, tgzMemberMode(t, result.Location, "README.md")) + assert.EqualValues(t, 0o640, tgzMemberMode(t, result.Location, fixtures.UnicodeName)) +} + +func TestArchivePermissions_AppliedBeforeAnythingIsVisible(t *testing.T) { + fixtures.RequirePOSIXModes(t) + spec := archiveSpec(artifact.PackagingTar) + spec = spec.withPermissions("0400") + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + archivePath := filepath.Join(repo.root, "team", "app", "1.0.0.tar") + assert.Equal(t, fs.FileMode(0o400), modeOf(t, archivePath), "even an interrupted publication never exposes a wrongly-permissioned archive") +} + +// An explicit permission request the host cannot enforce fails before anything is written. +func TestArchivePermissions_UnsupportedHostFailsExplicitly(t *testing.T) { + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0600") + repo := newRepo(t, spec) + repo.hostModes = false + + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + capErr, ok := artifact.AsCapabilityError(err) + require.True(t, ok) + assert.Equal(t, "permissions/archive-file-mode", capErr.Capability) + assert.Contains(t, capErr.Reason, "0600") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "nothing was created before the capability check") +} + +// Without an explicit request, a host that cannot enforce modes publishes with the host default. +func TestArchivePermissions_DefaultWorksWhereModesCannotBeEnforced(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + repo.hostModes = false + result, _ := publishFilePackage(t, repo) + assert.FileExists(t, result.Location) + assertIntact(t, repo, defaultRef()) +} + +// On the real host the behavior matches its capability: Windows rejects an explicit request, +// POSIX hosts honor it. +func TestArchivePermissions_RealHostBehavior(t *testing.T) { + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0600") + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "Windows cannot enforce POSIX archive permissions") + return + } + require.NoError(t, err) + assert.Equal(t, fs.FileMode(0o600), modeOf(t, filepath.Join(repo.root, "team", "app", "1.0.0.tgz"))) +} + +func TestPermissions_InvalidConfigurationIsRejected(t *testing.T) { + for _, bad := range []string{"preserve", "0o600", "9999", "4755", "6", "rw-------", "600.0"} { + spec := archiveSpec(artifact.PackagingTgz).withPermissions(bad).withRoot(filepath.Join(t.TempDir(), "r")) + _, err := newRepository("x", spec.kind, spec.config(t), nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, bad) + } + dir := dirSpec().withPermissions("0600").withRoot(filepath.Join(t.TempDir(), "r")) + _, err := newRepository("x", dir.kind, dir.config(t), nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "permissions apply only to local archive files") +} + +func TestCapabilities_AreHonest(t *testing.T) { + dir := newRepo(t, dirSpec()) + archive := newRepo(t, archiveSpec(artifact.PackagingTgz)) + + dirCaps, archiveCaps := dir.Capabilities(), archive.Capabilities() + assert.Equal(t, []artifact.Packaging{artifact.PackagingNone}, dirCaps.Packagings) + assert.ElementsMatch(t, []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip}, archiveCaps.Packagings) + for _, caps := range []artifact.Capabilities{dirCaps, archiveCaps} { + assert.True(t, caps.Read && caps.Write && caps.EnumerateNamed && caps.EnumerateCollection && caps.ConditionalUpdate) + assert.Equal(t, artifact.DeleteObject, caps.Delete) + assert.Equal(t, []artifact.LockScope{artifact.LockScopeHost}, caps.LockScopes, "host scope only") + assert.Equal(t, []artifact.SubjectKind{artifact.SubjectRecord}, caps.SubjectKinds) + assert.Empty(t, caps.EvidenceFormats, "no verifier is configured, so no evidence format is advertised") + } + assert.Equal(t, hostEnforcesModes(), dirCaps.PreservesModes) + assert.True(t, archiveCaps.PreservesModes, "member modes live inside the archive on every host") + + dir.hostModes = false + assert.False(t, dir.Capabilities().PreservesModes) + assert.Equal(t, dir.Capabilities(), dir.Capabilities(), "capabilities are stable") +} diff --git a/pkg/artifact/local/publish.go b/pkg/artifact/local/publish.go new file mode 100644 index 00000000000..d062368890e --- /dev/null +++ b/pkg/artifact/local/publish.go @@ -0,0 +1,375 @@ +package local + +import ( + "context" + "errors" + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// publishPlan is what the admission decision and the existing state say about one publication. +type publishPlan struct { + admission layout.Admission + current *layout.Reference + prior *layout.Prior + owned []string + abandoned string + oldOp *layout.Operation + expected *artifact.PinnedReference +} + +// Publish implements [artifact.Repository] with the publication sequence of the contracts: +// +// 1. Hold the reference lock (the service's, or one taken here) and the resolved-location lock. +// 2. Read the current reference and admit the publication: conflict, force, unchanged, or resume. +// 3. Reserve the location and record the pending operation journal and pending reference before +// any payload path changes. +// 4. Stage the payload and sidecars on the same filesystem, write the immutable Record, install +// the payload, and write the representation. +// 5. Complete the reference last, with an atomic replacement. +// 6. Clean up only the temporary resources this operation owns. +// +// A failure after step 3 leaves the reference pending: unavailable to readers, repaired by a retry +// with the same intent. The caller reconciles an ambiguous response by reading the reference. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (result *artifact.PublishResult, err error) { + defer perf.Track(nil, "local.Repository.Publish")() + + job, err := r.newJob(prepared) + if err != nil { + return nil, err + } + t, err := r.openTree(true) + if err != nil { + return nil, err + } + defer t.Close() + var held []*fileLock + defer func() { + if releaseErr := releaseAll(ctx, held...); err == nil { + err = releaseErr + } + }() + if options.Lock == nil { + lock, lockErr := r.acquire(ctx, prepared.Record.Name+":"+prepared.Record.Version, r.settings.owner, job.opID) + if lockErr != nil { + return nil, lockErr + } + held = append(held, lock) + } + return r.publishLocked(ctx, t, job, &options, &held) +} + +// publishLocked runs the publication while the reference lock is held. +func (r *Repository) publishLocked(ctx context.Context, t *tree, job *publishJob, options *artifact.PublishOptions, held *[]*fileLock) (*artifact.PublishResult, error) { + plan, err := r.admit(t, job, options) + if err != nil { + return nil, err + } + if plan.admission == layout.AdmitUnchanged { + return r.unchanged(t, job, plan.current) + } + if err := r.resolveLocation(t, job, plan); err != nil { + return nil, err + } + locationLock, err := r.acquire(ctx, locationResource+layout.FoldPath(job.location), r.settings.owner, job.opID) + if err != nil { + return nil, err + } + *held = append(*held, locationLock) + pending, err := r.begin(ctx, t, job, plan) + if err != nil { + return nil, err + } + result, err := r.build(ctx, t, job, plan, pending) + if err != nil && !isCrash(err) && !job.installing { + _ = t.removeAll(job.staging) + } + return result, err +} + +// admit reads the current reference and applies the admission rules. +func (r *Repository) admit(t *tree, job *publishJob, options *artifact.PublishOptions) (*publishPlan, error) { + current, err := t.loadReference(job.key) + if err != nil { + return nil, err + } + input := layout.AdmitInput{ + Current: current, IncomingRecord: job.recordDigest, IncomingIntent: job.prepared.IntentDigest, + ExpectedPrior: options.ExpectedPrior, Force: options.Force, + } + if current != nil && current.Available() { + record, _, loadErr := t.loadRecord(current.RecordDigest) + if loadErr != nil { + return nil, loadErr + } + if intent, intentErr := record.IntentDigest(); intentErr == nil { + input.CurrentIntent = intent + } + } + admission, err := layout.Admit(&input) + if err != nil { + return nil, err + } + return &publishPlan{admission: admission, current: current, expected: options.ExpectedPrior}, nil +} + +// unchanged returns the existing artifact for an identical republication and tidies leftovers of +// the operation that completed it. +func (r *Repository) unchanged(t *tree, job *publishJob, current *layout.Reference) (*artifact.PublishResult, error) { + record, _, err := t.loadRecord(current.RecordDigest) + if err != nil { + return nil, err + } + rep, err := t.loadRepresentation(job.key) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != current.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is completed but has no matching representation", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) + } + r.tidyOperation(t, current.OperationID) + return r.result(t, record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome()), nil +} + +// tidyOperation removes the journal and staging directory of a finished operation. It is +// best-effort: a leftover is harmless and is retried by the next publication. +func (r *Repository) tidyOperation(t *tree, operationID string) { + if operationID == "" { + return + } + _ = t.removeAll(layout.StagingPath(operationID)) + _ = t.removeAll(layout.OperationPath(operationID)) +} + +// resolveLocation decides where the payload lives. A resumed publication reuses the location in +// its journal; otherwise the key_pattern is rendered exactly once, here, from the Record's +// captured creation time. The result is persisted; it is never rendered again. +func (r *Repository) resolveLocation(t *tree, job *publishJob, plan *publishPlan) error { + rep, err := t.loadRepresentation(job.key) + if err != nil { + return err + } + if err := r.loadPendingState(t, plan, rep); err != nil { + return err + } + if plan.admission.Resuming() && plan.oldOp != nil && plan.oldOp.IntentDigest == job.prepared.IntentDigest && plan.oldOp.Packaging == job.prepared.Packaging { + job.location = plan.oldOp.Location + return nil + } + if plan.oldOp != nil { + plan.abandoned = plan.oldOp.Location + } + job.location, err = layout.RenderLocation(r.keyPattern, &layout.PatternInput{ + Name: job.record.Name, Version: job.record.Version, Kind: job.record.Kind, CreatedAt: job.record.CreatedAt.Time(), + }, job.prepared.Packaging) + return err +} + +// loadPendingState records the locations this reference already owns and the content a +// publication replaces. The interrupted operation's journal is read when one is pending. +func (r *Repository) loadPendingState(t *tree, plan *publishPlan, rep *layout.Representation) error { + if rep != nil { + plan.owned = append(plan.owned, rep.Location) + } + if plan.current != nil && !plan.current.Available() { + oldOp, err := t.loadOperation(plan.current.OperationID) + if err != nil { + return err + } + plan.oldOp = oldOp + } + plan.prior = priorState(plan, rep) + if plan.oldOp != nil { + plan.owned = append(plan.owned, plan.oldOp.Location) + if plan.oldOp.Prior != nil { + plan.owned = append(plan.owned, plan.oldOp.Prior.Location) + } + } + return nil +} + +// priorState describes the content a publication replaces, so a later cleanup can retire it. +func priorState(plan *publishPlan, rep *layout.Representation) *layout.Prior { + if plan.admission == layout.AdmitReplace { + prior := &layout.Prior{RecordDigest: plan.current.RecordDigest} + if rep != nil && rep.RecordDigest == prior.RecordDigest { + prior.Location = rep.Location + } + return prior + } + if !plan.admission.Resuming() { + return nil + } + if plan.oldOp != nil && plan.oldOp.Prior != nil { + prior := *plan.oldOp.Prior + return &prior + } + if plan.current.PriorRecordDigest == "" { + return nil + } + prior := &layout.Prior{RecordDigest: plan.current.PriorRecordDigest} + if rep != nil && rep.RecordDigest == prior.RecordDigest { + prior.Location = rep.Location + } + return prior +} + +// begin reserves the location, writes the operation journal, and marks the reference pending. +// It holds the repository-wide claims lock so two references can never both reserve overlapping +// locations, and it never changes a payload path. +func (r *Repository) begin(ctx context.Context, t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + claims, err := r.acquire(ctx, claimsResource, r.settings.owner, job.opID) + if err != nil { + return nil, err + } + pending, beginErr := r.reserve(t, job, plan) + if releaseErr := claims.Release(context.WithoutCancel(ctx)); beginErr == nil { + beginErr = releaseErr + } + if beginErr != nil { + return nil, beginErr + } + if plan.oldOp != nil && plan.oldOp.OperationID != job.opID { + r.tidyOperation(t, plan.oldOp.OperationID) + } + return pending, r.settings.fault(faultBeforePayload) +} + +// reserve checks for location collisions and unmanaged content, then records the journal and the +// pending reference. +func (r *Repository) reserve(t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + if err := r.checkCollision(t, job, plan); err != nil { + return nil, err + } + journal := &layout.Operation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: r.kind, OperationID: job.opID, + Name: job.record.Name, Version: job.record.Version, IntentDigest: job.prepared.IntentDigest, + RecordDigest: job.recordDigest, CapturedAt: job.record.CreatedAt, Packaging: job.prepared.Packaging, + Location: job.location, Prior: plan.prior, Payload: job.payload, Documents: job.documents, Owner: r.settings.owner, + } + if plan.expected != nil { + journal.ExpectedPrior = plan.expected.ResolvedDigest + } + if err := t.writeOperation(journal); err != nil { + return nil, fmt.Errorf("%w: record operation %s: %w", errUtils.ErrArtifactUploadFailed, job.opID, err) + } + pending := layout.NewPendingReference(job.record.Name, job.record.Version, job.opID, job.prepared.IntentDigest, plan.current) + if err := t.writeReference(&pending); err != nil { + return nil, fmt.Errorf("%w: mark %s:%s pending: %w", errUtils.ErrArtifactUploadFailed, job.record.Name, job.record.Version, err) + } + return &pending, nil +} + +// build stages, writes, installs, and completes. It is the part of the sequence after the pending +// state is durable. +func (r *Repository) build(ctx context.Context, t *tree, job *publishJob, plan *publishPlan, pending *layout.Reference) (*artifact.PublishResult, error) { + if err := t.removeAll(job.staging); err != nil { + return nil, fmt.Errorf("%w: clear staging: %w", errUtils.ErrArtifactUploadFailed, err) + } + if err := r.store.stage(ctx, t, job); err != nil { + return nil, wrapWrite("stage", err) + } + if err := t.writeRecord(job.recordDigest, job.recordBytes); err != nil { + return nil, wrapWrite("write record", err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + job.installing = true + if err := r.store.install(t, job); err != nil { + return nil, wrapWrite("install payload", err) + } + rep := r.representation(job) + if err := t.writeRepresentation(rep); err != nil { + return nil, wrapWrite("write representation", err) + } + if err := r.settings.fault(faultAfterMetadata); err != nil { + return nil, err + } + completed := pending.Completed(job.recordDigest) + if err := t.writeReference(&completed); err != nil { + return nil, wrapWrite("complete reference", err) + } + if err := r.settings.fault(faultAfterCommit); err != nil { + return nil, err + } + r.retire(t, job, plan) + return r.result(t, job.record, job.recordDigest, rep, plan.admission.Outcome()), nil +} + +// wrapWrite classifies a storage failure. Context errors and already-typed artifact errors pass +// through so callers keep matching on them. +func wrapWrite(step string, err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isArtifactError(err) { + return err + } + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactUploadFailed, step, err) +} + +func isArtifactError(err error) bool { + for _, sentinel := range []error{ + errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactIntegrityFailed, + errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactUnsafePath, errUtils.ErrArtifactTooLarge, + errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactRecordInvalid, + } { + if errors.Is(err, sentinel) { + return true + } + } + return isCrash(err) +} + +// representation builds the representation record for the job. +func (r *Repository) representation(job *publishJob) *layout.Representation { + return &layout.Representation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: r.kind, Name: job.record.Name, Version: job.record.Version, + RecordDigest: job.recordDigest, OperationID: job.opID, Packaging: job.prepared.Packaging, Location: job.location, + Payload: job.payload, Documents: job.documents, RepresentationDigest: job.representationDigest(job.prepared.Packaging), + } +} + +// retire removes what the completed publication replaced and the temporary resources it owned. +// Everything here is best-effort: the reference is already complete, and a leftover is garbage, +// not an inconsistency. +func (r *Repository) retire(t *tree, job *publishJob, plan *publishPlan) { + _ = t.removeAll(job.staging) + if plan.abandoned != "" && layout.FoldPath(plan.abandoned) != layout.FoldPath(job.location) { + r.removePayload(t, plan.abandoned) + } + if plan.prior != nil { + if plan.prior.Location != "" && layout.FoldPath(plan.prior.Location) != layout.FoldPath(job.location) { + r.removePayload(t, plan.prior.Location) + } + if plan.prior.RecordDigest != "" && plan.prior.RecordDigest != job.recordDigest { + _ = t.removeAll(layout.RecordPath(plan.prior.RecordDigest)) + _ = t.removeAll(layout.SignatureDir(artifact.SubjectRecord, plan.prior.RecordDigest)) + } + } + _ = t.removeAll(layout.OperationPath(job.opID)) +} + +// removePayload deletes the payload paths this repository owns at a location it no longer uses. +func (r *Repository) removePayload(t *tree, location string) { + for _, owned := range layout.PayloadPaths(r.kind, location) { + _ = t.removeAll(owned) + } + t.removeEmptyParents(location) +} + +// result builds the publish result for a stored artifact. +func (r *Repository) result(t *tree, record *artifact.Record, recordDigest digest.Digest, rep *layout.Representation, outcome artifact.Outcome) *artifact.PublishResult { + locator := t.absolute(rep.Location) + return &artifact.PublishResult{ + Outcome: outcome, + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, recordDigest), + Representation: rep.Artifact(locator), + Location: locator, + LockScope: artifact.LockScopeHost, + } +} diff --git a/pkg/artifact/local/read.go b/pkg/artifact/local/read.go new file mode 100644 index 00000000000..e4342244a0d --- /dev/null +++ b/pkg/artifact/local/read.go @@ -0,0 +1,231 @@ +package local + +import ( + "context" + "errors" + "fmt" + "hash" + "io" + "io/fs" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// payloadStore abstracts how one repository kind lays a payload out on disk. +type payloadStore interface { + // supports reports whether the kind can store and read the packaging. + supports(packaging artifact.Packaging) bool + + // validate checks, before any state changes, that the staged blobs agree with the Record. + validate(job *publishJob) error + + // stage writes the payload, documents, and sidecars of a publication into its staging directory. + stage(ctx context.Context, t *tree, job *publishJob) error + + // install moves the staged payload into place at the resolved location, moving any previous + // content aside first. + install(t *tree, job *publishJob) error + + // streams opens the verified payload and document streams of a completed artifact. + streams(ctx context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) + + // document opens one distribution document without touching the payload. + document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) + + // verifySidecars checks the adjacent metadata copy against the stored Record bytes. + verifySidecars(t *tree, v *view) error + + // remove deletes the files exclusively owned by the artifact and returns what it removed. + remove(t *tree, v *view) ([]string, error) +} + +// Read implements [artifact.Repository]. It opens the verified streams of exactly the pinned +// Record. The reference must still name that Record; after the caller reads the streams, the +// reference generation is rechecked, so bytes that changed underneath the reader are reported +// instead of served under the old digest. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "local.Repository.Read")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + if err := r.store.verifySidecars(t, v); err != nil { + return nil, err + } + guard := r.guard(v.ref) + streams, err := r.store.streams(ctx, t, v, guard) + if err != nil { + closeStreams(streams) + return nil, err + } + return &artifact.Content{ + Record: v.record, + Representation: v.rep.Artifact(t.absolute(v.rep.Location)), + Streams: streams, + }, nil +} + +// missingRoot maps an absent repository root to not found and passes other errors through. +func (r *Repository) missingRoot(err error, ref artifact.Ref) error { + if errors.Is(err, errRootMissing) { + return fmt.Errorf("%w: %s (the repository directory %q does not exist)", errUtils.ErrArtifactNotFound, ref, r.root) + } + return err +} + +func closeStreams(streams []artifact.ContentStream) { + for index := range streams { + if streams[index].Reader != nil { + _ = streams[index].Reader.Close() + } + } +} + +// guard returns the generation recheck for a reader that observed ref. It reads the reference +// again and fails when the generation, state, or Record changed, because the payload location is +// mutable and a changed reference means the bytes may belong to a different version. +func (r *Repository) guard(observed *layout.Reference) func() error { + return func() error { + t, err := r.openTree(false) + if err != nil { + return fmt.Errorf("%w: the repository changed while reading %s:%s: %w", errUtils.ErrArtifactPublicationUnavailable, observed.Name, observed.Version, err) + } + defer t.Close() + current, err := t.loadReference(observed.Key()) + switch { + case err != nil: + return err + case current == nil: + return fmt.Errorf("%w: %s:%s was deleted while it was being read", errUtils.ErrArtifactNotFound, observed.Name, observed.Version) + case current.Generation != observed.Generation || !current.Available() || current.RecordDigest != observed.RecordDigest: + return fmt.Errorf("%w: %s:%s changed while it was being read (generation %d to %d); the bytes read are discarded rather than served under the old digest", + errUtils.ErrArtifactPublicationUnavailable, observed.Name, observed.Version, observed.Generation, current.Generation) + } + return nil + } +} + +// verifyingReader verifies size and digest as the content is read. At EOF it compares the digest +// and then runs the generation recheck, so a clean EOF means the bytes were intact and still +// belonged to the pinned reference. +type verifyingReader struct { + source io.ReadCloser + hasher hash.Hash + want digest.Digest + size int64 + read int64 + guard func() error + finished bool + failure error +} + +func newVerifyingReader(source io.ReadCloser, want digest.Digest, size int64, guard func() error) *verifyingReader { + return &verifyingReader{source: source, hasher: want.Algorithm().Hash(), want: want, size: size, guard: guard} +} + +// Read implements io.Reader. +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "local.verifyingReader.Read")() + + if v.failure != nil { + return 0, v.failure + } + n, err := v.source.Read(p) + v.read += int64(n) + _, _ = v.hasher.Write(p[:n]) + switch { + case v.read > v.size: + v.failure = fmt.Errorf("%w: content is larger than its declared %d bytes", errUtils.ErrArtifactIntegrityFailed, v.size) + return n, v.failure + case errors.Is(err, io.EOF) && !v.finished: + v.finished = true + if verr := v.finish(); verr != nil { + v.failure = verr + return n, verr + } + case err != nil && !errors.Is(err, io.EOF): + v.failure = fmt.Errorf("%w: %w", errUtils.ErrArtifactIntegrityFailed, err) + return n, v.failure + } + return n, err +} + +func (v *verifyingReader) finish() error { + if v.read != v.size { + return fmt.Errorf("%w: content is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, v.read, v.size) + } + if got := digest.NewDigest(v.want.Algorithm(), v.hasher); got != v.want { + return fmt.Errorf("%w: content digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, v.want) + } + if v.guard != nil { + return v.guard() + } + return nil +} + +// Close implements io.Closer. +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "local.verifyingReader.Close")() + + return v.source.Close() +} + +// openFile opens a payload or sidecar file, mapping a missing file to an integrity failure: the +// completed reference promised it exists. +func (t *tree) openFile(rel, what string) (io.ReadCloser, fs.FileInfo, error) { + file, err := t.Open(native(rel)) + if err != nil { + return nil, nil, fmt.Errorf("%w: %s %q is missing or unreadable: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, fmt.Errorf("%w: %s %q: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, nil, fmt.Errorf("%w: %s %q is not a regular file", errUtils.ErrArtifactIntegrityFailed, what, rel) + } + return file, info, nil +} + +// verifyBytes compares a stored sidecar with the expected bytes. +func (t *tree) verifyBytes(rel, what string, want []byte) error { + got, err := t.readBounded(rel, artifact.MaxRecordSize) + if err != nil { + return fmt.Errorf("%w: %s %q is missing or unreadable: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + if string(got) != string(want) { + return fmt.Errorf("%w: %s %q does not match the stored Record", errUtils.ErrArtifactIntegrityFailed, what, rel) + } + return nil +} + +// documentDescriptor returns the Record's descriptor for a role. +func documentDescriptor(record *artifact.Record, role artifact.DocumentRole) (*artifact.Document, bool) { + for index := range record.Documents { + if record.Documents[index].Role == role { + return &record.Documents[index], true + } + } + return nil, false +} + +// blobDescriptor converts a Record document to the OCI-style descriptor used in streams. +func blobDescriptor(doc *artifact.Document) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: doc.MediaType, Digest: doc.Digest, Size: doc.Size} +} diff --git a/pkg/artifact/local/recovery_test.go b/pkg/artifact/local/recovery_test.go new file mode 100644 index 00000000000..8434b5328f7 --- /dev/null +++ b/pkg/artifact/local/recovery_test.go @@ -0,0 +1,292 @@ +package local + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +var allSpecs = []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz), archiveSpec(artifact.PackagingZip)} + +func specName(spec *repoSpec) string { return spec.kind + "/" + string(spec.packaging) } + +func listNames(t *testing.T, repo *Repository) []string { + t.Helper() + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + names := make([]string, 0, len(page.Entries)) + for index := range page.Entries { + names = append(names, page.Entries[index].Name+":"+page.Entries[index].Version) + } + return names +} + +// An interruption before the commit never advertises partial success: the reference is +// unavailable (not found, not complete-looking), invisible to listing, and a same-intent retry +// repairs it completely. +func TestInterruptedPublication_RetryRepairs(t *testing.T) { + points := []faultPoint{faultBeforePayload, faultDuringPayload, faultAfterMetadata} + for _, spec := range allSpecs { + for _, point := range points { + t.Run(specName(spec)+"/"+string(point), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(point))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "a pending reference is unavailable to managed readers") + require.ErrorContains(t, err, prepared.OperationID, "the diagnostic names the owning operation") + assert.Empty(t, listNames(t, repo), "a pending reference is absent from search results") + _, err = repo.Pin(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + recordDigest, err := prepared.Record.RecordDigest() + require.NoError(t, err) + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: recordDigest}) + require.Error(t, err, "even the digest does not expose a pending record") + assert.True(t, errors.Is(err, errUtils.ErrArtifactPublicationUnavailable) || errors.Is(err, errUtils.ErrArtifactNotFound), "%v", err) + + retry, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retry.Outcome) + assertIntact(t, repo, defaultRef()) + assert.Equal(t, []string{testName + ":" + testVersion}, listNames(t, repo)) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(prepared.OperationID))), "the retry cleans the operation's staging") + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath(prepared.OperationID)))) + }) + } + } +} + +// assertIntact reads the artifact end to end and checks it verifies. +func assertIntact(t *testing.T, repo *Repository, r artifact.Ref) { + t.Helper() + pinned := resolve(t, repo, r) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NoError(t, readAll(t, content)) +} + +// A crash after the commit but before the response leaves a completed reference: a retry is a +// no-op, and the caller can prove success by reading it. +func TestInterruptedPublication_AfterCommitIsComplete(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterCommit))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + assertIntact(t, repo, defaultRef()) + digestWant, err := prepared.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, digestWant, resolve(t, repo, defaultRef()).RecordDigest) + + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a writer that expects the reference to be absent is stale") + current := resolve(t, repo, defaultRef()) + retry, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{ExpectedPrior: ¤t}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, retry.Outcome) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(prepared.OperationID))), "an unchanged republish tidies the finished operation") + }) + } +} + +// A different intent never steals a pending operation, with or without force. The same intent +// from a later invocation (new operation ID and preparation time) repairs it. +func TestPendingOperation_IsNeverStolen(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + first := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "first\n"}, opID: "op-first"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + other := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "different\n"}, opID: "op-other"}) + for _, force := range []bool{false, true} { + _, err := repo.Publish(context.Background(), other, artifact.PublishOptions{Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "force=%v", force) + require.ErrorContains(t, err, "op-first", "the recovery diagnostic includes the operation ID") + } + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the failed attempt did not disturb the pending state") + + later := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "first\n"}, opID: "op-later", at: capturedAt.Add(72 * time.Hour)}) + result, err := repo.Publish(context.Background(), later, artifact.PublishOptions{}) + require.NoError(t, err, "the same intent from a later invocation repairs the pending operation") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath("op-first"))), "the abandoned journal is retired") + }) + } +} + +// A failed or interrupted force replacement leaves the reference unavailable until the same-intent +// retry repairs it. The old digest never serves the replacement's bytes and is not found after. +func TestInterruptedForceReplacement(t *testing.T) { + for _, spec := range allSpecs { + for _, point := range []faultPoint{faultDuringPayload, faultAfterMetadata} { + t.Run(specName(spec)+"/"+string(point), func(t *testing.T) { + var armed faultHook + spec.opts = []Option{withFaults(func(p faultPoint) error { + if armed != nil { + return armed(p) + } + return nil + })} + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}, opID: "op-1"}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}) + armed = crashAt(point) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the interrupted replacement leaves the reference unavailable") + _, err = repo.Read(context.Background(), old) + require.Error(t, err, "the old digest is never served while the replacement is in flight") + assert.Empty(t, listNames(t, repo)) + + result, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true}) + require.NoError(t, err, "the same-intent retry repairs the reference") + assert.Equal(t, artifact.OutcomeReplaced, result.Outcome) + current := resolve(t, repo, defaultRef()) + assert.NotEqual(t, old.RecordDigest, current.RecordDigest) + assertIntact(t, repo, defaultRef()) + + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "retired bytes report not found, never the current bytes") + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: old.RecordDigest}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } + } +} + +// Old-digest reads never return the replacement's bytes, even when the physical location is the +// same, and the guard catches a reader that is mid-stream when the replacement lands. +func TestReplacementNeverServesNewBytesUnderOldDigest(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + content, err := repo.Read(context.Background(), old) + require.NoError(t, err) + defer content.Close() + + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + + err = readAll(t, content) + require.Error(t, err, "a reader that was open when the reference moved must not report success") + assert.True(t, errorsIsAny(err, errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactNotFound), "%v", err) + + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// The recovery path triggers only for the matching condition: without an interruption a changed +// intent is a plain conflict and nothing is repaired or replaced. +func TestNoRecoveryWithoutInterruption(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + changed := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}) + _, err := repo.Publish(context.Background(), changed, artifact.PublishOptions{ExpectedPrior: &old}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, old.RecordDigest, resolve(t, repo, defaultRef()).RecordDigest, "the stored artifact is untouched") + assertIntact(t, repo, defaultRef()) + + stale := artifact.PinnedReference{ResolvedDigest: digest.FromString("stale")} + _, err = repo.Publish(context.Background(), changed, artifact.PublishOptions{Force: true, ExpectedPrior: &stale}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected prior is a conflict even under force") + + _, err = repo.Publish(context.Background(), changed, artifact.PublishOptions{Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the reference exists but the writer expected none") +} + +// Identical republication is unchanged even when time and operation differ. +func TestRepublishIsUnchanged(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + before := resolve(t, repo, defaultRef()) + pinned := before + second := publish(t, repo, &prepareSpec{at: capturedAt.Add(48 * time.Hour), opID: "op-again"}, artifact.PublishOptions{ExpectedPrior: &pinned}) + assert.Equal(t, artifact.OutcomeUnchanged, second.Outcome) + assert.Equal(t, first.Pinned.RecordDigest, second.Pinned.RecordDigest, "the existing Record is returned unchanged") + assert.Equal(t, first.Location, second.Location) + assert.Equal(t, before.RecordDigest, resolve(t, repo, defaultRef()).RecordDigest) + }) + } +} + +func errorsIsAny(err error, targets ...error) bool { + for _, target := range targets { + if errors.Is(err, target) { + return true + } + } + return false +} + +// A failed publication before any state change (validation, unsupported packaging) writes nothing. +func TestRejectedPublicationWritesNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + wrong := prepare(t, newRepo(t, archiveSpec(artifact.PackagingTgz)), &prepareSpec{}) + _, err := repo.Publish(context.Background(), wrong, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a dir repository does not accept an archive payload") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "the root was not even created") + + _, err = repo.Publish(context.Background(), nil, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +// A same-intent retry under a different packaging cannot reuse the journaled location (its +// extension would disagree), so it publishes at a freshly rendered location and retires the +// abandoned attempt's partial payload instead of orphaning it. +func TestResumedPublicationRetiresAnAbandonedLocation(t *testing.T) { + var fault armedFault + root := filepath.Join(t.TempDir(), "repo") + tgz := archiveSpec(artifact.PackagingTgz).withRoot(root).withOpts(fault.option()) + repo := newRepo(t, tgz) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{opID: "op-tgz"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + abandoned := filepath.Join(root, "team", "app", "1.0.0.tgz") + assert.FileExists(t, abandoned, "the crashed attempt left its payload behind") + + zipRepo := newRepo(t, archiveSpec(artifact.PackagingZip).withRoot(root)) + result, err := zipRepo.Publish(context.Background(), prepare(t, zipRepo, &prepareSpec{opID: "op-zip"}), artifact.PublishOptions{}) + require.NoError(t, err, "the same intent repairs the pending reference whatever the packaging") + assert.Equal(t, filepath.Join(root, "team", "app", "1.0.0.zip"), result.Location) + assert.NoFileExists(t, abandoned, "the abandoned payload was retired, not orphaned") + assert.NoFileExists(t, abandoned+".metadata.json") + assertIntact(t, zipRepo, defaultRef()) + assert.Equal(t, []string{"team/app/1.0.0.zip", "team/app/1.0.0.zip.metadata.json"}, payloadFiles(t, root)) +} diff --git a/pkg/artifact/local/register.go b/pkg/artifact/local/register.go new file mode 100644 index 00000000000..75f5a3f2354 --- /dev/null +++ b/pkg/artifact/local/register.go @@ -0,0 +1,72 @@ +package local + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Register registers the local/dir and local/archive repository kinds in the process-wide +// registry. It is called explicitly by a composition root, never from init. Options such as +// [WithVerifier] apply to every repository the registered factories construct. It does not touch +// the legacy CI artifact registry, whose own "local/dir" store is a different thing. +func Register(opts ...Option) error { + defer perf.Track(nil, "local.Register")() + + return RegisterWith(artifact.DefaultRepositoryRegistry(), opts...) +} + +// RegisterWith registers the local kinds in the given registry. A kind that is already +// registered is an error, so double registration is caught instead of silently replaced. +func RegisterWith(registry *artifact.RepositoryRegistry, opts ...Option) error { + defer perf.Track(nil, "local.RegisterWith")() + + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + desc := Descriptor(kind, opts...) + if err := registry.Register(&desc, factoryFor(kind, opts)); err != nil { + return err + } + } + return nil +} + +// Descriptor returns the static descriptor of a local kind: the built-in descriptor refined with +// the capabilities the registered backend actually provides (deletion, enumeration, conditional +// updates, detached evidence for the verifier's formats). It needs no I/O. +func Descriptor(kind string, opts ...Option) artifact.KindDescriptor { + defer perf.Track(nil, "local.Descriptor")() + + desc, _ := artifact.BuiltinKind(kind) + applied := applySettings(opts) + desc.Capabilities = staticCapabilities(kind, hostEnforcesModes(), applied.verifier) + return *desc +} + +func factoryFor(kind string, opts []Option) artifact.RepositoryFactory { + return func(_ context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + repo, err := newRepository(alias, kind, cfg, opts) + if err != nil { + return nil, err + } + return repo, nil + } +} + +// Kinds lists the repository kinds this package registers. +func Kinds() []string { + defer perf.Track(nil, "local.Kinds")() + + return []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} +} + +// ensureKind rejects a configuration that names another kind. +func ensureKind(want string, cfg *schema.ArtifactRepositoryConfig) error { + if cfg.Kind != "" && cfg.Kind != want { + return fmt.Errorf("%w: configuration kind %q cannot construct a %q repository", errUtils.ErrArtifactConfigInvalid, cfg.Kind, want) + } + return nil +} diff --git a/pkg/artifact/local/register_test.go b/pkg/artifact/local/register_test.go new file mode 100644 index 00000000000..0be61b12ac9 --- /dev/null +++ b/pkg/artifact/local/register_test.go @@ -0,0 +1,126 @@ +package local + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestRegisterWith_RegistersBothKindsAsSeparateBackends(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + assert.Equal(t, []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive}, Kinds()) + for _, kind := range Kinds() { + assert.True(t, registry.IsRegistered(kind), kind) + } + + dirDesc, ok := registry.Lookup(artifact.RepositoryKindLocalDir) + require.True(t, ok) + archiveDesc, ok := registry.Lookup(artifact.RepositoryKindLocalArchive) + require.True(t, ok) + assert.Equal(t, []artifact.Packaging{artifact.PackagingNone}, dirDesc.Capabilities.Packagings) + assert.Contains(t, archiveDesc.Capabilities.Packagings, artifact.PackagingZip) + assert.Equal(t, artifact.DeleteObject, dirDesc.Capabilities.Delete, "the registered descriptor carries the backend's real capabilities") + assert.True(t, archiveDesc.AllowsPermissions) + assert.False(t, dirDesc.AllowsPermissions) + assert.Equal(t, artifact.PackagingTgz, archiveDesc.DefaultPackaging) + assert.Equal(t, []string{artifact.OptionPath}, archiveDesc.PathOptions) + + require.ErrorIs(t, RegisterWith(registry), errUtils.ErrArtifactConflict, "double registration is an error, not a silent replacement") +} + +func TestRegisteredFactories_ConstructRepositories(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + root := t.TempDir() + + dir, err := registry.New(context.Background(), "dir", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "dir")}, + }) + require.NoError(t, err) + archive, err := registry.New(context.Background(), "archive", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Permissions: "0640", Packaging: "zip", + Options: map[string]any{"path": filepath.Join(root, "archive"), "key_pattern": `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.zip`}, + }) + require.NoError(t, err) + assert.IsType(t, dirStore{}, dir.(*Repository).store) + assert.IsType(t, archiveStore{}, archive.(*Repository).store) + assert.Equal(t, artifact.PackagingZip, archive.(*Repository).packaging) + assert.EqualValues(t, 0o640, archive.(*Repository).archiveMode) + assert.True(t, archive.(*Repository).modeExplicit) + + _, statErr := filepath.Glob(filepath.Join(root, "*")) + require.NoError(t, statErr) + entries, _ := filepath.Glob(filepath.Join(root, "*")) + assert.Empty(t, entries, "construction creates no directories") + + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Packaging: "tgz", Options: map[string]any{"path": root}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a directory repository does not accept archive packaging") + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Permissions: "0600", Options: map[string]any{"path": root}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": root, "key_pattern": "../{{ .Name }}.tgz"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a key_pattern that escapes the root is rejected before anything is built") +} + +func TestRegisterWithVerifier_AdvertisesEvidenceFormats(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithVerifier(&artifacttest.FakeVerifier{}), WithOwner("custom-owner"))) + desc, _ := registry.Lookup(artifact.RepositoryKindLocalArchive) + assert.Equal(t, []artifact.EvidenceFormat{artifacttest.FakeEvidenceFormat}, desc.Capabilities.EvidenceFormats) + + repo, err := registry.New(context.Background(), "a", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": t.TempDir()}, + }) + require.NoError(t, err) + caps := repo.Capabilities() + assert.Equal(t, desc.Capabilities.EvidenceFormats, caps.EvidenceFormats, "static and constructed capabilities agree") + assert.Equal(t, "custom-owner", repo.(*Repository).settings.owner) +} + +func TestDescriptor_ReflectsHostAndVerifier(t *testing.T) { + plain := Descriptor(artifact.RepositoryKindLocalDir) + assert.Empty(t, plain.Capabilities.EvidenceFormats) + assert.NotNil(t, plain.NewOptions) + assert.Equal(t, hostEnforcesModes(), plain.Capabilities.PreservesModes) +} + +func TestDefaultRegistryRegistration(t *testing.T) { + artifact.ResetRepositoryKindsForTesting() + t.Cleanup(artifact.ResetRepositoryKindsForTesting) + require.NoError(t, Register()) + repo, err := artifact.NewRepository(context.Background(), "x", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": t.TempDir()}, + }) + require.NoError(t, err) + assert.IsType(t, &Repository{}, repo) + require.ErrorIs(t, Register(), errUtils.ErrArtifactConflict) +} + +func TestNewRepository_RejectsBadInputs(t *testing.T) { + _, err := NewDir("x", nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = NewDir("x", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": t.TempDir()}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a dir constructor refuses another kind's configuration") + _, err = NewArchive("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": ""}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = NewArchive("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": t.TempDir(), "unknown": 1}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + + repo, err := NewDir("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": "relative/root"}}) + require.NoError(t, err) + assert.True(t, filepath.IsAbs(repo.Root()), "a relative root is made absolute at construction") +} diff --git a/pkg/artifact/local/repository.go b/pkg/artifact/local/repository.go new file mode 100644 index 00000000000..38b4d329c0c --- /dev/null +++ b/pkg/artifact/local/repository.go @@ -0,0 +1,288 @@ +package local + +import ( + "context" + "fmt" + "io/fs" + "path/filepath" + "runtime" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Repository is a local directory or local archive artifact repository. Construct it with +// [NewDir] or [NewArchive]. It is safe for concurrent use: all coordination happens through +// OS file locks and atomic file replacement on disk. +type Repository struct { + alias string + kind string + root string + keyPattern string + packaging artifact.Packaging + store payloadStore + settings settings + + // archiveMode and modeExplicit hold the quoted-octal archive permissions and whether the + // configuration requested them explicitly. They are meaningful for archives only. + archiveMode fs.FileMode + modeExplicit bool + + // hostModes reports whether the host can enforce permission bits. It is a field so tests can + // exercise the unsupported-platform path on any host. + hostModes bool +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// NewDir returns a local/dir repository for the configuration. It performs no I/O: the root is +// created by the first write. +func NewDir(alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "local.NewDir")() + + return newRepository(alias, artifact.RepositoryKindLocalDir, cfg, opts) +} + +// NewArchive returns a local/archive repository for the configuration. It performs no I/O. +func NewArchive(alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "local.NewArchive")() + + return newRepository(alias, artifact.RepositoryKindLocalArchive, cfg, opts) +} + +func newRepository(alias, kind string, cfg *schema.ArtifactRepositoryConfig, opts []Option) (*Repository, error) { + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, ok := artifact.BuiltinKind(kind) + if !ok { + return nil, fmt.Errorf("%w: unknown repository kind %q", errUtils.ErrArtifactConfigInvalid, kind) + } + if err := ensureKind(kind, cfg); err != nil { + return nil, err + } + checked := *cfg + checked.Kind = kind + if err := artifact.ValidateRepositoryConfig(desc, &checked); err != nil { + return nil, err + } + options := desc.NewOptions() + if err := artifact.DecodeOptions(cfg.Options, options); err != nil { + return nil, err + } + root, keyPattern, err := rootAndPattern(options) + if err != nil { + return nil, err + } + repo := &Repository{ + alias: alias, kind: kind, root: root, keyPattern: keyPattern, + packaging: desc.EffectivePackaging(cfg.Packaging), settings: applySettings(opts), + archiveMode: artifact.DefaultArchivePermissions, hostModes: hostEnforcesModes(), + } + if err := repo.configureStore(cfg.Permissions); err != nil { + return nil, err + } + return repo, nil +} + +// configureStore selects the payload store of the kind and, for archives, parses the permissions. +// It records whether the permissions were explicit, because an explicit request that the host +// cannot enforce is an error while a default is not. +func (r *Repository) configureStore(permissions string) error { + if r.kind != artifact.RepositoryKindLocalArchive { + r.store = dirStore{repo: r} + return nil + } + r.store = archiveStore{repo: r} + if permissions == "" { + return nil + } + mode, err := artifact.ParsePermissions(permissions) + if err != nil { + return err + } + r.archiveMode, r.modeExplicit = mode, true + return nil +} + +// rootAndPattern extracts the absolute root and key pattern from decoded options. +func rootAndPattern(options artifact.Options) (string, string, error) { + var path, pattern string + switch typed := options.(type) { + case *artifact.LocalDirOptions: + path, pattern = typed.Path, typed.KeyPattern + case *artifact.LocalArchiveOptions: + path, pattern = typed.Path, typed.KeyPattern + } + root, err := filepath.Abs(filepath.Clean(path)) + if err != nil { + return "", "", fmt.Errorf("%w: repository path %q: %w", errUtils.ErrArtifactConfigInvalid, path, err) + } + return root, pattern, nil +} + +// hostEnforcesModes reports whether the host honors Unix permission bits. Windows maps modes onto +// a read-only attribute and ACLs, so it cannot enforce an arbitrary mode. +func hostEnforcesModes() bool { + return runtime.GOOS != "windows" && runtime.GOOS != "plan9" +} + +// Root returns the absolute repository root. +func (r *Repository) Root() string { + defer perf.Track(nil, "local.Repository.Root")() + + return r.root +} + +// Capabilities implements [artifact.Repository]. It is computed from configuration alone. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "local.Repository.Capabilities")() + + return staticCapabilities(r.kind, r.hostModes, r.settings.verifier) +} + +// staticCapabilities describes a local kind without constructing a repository. +func staticCapabilities(kind string, hostModes bool, verifier artifact.Verifier) artifact.Capabilities { + desc, _ := artifact.BuiltinKind(kind) + caps := desc.Capabilities + caps.Read, caps.Write = true, true + caps.EnumerateNamed, caps.EnumerateCollection = true, true + caps.Delete = artifact.DeleteObject + caps.ConditionalUpdate = true + caps.PreservesModes = hostModes || kind == artifact.RepositoryKindLocalArchive + caps.SubjectKinds = []artifact.SubjectKind{artifact.SubjectRecord} + caps.LockScopes = []artifact.LockScope{artifact.LockScopeHost} + caps.EvidenceFormats = nil + if verifier != nil { + caps.EvidenceFormats = append(caps.EvidenceFormats, verifier.Formats()...) + } + return caps +} + +// pinned builds the immutable reference for a stored Record. +func (r *Repository) pinned(ref artifact.Ref, record *artifact.Record, recordDigest digest.Digest) artifact.PinnedReference { + return artifact.PinnedReference{ + Ref: ref, + Repository: r.root, + ResolvedDigest: recordDigest, + RecordDigest: recordDigest, + ContentDigest: record.ContentDigest, + } +} + +// view is a consistent, verified read of one completed artifact: its reference, Record, and +// representation. +type view struct { + ref *layout.Reference + record *artifact.Record + recordBytes []byte + rep *layout.Representation +} + +// unavailable builds the error for a reference that exists but cannot be used right now. +func (r *Repository) unavailable(ref *layout.Reference) error { + return fmt.Errorf("%w: %s:%s has a pending publication by operation %s; retry the publication with the same intent to repair it", + errUtils.ErrArtifactPublicationUnavailable, ref.Name, ref.Version, ref.OperationID) +} + +// resolved is a completed reference together with the verified Record it names. +type resolved struct { + ref *layout.Reference + record *artifact.Record + recordBytes []byte +} + +// resolveRef finds the reference and Record a request names. A version reference must be +// completed. A digest reference checks the Record's embedded name and succeeds only while that +// Record is still the current completed one: retired digests are not found, never redirected to +// the current bytes. +func (r *Repository) resolveRef(t *tree, ref artifact.Ref) (*resolved, error) { + if err := ref.Validate(); err != nil { + return nil, err + } + if ref.IsDigest() { + return r.resolveDigest(t, ref) + } + stored, err := t.loadReference(layout.ReferenceKey(ref.Name, ref.Version)) + if err != nil { + return nil, err + } + if stored == nil { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + if !stored.Available() { + return nil, r.unavailable(stored) + } + record, raw, err := t.loadRecord(stored.RecordDigest) + if err != nil { + return nil, err + } + if err := matchesReference(stored, record); err != nil { + return nil, err + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +func (r *Repository) resolveDigest(t *tree, ref artifact.Ref) (*resolved, error) { + record, raw, err := t.loadRecord(ref.Digest) + if err != nil { + return nil, err + } + if record.Name != ref.Name { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + stored, err := t.loadReference(layout.ReferenceKey(record.Name, record.Version)) + if err != nil { + return nil, err + } + switch { + case stored == nil: + return nil, fmt.Errorf("%w: %s (the reference was deleted)", errUtils.ErrArtifactNotFound, ref) + case !stored.Available(): + return nil, r.unavailable(stored) + case stored.RecordDigest != ref.Digest: + return nil, fmt.Errorf("%w: %s is retired; %s:%s now resolves to another record", errUtils.ErrArtifactNotFound, ref, record.Name, record.Version) + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +// matchesReference checks that a Record agrees with the reference that names it. +func matchesReference(ref *layout.Reference, record *artifact.Record) error { + if record.Name != ref.Name || record.Version != ref.Version { + return fmt.Errorf("%w: record names %s:%s but its reference is %s:%s", errUtils.ErrArtifactIntegrityFailed, record.Name, record.Version, ref.Name, ref.Version) + } + return nil +} + +// openView loads the verified view of a pinned artifact. The pinned Record must still be the +// current completed Record of its reference. +func (r *Repository) openView(ctx context.Context, t *tree, pinned *artifact.PinnedReference) (*view, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + current, err := r.resolveRef(t, artifact.Ref{Name: pinned.Ref.Name, Digest: pinned.RecordDigest}) + if err != nil { + return nil, err + } + stored := current.ref + rep, err := t.loadRepresentation(stored.Key()) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != pinned.RecordDigest || rep.RepositoryKind != r.kind { + return nil, fmt.Errorf("%w: %s:%s has no valid %s representation for record %s", errUtils.ErrArtifactIntegrityFailed, stored.Name, stored.Version, r.kind, pinned.RecordDigest) + } + if !r.store.supports(rep.Packaging) { + return nil, fmt.Errorf("%w: %s:%s is stored as %q, which this repository cannot read", errUtils.ErrArtifactPackagingInvalid, stored.Name, stored.Version, rep.Packaging) + } + return &view{ref: stored, record: current.record, recordBytes: current.recordBytes, rep: rep}, nil +} diff --git a/pkg/artifact/local/reserve.go b/pkg/artifact/local/reserve.go new file mode 100644 index 00000000000..23f3e8316fa --- /dev/null +++ b/pkg/artifact/local/reserve.go @@ -0,0 +1,91 @@ +package local + +import ( + "fmt" + "slices" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// checkCollision rejects a location that another reference owns or that holds content this +// repository does not manage. A second reference with a colliding key_pattern therefore fails +// with a conflict and is never bound to the first reference's payload, and a user's own files at +// the location are never overwritten. The caller holds the claims lock, so no other publication +// can reserve between the check and the journal write. +func (r *Repository) checkCollision(t *tree, job *publishJob, plan *publishPlan) error { + claims, err := r.collectClaims(t, job.key) + if err != nil { + return err + } + candidate := layout.Claim{ + ReferenceKey: job.key, Name: job.record.Name, Version: job.record.Version, + OperationID: job.opID, Paths: layout.PayloadPaths(r.kind, job.location), + } + if other := layout.FindConflict(&candidate, claims); other != nil { + return fmt.Errorf("%w: location %q for %s:%s is already claimed by %s:%s%s; give the repository a key_pattern that keeps references apart", + errUtils.ErrArtifactConflict, job.location, job.record.Name, job.record.Version, other.Name, other.Version, pendingSuffix(other)) + } + return r.checkUnmanaged(t, &candidate, plan) +} + +func pendingSuffix(claim *layout.Claim) string { + if claim.OperationID == "" { + return "" + } + return " (pending operation " + claim.OperationID + ")" +} + +// collectClaims lists the paths owned by every other reference: the representation of each +// completed reference and the journal of each pending one. +func (r *Repository) collectClaims(t *tree, ownKey string) ([]layout.Claim, error) { + refs, err := t.listReferences() + if err != nil { + return nil, err + } + var claims []layout.Claim + for _, ref := range refs { + if ref.Key() == ownKey { + continue + } + if ref.Available() { + rep, err := t.loadRepresentation(ref.Key()) + if err != nil { + return nil, err + } + if rep != nil { + claims = append(claims, rep.Claim()) + } + continue + } + op, err := t.loadOperation(ref.OperationID) + if err != nil { + return nil, err + } + if op != nil { + claims = append(claims, op.Claim()) + } + } + return claims, nil +} + +// checkUnmanaged fails when a path the publication would write already exists and this reference +// does not own it. +func (r *Repository) checkUnmanaged(t *tree, candidate *layout.Claim, plan *publishPlan) error { + var owned []string + for _, location := range plan.owned { + for _, path := range layout.PayloadPaths(r.kind, location) { + owned = append(owned, layout.FoldPath(path)) + } + } + for _, path := range candidate.Paths { + exists, err := t.exists(path) + if err != nil { + return fmt.Errorf("%w: inspect %q: %w", errUtils.ErrArtifactUploadFailed, path, err) + } + if exists && !slices.Contains(owned, layout.FoldPath(path)) { + return fmt.Errorf("%w: %q already exists and is not managed by this repository; remove it or choose another key_pattern", errUtils.ErrArtifactConflict, path) + } + } + return nil +} diff --git a/pkg/artifact/local/resolve.go b/pkg/artifact/local/resolve.go new file mode 100644 index 00000000000..a4b024edc98 --- /dev/null +++ b/pkg/artifact/local/resolve.go @@ -0,0 +1,232 @@ +package local + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// defaultPageSize is the list page size when the query does not set one. +const defaultPageSize = 100 + +// Resolve implements [artifact.Repository]. A version reference resolves once to the digest of +// its immutable Record, which every later operation uses. A pending reference is unavailable; a +// digest reference checks the Record's embedded name and names only the current Record. +func (r *Repository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "local.Repository.Resolve")() + + if err := ctx.Err(); err != nil { + return artifact.PinnedReference{}, err + } + t, err := r.openTree(false) + if err != nil { + return artifact.PinnedReference{}, r.missingRoot(err, ref) + } + defer t.Close() + current, err := r.resolveRef(t, ref) + if err != nil { + return artifact.PinnedReference{}, err + } + return r.pinned(ref, current.record, current.ref.RecordDigest), nil +} + +// Pin implements [artifact.Repository]. It resolves the reference and lists the evidence +// descriptors known now. It takes no lock and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "local.Repository.Pin")() + + pinned, err := r.Resolve(ctx, ref) + if err != nil { + return pinned, err + } + t, err := r.openTree(false) + if err != nil { + return artifact.PinnedReference{}, err + } + defer t.Close() + pinned.Evidence = descriptorsOf(t.listEvidence(pinned.RecordDigest)) + return pinned, nil +} + +// Inspect implements [artifact.Repository]. It reads only managed records and the small adjacent +// metadata copy; it never opens the payload. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Inspect(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "local.Repository.Inspect")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + if err := r.store.verifySidecars(t, v); err != nil { + return nil, err + } + record := *v.record + inspection := &artifact.Inspection{ + Pinned: r.pinned(pinned.Ref, &record, v.ref.RecordDigest), + Record: &record, + Representation: v.rep.Artifact(t.absolute(v.rep.Location)), + } + inspection.Evidence = t.listEvidence(v.ref.RecordDigest) + return inspection, nil +} + +// OpenDocument implements [artifact.Repository] without reading the payload. The reader verifies +// the document digest and size and rechecks the reference generation at EOF. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) OpenDocument(ctx context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "local.Repository.OpenDocument")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + doc, ok := documentDescriptor(v.record, role) + if !ok { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDocumentMissing, role) + } + reader, err := r.store.document(t, v, doc, r.guard(v.ref)) + if err != nil { + return nil, err + } + return &artifact.DocumentStream{Document: *doc, Reader: reader}, nil +} + +// List implements [artifact.Repository]. It enumerates managed completed references only: never +// arbitrary user files, pending publications, staging, or journals. Entries are sorted by name +// then version. +func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "local.Repository.List")() + + t, err := r.openTree(false) + if err != nil { + if errors.Is(err, errRootMissing) { + return &artifact.ListPage{}, nil + } + return nil, err + } + defer t.Close() + refs, err := t.listReferences() + if err != nil { + return nil, err + } + candidates := make([]listCandidate, 0, len(refs)) + for _, ref := range refs { + if ref.Available() && (query.Name == "" || ref.Name == query.Name) { + candidates = append(candidates, listCandidate{name: ref.Name, version: ref.Version, digest: ref.RecordDigest}) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].less(&candidates[j]) }) + return r.page(ctx, t, candidates, &query) +} + +// listCandidate is one completed reference awaiting its Record. +type listCandidate struct { + name string + version string + digest digest.Digest +} + +func (c *listCandidate) less(other *listCandidate) bool { + if c.name != other.name { + return c.name < other.name + } + return c.version < other.version +} + +// page loads the Records for one page. Label selectors need every Record, so they are applied +// before paging; without them only the page's Records are read. +func (r *Repository) page(ctx context.Context, t *tree, candidates []listCandidate, query *artifact.ListQuery) (*artifact.ListPage, error) { + size := query.PageSize + if size <= 0 { + size = defaultPageSize + } + if query.Cursor != "" { + start := sort.Search(len(candidates), func(i int) bool { return cursorAfter(&candidates[i], query.Cursor) }) + candidates = candidates[start:] + } + page := &artifact.ListPage{} + for index := range candidates { + if err := ctx.Err(); err != nil { + return nil, err + } + entry, matched, err := r.listEntry(t, &candidates[index], query.Labels) + if err != nil { + return nil, err + } + if !matched { + continue + } + if len(page.Entries) == size { + page.NextCursor = page.Entries[len(page.Entries)-1].Name + ":" + page.Entries[len(page.Entries)-1].Version + return page, nil + } + page.Entries = append(page.Entries, entry) + } + return page, nil +} + +// cursorAfter reports whether the candidate sorts strictly after the cursor's name and version. +func cursorAfter(candidate *listCandidate, cursor string) bool { + index := strings.LastIndex(cursor, ":") + if index < 0 { + return true + } + after := listCandidate{name: cursor[:index], version: cursor[index+1:]} + return after.less(candidate) +} + +func (r *Repository) listEntry(t *tree, candidate *listCandidate, labels []artifact.Label) (artifact.ListEntry, bool, error) { + record, _, err := t.loadRecord(candidate.digest) + if err != nil { + return artifact.ListEntry{}, false, err + } + if record.Name != candidate.name || record.Version != candidate.version { + return artifact.ListEntry{}, false, fmt.Errorf("%w: record %s describes %s:%s but its reference is %s:%s", + errUtils.ErrArtifactIntegrityFailed, candidate.digest, record.Name, record.Version, candidate.name, candidate.version) + } + for _, label := range labels { + if value, ok := record.Metadata.Labels[label.Key]; !ok || value != label.Value { + return artifact.ListEntry{}, false, nil + } + } + return artifact.ListEntry{ + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, candidate.digest), + Name: record.Name, + Version: record.Version, + Kind: record.Kind, + Description: record.Metadata.Description, + Labels: record.Metadata.Labels, + Documents: record.Documents, + }, true, nil +} + +// descriptorsOf lists evidence descriptors. +func descriptorsOf(evidence []artifact.SignatureEvidence) []ocispec.Descriptor { + out := make([]ocispec.Descriptor, 0, len(evidence)) + for index := range evidence { + out = append(out, evidence[index].Descriptor) + } + return out +} diff --git a/pkg/artifact/local/roundtrip_test.go b/pkg/artifact/local/roundtrip_test.go new file mode 100644 index 00000000000..429d9b33ad4 --- /dev/null +++ b/pkg/artifact/local/roundtrip_test.go @@ -0,0 +1,287 @@ +package local + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +// publishFilePackage publishes the shared "File package" fixture: nested files, an executable +// helper, an empty file, an empty directory, a confined symlink, a Unicode filename, and distinct +// package README and CHANGELOG documents. +func publishFilePackage(t *testing.T, repo *Repository) (*artifact.PublishResult, fixtures.FilePackage) { + t.Helper() + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: testName, Version: testVersion, Packaging: repo.packaging, + OperationID: "op-file-package", PreparedAt: capturedAt, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + result, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + return result, pkg +} + +func TestRoundTrip_EveryAllowedPackaging(t *testing.T) { + specs := []*repoSpec{ + dirSpec(), + archiveSpec(artifact.PackagingTar), + archiveSpec(artifact.PackagingTgz), + archiveSpec(artifact.PackagingZip), + } + for _, spec := range specs { + name := spec.kind + "/" + string(spec.packaging) + t.Run(name, func(t *testing.T) { + repo := newRepo(t, spec) + result, pkg := publishFilePackage(t, repo) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + assert.Equal(t, repo.packaging, result.Representation.Packaging) + + pinned := resolve(t, repo, defaultRef()) + assert.Equal(t, pinned.RecordDigest, pinned.ResolvedDigest) + assert.Equal(t, result.Pinned.RecordDigest, pinned.RecordDigest) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "the pulled tree matches the inventory, modes included") + + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), listInventoryPaths(content.Record)) + assert.NoDirExists(t, filepath.Join(out, ".atmos"), "the distribution directory never leaks into pulled user content") + assert.FileExists(t, filepath.Join(out, "README.md"), "a root README is ordinary payload") + fixtures.RequirePOSIXModes(t) + info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&0o111, "the executable bit survives the round trip") + }) + } +} + +func listInventoryPaths(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for index := range record.Inventory { + paths = append(paths, record.Inventory[index].Path) + } + sort.Strings(paths) + return paths +} + +func TestDirLayout_PayloadIsOrdinaryFilesWithAdjacentDistribution(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + location := filepath.Join(repo.root, "team", "app", "1.0.0") + assert.Equal(t, location, result.Location) + + assert.FileExists(t, filepath.Join(location, "README.md")) + data, err := os.ReadFile(filepath.Join(location, "nested", "deep", "file.txt")) + require.NoError(t, err) + assert.Equal(t, "deep content\n", string(data), "payload files are ordinary copied files") + assert.DirExists(t, filepath.Join(location, "emptydir")) + + dist := filepath.Join(location, ".atmos", "artifact") + for name, want := range map[string]string{"README.md": fixtures.PackageReadme, "CHANGELOG.md": fixtures.PackageChangelog} { + got, err := os.ReadFile(filepath.Join(dist, name)) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + } + manifest, err := os.ReadFile(filepath.Join(dist, "manifest.json")) + require.NoError(t, err) + record, err := artifact.DecodeRecord(manifest) + require.NoError(t, err) + assert.Equal(t, testName, record.Name) + + for _, managed := range []string{ + layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), + layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)), + layout.RecordPath(result.Pinned.RecordDigest), + } { + assert.FileExists(t, filepath.Join(repo.root, filepath.FromSlash(managed))) + } + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath("op-file-package"))), "staging is cleaned") + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath("op-file-package"))), "the finished journal is removed") +} + +func TestArchiveLayout_SidecarsAdjacentAndPortableCopiesEmbedded(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + archivePath := filepath.Join(repo.root, "team", "app", "1.0.0.tgz") + assert.Equal(t, archivePath, result.Location) + + assert.FileExists(t, archivePath) + metadata, err := os.ReadFile(archivePath + ".metadata.json") + require.NoError(t, err) + record, err := artifact.DecodeRecord(metadata) + require.NoError(t, err) + assert.Equal(t, result.Pinned.ContentDigest, record.ContentDigest) + for name, want := range map[string]string{"README.md": fixtures.PackageReadme, "CHANGELOG.md": fixtures.PackageChangelog} { + got, err := os.ReadFile(filepath.Join(archivePath+".docs", name)) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + } + + members := tgzMembers(t, archivePath) + assert.Contains(t, members, ".atmos/artifact/manifest.json", "the archive embeds a portable manifest") + assert.Contains(t, members, ".atmos/artifact/README.md") + assert.Equal(t, string(metadata), members[".atmos/artifact/manifest.json"], "the embedded manifest equals the adjacent metadata") +} + +// tgzMembers returns file member contents by name. +func tgzMembers(t *testing.T, archivePath string) map[string]string { + t.Helper() + file, err := os.Open(archivePath) + require.NoError(t, err) + defer file.Close() + gz, err := gzip.NewReader(file) + require.NoError(t, err) + members := map[string]string{} + reader := tar.NewReader(gz) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + if header.Typeflag == tar.TypeReg { + data, err := io.ReadAll(reader) + require.NoError(t, err) + members[header.Name] = string(data) + } + } + return members +} + +func TestDocumentsAreReadWithoutThePayload(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + // Make the main payload unreadable: documents and metadata must not need it. + if spec.kind == artifact.RepositoryKindLocalArchive { + require.NoError(t, os.Remove(result.Location)) + } else { + require.NoError(t, os.Remove(filepath.Join(result.Location, "README.md"))) + require.NoError(t, os.Remove(filepath.Join(result.Location, "nested", "deep", "file.txt"))) + } + + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.NoError(t, err) + data, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + require.NoError(t, stream.Reader.Close()) + assert.Equal(t, fixtures.PackageChangelog, string(data)) + + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + assert.Equal(t, testName, inspection.Record.Name) + + _, err = repo.OpenDocument(context.Background(), pinned, "unknown-role") + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing) + + content, err := repo.Read(context.Background(), pinned) + if err == nil { + defer content.Close() + err = readAll(t, content) + } + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "the payload itself is verified and reports the loss") + }) + } +} + +func TestPullReplacementRemovesStaleFilesInsideTheOutputOnly(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + parent := t.TempDir() + out := filepath.Join(parent, "out") + require.NoError(t, os.MkdirAll(filepath.Join(out, "stale"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(out, "stale", "old.txt"), []byte("old"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(out, "README.md"), []byte("old readme"), 0o644)) + sibling := filepath.Join(parent, "sibling.txt") + require.NoError(t, os.WriteFile(sibling, []byte("keep"), 0o644)) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.NotEqual(t, materialize.DestinationMatches, state) + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + + assert.NoFileExists(t, filepath.Join(out, "stale", "old.txt"), "stale files inside the output are removed, not merged") + assert.FileExists(t, sibling, "nothing outside the requested output root is touched") + got, err := os.ReadFile(filepath.Join(out, "README.md")) + require.NoError(t, err) + assert.Equal(t, fixtures.RootReadme, string(got)) + }) + } +} + +func TestSafeRelativeSymlinkRoundTrips(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTar)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + src := t.TempDir() + fixtures.WriteFile(t, src, "real/target.txt", "target\n", 0o644) + if !fixtures.Symlink(t, src, "alias.txt", "real/target.txt") { + t.Skip("symlinks are unavailable on this host") + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: testName, Version: testVersion, Packaging: repo.packaging, OperationID: "op-link", PreparedAt: capturedAt, + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + + content, err := repo.Read(context.Background(), resolve(t, repo, defaultRef())) + require.NoError(t, err) + defer content.Close() + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + target, err := os.Readlink(filepath.Join(out, "alias.txt")) + require.NoError(t, err) + assert.Equal(t, filepath.FromSlash("real/target.txt"), target, "a contained relative link stays a link") + }) + } +} + +func TestExecutableBitsSurviveAsOrdinaryFiles(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + info, err := os.Stat(filepath.Join(result.Location, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "the stored payload keeps the ordinary executable mode") + info, err = os.Stat(filepath.Join(result.Location, fixtures.UnicodeName)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o640), info.Mode().Perm()) + assert.True(t, repo.Capabilities().PreservesModes) +} diff --git a/pkg/artifact/local/service_test.go b/pkg/artifact/local/service_test.go new file mode 100644 index 00000000000..3e8b1854de3 --- /dev/null +++ b/pkg/artifact/local/service_test.go @@ -0,0 +1,146 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// newConfiguredService wires the registered local kinds through the configuration-driven +// repository table, the way a composition root does. +func newConfiguredService(t *testing.T) (*service.Service, string) { + t.Helper() + root := t.TempDir() + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "files": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "files")}}, + "packed": { + Kind: artifact.RepositoryKindLocalArchive, Packaging: "tgz", Permissions: "0600", + Options: map[string]any{"path": filepath.Join(root, "packed")}, + }, + }} + svc, err := service.New(service.FromConfig(cfg, registry), nil, + service.WithStagingRoot(t.TempDir()), service.WithRetry(2, time.Millisecond), service.WithSleep(func(context.Context, time.Duration) error { return nil })) + require.NoError(t, err) + return svc, root +} + +func TestService_DrivesBothLocalBackends(t *testing.T) { + svc, root := newConfiguredService(t) + ctx := context.Background() + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + + published, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, published.Outcome) + assert.Equal(t, artifact.LockScopeHost, published.LockScope) + assert.Equal(t, filepath.Join(root, "files", "team", "app", "1.0.0"), published.Location) + + again, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "an identical republication is a no-op") + + // Local directory to local archive: Record identity survives repackaging. + copied, err := svc.Copy(ctx, artifact.CopyRequest{Ref: defaultRef(), From: "files", To: "packed"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, copied.Outcome) + assert.Equal(t, filepath.Join(root, "packed", "team", "app", "1.0.0.tgz"), copied.Location) + + listed, err := svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + require.Len(t, listed.Artifacts, 2) + assert.Equal(t, listed.Artifacts[0].Pinned.RecordDigest, listed.Artifacts[1].Pinned.RecordDigest, "the same Record in both packagings") + + document, err := svc.ShowDocument(ctx, artifact.ShowDocumentRequest{Repository: "packed", Ref: defaultRef(), Role: artifact.DocumentReadme}) + require.NoError(t, err) + assert.Equal(t, fixtures.PackageReadme, string(document.Content)) + + out := filepath.Join(t.TempDir(), "pulled") + pulled, err := svc.Pull(ctx, artifact.PullRequest{Repository: "packed", Ref: defaultRef(), Destination: out}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, pulled.Outcome) + inspection, err := svc.Inspect(ctx, artifact.InspectRequest{Repository: "files", Ref: defaultRef()}) + require.NoError(t, err) + state, err := materialize.InspectDestination(out, inspection.Inspection.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), listInventoryPaths(inspection.Inspection.Record)) + + pin, err := svc.Pin(ctx, artifact.PinRequest{Repository: "packed", Ref: defaultRef()}) + require.NoError(t, err) + assert.Equal(t, inspection.Inspection.Pinned.RecordDigest, pin.Pinned.RecordDigest) + + _, err = svc.Delete(ctx, artifact.DeleteRequest{Repository: "files", Ref: defaultRef()}) + require.NoError(t, err) + listed, err = svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + require.Len(t, listed.Artifacts, 1) + assert.Equal(t, "packed", listed.Artifacts[0].Repository) + assert.NoDirExists(t, filepath.Join(root, "files", "team")) +} + +// The service reconciles a commit whose response was lost by reading the completed reference. +func TestService_ReconcilesALostResponseAfterCommit(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterCommit))) + repo := newRepo(t, spec) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"r": repo}}, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload\n", 0o644) + + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "r", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err, "the committed reference proves success") + var reconciled bool + for _, diagnostic := range result.Diagnostics { + reconciled = reconciled || diagnostic.Code == "reconciled" + } + assert.True(t, reconciled, "success was confirmed by reading the reference: %v", result.Diagnostics) + assertIntact(t, repo, defaultRef()) + }) + } +} + +// An interruption before the commit is surfaced as unavailable, and the service's own retry of the +// same intent repairs it. +func TestService_RetryRepairsAnInterruptedPublication(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + fault.hook = crashAt(faultAfterMetadata) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"r": repo}}, nil, + service.WithStagingRoot(t.TempDir()), service.WithRetry(1, time.Millisecond)) + require.NoError(t, err) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload\n", 0o644) + request := artifact.PublishRequest{Repository: "r", Source: src, Name: testName, Version: testVersion} + + _, err = svc.Publish(context.Background(), request) + require.Error(t, err) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + + result, err := svc.Publish(context.Background(), request) + require.NoError(t, err, "a later invocation with the same intent repairs the pending operation") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + _, statErr := os.Stat(filepath.Join(repo.root, ".atmos-artifacts", "v1", "operations")) + require.NoError(t, statErr) + assert.Empty(t, listFiles(t, filepath.Join(repo.root, ".atmos-artifacts", "v1", "operations"))) +} diff --git a/pkg/artifact/local/signature_test.go b/pkg/artifact/local/signature_test.go new file mode 100644 index 00000000000..d74f425b004 --- /dev/null +++ b/pkg/artifact/local/signature_test.go @@ -0,0 +1,302 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +func signedSpec(spec *repoSpec) *repoSpec { + return spec.withOpts(WithVerifier(&artifacttest.FakeVerifier{})) +} + +// recordSubject builds the Record signature subject the service uses for the default reference. +func recordSubject(t *testing.T, repo *Repository) artifact.SignatureSubject { + t.Helper() + inspection, err := repo.Inspect(context.Background(), resolve(t, repo, defaultRef())) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + return artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} +} + +// snapshot reads every file below dir into a map for before/after comparison. +func snapshot(t *testing.T, dir string, skip func(rel string) bool) map[string]string { + t.Helper() + out := map[string]string{} + for _, rel := range listFiles(t, dir) { + if skip != nil && skip(rel) { + continue + } + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + out[rel] = string(data) + } + return out +} + +func notEvidence(rel string) bool { + return len(rel) >= len(layout.ManagedRoot+"/signatures") && rel[:len(layout.ManagedRoot+"/signatures")] == layout.ManagedRoot+"/signatures" +} + +func TestSignature_StorageNeverMutatesTheSignedRecord(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, signedSpec(spec)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo) + assert.Equal(t, pinned.RecordDigest, subject.Digest) + before := snapshot(t, repo.root, nil) + + signer := newSigner() + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) + require.Len(t, evidence, 1) + assert.Equal(t, layout.SignaturePath(artifact.SubjectRecord, subject.Digest, evidence[0].Descriptor.Digest), evidence[0].Locator) + assert.FileExists(t, filepath.Join(repo.root, filepath.FromSlash(evidence[0].Locator))) + + after := snapshot(t, repo.root, nil) + added := 0 + for rel, content := range after { + if notEvidence(rel) { + added++ + continue + } + assert.Equal(t, before[rel], content, "%s must be byte-identical after signing", rel) + } + assert.Equal(t, 1, added, "exactly one detached evidence file was added") + assert.Len(t, after, len(before)+1) + + again := resolve(t, repo, defaultRef()) + assert.Equal(t, pinned.RecordDigest, again.RecordDigest, "the signed Record, and so its digest, did not change") + assert.Equal(t, result.Pinned.RecordDigest, again.RecordDigest) + assertIntact(t, repo, defaultRef()) + + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.Len(t, inspection.Evidence, 1) + assert.Equal(t, evidence[0].Descriptor.Digest, inspection.Evidence[0].Descriptor.Digest) + pin, err := repo.Pin(context.Background(), defaultRef()) + require.NoError(t, err) + require.Len(t, pin.Evidence, 1, "a pin carries the evidence descriptors known now") + + verified, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + assert.NotEmpty(t, verified.PolicyFingerprint) + _, err = repo.Verify(context.Background(), subject, signer.UntrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + } +} + +// Signing an archive after publication leaves the archive bytes untouched: the evidence covers the +// Record that describes it, and no claim is made about the original archive bytes. +func TestSignature_ArchiveBytesAreNeverRewritten(t *testing.T) { + repo := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + result, _ := publishFilePackage(t, repo) + archiveBefore, err := os.ReadFile(result.Location) + require.NoError(t, err) + + _, err = repo.Sign(context.Background(), recordSubject(t, repo), newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + + archiveAfter, err := os.ReadFile(result.Location) + require.NoError(t, err) + assert.Equal(t, archiveBefore, archiveAfter) + assert.Equal(t, digest.FromBytes(archiveBefore), result.Representation.RepresentationDigest, "the representation checksum still describes the unsigned archive") +} + +func TestSignature_VerifyFailsClosed(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + signer := newSigner() + + result, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "no evidence is never verified") + require.NotNil(t, result) + assert.Equal(t, artifact.VerificationFailed, result.Status) + + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + // A malformed file and a file bound to another subject are reported and ignored; they can + // neither break nor override the valid evidence. + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + junk := digest.FromString("junk") + require.NoError(t, os.WriteFile(filepath.Join(dir, junk.Encoded()+".json"), []byte("not json"), 0o644)) + foreign := digest.FromString("foreign subject") + evidence, err := layout.NewEvidence(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: foreign}, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte("{}")}) + require.NoError(t, err) + raw, err := evidence.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, evidence.Digest().Encoded()+".json"), raw, 0o644)) + + verified, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + codes := map[string]bool{} + for _, diagnostic := range verified.Diagnostics { + codes[diagnostic.Code] = true + } + assert.True(t, codes["malformed-evidence"], "malformed evidence is bounded and reported: %v", verified.Diagnostics) + assert.True(t, codes["subject-mismatch"], "foreign evidence is reported: %v", verified.Diagnostics) + + // Evidence whose bytes were edited no longer matches its file name and is ignored. + files := listFiles(t, dir) + for _, name := range files { + if name != junk.Encoded()+".json" && name != evidence.Digest().Encoded()+".json" { + tamper(t, dir, name, appendJunk) + } + } + _, err = repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "tampered evidence cannot verify") +} + +func TestSignature_SubjectBinding(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + signer := newSigner() + + t.Run("unsupported subject kind", func(t *testing.T) { + other := subject + other.Kind = artifact.SubjectOCIRoot + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, err = repo.Verify(context.Background(), other, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("unknown record", func(t *testing.T) { + other := subject + other.Digest = digest.FromString("missing") + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + t.Run("wrong declared size", func(t *testing.T) { + other := subject + other.Size++ + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) + t.Run("nil signer", func(t *testing.T) { + _, err := repo.Sign(context.Background(), subject, nil, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("format the repository cannot verify", func(t *testing.T) { + _, err := repo.Sign(context.Background(), subject, &foreignSigner{}, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("signer produces nothing", func(t *testing.T) { + _, err := repo.Sign(context.Background(), subject, &nilSigner{}, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) +} + +type foreignSigner struct{} + +func (*foreignSigner) Formats() []artifact.EvidenceFormat { return []artifact.EvidenceFormat{"other"} } + +func (*foreignSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + return &artifact.EvidencePayload{Format: "other", MediaType: "x", Data: []byte("x")}, nil +} + +type nilSigner struct{ foreignSigner } + +func (*nilSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + return nil, nil +} + +func TestSignature_WithoutVerifierIsACapabilityError(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "no advertised evidence format means signing is unavailable, not a silent no-op") + _, err = repo.Verify(context.Background(), subject, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestSignature_ExportAndImportEvidenceAcrossRepositories(t *testing.T) { + source := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + publish(t, source, &prepareSpec{}, artifact.PublishOptions{}) + sourcePinned := resolve(t, source, defaultRef()) + subject := recordSubject(t, source) + signer := newSigner() + _, err := source.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + blobs, err := source.ReadEvidence(context.Background(), &sourcePinned) + require.NoError(t, err) + require.Len(t, blobs, 1) + + // The destination stores the same Record in a different packaging. The Record, and so the + // signed subject, is identical, so imported evidence verifies there without re-signing. + destination := newRepo(t, signedSpec(dirSpec())) + publish(t, destination, &prepareSpec{}, artifact.PublishOptions{}) + destPinned := resolve(t, destination, defaultRef()) + assert.Equal(t, subject.Digest, destPinned.RecordDigest, "packaging does not change Record identity") + _, err = destination.WriteEvidence(context.Background(), &destPinned, blobs, nil) + require.NoError(t, err) + verified, err := destination.Verify(context.Background(), recordSubject(t, destination), signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + + different := newRepo(t, signedSpec(dirSpec())) + publish(t, different, &prepareSpec{files: map[string]string{"data.txt": "other content"}}, artifact.PublishOptions{}) + differentPinned := resolve(t, different, defaultRef()) + _, err = different.WriteEvidence(context.Background(), &differentPinned, blobs, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid, "evidence for another Record is never imported") + + imported, err := source.WriteEvidence(context.Background(), &sourcePinned, blobs, nil) + require.NoError(t, err) + require.Len(t, imported, 1) + assert.Equal(t, blobs[0].Evidence.Descriptor.Digest, imported[0].Descriptor.Digest, "re-import is idempotent") + again, err := source.ReadEvidence(context.Background(), &sourcePinned) + require.NoError(t, err) + assert.Len(t, again, 1) + + corrupted := blobs[0] + corrupted.Data = append([]byte(nil), corrupted.Data...) + corrupted.Data[0] ^= 0xff + _, err = source.WriteEvidence(context.Background(), &sourcePinned, []artifact.EvidenceBlob{corrupted}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestSignature_EvidenceFollowsTheRecordLifecycle(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + oldSubject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + evidenceDir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, oldSubject.Digest))) + assert.DirExists(t, evidenceDir) + + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + assert.NoDirExists(t, evidenceDir, "evidence of a retired Record is retired with it") + _, err = repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a retired Record cannot be signed") + + current := resolve(t, repo, defaultRef()) + currentSubject := recordSubject(t, repo) + _, err = repo.Sign(context.Background(), currentSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + currentDir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, currentSubject.Digest))) + assert.DirExists(t, currentDir) + _, err = repo.Delete(context.Background(), current, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.NoDirExists(t, currentDir, "delete removes the evidence the artifact exclusively owns") +} diff --git a/pkg/artifact/local/state.go b/pkg/artifact/local/state.go new file mode 100644 index 00000000000..3054d3b9fab --- /dev/null +++ b/pkg/artifact/local/state.go @@ -0,0 +1,167 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// readManaged reads a managed document, mapping a missing file to (nil, nil). +func (t *tree) readManaged(rel string, limit int64) ([]byte, error) { + raw, err := t.readBounded(rel, limit) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, nil + case err != nil: + return nil, fmt.Errorf("%w: read %s: %w", errUtils.ErrArtifactIntegrityFailed, rel, err) + } + return raw, nil +} + +// loadReference returns the stored reference, or nil when it does not exist. +func (t *tree) loadReference(key string) (*layout.Reference, error) { + raw, err := t.readManaged(layout.ReferencePath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + ref, err := layout.DecodeReference(raw) + if err != nil { + return nil, fmt.Errorf("reference %s: %w", key, err) + } + if ref.Key() != key { + return nil, fmt.Errorf("%w: reference file %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, ref.Name, ref.Version) + } + return ref, nil +} + +// loadRepresentation returns the stored representation, or nil when it does not exist. +func (t *tree) loadRepresentation(key string) (*layout.Representation, error) { + raw, err := t.readManaged(layout.RepresentationPath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + rep, err := layout.DecodeRepresentation(raw) + if err != nil { + return nil, fmt.Errorf("representation %s: %w", key, err) + } + if layout.ReferenceKey(rep.Name, rep.Version) != key { + return nil, fmt.Errorf("%w: representation file %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, rep.Name, rep.Version) + } + return rep, nil +} + +// loadOperation returns the stored journal, or nil when it does not exist. +func (t *tree) loadOperation(operationID string) (*layout.Operation, error) { + raw, err := t.readManaged(layout.OperationPath(operationID), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + op, err := layout.DecodeOperation(raw) + if err != nil { + return nil, fmt.Errorf("operation %s: %w", operationID, err) + } + if op.OperationID != operationID { + return nil, fmt.Errorf("%w: operation journal for %q describes %q", errUtils.ErrArtifactIntegrityFailed, operationID, op.OperationID) + } + return op, nil +} + +// loadRecord reads and verifies the immutable Record with the given digest. A missing Record is +// not found; a Record whose bytes do not hash to its file name, or that is not canonical, fails +// integrity. +func (t *tree) loadRecord(recordDigest digest.Digest) (*artifact.Record, []byte, error) { + if err := artifact.ValidateDigest(recordDigest); err != nil { + return nil, nil, err + } + raw, err := t.readManaged(layout.RecordPath(recordDigest), artifact.MaxRecordSize) + if err != nil { + return nil, nil, err + } + if raw == nil { + return nil, nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, recordDigest) + } + if got := digest.FromBytes(raw); got != recordDigest { + return nil, nil, fmt.Errorf("%w: record file %s holds bytes that hash to %s", errUtils.ErrArtifactIntegrityFailed, recordDigest, got) + } + record, err := artifact.DecodeRecord(raw) + if err != nil { + return nil, nil, fmt.Errorf("%w: record %s: %w", errUtils.ErrArtifactIntegrityFailed, recordDigest, err) + } + return record, raw, nil +} + +// writeReference replaces the reference record atomically. +func (t *tree) writeReference(ref *layout.Reference) error { + raw, err := ref.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.ReferencePath(ref.Key()), raw) +} + +// writeRepresentation replaces the representation record atomically. +func (t *tree) writeRepresentation(rep *layout.Representation) error { + raw, err := rep.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.RepresentationPath(layout.ReferenceKey(rep.Name, rep.Version)), raw) +} + +// writeOperation replaces the operation journal atomically. +func (t *tree) writeOperation(op *layout.Operation) error { + raw, err := op.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.OperationPath(op.OperationID), raw) +} + +// writeRecord stores the immutable canonical Record. An existing file with identical bytes is +// left alone; different bytes under the same digest name are corruption. +func (t *tree) writeRecord(recordDigest digest.Digest, canonical []byte) error { + existing, err := t.readManaged(layout.RecordPath(recordDigest), artifact.MaxRecordSize) + if err != nil { + return err + } + if existing != nil && string(existing) != string(canonical) { + return fmt.Errorf("%w: record file %s already exists with different bytes", errUtils.ErrArtifactIntegrityFailed, recordDigest) + } + if existing != nil { + return nil + } + return t.writeAtomic(layout.RecordPath(recordDigest), canonical) +} + +// listReferences returns every stored reference, sorted by file name, failing on a corrupt one. +func (t *tree) listReferences() ([]*layout.Reference, error) { + entries, err := fs.ReadDir(t.FS(), layout.ReferencesDir()) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, nil + case err != nil: + return nil, fmt.Errorf("%w: list references: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + refs := make([]*layout.Reference, 0, len(entries)) + for _, entry := range entries { + key, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + continue + } + ref, err := t.loadReference(key) + if err != nil { + return nil, err + } + if ref != nil { + refs = append(refs, ref) + } + } + return refs, nil +} diff --git a/pkg/artifact/local/verifier_test.go b/pkg/artifact/local/verifier_test.go new file mode 100644 index 00000000000..14ae02a0a2d --- /dev/null +++ b/pkg/artifact/local/verifier_test.go @@ -0,0 +1,144 @@ +package local + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// stubVerifier returns a canned verifier outcome. +type stubVerifier struct { + result *artifact.VerificationResult + err error + seen int +} + +func (*stubVerifier) Formats() []artifact.EvidenceFormat { + return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle} +} + +func (s *stubVerifier) Verify(_ context.Context, _ artifact.SignatureSubject, evidence []artifact.EvidenceBlob, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + s.seen = len(evidence) + return s.result, s.err +} + +type fixedSigner struct{ data []byte } + +func (*fixedSigner) Formats() []artifact.EvidenceFormat { + return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle} +} + +func (f *fixedSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + return &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: f.data}, nil +} + +func signedWith(t *testing.T, verifier artifact.Verifier) (*Repository, artifact.SignatureSubject) { + t.Helper() + repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(verifier)}}) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, &fixedSigner{data: []byte(`{"bundle":1}`)}, artifact.SignOptions{}) + require.NoError(t, err) + return repo, subject +} + +var trustedPolicy = artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: "who", Issuer: "https://issuer.example.com"}}}}, +} + +func TestVerify_TranslatesVerifierOutcomes(t *testing.T) { + tests := []struct { + name string + result *artifact.VerificationResult + err error + wantErr error + wantStatus artifact.VerificationStatus + }{ + {"verified", &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil, nil, artifact.VerificationVerified}, + {"unchecked passes through without a verdict", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil, nil, artifact.VerificationUnchecked}, + {"failed status without an error still fails", &artifact.VerificationResult{Status: artifact.VerificationFailed}, nil, errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + {"verifier error is wrapped as a verification failure", nil, errors.New("crypto exploded"), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + {"verification sentinel is kept", &artifact.VerificationResult{Status: artifact.VerificationFailed}, fmt.Errorf("%w: policy", errUtils.ErrArtifactVerificationFailed), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := &stubVerifier{result: tt.result, err: tt.err} + repo, subject := signedWith(t, stub) + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + require.NotNil(t, result) + assert.Equal(t, tt.wantStatus, result.Status) + assert.Equal(t, subject, result.Subject) + assert.NotEmpty(t, result.PolicyFingerprint, "the applied policy is always fingerprinted") + assert.Equal(t, 1, stub.seen) + }) + } +} + +func TestVerify_ReportsDiagnosticsAlongsideTheVerifier(t *testing.T) { + stub := &stubVerifier{result: &artifact.VerificationResult{ + Status: artifact.VerificationVerified, PolicyFingerprint: "verifier-fingerprint", + Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityInfo, Code: "from-verifier", Message: "ok"}}, + }} + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + require.NoError(t, os.WriteFile(filepath.Join(dir, "note.txt"), []byte("x"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "zz.json"), []byte("{}"), 0o644)) + require.NoError(t, os.Mkdir(filepath.Join(dir, "sub.json"), 0o755)) + + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.NoError(t, err) + assert.Equal(t, "verifier-fingerprint", result.PolicyFingerprint, "the verifier's own fingerprint is kept") + codes := map[string]bool{} + for _, diagnostic := range result.Diagnostics { + codes[diagnostic.Code] = true + } + assert.True(t, codes["from-verifier"]) + assert.True(t, codes["malformed-evidence"], "a file whose name is not an evidence digest is reported, not trusted") +} + +func TestVerify_BoundsTheNumberOfEvidenceFilesRead(t *testing.T) { + stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + repo, subject := signedWith(t, stub) + for index := range maxEvidenceFiles + 5 { + evidence, err := layout.NewEvidence(subject, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte(fmt.Sprintf("bundle-%d", index))}) + require.NoError(t, err) + raw, err := evidence.Marshal() + require.NoError(t, err) + path := filepath.Join(repo.root, filepath.FromSlash(layout.SignaturePath(artifact.SubjectRecord, subject.Digest, evidence.Digest()))) + require.NoError(t, os.WriteFile(path, raw, 0o644)) + } + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.NoError(t, err) + assert.Equal(t, maxEvidenceFiles, stub.seen, "the verifier never sees more than the bound") + truncated := false + for _, diagnostic := range result.Diagnostics { + truncated = truncated || diagnostic.Code == "evidence-truncated" + } + assert.True(t, truncated, "the cap is reported, not silent") +} + +func TestSign_RejectsOversizeEvidence(t *testing.T) { + repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(&stubVerifier{})}}) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, &fixedSigner{data: make([]byte, artifact.MaxEvidenceSize+1)}, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + _, err = repo.Sign(context.Background(), subject, &fixedSigner{data: nil}, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "empty evidence is invalid") +} From 9a4a5fc880fd43d99ed77dd08d231f5867dce13b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 8 Nov 2026 16:47:29 -0600 Subject: [PATCH 03/43] fix(artifacts): make local backends and tests Windows-safe, quiet CodeQL on the S3 fake - Retry opening local blob objects (and their attribute files) briefly on Windows. A reader that opens an object while a writer renames over it gets a sharing violation there, which made the concurrent compare-and-swap conformance case fail with a lost update. Missing files still return immediately. - Skip the archive file-mode setting in the configured-service test on Windows, where the repository rejects it by design, and close the reader before a directory replacement on Windows, which cannot rename a directory that holds open files. - Exclude the fake S3 server used only by tests from CodeQL. It returns stored request bodies by design, which the reflected-XSS query reports for any HTTP handler. Co-Authored-By: Claude Sonnet 5.5 --- .github/codeql/codeql-config.yml | 7 +++++ .github/workflows/codeql.yml | 1 + pkg/artifact/blob/local/attrs.go | 2 +- pkg/artifact/blob/local/etag.go | 2 +- pkg/artifact/blob/local/fsutil.go | 38 ++++++++++++++++++++++++ pkg/artifact/blob/local/fsutil_test.go | 41 ++++++++++++++++++++++++++ pkg/artifact/blob/local/get.go | 4 +-- pkg/artifact/local/recovery_test.go | 17 +++++++++-- pkg/artifact/local/service_test.go | 15 ++++++---- 9 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 .github/codeql/codeql-config.yml create mode 100644 pkg/artifact/blob/local/fsutil_test.go diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 00000000000..ea42d408fbd --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,7 @@ +name: "Atmos CodeQL configuration" + +# Test doubles that run only under `go test` and are never linked into the atmos binary. +# The fake S3 server deliberately returns stored request bodies and client-chosen content +# types, which the reflected-XSS query reports for any HTTP handler. +paths-ignore: + - pkg/artifact/blob/s3/s3test diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ccd026c124c..f0580e3e157 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -62,6 +62,7 @@ jobs: uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} + config-file: ./.github/codeql/codeql-config.yml # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. diff --git a/pkg/artifact/blob/local/attrs.go b/pkg/artifact/blob/local/attrs.go index 1ff218480ad..493c009ed8b 100644 --- a/pkg/artifact/blob/local/attrs.go +++ b/pkg/artifact/blob/local/attrs.go @@ -26,7 +26,7 @@ func attrsPath(objectPath string) string { // readAttrs loads the attribute file of objectPath; a missing file yields empty attributes. func readAttrs(objectPath string) (attrs, error) { var a attrs - data, err := os.ReadFile(attrsPath(objectPath)) + data, err := readFile(attrsPath(objectPath)) if errors.Is(err, os.ErrNotExist) { return a, nil } diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go index f1fca0c3e39..e09e04f70ec 100644 --- a/pkg/artifact/blob/local/etag.go +++ b/pkg/artifact/blob/local/etag.go @@ -21,7 +21,7 @@ func etagOf(path string, info os.FileInfo) (string, error) { if info.Size() > contentETagLimit { return weakETag(info), nil } - file, err := os.Open(path) + file, err := openFile(path) if err != nil { return "", err } diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go index 1b480ed877a..94cdf092de1 100644 --- a/pkg/artifact/blob/local/fsutil.go +++ b/pkg/artifact/blob/local/fsutil.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/hex" "errors" + "io" "os" "path/filepath" "runtime" @@ -13,6 +14,9 @@ import ( const ( renameAttempts = 8 renameBackoff = 5 * time.Millisecond + + // The openAttempts constant bounds the Windows retry for reads that race a concurrent rename-over. + openAttempts = 20 ) func mkdirAll(dir string, perm os.FileMode) error { @@ -70,6 +74,40 @@ func renameReplace(src, dst string) error { return err } +// openFile opens path for reading. Windows refuses to open a file with a sharing violation while a +// concurrent writer is renaming over it, so it retries briefly there. A missing file is reported +// immediately because absence is a normal answer. +func openFile(path string) (*os.File, error) { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = openAttempts + } + var ( + file *os.File + err error + ) + for i := range attempts { + file, err = os.Open(path) + if err == nil || errors.Is(err, os.ErrNotExist) { + return file, err + } + if i < attempts-1 { + time.Sleep(renameBackoff) + } + } + return nil, err +} + +// readFile returns the contents of path, retrying a transient Windows sharing violation like openFile. +func readFile(path string) ([]byte, error) { + file, err := openFile(path) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(file) +} + // removeEmptyDirs removes empty directories from dir up to, but never including, root. func removeEmptyDirs(root, dir string) { for dir != root && dir != "." && dir != string(filepath.Separator) && len(dir) > len(root) { diff --git a/pkg/artifact/blob/local/fsutil_test.go b/pkg/artifact/blob/local/fsutil_test.go new file mode 100644 index 00000000000..31fb9031381 --- /dev/null +++ b/pkg/artifact/blob/local/fsutil_test.go @@ -0,0 +1,41 @@ +package local + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenFile_ReturnsExistingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "object") + require.NoError(t, os.WriteFile(path, []byte("payload"), 0o600)) + + file, err := openFile(path) + require.NoError(t, err) + defer file.Close() + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "payload", string(data)) +} + +func TestOpenFile_MissingFileReportsNotExistWithoutRetrying(t *testing.T) { + file, err := openFile(filepath.Join(t.TempDir(), "absent")) + require.ErrorIs(t, err, os.ErrNotExist) + assert.Nil(t, file) +} + +func TestReadFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "attrs") + require.NoError(t, os.WriteFile(path, []byte(`{"a":1}`), 0o600)) + + data, err := readFile(path) + require.NoError(t, err) + assert.JSONEq(t, `{"a":1}`, string(data)) + + _, err = readFile(filepath.Join(t.TempDir(), "absent")) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go index 923b95266ec..adec86c45cf 100644 --- a/pkg/artifact/blob/local/get.go +++ b/pkg/artifact/blob/local/get.go @@ -37,7 +37,7 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo return nil, err } target := s.path(key) - file, err := os.Open(target) + file, err := openFile(target) if err != nil { return nil, wrapIO(blob.OpGet, key, err) } @@ -83,7 +83,7 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b return nil, err } target := s.path(key) - file, err := os.Open(target) + file, err := openFile(target) if err != nil { return nil, wrapIO(blob.OpStat, key, err) } diff --git a/pkg/artifact/local/recovery_test.go b/pkg/artifact/local/recovery_test.go index 8434b5328f7..8d3ed42bb64 100644 --- a/pkg/artifact/local/recovery_test.go +++ b/pkg/artifact/local/recovery_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "testing" "time" @@ -195,11 +196,21 @@ func TestReplacementNeverServesNewBytesUnderOldDigest(t *testing.T) { require.NoError(t, err) defer content.Close() + // Windows cannot rename a directory that contains open files, so a directory replacement + // cannot land under a mid-stream reader there. Close the reader first and keep the + // old-digest assertions; the mid-stream guard is exercised on POSIX hosts. + midStream := runtime.GOOS != "windows" || spec.kind != artifact.RepositoryKindLocalDir + if !midStream { + require.NoError(t, content.Close()) + } + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) - err = readAll(t, content) - require.Error(t, err, "a reader that was open when the reference moved must not report success") - assert.True(t, errorsIsAny(err, errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactNotFound), "%v", err) + if midStream { + err = readAll(t, content) + require.Error(t, err, "a reader that was open when the reference moved must not report success") + assert.True(t, errorsIsAny(err, errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactNotFound), "%v", err) + } _, err = repo.Read(context.Background(), old) require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) diff --git a/pkg/artifact/local/service_test.go b/pkg/artifact/local/service_test.go index 3e8b1854de3..2ec6603cc27 100644 --- a/pkg/artifact/local/service_test.go +++ b/pkg/artifact/local/service_test.go @@ -25,12 +25,17 @@ func newConfiguredService(t *testing.T) (*service.Service, string) { root := t.TempDir() registry := artifact.NewRepositoryRegistry() require.NoError(t, RegisterWith(registry)) + packed := schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Packaging: "tgz", + Options: map[string]any{"path": filepath.Join(root, "packed")}, + } + // Windows cannot enforce archive file modes, and the repository rejects the setting there. + if fixtures.POSIXModes() { + packed.Permissions = "0600" + } cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ - "files": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "files")}}, - "packed": { - Kind: artifact.RepositoryKindLocalArchive, Packaging: "tgz", Permissions: "0600", - Options: map[string]any{"path": filepath.Join(root, "packed")}, - }, + "files": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "files")}}, + "packed": packed, }} svc, err := service.New(service.FromConfig(cfg, registry), nil, service.WithStagingRoot(t.TempDir()), service.WithRetry(2, time.Millisecond), service.WithSleep(func(context.Context, time.Duration) error { return nil })) From 38c29ffaaf1c1e978ae1f8b0e7d040df0b16b159 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 9 Nov 2026 06:43:30 -0600 Subject: [PATCH 04/43] fix(artifacts): drop the S3 fake's response-writer pass-through that CodeQL flags CodeQL reports reflected XSS and stack-trace exposure at the fake S3 server's statusWriter.Write, reached from unrelated code that calls Write on an io.Writer. The override only defaulted the recorded status to 200, which status() already does, so remove it and let the embedded writer's Write be promoted. Also drop the CodeQL config added earlier, since paths-ignore does not filter Go alerts. Co-Authored-By: Claude Sonnet 5.5 --- .github/codeql/codeql-config.yml | 7 ------- .github/workflows/codeql.yml | 1 - pkg/artifact/blob/s3/s3test/server.go | 12 ++---------- 3 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 .github/codeql/codeql-config.yml diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml deleted file mode 100644 index ea42d408fbd..00000000000 --- a/.github/codeql/codeql-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: "Atmos CodeQL configuration" - -# Test doubles that run only under `go test` and are never linked into the atmos binary. -# The fake S3 server deliberately returns stored request bodies and client-chosen content -# types, which the reflected-XSS query reports for any HTTP handler. -paths-ignore: - - pkg/artifact/blob/s3/s3test diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f0580e3e157..ccd026c124c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -62,7 +62,6 @@ jobs: uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} - config-file: ./.github/codeql/codeql-config.yml # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go index d45d3202cca..5bd9249abbe 100644 --- a/pkg/artifact/blob/s3/s3test/server.go +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -199,7 +199,8 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request, key, op string) writeError(w, http.StatusNotImplemented, "NotImplemented", "operation is not supported by the fake") } -// statusWriter remembers the status code written. +// statusWriter remembers the status code written. It deliberately has no Write method of its own: +// the embedded writer's Write is promoted, and status reports 200 when no header was written. type statusWriter struct { http.ResponseWriter code int @@ -215,15 +216,6 @@ func (w *statusWriter) WriteHeader(code int) { w.ResponseWriter.WriteHeader(code) } -func (w *statusWriter) Write(p []byte) (int, error) { - defer perf.Track(nil, "s3test.statusWriter.Write")() - - if w.code == 0 { - w.code = http.StatusOK - } - return w.ResponseWriter.Write(p) -} - func (w *statusWriter) status() int { if w.code == 0 { return http.StatusOK From 3dc69d167884024892367322f1c802a219a87356 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 15:46:02 -0600 Subject: [PATCH 05/43] fix: tighten S3 test server request handling --- pkg/artifact/blob/s3/s3test/chunked.go | 9 ++++++++- pkg/artifact/blob/s3/s3test/multipart.go | 4 ++-- pkg/artifact/blob/s3/s3test/record.go | 11 ++++++++++- pkg/artifact/blob/s3/s3test/server.go | 2 +- pkg/artifact/blob/s3/s3test/server_test.go | 9 +++++++++ 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/blob/s3/s3test/chunked.go b/pkg/artifact/blob/s3/s3test/chunked.go index a0142da0d47..27200366109 100644 --- a/pkg/artifact/blob/s3/s3test/chunked.go +++ b/pkg/artifact/blob/s3/s3test/chunked.go @@ -3,12 +3,15 @@ package s3test import ( "bufio" "bytes" + "errors" "io" "net/http" "strconv" "strings" ) +var errInvalidChunkTerminator = errors.New("invalid chunk terminator") + // hexBase is the radix of aws-chunked chunk sizes. const hexBase = 16 @@ -50,8 +53,12 @@ func decodeAWSChunked(raw []byte) ([]byte, error) { if _, err := io.CopyN(&out, reader, size); err != nil { return nil, err } - if _, err := reader.Discard(2); err != nil { + var terminator [2]byte + if _, err := io.ReadFull(reader, terminator[:]); err != nil { return nil, err } + if terminator != [2]byte{'\r', '\n'} { + return nil, errInvalidChunkTerminator + } } } diff --git a/pkg/artifact/blob/s3/s3test/multipart.go b/pkg/artifact/blob/s3/s3test/multipart.go index 0263d624fa3..af32fce04d0 100644 --- a/pkg/artifact/blob/s3/s3test/multipart.go +++ b/pkg/artifact/blob/s3/s3test/multipart.go @@ -182,11 +182,11 @@ func (s *Server) assemble(up *upload, req completeRequest) (*object, string) { return obj, "" } -func (s *Server) abortUpload(w http.ResponseWriter, r *http.Request) { +func (s *Server) abortUpload(w http.ResponseWriter, r *http.Request, key string) { s.mu.Lock() defer s.mu.Unlock() id := r.URL.Query().Get("uploadId") - if _, ok := s.uploads[id]; !ok { + if up, ok := s.uploads[id]; !ok || up.key != key { writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") return } diff --git a/pkg/artifact/blob/s3/s3test/record.go b/pkg/artifact/blob/s3/s3test/record.go index b7e21b95bf9..f7836be54fb 100644 --- a/pkg/artifact/blob/s3/s3test/record.go +++ b/pkg/artifact/blob/s3/s3test/record.go @@ -85,7 +85,16 @@ func (s *Server) Requests() []Request { s.rec.mu.Lock() defer s.rec.mu.Unlock() - return append([]Request(nil), s.rec.requests...) + requests := append([]Request(nil), s.rec.requests...) + for i := range requests { + query := make(url.Values, len(requests[i].Query)) + for key, values := range requests[i].Query { + query[key] = append([]string(nil), values...) + } + requests[i].Query = query + requests[i].Header = requests[i].Header.Clone() + } + return requests } // RequestsFor returns the recorded requests of one operation. diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go index 5bd9249abbe..945e0937fed 100644 --- a/pkg/artifact/blob/s3/s3test/server.go +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -190,7 +190,7 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request, key, op string) OpCreateMultipartUpload: func() { s.createUpload(w, r, key) }, OpUploadPart: func() { s.uploadPart(w, r, key) }, OpCompleteMultipartUpload: func() { s.completeUpload(w, r, key) }, - OpAbortMultipartUpload: func() { s.abortUpload(w, r) }, + OpAbortMultipartUpload: func() { s.abortUpload(w, r, key) }, } if handler, ok := handlers[op]; ok { handler() diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 4d5e4b8bdca..b69230ae66e 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -173,6 +173,9 @@ func TestMultipartMinimumPartSizeAndAbort(t *testing.T) { }) assert.Equal(t, "InvalidPart", apiCode(t, err), "a non-final part below the minimum is rejected") + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("wrong"), UploadId: created.UploadId}) + assert.Equal(t, "NoSuchUpload", apiCode(t, err)) + assert.Len(t, srv.OpenUploads(), 1) _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId}) require.NoError(t, err) assert.Equal(t, 1, srv.AbortedUploads()) @@ -324,6 +327,10 @@ func TestRequestLogRecordsOrderAndHeaders(t *testing.T) { assert.Equal(t, "a/b", requests[0].Key) assert.Equal(t, "*", requests[0].Header.Get("If-None-Match")) assert.Equal(t, http.StatusOK, requests[0].Status) + requests[0].Header.Set("If-None-Match", "changed") + requests[0].Query.Set("injected", "changed") + assert.Equal(t, "*", srv.Requests()[0].Header.Get("If-None-Match")) + assert.Empty(t, srv.Requests()[0].Query.Get("injected")) assert.Equal(t, OpHeadObject, requests[1].Op) assert.Len(t, srv.RequestsFor(OpHeadObject), 1) srv.ResetRequests() @@ -337,6 +344,8 @@ func TestDecodeAWSChunked(t *testing.T) { assert.Equal(t, "hello world", string(got)) _, err = decodeAWSChunked([]byte("zz\r\n")) require.Error(t, err) + _, err = decodeAWSChunked([]byte("5\r\nhelloXX0\r\n")) + require.Error(t, err) _, err = decodeAWSChunked([]byte("5\r\nhel")) require.Error(t, err) } From 0a8a176e6e4fc7cf8f99321dadb7df559b8da35b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 15:56:10 -0600 Subject: [PATCH 06/43] fix: validate blob inputs and preserve local names --- pkg/artifact/blob/blobtest/cases_basic.go | 3 +++ .../blob/blobtest/cases_conditional.go | 10 ++++--- pkg/artifact/blob/blobtest/conformance.go | 1 - pkg/artifact/blob/local/list.go | 2 +- pkg/artifact/blob/local/options.go | 8 ++++++ pkg/artifact/blob/local/put.go | 7 +++++ pkg/artifact/blob/local/store.go | 15 +++++++---- pkg/artifact/blob/local/store_test.go | 11 ++++++++ pkg/artifact/blob/s3/put.go | 3 +++ pkg/artifact/blob/s3/read.go | 6 ++++- pkg/artifact/blob/s3/s3test/cond.go | 10 ++++++- pkg/artifact/blob/s3/s3test/doc.go | 2 +- pkg/artifact/blob/s3/s3test/objects.go | 2 +- pkg/artifact/blob/s3/s3test/options.go | 7 ++++- pkg/artifact/blob/s3/s3test/server_test.go | 17 ++++++++++++ pkg/artifact/blob/s3/store_test.go | 15 +++++++++++ pkg/ci/artifact/local/store.go | 2 +- .../artifact/local/transport_parity_test.go | 26 ++++++++++++++++++- 18 files changed, 130 insertions(+), 17 deletions(-) diff --git a/pkg/artifact/blob/blobtest/cases_basic.go b/pkg/artifact/blob/blobtest/cases_basic.go index 14176f2bbb8..602d8f8ca48 100644 --- a/pkg/artifact/blob/blobtest/cases_basic.go +++ b/pkg/artifact/blob/blobtest/cases_basic.go @@ -110,6 +110,9 @@ func (c *conformance) attributes(t *testing.T) { require.NoError(t, err) assert.Empty(t, stat.Tags) assert.Empty(t, stat.Metadata) + if caps.Metadata { + assert.NotEqual(t, "application/json", stat.ContentType) + } tooMany := map[string]string{} for _, key := range []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} { diff --git a/pkg/artifact/blob/blobtest/cases_conditional.go b/pkg/artifact/blob/blobtest/cases_conditional.go index 45d3e50e240..099f8c0e631 100644 --- a/pkg/artifact/blob/blobtest/cases_conditional.go +++ b/pkg/artifact/blob/blobtest/cases_conditional.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io" "strconv" "sync" "sync/atomic" @@ -162,10 +163,13 @@ func incrementWithRetry(t *testing.T, store blob.Store) { t.Errorf("get: %v", err) return } - raw := make([]byte, counterBuffer) - n, _ := object.Body.Read(raw) + raw, readErr := io.ReadAll(object.Body) _ = object.Body.Close() - current, err := strconv.Atoi(string(raw[:n])) + if readErr != nil { + t.Errorf("read counter: %v", readErr) + return + } + current, err := strconv.Atoi(string(raw)) if err != nil { continue } diff --git a/pkg/artifact/blob/blobtest/conformance.go b/pkg/artifact/blob/blobtest/conformance.go index ce1c83ba81b..f06da218ac9 100644 --- a/pkg/artifact/blob/blobtest/conformance.go +++ b/pkg/artifact/blob/blobtest/conformance.go @@ -24,7 +24,6 @@ const ( sourceChunk = 1024 maxRetries = 500 - counterBuffer = 16 listPageSize = 10 listPageGuard = 100 listKeyCount = 53 diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go index 4752c999431..8909efa6b94 100644 --- a/pkg/artifact/blob/local/list.go +++ b/pkg/artifact/blob/local/list.go @@ -158,7 +158,7 @@ func validatePrefix(prefix string) error { return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix contains a relative segment", errUtils.ErrBlobInvalid)) } } - if strings.ContainsRune(prefix, '\\') || strings.HasPrefix(prefix, "/") { + if strings.ContainsRune(prefix, '\\') || strings.ContainsRune(prefix, 0) || strings.HasPrefix(prefix, "/") { return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix must be a relative slash path", errUtils.ErrBlobInvalid)) } return nil diff --git a/pkg/artifact/blob/local/options.go b/pkg/artifact/blob/local/options.go index 0b9a231d802..513f3361a36 100644 --- a/pkg/artifact/blob/local/options.go +++ b/pkg/artifact/blob/local/options.go @@ -36,3 +36,11 @@ func WithLockAllWrites() Option { return func(s *Store) { s.lockAll = true } } + +// WithLiteralBackslashes permits literal backslashes on filesystems where they are not separators. +// This supports legacy local artifact names; other key validation remains in effect. +func WithLiteralBackslashes() Option { + defer perf.Track(nil, "blob.local.WithLiteralBackslashes")() + + return func(s *Store) { s.allowBackslash = true } +} diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index aabee729e3a..cf79cb21947 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" @@ -57,6 +58,9 @@ func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOption if err := s.validate(blob.OpPut, key); err != nil { return err } + if opts.Size < blob.UnknownSize || opts.Size == math.MaxInt64 { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid or unverifiable size %d", errUtils.ErrBlobInvalid, opts.Size)) + } if err := blob.ValidateTags(opts.Tags); err != nil { return err } @@ -109,6 +113,9 @@ func (s *Store) stageAttrs(pending *staged, target string, a attrs) error { // copyBody streams body into dst, enforcing a declared size and honoring cancellation. func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (int64, error) { + if size == math.MaxInt64 { + return 0, fmt.Errorf("%w: size %d is unverifiable", errUtils.ErrBlobInvalid, size) + } reader := io.Reader(&ctxReader{ctx: ctx, r: body}) if size >= 0 { reader = io.LimitReader(reader, size+1) diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index 4f032e91ce6..034a2e77d7c 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -27,10 +27,11 @@ const ( // Store is a filesystem [blob.Store]. type Store struct { - root string - dirPerm os.FileMode - filePerm os.FileMode - lockAll bool + root string + dirPerm os.FileMode + filePerm os.FileMode + lockAll bool + allowBackslash bool } // New returns a Store rooted at root. The directory is created on first write. @@ -79,7 +80,11 @@ func (s *Store) Capabilities() blob.Capabilities { // validate rejects keys that are unsafe on a filesystem or collide with the store's own files. func (s *Store) validate(op, key string) error { - if err := blob.ValidateKey(key); err != nil { + validationKey := key + if s.allowBackslash && os.PathSeparator != '\\' { + validationKey = strings.ReplaceAll(key, "\\", "_") + } + if err := blob.ValidateKey(validationKey); err != nil { return err } last := key[strings.LastIndexByte(key, '/')+1:] diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 19c895eae0c..c3d4fa5685f 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -3,6 +3,7 @@ package local import ( "bytes" "context" + "math" "os" "path/filepath" "testing" @@ -142,3 +143,13 @@ func TestCanceledContextFailsFast(t *testing.T) { require.ErrorIs(t, err, context.Canceled) assert.NoDirExists(t, store.Root(), "nothing is created for a canceled operation") } + +func TestInvalidSizesAndNULPrefix(t *testing.T) { + store := newStore(t) + for _, size := range []int64{-2, math.MaxInt64} { + _, err := store.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Size: size}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + } + _, err := store.List(context.Background(), blob.ListOptions{Prefix: "x\x00y"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index b138e7d278b..78e8a723159 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -55,6 +55,9 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) ( if err := s.validateKey(key); err != nil { return nil, err } + if opts.Size < blob.UnknownSize { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid size %d", errUtils.ErrBlobInvalid, opts.Size)) + } tags := blob.MergeTags(s.cfg.tags, opts.Tags) if err := blob.ValidateTags(tags); err != nil { return nil, err diff --git a/pkg/artifact/blob/s3/read.go b/pkg/artifact/blob/s3/read.go index 517958f6d4c..e96d2e62c34 100644 --- a/pkg/artifact/blob/s3/read.go +++ b/pkg/artifact/blob/s3/read.go @@ -3,6 +3,7 @@ package s3 import ( "context" "fmt" + "math" "strconv" "strings" @@ -30,6 +31,9 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo if opts.Offset < 0 { return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: negative offset", errUtils.ErrBlobInvalid)) } + if opts.Length > 0 && opts.Offset > math.MaxInt64-(opts.Length-1) { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: range end overflows", errUtils.ErrBlobInvalid)) + } api, err := s.client(ctx) if err != nil { return nil, err @@ -56,7 +60,7 @@ func rangeHeader(opts blob.GetOptions) *string { return nil } if opts.Length > 0 { - return aws.String(fmt.Sprintf("bytes=%d-%d", opts.Offset, opts.Offset+opts.Length-1)) + return aws.String(fmt.Sprintf("bytes=%d-%d", opts.Offset, opts.Offset+(opts.Length-1))) } return aws.String(fmt.Sprintf("bytes=%d-", opts.Offset)) } diff --git a/pkg/artifact/blob/s3/s3test/cond.go b/pkg/artifact/blob/s3/s3test/cond.go index c618a8497d7..d9e472d12eb 100644 --- a/pkg/artifact/blob/s3/s3test/cond.go +++ b/pkg/artifact/blob/s3/s3test/cond.go @@ -43,13 +43,21 @@ func enforce(ifNoneMatch, ifMatch string, current *object) conditionResult { switch { case current == nil: return conditionMissing - case ifMatch != "*" && !etagMatches(ifMatch, current.etag): + case ifMatch != "*" && !strongETagMatches(ifMatch, current.etag): return conditionFailed default: return conditionOK } } +// strongETagMatches rejects weak validators for If-Match. +func strongETagMatches(header, etag string) bool { + if strings.HasPrefix(strings.TrimSpace(header), "W/") || strings.HasPrefix(strings.TrimSpace(etag), "W/") { + return false + } + return etagMatches(header, etag) +} + // etagMatches compares ETags ignoring quotes and the weak prefix. func etagMatches(header, etag string) bool { return normalizeETag(header) == normalizeETag(etag) diff --git a/pkg/artifact/blob/s3/s3test/doc.go b/pkg/artifact/blob/s3/s3test/doc.go index d2c359c8d3d..e869b90bf6d 100644 --- a/pkg/artifact/blob/s3/s3test/doc.go +++ b/pkg/artifact/blob/s3/s3test/doc.go @@ -6,7 +6,7 @@ // - Conditional PUT, multipart completion, and DELETE with If-None-Match and If-Match, answering // 412 on a lost race and 404 for If-Match on a missing key, plus endpoints that ignore the // headers or answer 501. -// - ETags (MD5 for single puts, MD5-of-MD5s with a part count for multipart uploads). +// - ETags (truncated SHA-256 for single puts, and a digest with a part count for multipart uploads). // - Multipart create, upload-part, complete, and abort, with tracking of open and aborted uploads // and of concurrent part uploads. // - Object tags (the x-amz-tagging header and GetObjectTagging) and user metadata. diff --git a/pkg/artifact/blob/s3/s3test/objects.go b/pkg/artifact/blob/s3/s3test/objects.go index bfb93c71c73..9b03d701625 100644 --- a/pkg/artifact/blob/s3/s3test/objects.go +++ b/pkg/artifact/blob/s3/s3test/objects.go @@ -153,7 +153,7 @@ func parseRange(header string, size int64) (byteRange, bool) { return byteRange{}, false } start, err := strconv.ParseInt(first, decimalBase, bitSize64) - if err != nil || start > size || (start == size && size > 0) { + if err != nil || start < 0 || start >= size { return byteRange{}, false } end, ok := rangeEnd(last, start, size) diff --git a/pkg/artifact/blob/s3/s3test/options.go b/pkg/artifact/blob/s3/s3test/options.go index 4ae1657a1cb..ef87c1513a5 100644 --- a/pkg/artifact/blob/s3/s3test/options.go +++ b/pkg/artifact/blob/s3/s3test/options.go @@ -62,7 +62,12 @@ func WithConditionalBehavior(behavior ConditionalBehavior) Option { func WithPageCap(n int) Option { defer perf.Track(nil, "s3test.WithPageCap")() - return func(s *settings) { s.pageCap = n } + return func(s *settings) { + if n <= 0 { + panic("s3test: page cap must be positive") + } + s.pageCap = n + } } // WithMinPartSize rejects completion when a non-final part is smaller than n bytes, as S3 does diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index b69230ae66e..c11e11aba95 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -375,3 +375,20 @@ func TestEmulatorFromEnvSkipsWhenUnset(t *testing.T) { emulator := EmulatorFromEnv(t) assert.Equal(t, Emulator{Endpoint: "http://localhost:4566", Bucket: "b", Region: "us-east-1"}, emulator) } + +func TestWeakIfMatchAndEmptyRange(t *testing.T) { + srv := NewServer(t) + first, err := putObject(t, srv.Client(), srv, "k", "value") + require.NoError(t, err) + _, err = putObject(t, srv.Client(), srv, "k", "new", func(in *s3.PutObjectInput) { in.IfMatch = aws.String("W/" + aws.ToString(first.ETag)) }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + srv.Seed("empty", nil, nil) + _, err = srv.Client().GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("empty"), Range: aws.String("bytes=0-")}) + assert.Equal(t, "InvalidRange", apiCode(t, err)) +} + +func TestPageCapMustBePositive(t *testing.T) { + for _, cap := range []int{0, -1} { + assert.Panics(t, func() { NewServer(t, WithPageCap(cap)) }) + } +} diff --git a/pkg/artifact/blob/s3/store_test.go b/pkg/artifact/blob/s3/store_test.go index 0ee1556657d..1ea21f67ab1 100644 --- a/pkg/artifact/blob/s3/store_test.go +++ b/pkg/artifact/blob/s3/store_test.go @@ -3,8 +3,12 @@ package s3 import ( "context" "fmt" + "math" + "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -145,3 +149,14 @@ func TestListDefaultsToServerPageSizeWhenNoLimit(t *testing.T) { assert.Empty(t, page.NextCursor) assert.NotContains(t, fake.RequestsFor(s3test.OpListObjectsV2)[0].Query, "max-keys") } + +func TestInvalidSizeAndOverflowingRangeFailBeforeRequest(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", strings.NewReader(""), blob.PutOptions{Size: -2}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = s.Get(context.Background(), "k", blob.GetOptions{Offset: math.MaxInt64, Length: 2}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) + assert.Equal(t, "bytes=9223372036854775807-9223372036854775807", aws.ToString(rangeHeader(blob.GetOptions{Offset: math.MaxInt64, Length: 1}))) +} diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index 52ec2f9f8f2..5fb57a7265b 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -77,7 +77,7 @@ func (s *Store) transport() (*blobLocal.Store, error) { if root == "" { root = "." } - return blobLocal.New(root) + return blobLocal.New(root, blobLocal.WithLiteralBackslashes()) } // key maps a validated artifact name to the transport key. Names are cleaned the same way the diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go index 192ae4583e9..176c0297643 100644 --- a/pkg/ci/artifact/local/transport_parity_test.go +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "io" "os" "path/filepath" "sort" @@ -58,7 +59,7 @@ func TestTransportKeepsTheLegacyOnDiskLayout(t *testing.T) { require.NoError(t, err) assert.Equal(t, want, sidecar, "the sidecar is byte-identical to the legacy encoding") if info, statErr := os.Stat(filepath.Join(root, "dev", "vpc", "abc.tar")); statErr == nil && filepath.Separator == '/' { - assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()&0o644) + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()) } } @@ -176,3 +177,26 @@ func TestListSkipsOrphanSidecarsAndFiltersByQuery(t *testing.T) { assert.Equal(t, int64(2), prod[0].Size) assert.Equal(t, "prod", prod[0].Metadata.Stack) } + +func TestLegacyLiteralBackslashName(t *testing.T) { + if filepath.Separator == '\\' { + t.Skip("backslash is a path separator") + } + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + name := `literal\name` + require.NoError(t, store.Upload(ctx, name, strings.NewReader("payload"), 7, nil)) + assert.FileExists(t, filepath.Join(root, name)) + reader, _, err := store.Download(ctx, name) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "payload", string(data)) + listed, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, name, listed[0].Name) + require.NoError(t, store.Delete(ctx, name)) +} From 860b45212650efb134265003f34a6bfbfd583e96 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:10:39 -0600 Subject: [PATCH 07/43] fix: harden artifact recovery and conditional operations --- pkg/artifact/blob/blob_test.go | 6 ++ pkg/artifact/blob/blobtest/doc.go | 4 +- pkg/artifact/blob/local/etag.go | 14 +++- pkg/artifact/blob/local/fsutil.go | 2 +- pkg/artifact/blob/local/get.go | 16 ++-- pkg/artifact/blob/local/lock.go | 20 +++-- pkg/artifact/blob/local/open_other.go | 7 ++ pkg/artifact/blob/local/open_unix.go | 13 ++++ pkg/artifact/blob/local/open_unix_test.go | 24 ++++++ pkg/artifact/blob/local/put.go | 8 +- pkg/artifact/blob/local/store_test.go | 38 ++++++++++ pkg/artifact/blob/s3/classify.go | 3 +- pkg/artifact/blob/s3/conditional.go | 45 +++++++++-- pkg/artifact/blob/s3/conditional_test.go | 61 +++++++++++++-- pkg/artifact/blob/s3/multipart.go | 2 +- pkg/artifact/blob/s3/options.go | 2 +- pkg/artifact/blob/s3/put.go | 3 + pkg/artifact/blob/s3/s3test/objects.go | 5 +- pkg/artifact/blob/s3/s3test/server_test.go | 6 ++ pkg/artifact/blob/s3/store_test.go | 19 +++++ pkg/artifact/blob/tags.go | 4 +- pkg/artifact/layout/admit.go | 9 +++ pkg/artifact/layout/layout_test.go | 20 ++++- pkg/artifact/layout/location.go | 13 ++++ pkg/artifact/layout/paths.go | 2 +- pkg/artifact/layout/records.go | 14 +++- pkg/artifact/local/edge_test.go | 2 +- pkg/artifact/local/evidence.go | 75 +++++++++++++++---- pkg/artifact/local/lock_test.go | 24 ++++++ pkg/artifact/local/publish.go | 10 +++ pkg/artifact/local/recovery_test.go | 17 ++++- pkg/artifact/local/roundtrip_test.go | 10 ++- pkg/artifact/local/verifier_test.go | 39 +++++++--- .../artifact/local/transport_parity_test.go | 6 ++ pkg/ci/artifact/s3/store.go | 4 +- pkg/ci/artifact/s3/transport_parity_test.go | 9 +++ 36 files changed, 479 insertions(+), 77 deletions(-) create mode 100644 pkg/artifact/blob/local/open_other.go create mode 100644 pkg/artifact/blob/local/open_unix.go create mode 100644 pkg/artifact/blob/local/open_unix_test.go diff --git a/pkg/artifact/blob/blob_test.go b/pkg/artifact/blob/blob_test.go index 5291657b905..10384840652 100644 --- a/pkg/artifact/blob/blob_test.go +++ b/pkg/artifact/blob/blob_test.go @@ -177,3 +177,9 @@ func (s *listOnlyStore) List(_ context.Context, opts ListOptions) (*ListPage, er page := s.pages[len(s.calls)-1] return &page, nil } + +func TestValidateTagsRejectsInvalidUTF8(t *testing.T) { + for _, tags := range []map[string]string{{"\xff": "value"}, {"key": "\xff"}} { + require.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid) + } +} diff --git a/pkg/artifact/blob/blobtest/doc.go b/pkg/artifact/blob/blobtest/doc.go index 22158d75804..2d8ed32e6ac 100644 --- a/pkg/artifact/blob/blobtest/doc.go +++ b/pkg/artifact/blob/blobtest/doc.go @@ -1,4 +1,4 @@ // Package blobtest is the shared conformance suite for [blob.Store] implementations. Every backend -// runs [RunStoreConformance] from its own tests; behavior a backend does not advertise through -// [blob.Capabilities] is asserted to fail clearly instead of being skipped. +// runs [RunStoreConformance] from its own tests; unsupported behavior is checked for clear errors where the suite can assert them; other +// cases absent from [blob.Capabilities] are skipped. package blobtest diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go index e09e04f70ec..edcdc7aebfd 100644 --- a/pkg/artifact/blob/local/etag.go +++ b/pkg/artifact/blob/local/etag.go @@ -18,18 +18,26 @@ const hexBase = 16 // etagOf returns the ETag of the file at path described by info. func etagOf(path string, info os.FileInfo) (string, error) { - if info.Size() > contentETagLimit { - return weakETag(info), nil - } file, err := openFile(path) if err != nil { return "", err } defer file.Close() + return etagOpen(file, info) +} + +// etagOpen hashes the opened file and restores its position for the caller. +func etagOpen(file *os.File, info os.FileInfo) (string, error) { + if info.Size() > contentETagLimit { + return weakETag(info), nil + } hash := sha256.New() if _, err := io.Copy(hash, file); err != nil { return "", err } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return "", err + } return contentETag(hash.Sum(nil)), nil } diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go index 94cdf092de1..1e710a271b0 100644 --- a/pkg/artifact/blob/local/fsutil.go +++ b/pkg/artifact/blob/local/fsutil.go @@ -87,7 +87,7 @@ func openFile(path string) (*os.File, error) { err error ) for i := range attempts { - file, err = os.Open(path) + file, err = openRead(path) if err == nil || errors.Is(err, os.ErrNotExist) { return file, err } diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go index adec86c45cf..844de9505bb 100644 --- a/pkg/artifact/blob/local/get.go +++ b/pkg/artifact/blob/local/get.go @@ -41,7 +41,7 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo if err != nil { return nil, wrapIO(blob.OpGet, key, err) } - info, err := s.describe(key, target, file, false) + info, err := s.describe(blob.OpGet, key, target, file, false) if err != nil { _ = file.Close() return nil, err @@ -88,25 +88,25 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b return nil, wrapIO(blob.OpStat, key, err) } defer file.Close() - return s.describe(key, target, file, opts.IncludeTags) + return s.describe(blob.OpStat, key, target, file, opts.IncludeTags) } // describe builds ObjectInfo for an open regular file. A directory is not an object. -func (s *Store) describe(key, target string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { +func (s *Store) describe(op, key, target string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { stat, err := file.Stat() if err != nil { - return nil, wrapIO(blob.OpStat, key, err) + return nil, wrapIO(op, key, err) } if !stat.Mode().IsRegular() { - return nil, blob.NewError(blob.KindNotFound, blob.OpStat, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) + return nil, blob.NewError(blob.KindNotFound, op, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) } - etag, err := etagOf(target, stat) + etag, err := etagOpen(file, stat) if err != nil { - return nil, wrapIO(blob.OpStat, key, err) + return nil, wrapIO(op, key, err) } a, err := readAttrs(target) if err != nil { - return nil, blob.NewError(blob.KindOther, blob.OpStat, key, fmt.Errorf("read attributes: %w", err)) + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("read attributes: %w", err)) } info := &blob.ObjectInfo{Key: key, Size: stat.Size(), ETag: etag, ContentType: a.ContentType, LastModified: stat.ModTime(), Metadata: a.Metadata} if withTags { diff --git a/pkg/artifact/blob/local/lock.go b/pkg/artifact/blob/local/lock.go index a2c1d34a0ff..33a2d02586d 100644 --- a/pkg/artifact/blob/local/lock.go +++ b/pkg/artifact/blob/local/lock.go @@ -18,35 +18,39 @@ const lockRetry = 10 * time.Millisecond // serialize with each other without relying on the file lock alone. var rootMutexes sync.Map -func rootMutex(root string) *sync.Mutex { +func rootMutex(root string) chan struct{} { key := root if abs, err := filepath.Abs(root); err == nil { key = abs } - mutex, _ := rootMutexes.LoadOrStore(key, &sync.Mutex{}) - return mutex.(*sync.Mutex) + mutex, _ := rootMutexes.LoadOrStore(key, make(chan struct{}, 1)) + return mutex.(chan struct{}) } // lockRoot takes the per-root lock (in-process mutex, then OS file lock) and returns its release. func (s *Store) lockRoot(ctx context.Context, op, key string) (func(), error) { mutex := rootMutex(s.root) - mutex.Lock() + select { + case mutex <- struct{}{}: + case <-ctx.Done(): + return nil, blob.NewError(blob.KindOther, op, key, ctx.Err()) + } if err := ctx.Err(); err != nil { - mutex.Unlock() + <-mutex return nil, blob.NewError(blob.KindOther, op, key, err) } if err := mkdirAll(s.root, s.dirPerm); err != nil { - mutex.Unlock() + <-mutex return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("create lock directory: %w", err)) } fileLock := flock.New(filepath.Join(s.root, lockName)) if _, err := fileLock.TryLockContext(ctx, lockRetry); err != nil { - mutex.Unlock() + <-mutex return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("acquire root lock: %w", err)) } return func() { _ = fileLock.Unlock() - mutex.Unlock() + <-mutex }, nil } diff --git a/pkg/artifact/blob/local/open_other.go b/pkg/artifact/blob/local/open_other.go new file mode 100644 index 00000000000..0834bd8881a --- /dev/null +++ b/pkg/artifact/blob/local/open_other.go @@ -0,0 +1,7 @@ +//go:build !unix + +package local + +import "os" + +func openRead(path string) (*os.File, error) { return os.Open(path) } diff --git a/pkg/artifact/blob/local/open_unix.go b/pkg/artifact/blob/local/open_unix.go new file mode 100644 index 00000000000..069da378cc1 --- /dev/null +++ b/pkg/artifact/blob/local/open_unix.go @@ -0,0 +1,13 @@ +//go:build unix + +package local + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func openRead(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0) +} diff --git a/pkg/artifact/blob/local/open_unix_test.go b/pkg/artifact/blob/local/open_unix_test.go new file mode 100644 index 00000000000..5d8f3f27359 --- /dev/null +++ b/pkg/artifact/blob/local/open_unix_test.go @@ -0,0 +1,24 @@ +//go:build unix + +package local + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestGetAndStatRejectFIFOWithoutBlocking(t *testing.T) { + store := newStore(t) + require.NoError(t, mkdirAll(store.Root(), DefaultDirPerm)) + require.NoError(t, unix.Mkfifo(filepath.Join(store.Root(), "fifo"), 0o600)) + _, err := store.Get(context.Background(), "fifo", blob.GetOptions{}) + require.True(t, blob.IsNotFound(err)) + _, err = store.Stat(context.Background(), "fifo", blob.StatOptions{}) + require.True(t, blob.IsNotFound(err)) +} diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index cf79cb21947..50b5db4f9f1 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -29,6 +29,9 @@ type staged struct { func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { defer perf.Track(nil, "blob.local.Put")() + if body == nil { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: nil body", errUtils.ErrBlobInvalid)) + } if err := s.validatePut(ctx, key, opts); err != nil { return nil, err } @@ -132,9 +135,12 @@ func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (i // checkCondition evaluates the put condition against the current object. Callers hold the lock. func (s *Store) checkCondition(key, target string, cond blob.Condition) error { - if !cond.IsConditional() { + if cond.Kind == blob.ConditionNone { return nil } + if cond.Kind != blob.ConditionIfAbsent && cond.Kind != blob.ConditionIfMatch { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: unknown condition %d", errUtils.ErrBlobInvalid, cond.Kind)) + } state, err := inspect(target) if err != nil { return wrapIO(blob.OpPut, key, err) diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index c3d4fa5685f..a69b7a54f51 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -3,6 +3,8 @@ package local import ( "bytes" "context" + "crypto/sha256" + "io" "math" "os" "path/filepath" @@ -153,3 +155,39 @@ func TestInvalidSizesAndNULPrefix(t *testing.T) { _, err := store.List(context.Background(), blob.ListOptions{Prefix: "x\x00y"}) require.ErrorIs(t, err, errUtils.ErrBlobInvalid) } + +func TestDescribeHashesOpenedFile(t *testing.T) { + store := newStore(t) + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + path := filepath.Join(store.Root(), "k") + require.NoError(t, os.WriteFile(path, []byte("old"), 0o644)) + file, err := openFile(path) + require.NoError(t, err) + defer file.Close() + require.NoError(t, os.Rename(path, path+".old")) + require.NoError(t, os.WriteFile(path, []byte("new"), 0o644)) + info, err := store.describe(blob.OpGet, "k", path, file, false) + require.NoError(t, err) + sum := sha256.Sum256([]byte("old")) + assert.Equal(t, contentETag(sum[:]), info.ETag) + body, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "old", string(body)) +} + +func TestRootLockWaitRespectsCancellation(t *testing.T) { + store := newStore(t) + release, err := store.lockRoot(context.Background(), blob.OpPut, "k") + require.NoError(t, err) + defer release() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = store.lockRoot(ctx, blob.OpPut, "k") + require.ErrorIs(t, err, context.Canceled) +} + +func TestPutRejectsUnknownCondition(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Condition: blob.Condition{Kind: blob.ConditionKind(999)}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} diff --git a/pkg/artifact/blob/s3/classify.go b/pkg/artifact/blob/s3/classify.go index a21bfd97f1a..1ad3c2d5ed2 100644 --- a/pkg/artifact/blob/s3/classify.go +++ b/pkg/artifact/blob/s3/classify.go @@ -111,10 +111,9 @@ func rejectsCondition(status int, code string) bool { } // ambiguousStatus reports server errors after which a mutation may or may not have been applied. -// 503 (throttling) and 501 (not implemented) are definite rejections. +// 501 (not implemented) is a definite rejection. func ambiguousStatus(status int) bool { return status >= http.StatusInternalServerError && - status != http.StatusServiceUnavailable && status != http.StatusNotImplemented } diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go index ac22a2b5a1f..27aa8e6bc64 100644 --- a/pkg/artifact/blob/s3/conditional.go +++ b/pkg/artifact/blob/s3/conditional.go @@ -5,6 +5,8 @@ import ( "context" "crypto/rand" "encoding/hex" + "fmt" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -16,8 +18,12 @@ import ( // probePrefix holds the short-lived objects the conditional-write probe creates. const probePrefix = ".atmos-blob-probe/" +const probeOperation = "probe" + +const probeCleanupTimeout = 5 * time.Second + // ensureConditional makes sure a conditional operation is safe to send. In probe mode it first -// proves, once per store, that the endpoint enforces If-None-Match; an endpoint that silently +// proves, once per store, that the endpoint enforces If-None-Match and If-Match; an endpoint that silently // ignores the header would otherwise turn create-if-absent into an unconditional overwrite. func (s *Store) ensureConditional(ctx context.Context, key string, cond blob.Condition) error { if !cond.IsConditional() || s.cfg.conditional != ConditionalProbe { @@ -61,10 +67,10 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error { } // runProbe performs the double create and cleans up the probe object. -func (s *Store) runProbe(ctx context.Context, api API) error { +func (s *Store) runProbe(ctx context.Context, api API) (resultErr error) { var raw [8]byte if _, err := rand.Read(raw[:]); err != nil { - return blob.NewError(blob.KindOther, "probe", "", err) + return blob.NewError(blob.KindOther, probeOperation, "", err) } key := probePrefix + hex.EncodeToString(raw[:]) objectKey := s.objectKey(key) @@ -75,20 +81,45 @@ func (s *Store) runProbe(ctx context.Context, api API) error { Body: bytes.NewReader([]byte("atmos conditional write probe")), IfNoneMatch: aws.String("*"), }, noRetry) - return classify(request{op: "probe", key: key, mutating: true, conditional: true}, err) + return classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err) } if err := create(); err != nil { return err } defer func() { - _, _ = api.DeleteObject(context.WithoutCancel(ctx), &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}) + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), probeCleanupTimeout) + defer cancel() + _, err := api.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}, noRetry) + if err != nil { + resultErr = blob.NewError(blob.KindOther, probeOperation, key, fmt.Errorf("cleanup probe object: %w", err)) + } }() err := create() if blob.IsPreconditionFailed(err) { - return nil + return s.probeIfMatch(ctx, api, key, objectKey) } if err == nil { - return unsupportedConditional("probe", key, "the endpoint accepted a second create-if-absent for the same key, so it ignores If-None-Match") + return unsupportedConditional(probeOperation, key, "the endpoint accepted a second create-if-absent for the same key, so it ignores If-None-Match") } return err } + +// probeIfMatch verifies that stale validators cannot overwrite or delete the probe object. +func (s *Store) probeIfMatch(ctx context.Context, api API, key, objectKey string) error { + stale := aws.String(`"atmos-probe-nonmatching-etag"`) + _, err := api.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey), Body: bytes.NewReader([]byte("unexpected")), IfMatch: stale}, noRetry) + if err = classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err); !blob.IsPreconditionFailed(err) { + if err == nil { + return unsupportedConditional(probeOperation, key, "the endpoint ignored If-Match on put") + } + return err + } + _, err = api.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey), IfMatch: stale}, noRetry) + if err = classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err); !blob.IsPreconditionFailed(err) { + if err == nil { + return unsupportedConditional(probeOperation, key, "the endpoint ignored If-Match on delete") + } + return err + } + return nil +} diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go index 414a6528508..669fc903e07 100644 --- a/pkg/artifact/blob/s3/conditional_test.go +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -7,6 +7,8 @@ import ( "net/http" "testing" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -123,12 +125,12 @@ func TestProbeAcceptsEnforcingEndpointOnce(t *testing.T) { require.NoError(t, err) puts := fake.RequestsFor(s3test.OpPutObject) - assert.Len(t, puts, 4, "two probe creates plus the two caller puts; the probe runs once") + assert.Len(t, puts, 5, "three probe puts plus the two caller puts; the probe runs once") assert.Equal(t, []string{"lock", "lock2"}, fake.Keys(), "the probe object was cleaned up") - assert.Equal(t, 1, countPrefix(puts, probePrefix)/2, "exactly one probe key was created twice") + assert.Equal(t, 1, countPrefix(puts, probePrefix)/3, "exactly one probe key was created twice") require.NoError(t, s.ProbeConditionalWrites(context.Background())) - assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 4, "explicit probe reuses the cached verdict") + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 5, "explicit probe reuses the cached verdict") } func countPrefix(requests []s3test.Request, prefix string) int { @@ -184,13 +186,13 @@ func TestServerErrorOnConditionalMutationIsAmbiguous(t *testing.T) { assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) } -func TestThrottlingIsNotAmbiguous(t *testing.T) { +func TestServiceUnavailableIsAmbiguous(t *testing.T) { fake := newFake(t) fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusServiceUnavailable, Code: "SlowDown", Times: -1}) s := newTestStore(t, fake) _, err := putIfAbsent(s, "lock", "a") require.Error(t, err) - assert.False(t, blob.IsAmbiguous(err), "503 means the request was rejected, not half applied") + assert.True(t, blob.IsAmbiguous(err), "an endpoint may apply a mutation before responding 503") } func TestUnconditionalPutRetriesTransientServerError(t *testing.T) { @@ -257,3 +259,52 @@ func TestServiceCodesMapToKinds(t *testing.T) { }) } } + +func TestProbeCleanupFailureIsReturnedAndNotCached(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: http.StatusForbidden, Code: "AccessDenied", Skip: 1}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + err := s.ProbeConditionalWrites(context.Background()) + require.ErrorContains(t, err, "cleanup probe object") + fake.ClearFaults() + require.NoError(t, s.ProbeConditionalWrites(context.Background())) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 6) +} + +type ignoresIfMatch struct { + API + put, delete bool +} + +func (i ignoresIfMatch) PutObject(ctx context.Context, in *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) { + copy := *in + if i.put { + copy.IfMatch = nil + } + return i.API.PutObject(ctx, ©, opts...) +} + +func (i ignoresIfMatch) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { + copy := *in + if i.delete { + copy.IfMatch = nil + } + return i.API.DeleteObject(ctx, ©, opts...) +} + +func TestProbeRejectsIgnoredIfMatch(t *testing.T) { + for _, tc := range []struct { + name string + put, delete bool + }{{"put", true, false}, {"delete", false, true}} { + t.Run(tc.name, func(t *testing.T) { + fake := newFake(t) + api := ignoresIfMatch{API: fake.Client(), put: tc.put, delete: tc.delete} + s, err := NewFromClient(api, fake.Bucket(), WithConditionalMode(ConditionalProbe)) + require.NoError(t, err) + err = s.ProbeConditionalWrites(context.Background()) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Empty(t, fake.Keys()) + }) + } +} diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go index ad7d1ddccd6..c5bb4d282b7 100644 --- a/pkg/artifact/blob/s3/multipart.go +++ b/pkg/artifact/blob/s3/multipart.go @@ -41,7 +41,7 @@ func (s *Store) putMultipart(ctx context.Context, plan *putPlan, first, carry [] Metadata: plan.opts.Metadata, Tagging: optionalString(plan.tags), } - created, err := plan.api.CreateMultipartUpload(ctx, in, skipOptionalChecksums) + created, err := plan.api.CreateMultipartUpload(ctx, in, skipOptionalChecksums, noRetry) if err != nil { return nil, classify(plan.request(false), err) } diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go index 7540ffcf237..39bcea99d86 100644 --- a/pkg/artifact/blob/s3/options.go +++ b/pkg/artifact/blob/s3/options.go @@ -29,7 +29,7 @@ const ( ConditionalAuto ConditionalMode = iota // ConditionalProbe is like Auto, but first proves once per store that the endpoint enforces - // If-None-Match (see [Store.ProbeConditionalWrites]). Use it for third-party endpoints that + // If-None-Match and If-Match on writes and deletes (see [Store.ProbeConditionalWrites]). Use it for endpoints that // might silently ignore the headers. ConditionalProbe diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index 78e8a723159..644958f9af8 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -36,6 +36,9 @@ func (p *putPlan) request(mutating bool) request { func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { defer perf.Track(nil, "blob.s3.Put")() + if body == nil { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: nil body", errUtils.ErrBlobInvalid)) + } plan, err := s.planPut(ctx, key, opts) if err != nil { return nil, err diff --git a/pkg/artifact/blob/s3/s3test/objects.go b/pkg/artifact/blob/s3/s3test/objects.go index 9b03d701625..a4c630356bb 100644 --- a/pkg/artifact/blob/s3/s3test/objects.go +++ b/pkg/artifact/blob/s3/s3test/objects.go @@ -169,7 +169,10 @@ func rangeEnd(last string, start, size int64) (int64, bool) { if err != nil || through < start { return 0, false } - return min(through+1, size), true + if through >= size { + return size, true + } + return through + 1, true } func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, key string) { diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index c11e11aba95..a7d32aa8f24 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -392,3 +392,9 @@ func TestPageCapMustBePositive(t *testing.T) { assert.Panics(t, func() { NewServer(t, WithPageCap(cap)) }) } } + +func TestRangeEndMaxIntDoesNotOverflow(t *testing.T) { + got, ok := parseRange("bytes=0-9223372036854775807", 10) + require.True(t, ok) + assert.Equal(t, int64(10), got.end) +} diff --git a/pkg/artifact/blob/s3/store_test.go b/pkg/artifact/blob/s3/store_test.go index 1ea21f67ab1..fed3c6df374 100644 --- a/pkg/artifact/blob/s3/store_test.go +++ b/pkg/artifact/blob/s3/store_test.go @@ -1,6 +1,7 @@ package s3 import ( + "bytes" "context" "fmt" "math" @@ -160,3 +161,21 @@ func TestInvalidSizeAndOverflowingRangeFailBeforeRequest(t *testing.T) { assert.Empty(t, fake.Requests()) assert.Equal(t, "bytes=9223372036854775807-9223372036854775807", aws.ToString(rangeHeader(blob.GetOptions{Offset: math.MaxInt64, Length: 1}))) } + +func TestNilPutBodyIsInvalid(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) +} + +func TestMultipartCreateIsNotRetriedAfterLostResponse(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake, WithPartSize(smallPart)) + data := bytes.Repeat([]byte("x"), int(smallPart)+1) + _, err := s.Put(context.Background(), "k", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.Error(t, err) + assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) +} diff --git a/pkg/artifact/blob/tags.go b/pkg/artifact/blob/tags.go index 655796a045c..662dd223ead 100644 --- a/pkg/artifact/blob/tags.go +++ b/pkg/artifact/blob/tags.go @@ -39,10 +39,10 @@ func ValidateTags(tags map[string]string) error { return tagError(fmt.Sprintf("%d tags exceed the limit of %d", len(tags), MaxTags)) } for key, value := range tags { - if key == "" || utf8.RuneCountInString(key) > MaxTagKeyChars { + if key == "" || !utf8.ValidString(key) || utf8.RuneCountInString(key) > MaxTagKeyChars { return tagError(fmt.Sprintf("tag key %q must be 1-%d characters", key, MaxTagKeyChars)) } - if utf8.RuneCountInString(value) > MaxTagValueChars { + if !utf8.ValidString(value) || utf8.RuneCountInString(value) > MaxTagValueChars { return tagError(fmt.Sprintf("tag %q value exceeds %d characters", key, MaxTagValueChars)) } } diff --git a/pkg/artifact/layout/admit.go b/pkg/artifact/layout/admit.go index e1c21179439..40ce78be7d5 100644 --- a/pkg/artifact/layout/admit.go +++ b/pkg/artifact/layout/admit.go @@ -92,6 +92,15 @@ func Admit(in *AdmitInput) (Admission, error) { return AdmitCreate, nil } if in.Current.State == StatePending { + if in.Current.IntentDigest == in.IncomingIntent { + var expected digest.Digest + if in.ExpectedPrior != nil { + expected = in.ExpectedPrior.ResolvedDigest + } + if expected != in.Current.PriorRecordDigest { + return "", fmt.Errorf("%w: pending publication expected prior %s, caller expected %s", errUtils.ErrArtifactConflict, in.Current.PriorRecordDigest, expected) + } + } return admitPending(in) } if err := checkExpectedPrior(in.Current, in.ExpectedPrior); err != nil { diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go index 3f27c2e7592..a41cca3a4a5 100644 --- a/pkg/artifact/layout/layout_test.go +++ b/pkg/artifact/layout/layout_test.go @@ -436,7 +436,9 @@ func TestAdmit(t *testing.T) { {"stale expected prior", AdmitInput{Current: &completed, IncomingRecord: digestC, ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digestC}, Force: true}, "", "", errUtils.ErrArtifactConflict}, {"exists but writer expected none", AdmitInput{Current: &completed, IncomingRecord: digestC, Force: true}, "", "", errUtils.ErrArtifactConflict}, {"same-intent pending first publication resumes", AdmitInput{Current: &pendingFirst, IncomingIntent: digestA}, AdmitResumeCreate, artifact.OutcomeCreated, nil}, - {"same-intent pending replacement resumes", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA}, AdmitResumeReplace, artifact.OutcomeReplaced, nil}, + {"same-intent pending replacement resumes", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, AdmitResumeReplace, artifact.OutcomeReplaced, nil}, + {"stale pending replacement", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA}, "", "", errUtils.ErrArtifactConflict}, + {"unexpected prior for pending creation", AdmitInput{Current: &pendingFirst, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, {"different-intent pending names the operation", AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}, "", "", errUtils.ErrArtifactPublicationUnavailable}, {"force never steals a different pending intent", AdmitInput{Current: &pendingReplace, IncomingIntent: digestC, Force: true}, "", "", errUtils.ErrArtifactPublicationUnavailable}, } @@ -457,3 +459,19 @@ func TestAdmit(t *testing.T) { _, err := Admit(&AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}) require.ErrorContains(t, err, "op-1", "the recovery diagnostic includes the owning operation ID") } + +func TestPortableWindowsNames(t *testing.T) { + for _, name := range []string{"CON", "aux.txt", "com1", "lPt9.tar", "trailing.", "trailing "} { + _, err := CleanLocation("team/" + name) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, name) + } + for _, id := range []string{"CON", "aux.txt", "COM1", "lpt9.tar"} { + assert.NotEqual(t, id, OperationFileName(id)) + } + assert.Equal(t, "COM10", OperationFileName("COM10")) +} + +func TestOperationRejectsUnsafePriorLocation(t *testing.T) { + op := &Operation{SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op", Name: "team/app", Version: "1", CapturedAt: artifact.NewTimestamp(time.Now()), Location: "safe", Payload: []ocispec.Descriptor{}, Documents: []ocispec.Descriptor{}, Prior: &Prior{RecordDigest: digestA, Location: artifact.ReservedNamespace + "/unsafe"}} + require.Error(t, op.Validate()) +} diff --git a/pkg/artifact/layout/location.go b/pkg/artifact/layout/location.go index 120929d73f8..8236ff92c68 100644 --- a/pkg/artifact/layout/location.go +++ b/pkg/artifact/layout/location.go @@ -163,6 +163,8 @@ func CleanLocation(location string) (string, error) { switch { case segment == "" || segment == "." || segment == "..": return "", fmt.Errorf("%w: location %q has an empty, '.', or '..' segment", errUtils.ErrArtifactUnsafePath, location) + case strings.HasSuffix(segment, ".") || strings.HasSuffix(segment, " ") || windowsDeviceName(segment): + return "", fmt.Errorf("%w: location %q contains a Windows-reserved name", errUtils.ErrArtifactUnsafePath, location) case strings.EqualFold(segment, artifact.ReservedNamespace): return "", fmt.Errorf("%w: location %q enters the reserved %s namespace", errUtils.ErrArtifactUnsafePath, location, artifact.ReservedNamespace) } @@ -185,6 +187,17 @@ func checkLocationShape(location string) error { return nil } +// windowsDeviceName identifies reserved device basenames, including names with extensions. +func windowsDeviceName(segment string) bool { + base, _, _ := strings.Cut(segment, ".") + base = strings.ToUpper(base) + switch base { + case "CON", "PRN", "AUX", "NUL": + return true + } + return len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9' +} + func isControl(r rune) bool { return r < ' ' || r == deleteRune } diff --git a/pkg/artifact/layout/paths.go b/pkg/artifact/layout/paths.go index d9fcef6918f..13a58bfa20a 100644 --- a/pkg/artifact/layout/paths.go +++ b/pkg/artifact/layout/paths.go @@ -103,7 +103,7 @@ func RepresentationPath(key string) string { func OperationFileName(operationID string) string { defer perf.Track(nil, "layout.OperationFileName")() - if len(operationID) <= maxSafeOperationID && safeOperationID.MatchString(operationID) { + if len(operationID) <= maxSafeOperationID && safeOperationID.MatchString(operationID) && !windowsDeviceName(operationID) { return operationID } sum := sha256.Sum256([]byte(operationID)) diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go index 5365a3fd1bc..f7f35be1bd5 100644 --- a/pkg/artifact/layout/records.go +++ b/pkg/artifact/layout/records.go @@ -383,9 +383,21 @@ func (o *Operation) validateState() error { if o.Payload == nil || o.Documents == nil { return invalid(kindOperation, "payload and documents lists are required (use empty lists)") } - if o.Prior != nil && o.Prior.RecordDigest == "" { + return validatePrior(o.Prior) +} + +func validatePrior(prior *Prior) error { + if prior == nil { + return nil + } + if prior.RecordDigest == "" { return invalid(kindOperation, "prior needs a record_digest") } + if prior.Location != "" { + if _, err := CleanLocation(prior.Location); err != nil { + return invalid(kindOperation, "prior location %q is not a valid relative location", prior.Location) + } + } return nil } diff --git a/pkg/artifact/local/edge_test.go b/pkg/artifact/local/edge_test.go index 80da4df2942..36be45ce9ad 100644 --- a/pkg/artifact/local/edge_test.go +++ b/pkg/artifact/local/edge_test.go @@ -196,7 +196,7 @@ func TestPendingReference_RepairsWithoutItsJournal(t *testing.T) { require.ErrorIs(t, err, errCrash) require.NoError(t, os.RemoveAll(filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations"))) - retry, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true}) + retry, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) require.NoError(t, err) assert.Equal(t, artifact.OutcomeReplaced, retry.Outcome) assertIntact(t, repo, defaultRef()) diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index 97f09c86d1a..3528eb96034 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -15,9 +15,10 @@ import ( "github.com/cloudposse/atmos/pkg/perf" ) -// maxEvidenceFiles bounds how many evidence files one verification or export reads. More than -// this is reported, not silently dropped. -const maxEvidenceFiles = 64 +// maxMalformedEvidence bounds diagnostics and reads for invalid evidence entries. +const maxMalformedEvidence = 64 + +var errTooManyMalformedEvidence = errors.New("too many malformed evidence files") // loadedEvidence is one stored evidence file that parsed and matched its file name. type loadedEvidence struct { @@ -27,21 +28,20 @@ type loadedEvidence struct { // loadEvidence reads the evidence stored for a Record subject. Malformed or oversize files are // reported as diagnostics and skipped; they can never override valid matching evidence. -func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic) { +func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic, error) { dir := layout.SignatureDir(artifact.SubjectRecord, subject) entries, err := fs.ReadDir(t.FS(), dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, nil + } if err != nil { - return nil, nil + return nil, nil, fmt.Errorf("read evidence directory: %w", err) } var ( loaded []loadedEvidence diags []artifact.Diagnostic ) for _, entry := range entries { - if len(loaded) >= maxEvidenceFiles { - diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: fmt.Sprintf("more than %d evidence files; the rest were not read", maxEvidenceFiles)}) - break - } stem, ok := strings.CutSuffix(entry.Name(), ".json") if entry.IsDir() || !ok { continue @@ -49,12 +49,16 @@ func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact locator := dir + "/" + entry.Name() item, problem := t.readEvidenceFile(locator, stem) if problem != nil { + if len(diags) >= maxMalformedEvidence { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many malformed evidence files; remaining entries were not read"}) + return nil, diags, errTooManyMalformedEvidence + } diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}) continue } loaded = append(loaded, loadedEvidence{evidence: item, locator: locator}) } - return loaded, diags + return loaded, diags, nil } func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, error) { @@ -71,7 +75,7 @@ func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, er // listEvidence returns the stored evidence descriptors for a subject, skipping malformed files. func (t *tree) listEvidence(subject digest.Digest) []artifact.SignatureEvidence { - loaded, _ := t.loadEvidence(subject) + loaded, _, _ := t.loadEvidence(subject) out := make([]artifact.SignatureEvidence, 0, len(loaded)) for index := range loaded { out = append(out, loaded[index].evidence.Reference(loaded[index].locator)) @@ -185,9 +189,15 @@ func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubje if _, err := r.requireCurrentRecord(t, subject.Digest); err != nil { return nil, err } - loaded, diags := t.loadEvidence(subject.Digest) + loaded, diags, loadErr := t.loadEvidence(subject.Digest) + if loadErr != nil { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-read-failed", Message: loadErr.Error()}) + } blobs := evidenceBlobs(subject, loaded, &diags) result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed, Diagnostics: diags} + if loadErr != nil { + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, loadErr) + } if fingerprint, fingerprintErr := policy.Fingerprint(); fingerprintErr == nil { result.PolicyFingerprint = fingerprint } @@ -231,7 +241,7 @@ func (r *Repository) finishVerification(base, verified *artifact.VerificationRes return result, verifyErr case verifyErr != nil: return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) - case result.Status == artifact.VerificationFailed: + case result.Status != artifact.VerificationVerified: return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) } return result, nil @@ -254,7 +264,10 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { return nil, err } - loaded, _ := t.loadEvidence(pinned.RecordDigest) + loaded, _, err := t.loadEvidence(pinned.RecordDigest) + if err != nil { + return nil, err + } blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) for index := range loaded { blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) @@ -265,7 +278,7 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe // WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this // repository already holds. The evidence must bind to the pinned Record and carry matching bytes; // importing never changes the Record or the reference. -func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, _ artifact.LockHandle) ([]artifact.SignatureEvidence, error) { +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, lock artifact.LockHandle) (stored []artifact.SignatureEvidence, err error) { defer perf.Track(nil, "local.Repository.WriteEvidence")() if err := ctx.Err(); err != nil { @@ -276,9 +289,25 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR return nil, r.missingRoot(err, pinned.Ref) } defer t.Close() + held, err := r.lockEvidenceReference(ctx, t, pinned, lock) + if err != nil { + return nil, err + } + if held != nil { + defer func() { + if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { + err = releaseErr + } + }() + } if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { return nil, err } + return r.writeEvidenceLocked(t, pinned, evidence) +} + +// writeEvidenceLocked validates and stores each evidence blob under the reference lock. +func (r *Repository) writeEvidenceLocked(t *tree, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob) ([]artifact.SignatureEvidence, error) { stored := make([]artifact.SignatureEvidence, 0, len(evidence)) for index := range evidence { blob := &evidence[index] @@ -300,3 +329,19 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR } return stored, nil } + +// lockEvidenceReference reuses a matching caller lock or acquires the reference lock. +func (r *Repository) lockEvidenceReference(ctx context.Context, t *tree, pinned *artifact.PinnedReference, lock artifact.LockHandle) (*fileLock, error) { + record, _, err := t.loadRecord(pinned.RecordDigest) + if err != nil { + return nil, err + } + resource := record.Name + ":" + record.Version + if lock != nil { + if lock.Resource() != resource { + return nil, fmt.Errorf("%w: supplied lock does not cover reference %s", errUtils.ErrArtifactConflict, resource) + } + return nil, nil + } + return r.acquire(ctx, resource, r.settings.owner, "") +} diff --git a/pkg/artifact/local/lock_test.go b/pkg/artifact/local/lock_test.go index ff4c5e6cc8f..472d1c10cae 100644 --- a/pkg/artifact/local/lock_test.go +++ b/pkg/artifact/local/lock_test.go @@ -378,3 +378,27 @@ func TestLock_ReleasesWhenItsOwnerProcessExits(t *testing.T) { func lockFile(repo *Repository, resource string) string { return filepath.Join(repo.root, filepath.FromSlash(layout.LockPath(resource))) } + +func TestPublishRejectsLockForAnotherReference(t *testing.T) { + repo := newRepo(t, dirSpec()) + held, err := repo.Lock(context.Background(), lockRequest("other:1.0.0")) + require.NoError(t, err) + defer held.Release(context.Background()) + _, err = repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{Lock: held}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) +} + +func TestWriteEvidenceHonorsReferenceLock(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + held, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + _, err = repo.WriteEvidence(context.Background(), &pinned, nil, held) + require.NoError(t, err) + require.NoError(t, held.Release(context.Background())) +} diff --git a/pkg/artifact/local/publish.go b/pkg/artifact/local/publish.go index d062368890e..8c04eac961f 100644 --- a/pkg/artifact/local/publish.go +++ b/pkg/artifact/local/publish.go @@ -55,6 +55,9 @@ func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArt err = releaseErr } }() + if options.Lock != nil && options.Lock.Resource() != prepared.Record.Name+":"+prepared.Record.Version { + return nil, fmt.Errorf("%w: supplied lock does not cover reference %s:%s", errUtils.ErrArtifactConflict, prepared.Record.Name, prepared.Record.Version) + } if options.Lock == nil { lock, lockErr := r.acquire(ctx, prepared.Record.Name+":"+prepared.Record.Version, r.settings.owner, job.opID) if lockErr != nil { @@ -244,6 +247,13 @@ func (r *Repository) begin(ctx context.Context, t *tree, job *publishJob, plan * // reserve checks for location collisions and unmanaged content, then records the journal and the // pending reference. func (r *Repository) reserve(t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + existing, err := t.loadOperation(job.opID) + if err != nil { + return nil, err + } + if existing != nil && layout.ReferenceKey(existing.Name, existing.Version) != job.key { + return nil, fmt.Errorf("%w: operation %s belongs to %s:%s", errUtils.ErrArtifactConflict, job.opID, existing.Name, existing.Version) + } if err := r.checkCollision(t, job, plan); err != nil { return nil, err } diff --git a/pkg/artifact/local/recovery_test.go b/pkg/artifact/local/recovery_test.go index 8d3ed42bb64..0b9d36d0eef 100644 --- a/pkg/artifact/local/recovery_test.go +++ b/pkg/artifact/local/recovery_test.go @@ -168,7 +168,8 @@ func TestInterruptedForceReplacement(t *testing.T) { require.Error(t, err, "the old digest is never served while the replacement is in flight") assert.Empty(t, listNames(t, repo)) - result, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true}) + repo = newRepo(t, spec.withRoot(repo.root)) + result, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) require.NoError(t, err, "the same-intent retry repairs the reference") assert.Equal(t, artifact.OutcomeReplaced, result.Outcome) current := resolve(t, repo, defaultRef()) @@ -301,3 +302,17 @@ func TestResumedPublicationRetiresAnAbandonedLocation(t *testing.T) { assertIntact(t, zipRepo, defaultRef()) assert.Equal(t, []string{"team/app/1.0.0.zip", "team/app/1.0.0.zip.metadata.json"}, payloadFiles(t, root)) } + +func TestPendingOperationIDCannotBeReusedForAnotherReference(t *testing.T) { + var fault armedFault + repo := newRepo(t, dirSpec().withOpts(fault.option())) + fault.hook = crashAt(faultBeforePayload) + first := prepare(t, repo, &prepareSpec{name: "team/first", opID: "shared-op"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + second := prepare(t, repo, &prepareSpec{name: "team/second", opID: "shared-op"}) + _, err = repo.Publish(context.Background(), second, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: "team/first", Version: testVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) +} diff --git a/pkg/artifact/local/roundtrip_test.go b/pkg/artifact/local/roundtrip_test.go index 429d9b33ad4..84db8c4796d 100644 --- a/pkg/artifact/local/roundtrip_test.go +++ b/pkg/artifact/local/roundtrip_test.go @@ -72,10 +72,12 @@ func TestRoundTrip_EveryAllowedPackaging(t *testing.T) { assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), listInventoryPaths(content.Record)) assert.NoDirExists(t, filepath.Join(out, ".atmos"), "the distribution directory never leaks into pulled user content") assert.FileExists(t, filepath.Join(out, "README.md"), "a root README is ordinary payload") - fixtures.RequirePOSIXModes(t) - info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) - require.NoError(t, err) - assert.NotZero(t, info.Mode()&0o111, "the executable bit survives the round trip") + t.Run("POSIX modes", func(t *testing.T) { + fixtures.RequirePOSIXModes(t) + info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&0o111, "the executable bit survives the round trip") + }) }) } } diff --git a/pkg/artifact/local/verifier_test.go b/pkg/artifact/local/verifier_test.go index 14ae02a0a2d..a3c92bd1840 100644 --- a/pkg/artifact/local/verifier_test.go +++ b/pkg/artifact/local/verifier_test.go @@ -66,7 +66,7 @@ func TestVerify_TranslatesVerifierOutcomes(t *testing.T) { wantStatus artifact.VerificationStatus }{ {"verified", &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil, nil, artifact.VerificationVerified}, - {"unchecked passes through without a verdict", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil, nil, artifact.VerificationUnchecked}, + {"unchecked is not verified", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil, errUtils.ErrArtifactVerificationFailed, artifact.VerificationUnchecked}, {"failed status without an error still fails", &artifact.VerificationResult{Status: artifact.VerificationFailed}, nil, errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, {"verifier error is wrapped as a verification failure", nil, errors.New("crypto exploded"), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, {"verification sentinel is kept", &artifact.VerificationResult{Status: artifact.VerificationFailed}, fmt.Errorf("%w: policy", errUtils.ErrArtifactVerificationFailed), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, @@ -112,10 +112,10 @@ func TestVerify_ReportsDiagnosticsAlongsideTheVerifier(t *testing.T) { assert.True(t, codes["malformed-evidence"], "a file whose name is not an evidence digest is reported, not trusted") } -func TestVerify_BoundsTheNumberOfEvidenceFilesRead(t *testing.T) { +func TestVerify_ReadsAllValidEvidence(t *testing.T) { stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} repo, subject := signedWith(t, stub) - for index := range maxEvidenceFiles + 5 { + for index := range maxMalformedEvidence + 5 { evidence, err := layout.NewEvidence(subject, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte(fmt.Sprintf("bundle-%d", index))}) require.NoError(t, err) raw, err := evidence.Marshal() @@ -125,12 +125,8 @@ func TestVerify_BoundsTheNumberOfEvidenceFilesRead(t *testing.T) { } result, err := repo.Verify(context.Background(), subject, trustedPolicy) require.NoError(t, err) - assert.Equal(t, maxEvidenceFiles, stub.seen, "the verifier never sees more than the bound") - truncated := false - for _, diagnostic := range result.Diagnostics { - truncated = truncated || diagnostic.Code == "evidence-truncated" - } - assert.True(t, truncated, "the cap is reported, not silent") + assert.Equal(t, maxMalformedEvidence+6, stub.seen, "all valid evidence reaches the verifier") + assert.Equal(t, artifact.VerificationVerified, result.Status) } func TestSign_RejectsOversizeEvidence(t *testing.T) { @@ -142,3 +138,28 @@ func TestSign_RejectsOversizeEvidence(t *testing.T) { _, err = repo.Sign(context.Background(), subject, &fixedSigner{data: nil}, artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "empty evidence is invalid") } + +func TestVerifyRejectsUnreadableEvidenceDirectory(t *testing.T) { + stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + require.NoError(t, os.RemoveAll(dir)) + require.NoError(t, os.WriteFile(dir, []byte("not a directory"), 0o644)) + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.Contains(t, result.Diagnostics[0].Code, "evidence-read-failed") + assert.Zero(t, stub.seen) +} + +func TestVerifyFailsClosedOnTooManyMalformedEvidenceFiles(t *testing.T) { + stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + for i := range maxMalformedEvidence + 1 { + require.NoError(t, os.WriteFile(filepath.Join(dir, fmt.Sprintf("bad-%03d.json", i)), []byte("bad"), 0o644)) + } + _, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.ErrorIs(t, err, errTooManyMalformedEvidence) + assert.Zero(t, stub.seen) +} diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go index 176c0297643..524d1b12bc3 100644 --- a/pkg/ci/artifact/local/transport_parity_test.go +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -86,6 +86,12 @@ func TestBundledStoreTarBytesAndChecksumSurviveTheTransport(t *testing.T) { require.Len(t, got, 2) assert.Equal(t, "plan.tfplan", got[0].Name) assert.Equal(t, "lock.hcl", got[1].Name) + for index, want := range []string{"plan-bytes", "lock"} { + data, readErr := io.ReadAll(got[index].Data) + require.NoError(t, readErr) + require.NoError(t, got[index].Data.Close()) + assert.Equal(t, want, string(data)) + } } // TestNamesAreCleanedLikeTheFilesystemDid pins that equivalent spellings address one artifact. diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index 716eb8451c1..8c392c5b108 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -309,8 +309,8 @@ func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.Arti // Remove prefix to get relative key. relKey := key - if s.prefix != "" && len(key) > len(s.prefix)+1 { - relKey = key[len(s.prefix)+1:] + if prefix := s.fullKey(""); s.prefix != "" { + relKey = strings.TrimPrefix(key, prefix+"/") } // Try to load metadata. diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go index 7f6a0d3d78f..73f98df023f 100644 --- a/pkg/ci/artifact/s3/transport_parity_test.go +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "strings" "testing" "time" @@ -62,6 +63,10 @@ func TestAdapterLegacyKeyQuirksArePreserved(t *testing.T) { store, fake := newFakeStore(t, "pre/fix/") require.NoError(t, store.Upload(context.Background(), "a//b/../c.tar", strings.NewReader("x"), 1, nil)) assert.Equal(t, []string{"pre/fix/a/c.tar"}, fake.Keys(), "path.Join cleaning of prefix and key is unchanged; nil metadata writes no sidecar") + listed, err := store.List(context.Background(), artifact.Query{All: true}) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "a/c.tar", listed[0].Name) } func TestAdapterRoundTripWithoutPrefix(t *testing.T) { @@ -155,6 +160,10 @@ func TestAdapterBundledStoreChecksumAndTarSurviveTheTransport(t *testing.T) { require.NoError(t, err) require.Len(t, got, 1) assert.Equal(t, "plan.tfplan", got[0].Name) + data, err := io.ReadAll(got[0].Data) + require.NoError(t, err) + require.NoError(t, got[0].Data.Close()) + assert.Equal(t, "plan-bytes", string(data)) } func TestAdapterLargeUploadStreamsAsMultipartWithLegacyClient(t *testing.T) { From ea77e1a8b336a80ceb8a4a8aa5ce6c7d103e0f89 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:13:22 -0600 Subject: [PATCH 08/43] fix: bound evidence scanning and test retry paths --- pkg/artifact/blob/local/fsutil.go | 6 ++- pkg/artifact/blob/local/fsutil_test.go | 19 ++++++++ pkg/artifact/blob/s3/put.go | 5 +- pkg/artifact/blob/s3/store_test.go | 4 ++ pkg/artifact/local/evidence.go | 53 +++++++++++++-------- pkg/ci/artifact/s3/endpoint_request_test.go | 26 ++++++++++ 6 files changed, 90 insertions(+), 23 deletions(-) diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go index 1e710a271b0..91cf0622118 100644 --- a/pkg/artifact/blob/local/fsutil.go +++ b/pkg/artifact/blob/local/fsutil.go @@ -82,12 +82,16 @@ func openFile(path string) (*os.File, error) { if runtime.GOOS == "windows" { attempts = openAttempts } + return openFileWith(path, attempts, openRead) +} + +func openFileWith(path string, attempts int, open func(string) (*os.File, error)) (*os.File, error) { var ( file *os.File err error ) for i := range attempts { - file, err = openRead(path) + file, err = open(path) if err == nil || errors.Is(err, os.ErrNotExist) { return file, err } diff --git a/pkg/artifact/blob/local/fsutil_test.go b/pkg/artifact/blob/local/fsutil_test.go index 31fb9031381..1392533a584 100644 --- a/pkg/artifact/blob/local/fsutil_test.go +++ b/pkg/artifact/blob/local/fsutil_test.go @@ -39,3 +39,22 @@ func TestReadFile(t *testing.T) { _, err = readFile(filepath.Join(t.TempDir(), "absent")) require.ErrorIs(t, err, os.ErrNotExist) } + +func TestOpenFileRetriesTransientFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "object") + require.NoError(t, os.WriteFile(path, []byte("payload"), 0o600)) + attempts := 0 + file, err := openFileWith(path, 2, func(path string) (*os.File, error) { + attempts++ + if attempts == 1 { + return nil, os.ErrPermission + } + return os.Open(path) + }) + require.NoError(t, err) + defer file.Close() + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "payload", string(data)) + assert.Equal(t, 2, attempts) +} diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index 644958f9af8..5a185fe753b 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -93,7 +93,10 @@ func effectivePartSize(size, partSize int64) int64 { if size <= 0 { return partSize } - needed := (size + MaxParts - 1) / MaxParts + needed := size / MaxParts + if size%MaxParts != 0 { + needed++ + } return max(partSize, needed) } diff --git a/pkg/artifact/blob/s3/store_test.go b/pkg/artifact/blob/s3/store_test.go index fed3c6df374..b07afcebfb2 100644 --- a/pkg/artifact/blob/s3/store_test.go +++ b/pkg/artifact/blob/s3/store_test.go @@ -179,3 +179,7 @@ func TestMultipartCreateIsNotRetriedAfterLostResponse(t *testing.T) { require.Error(t, err) assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) } + +func TestEffectivePartSizeDoesNotOverflow(t *testing.T) { + assert.Equal(t, int64(math.MaxInt64/MaxParts+1), effectivePartSize(math.MaxInt64, DefaultPartSize)) +} diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index 3528eb96034..326c945229b 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "strings" @@ -30,35 +31,45 @@ type loadedEvidence struct { // reported as diagnostics and skipped; they can never override valid matching evidence. func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic, error) { dir := layout.SignatureDir(artifact.SubjectRecord, subject) - entries, err := fs.ReadDir(t.FS(), dir) + file, err := t.Open(native(dir)) if errors.Is(err, fs.ErrNotExist) { return nil, nil, nil } if err != nil { - return nil, nil, fmt.Errorf("read evidence directory: %w", err) - } - var ( - loaded []loadedEvidence - diags []artifact.Diagnostic - ) - for _, entry := range entries { - stem, ok := strings.CutSuffix(entry.Name(), ".json") - if entry.IsDir() || !ok { - continue - } - locator := dir + "/" + entry.Name() - item, problem := t.readEvidenceFile(locator, stem) - if problem != nil { - if len(diags) >= maxMalformedEvidence { - diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many malformed evidence files; remaining entries were not read"}) + return nil, nil, fmt.Errorf("open evidence directory: %w", err) + } + defer file.Close() + var loaded []loadedEvidence + var diags []artifact.Diagnostic + ignored := 0 + for { + entries, readErr := file.ReadDir(maxMalformedEvidence) + for _, entry := range entries { + stem, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + ignored++ + } else { + locator := dir + "/" + entry.Name() + item, problem := t.readEvidenceFile(locator, stem) + if problem == nil { + loaded = append(loaded, loadedEvidence{evidence: item, locator: locator}) + continue + } + ignored++ + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}) + } + if ignored > maxMalformedEvidence { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many invalid evidence entries; remaining entries were not read"}) return nil, diags, errTooManyMalformedEvidence } - diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}) - continue } - loaded = append(loaded, loadedEvidence{evidence: item, locator: locator}) + if errors.Is(readErr, io.EOF) { + return loaded, diags, nil + } + if readErr != nil { + return nil, diags, fmt.Errorf("read evidence directory: %w", readErr) + } } - return loaded, diags, nil } func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, error) { diff --git a/pkg/ci/artifact/s3/endpoint_request_test.go b/pkg/ci/artifact/s3/endpoint_request_test.go index 8e1dcaf6a86..4cc13f31987 100644 --- a/pkg/ci/artifact/s3/endpoint_request_test.go +++ b/pkg/ci/artifact/s3/endpoint_request_test.go @@ -40,3 +40,29 @@ func TestIdentityEndpointUsesBucketPath(t *testing.T) { t.Fatal("SDK request did not reach the emulator endpoint") } } + +func TestAdapterIdentityResolverRoutesToEndpoint(t *testing.T) { + requests := make(chan *http.Request, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Clone(r.Context()) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + resolver := &fakeAuthContextResolver{awsResult: &artifact.AWSAuthConfig{Region: "us-east-1", EndpointURL: server.URL}} + backend, err := NewStore(artifact.StoreOptions{Identity: "deploy", Options: map[string]any{"bucket": "artifacts"}}) + require.NoError(t, err) + store := backend.(*Store) + store.SetAuthContext(resolver, "") + exists, err := store.Exists(t.Context(), "path/object") + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, "deploy", resolver.requestedIDName) + select { + case request := <-requests: + require.Equal(t, "/artifacts/path/object", request.URL.Path) + default: + t.Fatal("identity endpoint received no request") + } +} From ec3ab0ef8e4bf2c2e7e411dd74a8be92c6a0409a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:22:37 -0600 Subject: [PATCH 09/43] fix: preserve unmanaged files and support single upload fallback --- pkg/artifact/blob/s3/fallback.go | 58 ++++++++++++++++ pkg/artifact/blob/s3/multipart.go | 5 ++ pkg/artifact/blob/s3/options.go | 27 +++++--- pkg/artifact/local/publish.go | 24 ++++++- pkg/artifact/local/recovery_test.go | 29 ++++++++ pkg/artifact/local/unmanaged.go | 75 +++++++++++++++++++++ pkg/ci/artifact/s3/store.go | 2 +- pkg/ci/artifact/s3/transport_parity_test.go | 21 ++++++ 8 files changed, 228 insertions(+), 13 deletions(-) create mode 100644 pkg/artifact/blob/s3/fallback.go create mode 100644 pkg/artifact/local/unmanaged.go diff --git a/pkg/artifact/blob/s3/fallback.go b/pkg/artifact/blob/s3/fallback.go new file mode 100644 index 00000000000..e0d3afb15b9 --- /dev/null +++ b/pkg/artifact/blob/s3/fallback.go @@ -0,0 +1,58 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// putFallback preserves the already-consumed prefix, spooling only after multipart is rejected. +func (s *Store) putFallback(ctx context.Context, plan *putPlan, first, carry []byte, body io.Reader) (*blob.ObjectInfo, error) { + file, err := os.CreateTemp("", "atmos-blob-fallback-*") + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("create upload spool: %w", err)) + } + defer func() { + _ = file.Close() + _ = os.Remove(file.Name()) + }() + size, err := io.Copy(file, &fallbackReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("spool upload: %w", err)) + } + if plan.opts.Size >= 0 && size != plan.opts.Size { + return nil, sizeMismatch(plan.key, size, plan.opts.Size) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, err) + } + in := &s3.PutObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(plan.objectKey), Body: file, ContentLength: aws.Int64(size), ContentType: optionalString(plan.opts.ContentType), Metadata: plan.opts.Metadata, Tagging: optionalString(plan.tags)} + in.IfNoneMatch, in.IfMatch = conditionHeaders(plan.opts.Condition) + out, err := plan.api.PutObject(ctx, in, retryOptions(plan.opts.Condition.IsConditional())...) + if err != nil { + return nil, classify(plan.request(true), err) + } + return &blob.ObjectInfo{Key: plan.key, Size: size, ETag: aws.ToString(out.ETag)}, nil +} + +type fallbackReader struct { + ctx context.Context + reader io.Reader +} + +func (r *fallbackReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "s3.fallbackReader.Read")() + + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go index c5bb4d282b7..56a53a2ff92 100644 --- a/pkg/artifact/blob/s3/multipart.go +++ b/pkg/artifact/blob/s3/multipart.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "net/http" "sort" "sync" "time" @@ -43,6 +44,10 @@ func (s *Store) putMultipart(ctx context.Context, plan *putPlan, first, carry [] } created, err := plan.api.CreateMultipartUpload(ctx, in, skipOptionalChecksums, noRetry) if err != nil { + status, code := responseDetails(err) + if s.cfg.singleFallback && (status == http.StatusNotImplemented || code == "NotImplemented") { + return s.putFallback(ctx, plan, first, carry, body) + } return nil, classify(plan.request(false), err) } upload := &multipart{store: s, plan: plan, uploadID: aws.ToString(created.UploadId)} diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go index 39bcea99d86..d214ea4f5bf 100644 --- a/pkg/artifact/blob/s3/options.go +++ b/pkg/artifact/blob/s3/options.go @@ -39,15 +39,16 @@ const ( // settings holds the resolved configuration of a Store. type settings struct { - bucket string - prefix string - tags map[string]string - partSize int64 - concurrency int - conditional ConditionalMode - opaqueKeys bool - client ClientConfig - api API + bucket string + prefix string + tags map[string]string + partSize int64 + concurrency int + conditional ConditionalMode + opaqueKeys bool + singleFallback bool + client ClientConfig + api API } // Option configures a [Store]. @@ -174,3 +175,11 @@ func trimSlashes(s string) string { } return s } + +// WithSinglePutFallback retries an explicitly unsupported multipart upload as one PutObject. +// The remaining stream is spooled to disk only if the endpoint rejects multipart creation. +func WithSinglePutFallback() Option { + defer perf.Track(nil, "blob.s3.WithSinglePutFallback")() + + return func(s *settings) { s.singleFallback = true } +} diff --git a/pkg/artifact/local/publish.go b/pkg/artifact/local/publish.go index 8c04eac961f..d91aef7c29d 100644 --- a/pkg/artifact/local/publish.go +++ b/pkg/artifact/local/publish.go @@ -247,6 +247,9 @@ func (r *Repository) begin(ctx context.Context, t *tree, job *publishJob, plan * // reserve checks for location collisions and unmanaged content, then records the journal and the // pending reference. func (r *Repository) reserve(t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + if err := r.rejectUnmanagedPrior(t, job, plan); err != nil { + return nil, err + } existing, err := t.loadOperation(job.opID) if err != nil { return nil, err @@ -348,13 +351,28 @@ func (r *Repository) representation(job *publishJob) *layout.Representation { // Everything here is best-effort: the reference is already complete, and a leftover is garbage, // not an inconsistency. func (r *Repository) retire(t *tree, job *publishJob, plan *publishPlan) { - _ = t.removeAll(job.staging) - if plan.abandoned != "" && layout.FoldPath(plan.abandoned) != layout.FoldPath(job.location) { + removeStaging := true + if r.kind == artifact.RepositoryKindLocalDir && plan.prior != nil { + if record, _, err := t.loadRecord(plan.prior.RecordDigest); err != nil || checkOwnedDirectory(t, job.retiredDir()+"/payload", record) != nil { + removeStaging = false + } + } + if removeStaging { + _ = t.removeAll(job.staging) + } + if plan.abandoned != "" && layout.FoldPath(plan.abandoned) != layout.FoldPath(job.location) && r.kind != artifact.RepositoryKindLocalDir { r.removePayload(t, plan.abandoned) } if plan.prior != nil { if plan.prior.Location != "" && layout.FoldPath(plan.prior.Location) != layout.FoldPath(job.location) { - r.removePayload(t, plan.prior.Location) + if r.kind == artifact.RepositoryKindLocalDir { + if record, _, err := t.loadRecord(plan.prior.RecordDigest); err == nil { + _, _ = r.store.remove(t, &view{record: record, rep: &layout.Representation{Location: plan.prior.Location}}) + t.removeEmptyParents(plan.prior.Location) + } + } else { + r.removePayload(t, plan.prior.Location) + } } if plan.prior.RecordDigest != "" && plan.prior.RecordDigest != job.recordDigest { _ = t.removeAll(layout.RecordPath(plan.prior.RecordDigest)) diff --git a/pkg/artifact/local/recovery_test.go b/pkg/artifact/local/recovery_test.go index 0b9d36d0eef..daf7c459fea 100644 --- a/pkg/artifact/local/recovery_test.go +++ b/pkg/artifact/local/recovery_test.go @@ -316,3 +316,32 @@ func TestPendingOperationIDCannotBeReusedForAnotherReference(t *testing.T) { _, err = repo.Resolve(context.Background(), artifact.Ref{Name: "team/first", Version: testVersion}) require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) } + +func TestReplacementRejectsUnmanagedFiles(t *testing.T) { + repo := newRepo(t, dirSpec()) + first := publish(t, repo, &prepareSpec{files: map[string]string{"owned": "old"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + unmanaged := filepath.Join(first.Location, "user-file") + require.NoError(t, os.WriteFile(unmanaged, []byte("keep"), 0o644)) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"owned": "new"}, opID: "other"}) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + data, err := os.ReadFile(unmanaged) + require.NoError(t, err) + assert.Equal(t, "keep", string(data)) + assertIntact(t, repo, defaultRef()) +} + +func TestReplacementRecoveryWithChangedInventory(t *testing.T) { + var fault armedFault + repo := newRepo(t, dirSpec().withOpts(fault.option())) + publish(t, repo, &prepareSpec{files: map[string]string{"old-name": "old"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"new-name": "new"}, opID: "replacement"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.NoError(t, err) + assertIntact(t, repo, defaultRef()) +} diff --git a/pkg/artifact/local/unmanaged.go b/pkg/artifact/local/unmanaged.go new file mode 100644 index 00000000000..f5934efd206 --- /dev/null +++ b/pkg/artifact/local/unmanaged.go @@ -0,0 +1,75 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "path" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// rejectUnmanagedPrior refuses to retire a directory containing files outside its Record. +func (r *Repository) rejectUnmanagedPrior(t *tree, job *publishJob, plan *publishPlan) error { + if r.kind != artifact.RepositoryKindLocalDir || plan.prior == nil || plan.prior.Location == "" { + return nil + } + record, _, err := t.loadRecord(plan.prior.RecordDigest) + if err != nil { + return err + } + priorRecords := recordsForPrior(record, job, plan) + if err := checkOwnedDirectory(t, plan.prior.Location, priorRecords...); err != nil { + return err + } + locations := []string{job.retiredDir() + "/payload"} + if plan.oldOp != nil { + locations = append(locations, layout.StagingPath(plan.oldOp.OperationID)+"/retired/payload") + } + for _, location := range locations { + if err := checkOwnedDirectory(t, location, record); err != nil { + return err + } + } + return nil +} + +func recordsForPrior(record *artifact.Record, job *publishJob, plan *publishPlan) []*artifact.Record { + if plan.admission.Resuming() && layout.FoldPath(plan.prior.Location) == layout.FoldPath(job.location) { + return []*artifact.Record{record, job.record} + } + return []*artifact.Record{record} +} + +func checkOwnedDirectory(t *tree, location string, records ...*artifact.Record) error { + allowed := map[string]bool{location: true} + add := func(relative string) { + //nolint:forbidigo // io/fs paths are slash-separated on all platforms. + for name := path.Join(location, relative); name != location && name != "."; name = path.Dir(name) { + allowed[name] = true + } + } + for _, record := range records { + for _, entry := range record.Inventory { + add(entry.Path) + } + } + for _, name := range []string{layout.ManifestName, "README.md", "CHANGELOG.md"} { + add(layout.DistributionDir + "/" + name) + } + err := fs.WalkDir(t.FS(), location, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !allowed[name] { + return fmt.Errorf("%w: cannot replace %q: %q is not managed by the artifact", errUtils.ErrArtifactConflict, location, name) + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index 8c392c5b108..9e096f489f1 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -189,7 +189,7 @@ func (s *Store) transport(ctx context.Context) (*blobs3.Store, error) { return nil, err } s.transportOnce.Do(func() { - s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys()) + s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys(), blobs3.WithSinglePutFallback()) }) return s.transportStore, s.transportErr } diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go index 73f98df023f..28c4ee46e57 100644 --- a/pkg/ci/artifact/s3/transport_parity_test.go +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "strings" "testing" "time" @@ -210,3 +211,23 @@ func TestAdapterWrapsTransportFailuresInLegacySentinels(t *testing.T) { fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: 403, Code: "AccessDenied", Times: -1}) require.ErrorIs(t, store.Delete(ctx, "k.tar"), errUtils.ErrArtifactDeleteFailed) } + +func TestAdapterFallsBackWhenMultipartUnsupported(t *testing.T) { + store, fake := newFakeStore(t, "") + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) + body := bytes.Repeat([]byte("x"), 9<<20) + require.NoError(t, store.Upload(context.Background(), "big.tar", bytes.NewBuffer(body), int64(len(body)), nil)) + object, ok := fake.Object("big.tar") + require.True(t, ok) + assert.Equal(t, body, object.Data) + assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestAdapterDoesNotFallbackForOtherMultipartErrors(t *testing.T) { + store, fake := newFakeStore(t, "") + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusForbidden, Code: "AccessDenied"}) + body := bytes.Repeat([]byte("x"), 9<<20) + require.Error(t, store.Upload(context.Background(), "big.tar", bytes.NewReader(body), int64(len(body)), nil)) + assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) +} From 93e82cfaff1a1ef8c27a8d7129843d54a5cadc58 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:29:43 -0600 Subject: [PATCH 10/43] fix: confine local blob paths to their root --- pkg/artifact/blob/local/attrs.go | 4 +- pkg/artifact/blob/local/delete.go | 22 +++++-- pkg/artifact/blob/local/errors.go | 7 +- pkg/artifact/blob/local/etag.go | 4 +- pkg/artifact/blob/local/fsutil.go | 58 ----------------- pkg/artifact/blob/local/get.go | 27 +++++--- pkg/artifact/blob/local/list.go | 36 +++++----- pkg/artifact/blob/local/open_other.go | 2 + pkg/artifact/blob/local/open_unix.go | 2 + pkg/artifact/blob/local/put.go | 72 ++++++++++---------- pkg/artifact/blob/local/root.go | 94 +++++++++++++++++++++++++++ pkg/artifact/blob/local/store.go | 5 -- pkg/artifact/blob/local/store_test.go | 32 ++++++++- 13 files changed, 229 insertions(+), 136 deletions(-) create mode 100644 pkg/artifact/blob/local/root.go diff --git a/pkg/artifact/blob/local/attrs.go b/pkg/artifact/blob/local/attrs.go index 493c009ed8b..0e2069a9be4 100644 --- a/pkg/artifact/blob/local/attrs.go +++ b/pkg/artifact/blob/local/attrs.go @@ -24,9 +24,9 @@ func attrsPath(objectPath string) string { } // readAttrs loads the attribute file of objectPath; a missing file yields empty attributes. -func readAttrs(objectPath string) (attrs, error) { +func readAttrs(root *os.Root, objectPath string) (attrs, error) { var a attrs - data, err := readFile(attrsPath(objectPath)) + data, err := rootReadFile(root, attrsPath(objectPath)) if errors.Is(err, os.ErrNotExist) { return a, nil } diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go index 13ea2e2f9ea..c66b6585b40 100644 --- a/pkg/artifact/blob/local/delete.go +++ b/pkg/artifact/blob/local/delete.go @@ -26,26 +26,34 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) } defer release() - target := s.path(key) + root, err := s.openRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return wrapIO(blob.OpDelete, key, err) + } + defer root.Close() + target := filepath.FromSlash(key) if opts.IfMatch != "" { - if err := checkDeleteETag(key, target, opts.IfMatch); err != nil { + if err := checkDeleteETag(root, key, target, opts.IfMatch); err != nil { return err } } for _, path := range []string{target, attrsPath(target)} { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + if err := root.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return wrapIO(blob.OpDelete, key, err) } } - removeEmptyDirs(s.root, filepath.Dir(target)) + rootRemoveEmptyDirs(root, filepath.Dir(target)) return nil } // checkDeleteETag verifies the if-match token of a conditional delete. Callers hold the lock. -func checkDeleteETag(key, target, want string) error { - state, err := inspect(target) +func checkDeleteETag(root *os.Root, key, target, want string) error { + state, err := inspect(root, target) if err != nil { return wrapIO(blob.OpDelete, key, err) } - return matchETag(blob.OpDelete, key, state, want) + return matchETag(root, blob.OpDelete, key, state, want) } diff --git a/pkg/artifact/blob/local/errors.go b/pkg/artifact/blob/local/errors.go index b9ff8e33e5b..86fb1249497 100644 --- a/pkg/artifact/blob/local/errors.go +++ b/pkg/artifact/blob/local/errors.go @@ -7,7 +7,6 @@ import ( "io" "os" "path/filepath" - "strings" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/blob" @@ -36,9 +35,9 @@ func classifyCopy(key string, err error) error { // classifyCreate explains why a temporary file could not be created beside target. An existing // object occupying a parent path is a key conflict, which the filesystem cannot represent. -func (s *Store) classifyCreate(key, target string, err error) error { - for dir := filepath.Dir(target); strings.HasPrefix(dir, s.root) && dir != s.root; dir = filepath.Dir(dir) { - if info, statErr := os.Stat(dir); statErr == nil && !info.IsDir() { +func (s *Store) classifyCreate(root *os.Root, key, target string, err error) error { + for dir := filepath.Dir(target); dir != "."; dir = filepath.Dir(dir) { + if info, statErr := root.Stat(dir); statErr == nil && !info.IsDir() { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: a parent path is an existing object", errUtils.ErrBlobInvalid)) } } diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go index edcdc7aebfd..d0a74474fb8 100644 --- a/pkg/artifact/blob/local/etag.go +++ b/pkg/artifact/blob/local/etag.go @@ -17,8 +17,8 @@ const contentETagLimit = 4 << 20 const hexBase = 16 // etagOf returns the ETag of the file at path described by info. -func etagOf(path string, info os.FileInfo) (string, error) { - file, err := openFile(path) +func etagOf(root *os.Root, path string, info os.FileInfo) (string, error) { + file, err := rootOpenFile(root, path) if err != nil { return "", err } diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go index 91cf0622118..443c1b73f91 100644 --- a/pkg/artifact/blob/local/fsutil.go +++ b/pkg/artifact/blob/local/fsutil.go @@ -6,7 +6,6 @@ import ( "errors" "io" "os" - "path/filepath" "runtime" "time" ) @@ -23,30 +22,6 @@ func mkdirAll(dir string, perm os.FileMode) error { return os.MkdirAll(dir, perm) } -// createTemp creates an exclusive temporary file in dir, creating dir when needed. A concurrent -// delete may remove an empty directory between creation and use, so it retries a few times. -func createTemp(dir string, dirPerm, filePerm os.FileMode) (*os.File, error) { - var lastErr error - for range renameAttempts { - if err := mkdirAll(dir, dirPerm); err != nil { - return nil, err - } - name, err := tempName() - if err != nil { - return nil, err - } - file, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_EXCL, filePerm) - if err == nil { - return file, nil - } - if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - lastErr = err - } - return nil, lastErr -} - func tempName() (string, error) { var raw [8]byte if _, err := rand.Read(raw[:]); err != nil { @@ -55,25 +30,6 @@ func tempName() (string, error) { return tempPrefix + hex.EncodeToString(raw[:]), nil } -// renameReplace atomically moves src over dst. Windows rejects a rename while another handle is -// open on dst, so it retries briefly there. -func renameReplace(src, dst string) error { - attempts := 1 - if runtime.GOOS == "windows" { - attempts = renameAttempts - } - var err error - for i := range attempts { - if err = os.Rename(src, dst); err == nil { - return nil - } - if i < attempts-1 { - time.Sleep(renameBackoff) - } - } - return err -} - // openFile opens path for reading. Windows refuses to open a file with a sharing violation while a // concurrent writer is renaming over it, so it retries briefly there. A missing file is reported // immediately because absence is a normal answer. @@ -111,17 +67,3 @@ func readFile(path string) ([]byte, error) { defer file.Close() return io.ReadAll(file) } - -// removeEmptyDirs removes empty directories from dir up to, but never including, root. -func removeEmptyDirs(root, dir string) { - for dir != root && dir != "." && dir != string(filepath.Separator) && len(dir) > len(root) { - entries, err := os.ReadDir(dir) - if err != nil || len(entries) > 0 { - return - } - if os.Remove(dir) != nil { - return - } - dir = filepath.Dir(dir) - } -} diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go index 844de9505bb..1e67b49aed1 100644 --- a/pkg/artifact/blob/local/get.go +++ b/pkg/artifact/blob/local/get.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "path/filepath" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/blob" @@ -36,12 +37,17 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo if err := checkContext(ctx, blob.OpGet, key); err != nil { return nil, err } - target := s.path(key) - file, err := openFile(target) + root, err := s.openRoot(false) if err != nil { return nil, wrapIO(blob.OpGet, key, err) } - info, err := s.describe(blob.OpGet, key, target, file, false) + defer root.Close() + target := filepath.FromSlash(key) + file, err := rootOpenFile(root, target) + if err != nil { + return nil, wrapIO(blob.OpGet, key, err) + } + info, err := s.describe(root, blob.OpGet, key, file, false) if err != nil { _ = file.Close() return nil, err @@ -82,17 +88,22 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b if err := checkContext(ctx, blob.OpStat, key); err != nil { return nil, err } - target := s.path(key) - file, err := openFile(target) + root, err := s.openRoot(false) + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + defer root.Close() + target := filepath.FromSlash(key) + file, err := rootOpenFile(root, target) if err != nil { return nil, wrapIO(blob.OpStat, key, err) } defer file.Close() - return s.describe(blob.OpStat, key, target, file, opts.IncludeTags) + return s.describe(root, blob.OpStat, key, file, opts.IncludeTags) } // describe builds ObjectInfo for an open regular file. A directory is not an object. -func (s *Store) describe(op, key, target string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { +func (s *Store) describe(root *os.Root, op, key string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { stat, err := file.Stat() if err != nil { return nil, wrapIO(op, key, err) @@ -104,7 +115,7 @@ func (s *Store) describe(op, key, target string, file *os.File, withTags bool) ( if err != nil { return nil, wrapIO(op, key, err) } - a, err := readAttrs(target) + a, err := readAttrs(root, filepath.FromSlash(key)) if err != nil { return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("read attributes: %w", err)) } diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go index 8909efa6b94..eecd5a70492 100644 --- a/pkg/artifact/blob/local/list.go +++ b/pkg/artifact/blob/local/list.go @@ -6,7 +6,7 @@ import ( "fmt" "io/fs" "os" - "path/filepath" + "path" "sort" "strings" @@ -43,19 +43,27 @@ func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage // walk returns every object whose key starts with prefix, sorted by key. func (s *Store) walk(ctx context.Context, prefix string) ([]entry, error) { - start := s.root + root, err := s.openRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + defer root.Close() + start := "." if idx := strings.LastIndexByte(prefix, '/'); idx >= 0 { - start = s.path(prefix[:idx]) + start = prefix[:idx] } var entries []entry - err := filepath.WalkDir(start, func(path string, d fs.DirEntry, walkErr error) error { + err = fs.WalkDir(root.FS(), start, func(name string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return ignoreMissing(walkErr) } if err := ctx.Err(); err != nil { return err } - found, ok, err := s.objectAt(path, d, prefix) + found, ok, err := s.objectAt(name, d, prefix) if err != nil || !ok { return err } @@ -79,15 +87,11 @@ func ignoreMissing(err error) error { // objectAt returns the object stored at a walked path, or false for directories, bookkeeping // files, keys outside the prefix, and files that vanished. -func (s *Store) objectAt(path string, d fs.DirEntry, prefix string) (entry, bool, error) { - if d.IsDir() || reserved(path, s.root, d.Name()) { +func (s *Store) objectAt(name string, d fs.DirEntry, prefix string) (entry, bool, error) { + if d.IsDir() || reserved(name, d.Name()) { return entry{}, false, nil } - rel, err := filepath.Rel(s.root, path) - if err != nil { - return entry{}, false, err - } - key := filepath.ToSlash(rel) + key := name if !strings.HasPrefix(key, prefix) { return entry{}, false, nil } @@ -99,10 +103,10 @@ func (s *Store) objectAt(path string, d fs.DirEntry, prefix string) (entry, bool } // reserved reports whether a file is one of the store's own bookkeeping files. -func reserved(path, root, name string) bool { - return strings.HasPrefix(name, tempPrefix) || - strings.HasSuffix(name, attrsSuffix) || - (name == lockName && filepath.Dir(path) == root) +func reserved(name, base string) bool { + return strings.HasPrefix(base, tempPrefix) || + strings.HasSuffix(base, attrsSuffix) || + (base == lockName && path.Dir(name) == ".") } // paginate applies the delimiter roll-up, cursor, and limit to sorted entries. diff --git a/pkg/artifact/blob/local/open_other.go b/pkg/artifact/blob/local/open_other.go index 0834bd8881a..1932e726a2e 100644 --- a/pkg/artifact/blob/local/open_other.go +++ b/pkg/artifact/blob/local/open_other.go @@ -4,4 +4,6 @@ package local import "os" +const nonblockFlag = 0 + func openRead(path string) (*os.File, error) { return os.Open(path) } diff --git a/pkg/artifact/blob/local/open_unix.go b/pkg/artifact/blob/local/open_unix.go index 069da378cc1..19a2249c8e4 100644 --- a/pkg/artifact/blob/local/open_unix.go +++ b/pkg/artifact/blob/local/open_unix.go @@ -8,6 +8,8 @@ import ( "golang.org/x/sys/unix" ) +const nonblockFlag = unix.O_NONBLOCK + func openRead(path string) (*os.File, error) { return os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0) } diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index 50b5db4f9f1..a1ca2d32fd8 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -35,12 +35,17 @@ func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.P if err := s.validatePut(ctx, key, opts); err != nil { return nil, err } - target := s.path(key) - pending, err := s.stage(ctx, key, target, body, opts) + root, err := s.openRoot(true) + if err != nil { + return nil, wrapIO(blob.OpPut, key, err) + } + defer root.Close() + target := filepath.FromSlash(key) + pending, err := s.stage(ctx, root, key, body, opts) if err != nil { return nil, err } - defer pending.discard() + defer pending.discard(root) release, err := s.guard(ctx, blob.OpPut, key, opts.Condition.IsConditional()) if err != nil { @@ -48,13 +53,13 @@ func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.P } defer release() - if err := s.checkCondition(key, target, opts.Condition); err != nil { + if err := s.checkCondition(root, key, target, opts.Condition); err != nil { return nil, err } - if err := s.commit(key, target, pending, fromPut(opts).empty()); err != nil { + if err := s.commit(root, key, target, pending, fromPut(opts).empty()); err != nil { return nil, err } - return s.statCommitted(key, target, pending) + return s.statCommitted(root, key, target, pending) } func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOptions) error { @@ -74,30 +79,31 @@ func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOption } // stage writes the body (and attribute file, when needed) to temporary files next to target. -func (s *Store) stage(ctx context.Context, key, target string, body io.Reader, opts blob.PutOptions) (*staged, error) { - file, err := createTemp(filepath.Dir(target), s.dirPerm, s.filePerm) +func (s *Store) stage(ctx context.Context, root *os.Root, key string, body io.Reader, opts blob.PutOptions) (*staged, error) { + target := filepath.FromSlash(key) + file, name, err := rootCreateTemp(root, filepath.Dir(target), s.dirPerm, s.filePerm) if err != nil { - return nil, s.classifyCreate(key, target, err) + return nil, s.classifyCreate(root, key, target, err) } - pending := &staged{tempPath: file.Name()} + pending := &staged{tempPath: name} hash := sha256.New() written, copyErr := copyBody(ctx, io.MultiWriter(file, hash), body, opts.Size) closeErr := file.Close() if err := firstError(copyErr, closeErr); err != nil { - pending.discard() + pending.discard(root) return nil, classifyCopy(key, err) } pending.size = written pending.etag = contentETag(hash.Sum(nil)) - if err := s.stageAttrs(pending, target, fromPut(opts)); err != nil { - pending.discard() + if err := s.stageAttrs(root, pending, target, fromPut(opts)); err != nil { + pending.discard(root) return nil, wrapIO(blob.OpPut, key, err) } return pending, nil } // stageAttrs writes the attribute file to a temporary path when any attribute is set. -func (s *Store) stageAttrs(pending *staged, target string, a attrs) error { +func (s *Store) stageAttrs(root *os.Root, pending *staged, target string, a attrs) error { if a.empty() { return nil } @@ -105,11 +111,11 @@ func (s *Store) stageAttrs(pending *staged, target string, a attrs) error { if err != nil { return err } - file, err := createTemp(filepath.Dir(target), s.dirPerm, s.filePerm) + file, name, err := rootCreateTemp(root, filepath.Dir(target), s.dirPerm, s.filePerm) if err != nil { return err } - pending.attrsTemp = file.Name() + pending.attrsTemp = name _, writeErr := file.Write(data) return firstError(writeErr, file.Close()) } @@ -134,14 +140,14 @@ func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (i } // checkCondition evaluates the put condition against the current object. Callers hold the lock. -func (s *Store) checkCondition(key, target string, cond blob.Condition) error { +func (s *Store) checkCondition(root *os.Root, key, target string, cond blob.Condition) error { if cond.Kind == blob.ConditionNone { return nil } if cond.Kind != blob.ConditionIfAbsent && cond.Kind != blob.ConditionIfMatch { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: unknown condition %d", errUtils.ErrBlobInvalid, cond.Kind)) } - state, err := inspect(target) + state, err := inspect(root, target) if err != nil { return wrapIO(blob.OpPut, key, err) } @@ -151,7 +157,7 @@ func (s *Store) checkCondition(key, target string, cond blob.Condition) error { return preconditionFailed(blob.OpPut, key, "object already exists") } case blob.ConditionIfMatch: - return matchETag(blob.OpPut, key, state, cond.ETag) + return matchETag(root, blob.OpPut, key, state, cond.ETag) } return nil } @@ -164,8 +170,8 @@ type fileState struct { } // inspect reports whether a regular file exists at path. A missing path is not an error. -func inspect(path string) (fileState, error) { - info, err := os.Stat(path) +func inspect(root *os.Root, path string) (fileState, error) { + info, err := root.Stat(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return fileState{}, err } @@ -174,11 +180,11 @@ func inspect(path string) (fileState, error) { // matchETag fails with NotFound when the object is missing and with a precondition failure when // its ETag differs from want. -func matchETag(op, key string, state fileState, want string) error { +func matchETag(root *os.Root, op, key string, state fileState, want string) error { if !state.exists { return blob.NewError(blob.KindNotFound, op, key, nil) } - current, err := etagOf(state.path, state.info) + current, err := etagOf(root, state.path, state.info) if err != nil { return wrapIO(op, key, err) } @@ -190,22 +196,22 @@ func matchETag(op, key string, state fileState, want string) error { // commit moves the staged files into place: attributes first, so an object is only ever visible // together with its attributes or with stale ones until the data rename lands. -func (s *Store) commit(key, target string, pending *staged, noAttrs bool) error { +func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAttrs bool) error { switch { case pending.attrsTemp != "": - if err := renameReplace(pending.attrsTemp, attrsPath(target)); err != nil { + if err := rootRenameReplace(root, pending.attrsTemp, attrsPath(target)); err != nil { return wrapIO(blob.OpPut, key, err) } pending.attrsTemp = "" case noAttrs: - if err := os.Remove(attrsPath(target)); err != nil && !errors.Is(err, os.ErrNotExist) { + if err := root.Remove(attrsPath(target)); err != nil && !errors.Is(err, os.ErrNotExist) { return wrapIO(blob.OpPut, key, err) } } - if info, err := os.Stat(target); err == nil && info.IsDir() { + if info, err := root.Stat(target); err == nil && info.IsDir() { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) } - if err := renameReplace(pending.tempPath, target); err != nil { + if err := rootRenameReplace(root, pending.tempPath, target); err != nil { return wrapIO(blob.OpPut, key, err) } pending.tempPath = "" @@ -213,8 +219,8 @@ func (s *Store) commit(key, target string, pending *staged, noAttrs bool) error } // statCommitted describes the object that was just written. -func (s *Store) statCommitted(key, target string, pending *staged) (*blob.ObjectInfo, error) { - info, err := os.Stat(target) +func (s *Store) statCommitted(root *os.Root, key, target string, pending *staged) (*blob.ObjectInfo, error) { + info, err := root.Stat(target) if err != nil { return nil, wrapIO(blob.OpPut, key, err) } @@ -226,11 +232,11 @@ func (s *Store) statCommitted(key, target string, pending *staged) (*blob.Object } // discard removes any temporary files that were not committed. -func (p *staged) discard() { +func (p *staged) discard(root *os.Root) { if p.tempPath != "" { - _ = os.Remove(p.tempPath) + _ = root.Remove(p.tempPath) } if p.attrsTemp != "" { - _ = os.Remove(p.attrsTemp) + _ = root.Remove(p.attrsTemp) } } diff --git a/pkg/artifact/blob/local/root.go b/pkg/artifact/blob/local/root.go new file mode 100644 index 00000000000..8babc9f5c79 --- /dev/null +++ b/pkg/artifact/blob/local/root.go @@ -0,0 +1,94 @@ +package local + +import ( + "errors" + "io" + "os" + "path/filepath" + "runtime" + "time" +) + +// openRoot confines object operations to the configured directory, including through symlinks. +func (s *Store) openRoot(create bool) (*os.Root, error) { + if create { + if err := os.MkdirAll(s.root, s.dirPerm); err != nil { + return nil, err + } + } + return os.OpenRoot(s.root) +} + +func rootOpenFile(root *os.Root, name string) (*os.File, error) { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = openAttempts + } + return openFileWith(name, attempts, func(name string) (*os.File, error) { + return root.OpenFile(name, os.O_RDONLY|nonblockFlag, 0) + }) +} + +func rootReadFile(root *os.Root, name string) ([]byte, error) { + file, err := rootOpenFile(root, name) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(file) +} + +func rootCreateTemp(root *os.Root, dir string, dirPerm, filePerm os.FileMode) (*os.File, string, error) { + var lastErr error + for range renameAttempts { + if err := root.MkdirAll(dir, dirPerm); err != nil { + return nil, "", err + } + name, err := tempName() + if err != nil { + return nil, "", err + } + name = filepath.Join(dir, name) + file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, filePerm) + if err == nil { + return file, name, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, "", err + } + lastErr = err + } + return nil, "", lastErr +} + +func rootRenameReplace(root *os.Root, src, dst string) error { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = renameAttempts + } + var err error + for i := range attempts { + if err = root.Rename(src, dst); err == nil { + return nil + } + if i < attempts-1 { + time.Sleep(renameBackoff) + } + } + return err +} + +func rootRemoveEmptyDirs(root *os.Root, dir string) { + for dir != "." && dir != string(filepath.Separator) { + file, err := root.Open(dir) + if err != nil { + return + } + _, readErr := file.ReadDir(1) + _ = file.Close() + if !errors.Is(readErr, io.EOF) || root.Remove(dir) != nil { + return + } + dir = filepath.Dir(dir) + } +} diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index 034a2e77d7c..09268f94cb1 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -94,11 +94,6 @@ func (s *Store) validate(op, key string) error { return nil } -// path maps a validated key to its file path. -func (s *Store) path(key string) string { - return filepath.Join(s.root, filepath.FromSlash(key)) -} - // checkContext fails fast when ctx is already done. func checkContext(ctx context.Context, op, key string) error { if err := ctx.Err(); err != nil { diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index a69b7a54f51..15fb073f47f 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -8,6 +8,7 @@ import ( "math" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -166,7 +167,10 @@ func TestDescribeHashesOpenedFile(t *testing.T) { defer file.Close() require.NoError(t, os.Rename(path, path+".old")) require.NoError(t, os.WriteFile(path, []byte("new"), 0o644)) - info, err := store.describe(blob.OpGet, "k", path, file, false) + root, err := store.openRoot(false) + require.NoError(t, err) + defer root.Close() + info, err := store.describe(root, blob.OpGet, "k", file, false) require.NoError(t, err) sum := sha256.Sum256([]byte("old")) assert.Equal(t, contentETag(sum[:]), info.ETag) @@ -191,3 +195,29 @@ func TestPutRejectsUnknownCondition(t *testing.T) { _, err := store.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Condition: blob.Condition{Kind: blob.ConditionKind(999)}}) require.ErrorIs(t, err, errUtils.ErrBlobInvalid) } + +func TestSymlinksCannotEscapeRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires extra privileges on Windows") + } + store := newStore(t) + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + outside := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(outside, "object"), []byte("keep"), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(store.Root(), "team"))) + ctx := context.Background() + _, err := store.Put(ctx, "team/object", bytes.NewReader([]byte("overwrite")), blob.PutOptions{Size: 9}) + require.Error(t, err) + _, err = store.Get(ctx, "team/object", blob.GetOptions{}) + require.Error(t, err) + _, err = store.Stat(ctx, "team/object", blob.StatOptions{}) + require.Error(t, err) + require.Error(t, store.Delete(ctx, "team/object", blob.DeleteOptions{})) + page, listErr := store.List(ctx, blob.ListOptions{Prefix: "team/"}) + if listErr == nil { + assert.Empty(t, page.Objects) + } + data, err := os.ReadFile(filepath.Join(outside, "object")) + require.NoError(t, err) + assert.Equal(t, "keep", string(data)) +} From c1a62ef5290e4da5e192aa185af8e86927bc7aa8 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:32:21 -0600 Subject: [PATCH 11/43] perf: reuse local listing snapshots across pages --- pkg/artifact/blob/local/list.go | 28 ++++++++-- pkg/artifact/blob/local/snapshot.go | 77 +++++++++++++++++++++++++++ pkg/artifact/blob/local/store.go | 3 ++ pkg/artifact/blob/local/store_test.go | 31 +++++++++++ 4 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 pkg/artifact/blob/local/snapshot.go diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go index eecd5a70492..1150e6f1ffa 100644 --- a/pkg/artifact/blob/local/list.go +++ b/pkg/artifact/blob/local/list.go @@ -34,11 +34,31 @@ func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage if err := validatePrefix(opts.Prefix); err != nil { return nil, err } - entries, err := s.walk(ctx, opts.Prefix) - if err != nil { - return nil, blob.NewError(blob.KindOther, blob.OpList, opts.Prefix, err) + var entries []entry + var snapshotID string + if id, last, ok := parseSnapshotCursor(opts.Cursor); ok { + opts.Cursor = last + snapshotID = id + entries = s.lookupSnapshot(id, opts) + } + if entries == nil { + var err error + entries, err = s.walk(ctx, opts.Prefix) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpList, opts.Prefix, err) + } + snapshotID = "" + } + page := paginate(entries, opts) + if page.NextCursor != "" { + if snapshotID == "" { + snapshotID = s.rememberSnapshot(entries, opts) + } + if snapshotID != "" { + page.NextCursor = snapshotCursor(snapshotID, page.NextCursor) + } } - return paginate(entries, opts), nil + return page, nil } // walk returns every object whose key starts with prefix, sorted by key. diff --git a/pkg/artifact/blob/local/snapshot.go b/pkg/artifact/blob/local/snapshot.go new file mode 100644 index 00000000000..f04203ae65f --- /dev/null +++ b/pkg/artifact/blob/local/snapshot.go @@ -0,0 +1,77 @@ +package local + +import ( + "encoding/base64" + "strings" + "time" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +const ( + snapshotPrefix = "\x00atmos-local:" + snapshotLifetime = 5 * time.Minute + maxSnapshots = 8 +) + +type listSnapshot struct { + entries []entry + prefix, delimiter string + created time.Time +} + +// snapshotCursor stores the last ordering token as well as the cache key, so a cursor remains +// usable if the bounded cache is evicted or a caller uses a new Store. +func snapshotCursor(id, last string) string { + return snapshotPrefix + id + ":" + base64.RawURLEncoding.EncodeToString([]byte(last)) +} + +func parseSnapshotCursor(cursor string) (id, last string, ok bool) { + value, found := strings.CutPrefix(cursor, snapshotPrefix) + if !found { + return "", cursor, false + } + id, encoded, found := strings.Cut(value, ":") + if !found || id == "" { + return "", cursor, false + } + decoded, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return "", cursor, false + } + return id, string(decoded), true +} + +func (s *Store) lookupSnapshot(id string, opts blob.ListOptions) []entry { + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + snapshot, ok := s.snapshots[id] + if !ok || time.Since(snapshot.created) > snapshotLifetime || snapshot.prefix != opts.Prefix || snapshot.delimiter != opts.Delimiter { + return nil + } + return snapshot.entries +} + +func (s *Store) rememberSnapshot(entries []entry, opts blob.ListOptions) string { + id, err := tempName() + if err != nil { + return "" + } + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + if s.snapshots == nil { + s.snapshots = make(map[string]listSnapshot) + } + if len(s.snapshots) >= maxSnapshots { + var oldestID string + var oldest time.Time + for key, snapshot := range s.snapshots { + if oldestID == "" || snapshot.created.Before(oldest) { + oldestID, oldest = key, snapshot.created + } + } + delete(s.snapshots, oldestID) + } + s.snapshots[id] = listSnapshot{entries: entries, prefix: opts.Prefix, delimiter: opts.Delimiter, created: time.Now()} + return id +} diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index 09268f94cb1..0566b9eee09 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/blob" @@ -32,6 +33,8 @@ type Store struct { filePerm os.FileMode lockAll bool allowBackslash bool + snapshotMu sync.Mutex + snapshots map[string]listSnapshot } // New returns a Store rooted at root. The directory is created on first write. diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 15fb073f47f..8683a9134b5 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -221,3 +221,34 @@ func TestSymlinksCannotEscapeRoot(t *testing.T) { require.NoError(t, err) assert.Equal(t, "keep", string(data)) } + +func TestListingPagesUseSnapshotAndFreshListingWalksAgain(t *testing.T) { + store := newStore(t) + ctx := context.Background() + for _, key := range []string{"a", "b"} { + _, err := store.Put(ctx, key, bytes.NewReader([]byte(key)), blob.PutOptions{Size: 1}) + require.NoError(t, err) + } + first, err := store.List(ctx, blob.ListOptions{Limit: 1}) + require.NoError(t, err) + require.Len(t, first.Objects, 1) + assert.Equal(t, "a", first.Objects[0].Key) + require.NotEmpty(t, first.NextCursor) + require.NoError(t, store.Delete(ctx, "b", blob.DeleteOptions{})) + _, err = store.Put(ctx, "c", bytes.NewReader([]byte("c")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + second, err := store.List(ctx, blob.ListOptions{Limit: 1, Cursor: first.NextCursor}) + require.NoError(t, err) + require.Len(t, second.Objects, 1) + assert.Equal(t, "b", second.Objects[0].Key, "continuation uses the original walk") + fresh, err := store.List(ctx, blob.ListOptions{Limit: 2}) + require.NoError(t, err) + require.Len(t, fresh.Objects, 2) + assert.Equal(t, "c", fresh.Objects[1].Key, "a fresh listing sees filesystem changes") + other, err := New(store.Root()) + require.NoError(t, err) + resumed, err := other.List(ctx, blob.ListOptions{Limit: 1, Cursor: first.NextCursor}) + require.NoError(t, err) + require.Len(t, resumed.Objects, 1) + assert.Equal(t, "c", resumed.Objects[0].Key, "an evicted snapshot resumes from its ordering token") +} From 2a9278386763b8ac47fde9fc413943848ebae592 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:36:55 -0600 Subject: [PATCH 12/43] fix: preserve attributes on directory conflicts --- pkg/artifact/blob/local/put.go | 10 +++++++--- pkg/artifact/blob/local/store_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index a1ca2d32fd8..d4500513e66 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -197,6 +197,13 @@ func matchETag(root *os.Root, op, key string, state fileState, want string) erro // commit moves the staged files into place: attributes first, so an object is only ever visible // together with its attributes or with stale ones until the data rename lands. func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAttrs bool) error { + info, err := root.Stat(target) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return wrapIO(blob.OpPut, key, err) + } + if err == nil && info.IsDir() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) + } switch { case pending.attrsTemp != "": if err := rootRenameReplace(root, pending.attrsTemp, attrsPath(target)); err != nil { @@ -208,9 +215,6 @@ func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAtt return wrapIO(blob.OpPut, key, err) } } - if info, err := root.Stat(target); err == nil && info.IsDir() { - return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) - } if err := rootRenameReplace(root, pending.tempPath, target); err != nil { return wrapIO(blob.OpPut, key, err) } diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 8683a9134b5..86d1bdfe809 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -252,3 +252,18 @@ func TestListingPagesUseSnapshotAndFreshListingWalksAgain(t *testing.T) { require.Len(t, resumed.Objects, 1) assert.Equal(t, "c", resumed.Objects[0].Key, "an evicted snapshot resumes from its ordering token") } + +func TestPutDirectoryConflictPreservesExistingSidecar(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "prefix/child", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + sidecar := filepath.Join(store.Root(), "prefix"+attrsSuffix) + original := []byte(`{"content_type":"original"}`) + require.NoError(t, os.WriteFile(sidecar, original, 0o644)) + _, err = store.Put(ctx, "prefix", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, ContentType: "new"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + got, err := os.ReadFile(sidecar) + require.NoError(t, err) + assert.Equal(t, original, got) +} From cc0846da34e9bd7154d5af80892df9c84f28c79b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 20:54:20 -0600 Subject: [PATCH 13/43] test: honor Windows artifact filesystem semantics --- pkg/artifact/blob/local/store_test.go | 6 +++--- pkg/artifact/local/roundtrip_test.go | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 86d1bdfe809..57462ee8b6e 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -161,11 +161,11 @@ func TestDescribeHashesOpenedFile(t *testing.T) { store := newStore(t) require.NoError(t, os.MkdirAll(store.Root(), 0o755)) path := filepath.Join(store.Root(), "k") - require.NoError(t, os.WriteFile(path, []byte("old"), 0o644)) - file, err := openFile(path) + oldPath := filepath.Join(store.Root(), "old") + require.NoError(t, os.WriteFile(oldPath, []byte("old"), 0o644)) + file, err := openFile(oldPath) require.NoError(t, err) defer file.Close() - require.NoError(t, os.Rename(path, path+".old")) require.NoError(t, os.WriteFile(path, []byte("new"), 0o644)) root, err := store.openRoot(false) require.NoError(t, err) diff --git a/pkg/artifact/local/roundtrip_test.go b/pkg/artifact/local/roundtrip_test.go index 84db8c4796d..4b2b632d62c 100644 --- a/pkg/artifact/local/roundtrip_test.go +++ b/pkg/artifact/local/roundtrip_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "runtime" "sort" "testing" @@ -235,7 +236,19 @@ func TestPullReplacementRemovesStaleFilesInsideTheOutputOnly(t *testing.T) { state, err := materialize.InspectDestination(out, content.Record) require.NoError(t, err) assert.NotEqual(t, materialize.DestinationMatches, state) - require.NoError(t, materialize.Materialize(context.Background(), content, out)) + err = materialize.Materialize(context.Background(), content, out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + old, readErr := os.ReadFile(filepath.Join(out, "stale", "old.txt")) + require.NoError(t, readErr) + assert.Equal(t, "old", string(old), "unsupported replacement preserves the original output") + readme, readErr := os.ReadFile(filepath.Join(out, "README.md")) + require.NoError(t, readErr) + assert.Equal(t, "old readme", string(readme)) + assert.FileExists(t, sibling) + return + } + require.NoError(t, err) assert.NoFileExists(t, filepath.Join(out, "stale", "old.txt"), "stale files inside the output are removed, not merged") assert.FileExists(t, sibling, "nothing outside the requested output root is touched") From 7140fe2c68059d5a072cab15c7bb9f540e22c558 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 21:09:26 -0600 Subject: [PATCH 14/43] test: propagate range response read errors --- pkg/artifact/blob/s3/s3test/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index a7d32aa8f24..972c48219e5 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -219,8 +219,8 @@ func TestRanges(t *testing.T) { return "", err } defer out.Body.Close() - data, _ := io.ReadAll(out.Body) - return string(data), nil + data, err := io.ReadAll(out.Body) + return string(data), err } got, err := get("bytes=2-4") require.NoError(t, err) From a93f959e133d2f338ee4a309f2ab4f9faff80570 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 06:46:14 -0600 Subject: [PATCH 15/43] ci: record govulncheck resource pressure --- .github/scripts/govulncheck-monitor.sh | 40 ++++++++++++++++++++ .github/scripts/govulncheck-monitor.test.cjs | 20 ++++++++++ .github/workflows/codeql.yml | 4 +- 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/govulncheck-monitor.sh create mode 100644 .github/scripts/govulncheck-monitor.test.cjs diff --git a/.github/scripts/govulncheck-monitor.sh b/.github/scripts/govulncheck-monitor.sh new file mode 100644 index 00000000000..7826010a7dd --- /dev/null +++ b/.github/scripts/govulncheck-monitor.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Run the scanner unchanged while recording resource pressure for slow CI jobs. +set -u + +monitor() { + local sleeper="" + trap 'kill "$sleeper" 2>/dev/null || :; exit 0' TERM INT + while :; do + { + echo "--- govulncheck resources $(date -u +%FT%TZ) ---" + free -m || : + for resource in cpu memory io; do + if [[ -r "/proc/pressure/$resource" ]]; then + echo "$resource pressure:" + cat "/proc/pressure/$resource" || : + fi + done + # Show the largest processes and the scanner even if it is not among them. + ps -eo pid,ppid,stat,pcpu,rss,vsz,comm --sort=-rss | head -n 26 || : + for pid in $(pgrep -x govulncheck || :); do + echo "govulncheck pid $pid:" + awk '/^(State|VmRSS|VmSwap):/ { print }' "/proc/$pid/status" 2>/dev/null || : + done + } >&2 + sleep 60 & + sleeper=$! + wait "$sleeper" || : + sleeper="" + done +} + +monitor & +monitor_pid=$! +cleanup() { + kill "$monitor_pid" 2>/dev/null || : + wait "$monitor_pid" 2>/dev/null || : +} +trap cleanup EXIT + +"$@" diff --git a/.github/scripts/govulncheck-monitor.test.cjs b/.github/scripts/govulncheck-monitor.test.cjs new file mode 100644 index 00000000000..1591adb148d --- /dev/null +++ b/.github/scripts/govulncheck-monitor.test.cjs @@ -0,0 +1,20 @@ +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); +const { test } = require('node:test'); + +const script = path.join(__dirname, 'govulncheck-monitor.sh'); +const options = { skip: process.platform === 'win32' }; + +for (const status of [0, 23]) { + test(`preserves scanner output and exit status ${status}, and stops the monitor`, options, () => { + const result = spawnSync('bash', [script, 'bash', '-c', + `sleep 0.2; printf '%s' '{"version":"2.1.0"}'; printf 'scanner diagnostic\\n' >&2; exit ${status}`], + { encoding: 'utf8', timeout: 5000 }); + assert.ifError(result.error); + assert.equal(result.status, status); + assert.equal(result.stdout, '{"version":"2.1.0"}'); + assert.match(result.stderr, /govulncheck resources/); + assert.match(result.stderr, /scanner diagnostic/); + }); +} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ccd026c124c..e180ead5c83 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -204,7 +204,9 @@ jobs: # The target also removes exact duplicate stacks from the report: the # SARIF schema requires them to be unique and the upload rejects the # report otherwise. See internal/ci/vulncheck. - run: go tool mage ci:vulncheck govulncheck.sarif + # Diagnostics run separately on stderr; the scanner and SARIF output + # are unchanged. They help distinguish pressure from a stalled scan. + run: bash .github/scripts/govulncheck-monitor.sh go tool mage ci:vulncheck govulncheck.sarif - name: Upload SARIF file # Setup failures produce no report; preserve the original failure without From da4d947c063c7e68d0c9db05a127c9f7b1c7365b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 07:58:24 -0600 Subject: [PATCH 16/43] ci: measure scanner garbage collection tradeoff --- .github/workflows/codeql.yml | 4 ++++ internal/ci/vulncheck/scan.go | 13 +++++++++++++ internal/ci/vulncheck/scan_test.go | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e180ead5c83..7d46037adc5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -201,6 +201,10 @@ jobs: # (govulncheck's SSA construction parallelizes via goroutines, which # -p doesn't govern) while nearly tripling run time. GOMEMLIMIT: "32GiB" + # Controlled scanner-only comparison against the GOGC=100 baseline. + # GC traces report live heap and collection cost on scanner stderr. + ATMOS_VULNCHECK_GOGC: "50" + ATMOS_VULNCHECK_GCTRACE: "1" # The target also removes exact duplicate stacks from the report: the # SARIF schema requires them to be unique and the upload rejects the # report otherwise. See internal/ci/vulncheck. diff --git a/internal/ci/vulncheck/scan.go b/internal/ci/vulncheck/scan.go index 90a8cfc739c..4853e09d4d3 100644 --- a/internal/ci/vulncheck/scan.go +++ b/internal/ci/vulncheck/scan.go @@ -74,6 +74,19 @@ func scan(ctx context.Context, params *Params, run commandRunner) error { func runCommand(ctx context.Context, stdout, stderr io.Writer, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) // #nosec G204 -- fixed tool name, mage-target-controlled args. + // These optional CI experiment settings affect only the scanner child, not + // Mage, compilation, or other Go processes in the job. + cmd.Env = cmd.Environ() + if gc, ok := os.LookupEnv("ATMOS_VULNCHECK_GOGC"); ok && gc != "" { + cmd.Env = append(cmd.Env, "GOGC="+gc) + } + if trace, ok := os.LookupEnv("ATMOS_VULNCHECK_GCTRACE"); ok && trace == "1" { + debug, _ := os.LookupEnv("GODEBUG") + if debug != "" { + debug += "," + } + cmd.Env = append(cmd.Env, "GODEBUG="+debug+"gctrace=1") + } cmd.Stdout = stdout cmd.Stderr = stderr return cmd.Run() diff --git a/internal/ci/vulncheck/scan_test.go b/internal/ci/vulncheck/scan_test.go index 07ee609f68c..a90a70428bd 100644 --- a/internal/ci/vulncheck/scan_test.go +++ b/internal/ci/vulncheck/scan_test.go @@ -176,6 +176,25 @@ func TestRunCommand_MissingBinary(t *testing.T) { require.Error(t, err) } +func TestRunCommand_ScannerEnvironment(t *testing.T) { + if os.Getenv("VULNCHECK_ENV_HELPER") == "1" { + _, _ = io.WriteString(os.Stdout, os.Getenv("GOGC")+"|"+os.Getenv("GODEBUG")) + return + } + t.Setenv("VULNCHECK_ENV_HELPER", "1") + t.Setenv("GOGC", "100") + t.Setenv("GODEBUG", "invalidptr=1") + t.Setenv("ATMOS_VULNCHECK_GOGC", "50") + t.Setenv("ATMOS_VULNCHECK_GCTRACE", "1") + exe, err := os.Executable() + require.NoError(t, err) + var stdout, stderr bytes.Buffer + require.NoError(t, runCommand(context.Background(), &stdout, &stderr, exe, "-test.run=^TestRunCommand_ScannerEnvironment$")) + assert.Contains(t, stdout.String(), "50|invalidptr=1,gctrace=1") + assert.Equal(t, "100", os.Getenv("GOGC")) + assert.Equal(t, "invalidptr=1", os.Getenv("GODEBUG")) +} + func TestScan_ReportsMissingGovulncheck(t *testing.T) { // With nothing on PATH the real runner cannot find the tool, and Scan must // say so instead of writing an empty report. From b8134d9dee6d7592f40aada080a606c1a6343dac Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 12:26:50 -0600 Subject: [PATCH 17/43] fix: keep local blob content and attributes consistent --- pkg/artifact/blob/local/doc.go | 17 +- pkg/artifact/blob/local/etag.go | 25 +-- pkg/artifact/blob/local/get.go | 12 +- pkg/artifact/blob/local/lock.go | 19 ++- pkg/artifact/blob/local/options.go | 12 +- pkg/artifact/blob/local/put.go | 15 +- pkg/artifact/blob/local/store.go | 1 + pkg/artifact/blob/local/store_test.go | 19 ++- pkg/artifact/blob/local/transaction.go | 159 +++++++++++++++++ pkg/artifact/blob/local/transaction_test.go | 180 ++++++++++++++++++++ pkg/artifact/blob/store.go | 2 +- pkg/ci/artifact/local/store.go | 2 +- 12 files changed, 417 insertions(+), 46 deletions(-) create mode 100644 pkg/artifact/blob/local/transaction.go create mode 100644 pkg/artifact/blob/local/transaction_test.go diff --git a/pkg/artifact/blob/local/doc.go b/pkg/artifact/blob/local/doc.go index 6030b3b41ac..0f28761b014 100644 --- a/pkg/artifact/blob/local/doc.go +++ b/pkg/artifact/blob/local/doc.go @@ -1,18 +1,19 @@ // Package local is the filesystem implementation of [blob.Store]. // // Objects are plain files under a root directory, so a directory written here is readable with -// ordinary tools and matches the legacy CI "local/dir" layout byte for byte. Writes stream to a +// ordinary tools. Writes stream to a // temporary file in the destination directory and are renamed into place, so readers never see a // partial object. Content type, user metadata, and tags live in an optional ".atmos-blob.json" // attribute file that is written only when an attribute is supplied. // -// Conditional writes and deletes are emulated: the condition is checked and the rename performed -// while holding a per-root lock (an in-process mutex plus an OS file lock), so concurrent -// conditional operations from any process are linearizable. Unconditional writes do not take the -// lock unless [WithLockAllWrites] is set; callers that mix conditional and unconditional writers -// on one key must enable it. +// Managed reads, writes, and deletes hold a per-root process and file lock. A recoverable journal +// protects content and attributes across interrupted commits; recovery runs before the next managed +// operation. Direct filesystem edits and mixing [WithPlainFileCompatibility] with managed stores +// on one root are outside this coordination. Plain-file mode preserves the legacy CI layout and +// unlocked unconditional operations. The journal recovers interrupted processes, not power loss. // // A local object cannot share a path with a key prefix: "a" and "a/b" cannot both exist. ETags are -// opaque tokens: the SHA-256 of the content for small objects and a weak size/mtime token for -// objects over the content-hash limit. Listings do not populate ETag; call Stat for a token. +// opaque content-based SHA-256 tokens. Identical contents have the same token, including after an +// intervening write. Get and Stat stream the entire file to compute the token, using bounded memory; +// listings do not populate ETag. package local diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go index d0a74474fb8..69a5631a4e4 100644 --- a/pkg/artifact/blob/local/etag.go +++ b/pkg/artifact/blob/local/etag.go @@ -3,34 +3,22 @@ package local import ( "crypto/sha256" "encoding/hex" - "fmt" "io" "os" - "strconv" ) -// contentETagLimit is the largest object whose ETag is its SHA-256. Larger objects get a weak -// size/mtime token so a head request never reads gigabytes. -const contentETagLimit = 4 << 20 - -// hexBase formats sizes and times inside weak ETags. -const hexBase = 16 - -// etagOf returns the ETag of the file at path described by info. -func etagOf(root *os.Root, path string, info os.FileInfo) (string, error) { +// etagOf returns the content-based ETag of the file at path. +func etagOf(root *os.Root, path string) (string, error) { file, err := rootOpenFile(root, path) if err != nil { return "", err } defer file.Close() - return etagOpen(file, info) + return etagOpen(file) } // etagOpen hashes the opened file and restores its position for the caller. -func etagOpen(file *os.File, info os.FileInfo) (string, error) { - if info.Size() > contentETagLimit { - return weakETag(info), nil - } +func etagOpen(file *os.File) (string, error) { hash := sha256.New() if _, err := io.Copy(hash, file); err != nil { return "", err @@ -45,8 +33,3 @@ func etagOpen(file *os.File, info os.FileInfo) (string, error) { func contentETag(sum []byte) string { return `"` + hex.EncodeToString(sum) + `"` } - -// weakETag derives a token from size and modification time for objects over the content limit. -func weakETag(info os.FileInfo) string { - return fmt.Sprintf(`W/"%s-%s"`, strconv.FormatInt(info.Size(), hexBase), strconv.FormatInt(info.ModTime().UnixNano(), hexBase)) -} diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go index 1e67b49aed1..fe28bf61b21 100644 --- a/pkg/artifact/blob/local/get.go +++ b/pkg/artifact/blob/local/get.go @@ -42,6 +42,11 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo return nil, wrapIO(blob.OpGet, key, err) } defer root.Close() + release, err := s.guard(ctx, blob.OpGet, key, false) + if err != nil { + return nil, err + } + defer release() target := filepath.FromSlash(key) file, err := rootOpenFile(root, target) if err != nil { @@ -93,6 +98,11 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b return nil, wrapIO(blob.OpStat, key, err) } defer root.Close() + release, err := s.guard(ctx, blob.OpStat, key, false) + if err != nil { + return nil, err + } + defer release() target := filepath.FromSlash(key) file, err := rootOpenFile(root, target) if err != nil { @@ -111,7 +121,7 @@ func (s *Store) describe(root *os.Root, op, key string, file *os.File, withTags if !stat.Mode().IsRegular() { return nil, blob.NewError(blob.KindNotFound, op, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) } - etag, err := etagOpen(file, stat) + etag, err := etagOpen(file) if err != nil { return nil, wrapIO(op, key, err) } diff --git a/pkg/artifact/blob/local/lock.go b/pkg/artifact/blob/local/lock.go index 33a2d02586d..87f70833e7b 100644 --- a/pkg/artifact/blob/local/lock.go +++ b/pkg/artifact/blob/local/lock.go @@ -56,8 +56,23 @@ func (s *Store) lockRoot(ctx context.Context, op, key string) (func(), error) { // guard returns the lock release for a write or delete, or a no-op when no lock is needed. func (s *Store) guard(ctx context.Context, op, key string, conditional bool) (func(), error) { - if !conditional && !s.lockAll { + if s.plainFiles && !conditional && !s.lockAll { return func() {}, nil } - return s.lockRoot(ctx, op, key) + release, err := s.lockRoot(ctx, op, key) + if err != nil { + return nil, err + } + if !s.plainFiles { + root, openErr := s.openRoot(false) + if openErr == nil { + openErr = s.recoverCommit(root) + _ = root.Close() + } + if openErr != nil { + release() + return nil, wrapIO(op, key, openErr) + } + } + return release, nil } diff --git a/pkg/artifact/blob/local/options.go b/pkg/artifact/blob/local/options.go index 513f3361a36..3d8f298f08c 100644 --- a/pkg/artifact/blob/local/options.go +++ b/pkg/artifact/blob/local/options.go @@ -29,14 +29,22 @@ func WithFilePerm(perm os.FileMode) Option { return func(s *Store) { s.filePerm = perm } } -// WithLockAllWrites serializes unconditional writes and deletes with conditional ones through the -// per-root lock. Enable it when one key is written both conditionally and unconditionally. +// WithLockAllWrites serializes unconditional writes and deletes with conditional ones. Managed +// stores already do this; it can also be used with plain-file compatibility. func WithLockAllWrites() Option { defer perf.Track(nil, "blob.local.WithLockAllWrites")() return func(s *Store) { s.lockAll = true } } +// WithPlainFileCompatibility preserves the legacy CI layout and unlocked unconditional operations. +// Do not mix this mode with managed stores on the same root when consistency is required. +func WithPlainFileCompatibility() Option { + defer perf.Track(nil, "local.WithPlainFileCompatibility")() + + return func(s *Store) { s.plainFiles = true } +} + // WithLiteralBackslashes permits literal backslashes on filesystems where they are not separators. // This supports legacy local artifact names; other key validation remains in effect. func WithLiteralBackslashes() Option { diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index d4500513e66..d181555df94 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -22,6 +22,7 @@ type staged struct { attrsTemp string size int64 etag string + preserve bool } // Put implements [blob.Store]. The body is streamed to a temporary file and renamed into place, @@ -184,7 +185,7 @@ func matchETag(root *os.Root, op, key string, state fileState, want string) erro if !state.exists { return blob.NewError(blob.KindNotFound, op, key, nil) } - current, err := etagOf(root, state.path, state.info) + current, err := etagOf(root, state.path) if err != nil { return wrapIO(op, key, err) } @@ -204,6 +205,9 @@ func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAtt if err == nil && info.IsDir() { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) } + if !s.plainFiles { + return s.commitManaged(root, key, pending, noAttrs, rootRenameReplace) + } switch { case pending.attrsTemp != "": if err := rootRenameReplace(root, pending.attrsTemp, attrsPath(target)); err != nil { @@ -228,15 +232,14 @@ func (s *Store) statCommitted(root *os.Root, key, target string, pending *staged if err != nil { return nil, wrapIO(blob.OpPut, key, err) } - etag := pending.etag - if info.Size() > contentETagLimit { - etag = weakETag(info) - } - return &blob.ObjectInfo{Key: key, Size: pending.size, ETag: etag, LastModified: info.ModTime()}, nil + return &blob.ObjectInfo{Key: key, Size: pending.size, ETag: pending.etag, LastModified: info.ModTime()}, nil } // discard removes any temporary files that were not committed. func (p *staged) discard(root *os.Root) { + if p.preserve { + return + } if p.tempPath != "" { _ = root.Remove(p.tempPath) } diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index 0566b9eee09..d2809362dbf 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -32,6 +32,7 @@ type Store struct { dirPerm os.FileMode filePerm os.FileMode lockAll bool + plainFiles bool allowBackslash bool snapshotMu sync.Mutex snapshots map[string]listSnapshot diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 57462ee8b6e..e97601dffe6 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,7 +41,7 @@ func TestNewRequiresRoot(t *testing.T) { } func TestWritesPlainFilesWithoutExtraArtifacts(t *testing.T) { - store := newStore(t) + store := newStore(t, WithPlainFileCompatibility()) _, err := store.Put(context.Background(), "stack/component/sha.tar", bytes.NewReader([]byte("tar")), blob.PutOptions{Size: 3}) require.NoError(t, err) @@ -109,15 +110,25 @@ func TestObjectAndPrefixCannotShareAPath(t *testing.T) { assert.True(t, blob.IsNotFound(err), "a prefix is not an object") } -func TestLargeObjectGetsWeakETag(t *testing.T) { +func TestLargeObjectContentETag(t *testing.T) { store := newStore(t) - big := bytes.Repeat([]byte("x"), contentETagLimit+1) + big := bytes.Repeat([]byte("x"), (4<<20)+1) info, err := store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big))}) require.NoError(t, err) - assert.Contains(t, info.ETag, `W/"`) + stamp := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(filepath.Join(store.Root(), "big"), stamp, stamp)) stat, err := store.Stat(context.Background(), "big", blob.StatOptions{}) require.NoError(t, err) assert.Equal(t, info.ETag, stat.ETag) + big[len(big)-1] = 'y' + _, err = store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big))}) + require.NoError(t, err) + require.NoError(t, os.Chtimes(filepath.Join(store.Root(), "big"), stamp, stamp)) + updated, err := store.Stat(context.Background(), "big", blob.StatOptions{}) + require.NoError(t, err) + assert.NotEqual(t, stat.ETag, updated.ETag) + _, err = store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big)), Condition: blob.IfMatch(stat.ETag)}) + assert.ErrorIs(t, err, errUtils.ErrBlobPreconditionFailed) } func TestListRejectsEscapingPrefix(t *testing.T) { diff --git a/pkg/artifact/blob/local/transaction.go b/pkg/artifact/blob/local/transaction.go new file mode 100644 index 00000000000..0db9ac2dc5c --- /dev/null +++ b/pkg/artifact/blob/local/transaction.go @@ -0,0 +1,159 @@ +package local + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// A root has at most one transaction because managed operations hold its lock. +// The marker is hidden by the existing temporary-file listing rule. +const transactionName = tempPrefix + "transaction" + +type transaction struct { + Target string `json:"target"` + DataTemp string `json:"data_temp"` + AttrsTemp string `json:"attrs_temp,omitempty"` + HadAttrs bool `json:"had_attrs"` + OldAttrs []byte `json:"old_attrs,omitempty"` +} + +// commitManaged records the old attributes before publishing either replacement. The presence of +// DataTemp distinguishes a failed/incomplete data rename from a completed one during recovery. +func (s *Store) commitManaged(root *os.Root, key string, pending *staged, noAttrs bool, rename func(*os.Root, string, string) error) error { + target := filepath.FromSlash(key) + if err := s.writeTransaction(root, target, pending); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.preserve = true + err := publishTransaction(root, target, pending, noAttrs, rename) + if err != nil { + if recoveryErr := s.recoverCommit(root); recoveryErr != nil { + return wrapIO(blob.OpPut, key, errors.Join(err, recoveryErr)) + } + pending.preserve = false + return wrapIO(blob.OpPut, key, err) + } + if err := s.recoverCommit(root); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.preserve = false + pending.tempPath, pending.attrsTemp = "", "" + return nil +} + +func (s *Store) writeTransaction(root *os.Root, target string, pending *staged) error { + if _, err := root.Stat(pending.tempPath); err != nil { + return err + } + old, err := rootReadFile(root, attrsPath(target)) + hadAttrs := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + txn := transaction{Target: target, DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp, HadAttrs: hadAttrs, OldAttrs: old} + data, err := json.Marshal(txn) + if err != nil { + return err + } + file, name, err := rootCreateTemp(root, ".", s.dirPerm, s.filePerm) + if err != nil { + return err + } + defer func() { _ = root.Remove(name) }() + _, writeErr := file.Write(data) + if err := firstError(writeErr, file.Close()); err != nil { + return err + } + return rootRenameReplace(root, name, transactionName) +} + +func publishTransaction(root *os.Root, target string, pending *staged, noAttrs bool, rename func(*os.Root, string, string) error) error { + var err error + if pending.attrsTemp != "" { + err = rename(root, pending.attrsTemp, attrsPath(target)) + } else if noAttrs { + err = root.Remove(attrsPath(target)) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + } + if err == nil { + err = rename(root, pending.tempPath, target) + } + return err +} + +// recoverCommit is idempotent: it restores the old attributes while the data temp still exists, +// and otherwise finishes cleanup after the data rename. Callers hold the root lock. +func (s *Store) recoverCommit(root *os.Root) error { + data, err := rootReadFile(root, transactionName) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var txn transaction + if err := json.Unmarshal(data, &txn); err != nil { + return fmt.Errorf("decode local transaction: %w", err) + } + if err := s.validateTransaction(&txn); err != nil { + return err + } + _, statErr := root.Stat(txn.DataTemp) + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + if statErr == nil { + if err := s.restoreAttrs(root, &txn); err != nil { + return err + } + } + if err := root.Remove(transactionName); err != nil { + return err + } + _ = root.Remove(txn.DataTemp) + s.removeAttrsTemp(root, &txn) + return nil +} + +func (s *Store) validateTransaction(txn *transaction) error { + if err := s.validate(blob.OpPut, filepath.ToSlash(txn.Target)); err != nil || + filepath.Dir(txn.DataTemp) != filepath.Dir(txn.Target) || !strings.HasPrefix(filepath.Base(txn.DataTemp), tempPrefix) { + return fmt.Errorf("%w: invalid local transaction", errUtils.ErrBlobInvalid) + } + return nil +} + +func (s *Store) removeAttrsTemp(root *os.Root, txn *transaction) { + if txn.AttrsTemp != "" && filepath.Dir(txn.AttrsTemp) == filepath.Dir(txn.Target) && strings.HasPrefix(filepath.Base(txn.AttrsTemp), tempPrefix) { + _ = root.Remove(txn.AttrsTemp) + } +} + +func (s *Store) restoreAttrs(root *os.Root, txn *transaction) error { + if !txn.HadAttrs { + err := root.Remove(attrsPath(txn.Target)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + file, name, err := rootCreateTemp(root, filepath.Dir(txn.Target), s.dirPerm, s.filePerm) + if err != nil { + return err + } + defer func() { _ = root.Remove(name) }() + _, writeErr := file.Write(txn.OldAttrs) + if err := firstError(writeErr, file.Close()); err != nil { + return err + } + return rootRenameReplace(root, name, attrsPath(txn.Target)) +} diff --git a/pkg/artifact/blob/local/transaction_test.go b/pkg/artifact/blob/local/transaction_test.go new file mode 100644 index 00000000000..8a6001037b6 --- /dev/null +++ b/pkg/artifact/blob/local/transaction_test.go @@ -0,0 +1,180 @@ +package local + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + defer root.Close() + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + defer pending.discard(root) + release, err := s.guard(ctx, blob.OpPut, "key", false) + require.NoError(t, err) + between, proceed := make(chan struct{}), make(chan struct{}) + commitDone := make(chan error, 1) + go func() { + defer release() + commitDone <- s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { + if err := rootRenameReplace(r, from, to); err != nil { + return err + } + if to == attrsPath("key") { + close(between) + <-proceed + } + return nil + }) + }() + <-between + type result struct { + body string + info *blob.ObjectInfo + err error + } + getDone, statDone := make(chan result, 1), make(chan result, 1) + go func() { + obj, err := s.Get(ctx, "key", blob.GetOptions{}) + if err != nil { + getDone <- result{err: err} + return + } + body, err := io.ReadAll(obj.Body) + _ = obj.Body.Close() + getDone <- result{body: string(body), info: &obj.ObjectInfo, err: err} + }() + go func() { + info, err := s.Stat(ctx, "key", blob.StatOptions{}) + statDone <- result{info: info, err: err} + }() + early := false + select { + case got := <-getDone: + t.Errorf("Get returned during an incomplete commit: %+v", got) + early = true + case got := <-statDone: + t.Errorf("Stat returned during an incomplete commit: %+v", got) + early = true + case <-time.After(100 * time.Millisecond): + } + close(proceed) + require.NoError(t, <-commitDone) + require.False(t, early) + got, stat := <-getDone, <-statDone + require.NoError(t, got.err) + require.NoError(t, stat.err) + assert.Equal(t, "new", got.body) + assert.Equal(t, "new", got.info.Metadata["version"]) + assert.Equal(t, "new", stat.info.Metadata["version"]) +} + +func TestManagedCommitFailureRestoresOldAttributes(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + defer root.Close() + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + defer pending.discard(root) + release, err := s.guard(ctx, blob.OpPut, "key", false) + require.NoError(t, err) + failure := errors.New("injected data rename failure") + err = s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { + if to == "key" { + return failure + } + return rootRenameReplace(r, from, to) + }) + release() + require.ErrorIs(t, err, failure) + obj, err := s.Get(ctx, "key", blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + body, err := io.ReadAll(obj.Body) + require.NoError(t, err) + assert.Equal(t, "old", string(body)) + assert.Equal(t, "old", obj.Metadata["version"]) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) +} + +func TestManagedRecoveryAfterInterruptedCommit(t *testing.T) { + for _, dataRenamed := range []bool{false, true} { + t.Run(map[bool]string{false: "before data rename", true: "after data rename"}[dataRenamed], func(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + oldAttrs, err := rootReadFile(root, attrsPath("key")) + require.NoError(t, err) + marker, err := json.Marshal(transaction{Target: "key", DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp, HadAttrs: true, OldAttrs: oldAttrs}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(s.root, transactionName), marker, 0o644)) + require.NoError(t, rootRenameReplace(root, pending.attrsTemp, attrsPath("key"))) + if dataRenamed { + require.NoError(t, rootRenameReplace(root, pending.tempPath, "key")) + } + require.NoError(t, root.Close()) + restarted, err := New(s.root) + require.NoError(t, err) + obj, err := restarted.Get(ctx, "key", blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + body, err := io.ReadAll(obj.Body) + require.NoError(t, err) + want := "old" + if dataRenamed { + want = "new" + } + assert.Equal(t, want, string(body)) + assert.Equal(t, want, obj.Metadata["version"]) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) + }) + } +} + +func TestManagedRecoveryRemovesNewAttributesWhenOldObjectHadNone(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + marker, err := json.Marshal(transaction{Target: "key", DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(s.root, transactionName), marker, 0o644)) + require.NoError(t, rootRenameReplace(root, pending.attrsTemp, attrsPath("key"))) + require.NoError(t, root.Close()) + info, err := s.Stat(ctx, "key", blob.StatOptions{}) + require.NoError(t, err) + assert.Empty(t, info.Metadata) + assert.NoFileExists(t, filepath.Join(s.root, attrsPath("key"))) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) +} diff --git a/pkg/artifact/blob/store.go b/pkg/artifact/blob/store.go index 317f067ba61..953ecb9b7e7 100644 --- a/pkg/artifact/blob/store.go +++ b/pkg/artifact/blob/store.go @@ -31,7 +31,7 @@ type Store interface { // with ErrBlobNotFound. Get(ctx context.Context, key string, opts GetOptions) (*Object, error) - // Stat returns object attributes without reading the body. A missing object fails with + // Stat returns object attributes. Some backends read the body to compute an ETag. A missing object fails with // ErrBlobNotFound. Stat(ctx context.Context, key string, opts StatOptions) (*ObjectInfo, error) diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index 5fb57a7265b..e3efe45afc4 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -77,7 +77,7 @@ func (s *Store) transport() (*blobLocal.Store, error) { if root == "" { root = "." } - return blobLocal.New(root, blobLocal.WithLiteralBackslashes()) + return blobLocal.New(root, blobLocal.WithLiteralBackslashes(), blobLocal.WithPlainFileCompatibility()) } // key maps a validated artifact name to the transport key. Names are cleaned the same way the From 8d280f880ea843a7f6d2614a9a77eed681490a1f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 16:13:44 -0600 Subject: [PATCH 18/43] fix(artifact): read S3 tags for the headed object version Stat with IncludeTags issued GetObjectTagging without a version, so an overwrite between HeadObject and the tag read could pair metadata from one write with tags from another. When HeadObject reports a real version ID, the tag read now targets that version. Unversioned buckets and the replaceable "null" version keep reading the current tags without a version-matching guarantee, and bucket versioning stays optional. A version-specific read needs s3:GetObjectVersionTagging. A denial fails the call instead of falling back to the current version's tags. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/blob/s3/doc.go | 4 + pkg/artifact/blob/s3/read.go | 30 ++++++- pkg/artifact/blob/s3/read_version_test.go | 97 +++++++++++++++++++++++ 3 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 pkg/artifact/blob/s3/read_version_test.go diff --git a/pkg/artifact/blob/s3/doc.go b/pkg/artifact/blob/s3/doc.go index e87a1bb0981..041de6e4550 100644 --- a/pkg/artifact/blob/s3/doc.go +++ b/pkg/artifact/blob/s3/doc.go @@ -10,6 +10,10 @@ // cannot enforce conditions fails the operation with a clear error, and an optional probe detects // endpoints that silently ignore the headers. // +// Stat with tags pins the tag read to the version HeadObject returned when the bucket is versioned, +// which needs s3:GetObjectVersionTagging in addition to s3:GetObjectTagging. Bucket versioning is +// not required; without it, tags come from the current object and are not version-matched. +// // Credentials are resolved lazily on first use. The generic store fails closed: a configured // identity that cannot be resolved is an error, never a silent fall back to ambient credentials. package s3 diff --git a/pkg/artifact/blob/s3/read.go b/pkg/artifact/blob/s3/read.go index e96d2e62c34..bc4100de55e 100644 --- a/pkg/artifact/blob/s3/read.go +++ b/pkg/artifact/blob/s3/read.go @@ -78,6 +78,13 @@ func objectSize(out *s3.GetObjectOutput) int64 { } // Stat implements [blob.Store]. +// +// With IncludeTags, the tags are read for the exact object version HeadObject returned when the +// bucket reports a real version ID, so metadata and tags describe the same object even if the key +// is overwritten between the two requests. That read requires s3:GetObjectVersionTagging; a denial +// fails the call rather than falling back to the current version's tags. Unversioned buckets and +// the replaceable "null" version read the current tags with s3:GetObjectTagging, so a concurrent +// overwrite can still pair metadata from one write with tags from another there. func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*blob.ObjectInfo, error) { defer perf.Track(nil, "blob.s3.Stat")() @@ -101,16 +108,25 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b Metadata: out.Metadata, } if opts.IncludeTags { - if info.Tags, err = s.tags(ctx, api, key); err != nil { + if info.Tags, err = s.tags(ctx, api, key, out.VersionId); err != nil { return nil, err } } return info, nil } -// tags loads the object tags. -func (s *Store) tags(ctx context.Context, api API, key string) (map[string]string, error) { - out, err := api.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))}) +// nullVersionID is the version ID S3 reports for objects written while versioning was never +// enabled or was suspended. A later write replaces that version in place, so it does not identify +// one immutable object. +const nullVersionID = "null" + +// tags loads the object tags, pinned to versionID when it names an immutable object version. +func (s *Store) tags(ctx context.Context, api API, key string, versionID *string) (map[string]string, error) { + in := &s3.GetObjectTaggingInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))} + if pinnedVersion(versionID) { + in.VersionId = versionID + } + out, err := api.GetObjectTagging(ctx, in) if err != nil { return nil, classify(request{op: "tags", key: key}, err) } @@ -120,3 +136,9 @@ func (s *Store) tags(ctx context.Context, api API, key string) (map[string]strin } return tags, nil } + +// pinnedVersion reports whether a HeadObject version ID identifies one immutable object version. +func pinnedVersion(versionID *string) bool { + id := aws.ToString(versionID) + return id != "" && id != nullVersionID +} diff --git a/pkg/artifact/blob/s3/read_version_test.go b/pkg/artifact/blob/s3/read_version_test.go new file mode 100644 index 00000000000..83e33248c12 --- /dev/null +++ b/pkg/artifact/blob/s3/read_version_test.go @@ -0,0 +1,97 @@ +package s3 + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// errTaggingDenied stands in for a version-specific tagging denial. +var errTaggingDenied = errors.New("tagging denied") + +// versionedAPI wraps a real API, reporting a fixed HeadObject version ID and recording the +// GetObjectTagging inputs. A non-nil taggingErr fails every tagging request. +type versionedAPI struct { + API + versionID *string + taggingErr error + tagging []*s3.GetObjectTaggingInput +} + +func (a *versionedAPI) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + out, err := a.API.HeadObject(ctx, in, optFns...) + if err != nil { + return nil, err + } + out.VersionId = a.versionID + return out, nil +} + +func (a *versionedAPI) GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + a.tagging = append(a.tagging, in) + if a.taggingErr != nil { + return nil, a.taggingErr + } + // The fake endpoint is unversioned, so serve the current tags for any requested version. + plain := *in + plain.VersionId = nil + return a.API.GetObjectTagging(ctx, &plain, optFns...) +} + +func TestStatPinsTagReadToTheHeadedVersion(t *testing.T) { + tests := []struct { + name string + versionID *string + want *string + }{ + {name: "versioned bucket pins the read", versionID: aws.String("3HL4kqtJlcpXroDTDmJ"), want: aws.String("3HL4kqtJlcpXroDTDmJ")}, + {name: "unversioned bucket reads current tags", versionID: nil, want: nil}, + {name: "empty version reads current tags", versionID: aws.String(""), want: nil}, + {name: "replaceable null version reads current tags", versionID: aws.String(nullVersionID), want: nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fake := newFake(t) + api := &versionedAPI{API: fake.Client(), versionID: tc.versionID} + s := newVersionedStore(t, fake, api) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"a": "b"}, info.Tags) + require.Len(t, api.tagging, 1) + assert.Equal(t, aws.ToString(tc.want), aws.ToString(api.tagging[0].VersionId)) + assert.Equal(t, tc.want == nil, api.tagging[0].VersionId == nil) + }) + } +} + +func TestStatDoesNotFallBackWhenVersionedTagReadFails(t *testing.T) { + fake := newFake(t) + api := &versionedAPI{API: fake.Client(), versionID: aws.String("v1")} + s := newVersionedStore(t, fake, api) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) + api.taggingErr = errTaggingDenied + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.ErrorIs(t, err, errTaggingDenied) + assert.Nil(t, info) + require.Len(t, api.tagging, 1, "a denied version-specific read must not retry against the current version") + assert.Equal(t, "v1", aws.ToString(api.tagging[0].VersionId)) +} + +// newVersionedStore returns a store whose requests go through the wrapping API. +func newVersionedStore(t *testing.T, fake *s3test.Server, api *versionedAPI) *Store { + t.Helper() + s, err := NewFromClient(api, fake.Bucket(), WithPartSize(smallPart)) + require.NoError(t, err) + return s +} From 01ec244bde63ca7a7a890b2a327af0be9263dd94 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 16:41:58 -0600 Subject: [PATCH 19/43] refactor(artifact): define the aws-chunked terminator sentinel centrally The S3 test endpoint declared its invalid chunk terminator error locally. It now uses ErrBlobInvalidChunkTerminator from errors/errors.go, and the decoder test asserts that sentinel instead of any error. Co-Authored-By: Claude Opus 5.5 --- errors/errors.go | 4 ++++ pkg/artifact/blob/s3/s3test/chunked.go | 7 +++---- pkg/artifact/blob/s3/s3test/server_test.go | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 34d1c43393a..caba6f75f31 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2163,6 +2163,10 @@ var ( // ErrBlobIdentityUnresolved is returned when a configured identity cannot be resolved to credentials. // The generic blob transport fails closed instead of falling back to ambient credentials. ErrBlobIdentityUnresolved = errors.New("blob identity could not be resolved") + + // ErrBlobInvalidChunkTerminator is returned by the S3 test endpoint when an aws-chunked request + // body has a chunk that is not followed by the required CRLF terminator. + ErrBlobInvalidChunkTerminator = errors.New("invalid aws-chunked chunk terminator") ) // ExitCodeError is a typed error that preserves subcommand exit codes. diff --git a/pkg/artifact/blob/s3/s3test/chunked.go b/pkg/artifact/blob/s3/s3test/chunked.go index 27200366109..560cab7343f 100644 --- a/pkg/artifact/blob/s3/s3test/chunked.go +++ b/pkg/artifact/blob/s3/s3test/chunked.go @@ -3,14 +3,13 @@ package s3test import ( "bufio" "bytes" - "errors" "io" "net/http" "strconv" "strings" -) -var errInvalidChunkTerminator = errors.New("invalid chunk terminator") + errUtils "github.com/cloudposse/atmos/errors" +) // hexBase is the radix of aws-chunked chunk sizes. const hexBase = 16 @@ -58,7 +57,7 @@ func decodeAWSChunked(raw []byte) ([]byte, error) { return nil, err } if terminator != [2]byte{'\r', '\n'} { - return nil, errInvalidChunkTerminator + return nil, errUtils.ErrBlobInvalidChunkTerminator } } } diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 972c48219e5..38e95f37f87 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -15,6 +15,8 @@ import ( "github.com/aws/smithy-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" ) func apiCode(t *testing.T, err error) string { @@ -345,7 +347,7 @@ func TestDecodeAWSChunked(t *testing.T) { _, err = decodeAWSChunked([]byte("zz\r\n")) require.Error(t, err) _, err = decodeAWSChunked([]byte("5\r\nhelloXX0\r\n")) - require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrBlobInvalidChunkTerminator) _, err = decodeAWSChunked([]byte("5\r\nhel")) require.Error(t, err) } From 452a683b9bf0919ece1c6a30103219318ee811eb Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 12 Nov 2026 22:10:52 -0600 Subject: [PATCH 20/43] refactor(artifact): reduce complexity flagged by code scanning Split functions that code scanning flagged for cyclomatic, cognitive, and nesting complexity into focused helpers. Behavior is unchanged: - layout.Admit: pending and completed references are now decided by admitPending and admitCompleted. The pending prior check still runs only when the intent matches. - local evidence loading: one directory entry is read by evidenceEntry. - local blob Delete and commit: the root-relative work is in deleteFromRoot and commitPlain. - local publish retire: removal of the replaced reference is in retirePrior and retirePriorPayload. The govulncheck monitor test now runs the script by name from its own directory, so no absolute path reaches the command line that CodeQL traced. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/govulncheck-monitor.test.cjs | 6 +-- pkg/artifact/blob/local/delete.go | 10 ++++- pkg/artifact/blob/local/put.go | 5 +++ pkg/artifact/layout/admit.go | 23 ++++++----- pkg/artifact/local/evidence.go | 35 +++++++++++------ pkg/artifact/local/publish.go | 40 +++++++++++++------- 6 files changed, 79 insertions(+), 40 deletions(-) diff --git a/.github/scripts/govulncheck-monitor.test.cjs b/.github/scripts/govulncheck-monitor.test.cjs index 1591adb148d..bbc431af514 100644 --- a/.github/scripts/govulncheck-monitor.test.cjs +++ b/.github/scripts/govulncheck-monitor.test.cjs @@ -1,16 +1,16 @@ const assert = require('node:assert/strict'); const { spawnSync } = require('node:child_process'); -const path = require('node:path'); const { test } = require('node:test'); -const script = path.join(__dirname, 'govulncheck-monitor.sh'); +// Run the script by its name from its own directory, so no absolute path reaches the command line. +const script = 'govulncheck-monitor.sh'; const options = { skip: process.platform === 'win32' }; for (const status of [0, 23]) { test(`preserves scanner output and exit status ${status}, and stops the monitor`, options, () => { const result = spawnSync('bash', [script, 'bash', '-c', `sleep 0.2; printf '%s' '{"version":"2.1.0"}'; printf 'scanner diagnostic\\n' >&2; exit ${status}`], - { encoding: 'utf8', timeout: 5000 }); + { cwd: __dirname, encoding: 'utf8', timeout: 5000 }); assert.ifError(result.error); assert.equal(result.status, status); assert.equal(result.stdout, '{"version":"2.1.0"}'); diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go index c66b6585b40..c5b8c192a7c 100644 --- a/pkg/artifact/blob/local/delete.go +++ b/pkg/artifact/blob/local/delete.go @@ -34,9 +34,15 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) return wrapIO(blob.OpDelete, key, err) } defer root.Close() + return deleteFromRoot(root, key, opts.IfMatch) +} + +// deleteFromRoot removes the object and its attributes after an optional ETag check, then prunes +// directories the removal left empty. +func deleteFromRoot(root *os.Root, key, ifMatch string) error { target := filepath.FromSlash(key) - if opts.IfMatch != "" { - if err := checkDeleteETag(root, key, target, opts.IfMatch); err != nil { + if ifMatch != "" { + if err := checkDeleteETag(root, key, target, ifMatch); err != nil { return err } } diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index d181555df94..41ae86b1032 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -208,6 +208,11 @@ func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAtt if !s.plainFiles { return s.commitManaged(root, key, pending, noAttrs, rootRenameReplace) } + return commitPlain(root, key, target, pending, noAttrs) +} + +// commitPlain installs a staged object in a plain-files store: attributes first, then the object. +func commitPlain(root *os.Root, key, target string, pending *staged, noAttrs bool) error { switch { case pending.attrsTemp != "": if err := rootRenameReplace(root, pending.attrsTemp, attrsPath(target)); err != nil { diff --git a/pkg/artifact/layout/admit.go b/pkg/artifact/layout/admit.go index 40ce78be7d5..b9cd8d91545 100644 --- a/pkg/artifact/layout/admit.go +++ b/pkg/artifact/layout/admit.go @@ -92,17 +92,13 @@ func Admit(in *AdmitInput) (Admission, error) { return AdmitCreate, nil } if in.Current.State == StatePending { - if in.Current.IntentDigest == in.IncomingIntent { - var expected digest.Digest - if in.ExpectedPrior != nil { - expected = in.ExpectedPrior.ResolvedDigest - } - if expected != in.Current.PriorRecordDigest { - return "", fmt.Errorf("%w: pending publication expected prior %s, caller expected %s", errUtils.ErrArtifactConflict, in.Current.PriorRecordDigest, expected) - } - } return admitPending(in) } + return admitCompleted(in) +} + +// admitCompleted decides publication over a completed reference. +func admitCompleted(in *AdmitInput) (Admission, error) { if err := checkExpectedPrior(in.Current, in.ExpectedPrior); err != nil { return "", err } @@ -115,11 +111,20 @@ func Admit(in *AdmitInput) (Admission, error) { return AdmitReplace, nil } +// admitPending decides publication over a pending reference: only the same intent, expecting the +// same prior, may resume it. func admitPending(in *AdmitInput) (Admission, error) { if in.Current.IntentDigest != in.IncomingIntent { return "", fmt.Errorf("%w: %s:%s is pending for operation %s with a different intent; recover or repair it with the same intent before publishing", errUtils.ErrArtifactPublicationUnavailable, in.Current.Name, in.Current.Version, in.Current.OperationID) } + var expected digest.Digest + if in.ExpectedPrior != nil { + expected = in.ExpectedPrior.ResolvedDigest + } + if expected != in.Current.PriorRecordDigest { + return "", fmt.Errorf("%w: pending publication expected prior %s, caller expected %s", errUtils.ErrArtifactConflict, in.Current.PriorRecordDigest, expected) + } if in.Current.PriorRecordDigest != "" { return AdmitResumeReplace, nil } diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index 326c945229b..7c2ede000bd 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -45,18 +45,14 @@ func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact for { entries, readErr := file.ReadDir(maxMalformedEvidence) for _, entry := range entries { - stem, ok := strings.CutSuffix(entry.Name(), ".json") - if entry.IsDir() || !ok { - ignored++ - } else { - locator := dir + "/" + entry.Name() - item, problem := t.readEvidenceFile(locator, stem) - if problem == nil { - loaded = append(loaded, loadedEvidence{evidence: item, locator: locator}) - continue - } - ignored++ - diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}) + item, diag, ok := t.evidenceEntry(dir, entry) + if ok { + loaded = append(loaded, item) + continue + } + ignored++ + if diag != nil { + diags = append(diags, *diag) } if ignored > maxMalformedEvidence { diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many invalid evidence entries; remaining entries were not read"}) @@ -72,6 +68,21 @@ func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact } } +// evidenceEntry reads one directory entry of an evidence directory. It reports ok for valid +// evidence; otherwise the entry is ignored, with a diagnostic when it looked like evidence. +func (t *tree) evidenceEntry(dir string, entry fs.DirEntry) (loadedEvidence, *artifact.Diagnostic, bool) { + stem, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + return loadedEvidence{}, nil, false + } + locator := dir + "/" + entry.Name() + item, problem := t.readEvidenceFile(locator, stem) + if problem != nil { + return loadedEvidence{}, &artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}, false + } + return loadedEvidence{evidence: item, locator: locator}, nil, true +} + func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, error) { want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) if err := want.Validate(); err != nil { diff --git a/pkg/artifact/local/publish.go b/pkg/artifact/local/publish.go index d91aef7c29d..1fa036beb4a 100644 --- a/pkg/artifact/local/publish.go +++ b/pkg/artifact/local/publish.go @@ -364,24 +364,36 @@ func (r *Repository) retire(t *tree, job *publishJob, plan *publishPlan) { r.removePayload(t, plan.abandoned) } if plan.prior != nil { - if plan.prior.Location != "" && layout.FoldPath(plan.prior.Location) != layout.FoldPath(job.location) { - if r.kind == artifact.RepositoryKindLocalDir { - if record, _, err := t.loadRecord(plan.prior.RecordDigest); err == nil { - _, _ = r.store.remove(t, &view{record: record, rep: &layout.Representation{Location: plan.prior.Location}}) - t.removeEmptyParents(plan.prior.Location) - } - } else { - r.removePayload(t, plan.prior.Location) - } - } - if plan.prior.RecordDigest != "" && plan.prior.RecordDigest != job.recordDigest { - _ = t.removeAll(layout.RecordPath(plan.prior.RecordDigest)) - _ = t.removeAll(layout.SignatureDir(artifact.SubjectRecord, plan.prior.RecordDigest)) - } + r.retirePrior(t, job, plan.prior) } _ = t.removeAll(layout.OperationPath(job.opID)) } +// retirePrior removes what the replaced reference left behind: its payload at a location the new +// publication no longer uses, and its record and evidence when the record changed. +func (r *Repository) retirePrior(t *tree, job *publishJob, prior *layout.Prior) { + if prior.Location != "" && layout.FoldPath(prior.Location) != layout.FoldPath(job.location) { + r.retirePriorPayload(t, prior) + } + if prior.RecordDigest != "" && prior.RecordDigest != job.recordDigest { + _ = t.removeAll(layout.RecordPath(prior.RecordDigest)) + _ = t.removeAll(layout.SignatureDir(artifact.SubjectRecord, prior.RecordDigest)) + } +} + +// retirePriorPayload removes the replaced payload. A local directory repository removes only the +// files its record owns, so unrelated files at that location survive. +func (r *Repository) retirePriorPayload(t *tree, prior *layout.Prior) { + if r.kind != artifact.RepositoryKindLocalDir { + r.removePayload(t, prior.Location) + return + } + if record, _, err := t.loadRecord(prior.RecordDigest); err == nil { + _, _ = r.store.remove(t, &view{record: record, rep: &layout.Representation{Location: prior.Location}}) + t.removeEmptyParents(prior.Location) + } +} + // removePayload deletes the payload paths this repository owns at a location it no longer uses. func (r *Repository) removePayload(t *tree, location string) { for _, owned := range layout.PayloadPaths(r.kind, location) { From dc882055a86909734081a34f81ceb2d13dc4c353 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:09:08 -0600 Subject: [PATCH 21/43] fix(artifact): harden S3 probe, multipart completion, and condition validation Register probe cleanup before the first PUT and keep both the probe verdict and a cleanup failure without caching either. Disable SDK retries for every multipart completion, propagate a source-returned io.ErrUnexpectedEOF instead of treating it as end of input, reject unknown condition kinds before any request, and narrow the documented probe guarantee to single-object writes. Clear the region variable in the emulator env test. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/blob/s3/conditional.go | 53 ++++++++++------ pkg/artifact/blob/s3/conditional_test.go | 73 ++++++++++++++++++++++ pkg/artifact/blob/s3/multipart.go | 28 +++++---- pkg/artifact/blob/s3/options.go | 4 +- pkg/artifact/blob/s3/put.go | 20 +++++- pkg/artifact/blob/s3/put_test.go | 18 ++++++ pkg/artifact/blob/s3/s3test/server_test.go | 1 + 7 files changed, 165 insertions(+), 32 deletions(-) diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go index 27aa8e6bc64..b45e1086eb6 100644 --- a/pkg/artifact/blob/s3/conditional.go +++ b/pkg/artifact/blob/s3/conditional.go @@ -5,6 +5,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "fmt" "time" @@ -44,9 +45,12 @@ func unsupportedConditionalFrom(key string, probeErr error) error { } // ProbeConditionalWrites verifies that the endpoint enforces If-None-Match by creating a throwaway -// object twice: the second create must be rejected. A definite answer is cached for the life of the -// store; a transient failure is returned without being cached. It returns an error matching -// ErrBlobConditionalUnsupported when the endpoint ignores or rejects the condition. +// object twice: the second create must be rejected. It then checks that stale If-Match validators +// are rejected on PutObject and DeleteObject. The guarantee covers those single-object requests +// only; conditional enforcement on CompleteMultipartUpload is not probed. A definite answer is +// cached for the life of the store; a transient failure is returned without being cached. It +// returns an error matching ErrBlobConditionalUnsupported when the endpoint ignores or rejects the +// condition. func (s *Store) ProbeConditionalWrites(ctx context.Context) error { defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")() @@ -59,21 +63,42 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error { if err != nil { return err } - err = s.runProbe(ctx, api) - if err == nil || blob.IsConditionalUnsupported(err) { - s.probed, s.probeError = true, err + verdict, cleanupErr := s.runProbe(ctx, api) + if cleanupErr != nil { + // A leftover probe object is a transient failure: neither verdict is cached, and both + // errors stay visible to callers. + return errors.Join(verdict, cleanupErr) } - return err + if verdict == nil || blob.IsConditionalUnsupported(verdict) { + s.probed, s.probeError = true, verdict + } + return verdict } -// runProbe performs the double create and cleans up the probe object. -func (s *Store) runProbe(ctx context.Context, api API) (resultErr error) { +// runProbe performs the double create and removes the probe object. The verdict is the probe +// result; cleanupErr reports a failure to delete the probe object. +func (s *Store) runProbe(ctx context.Context, api API) (verdict, cleanupErr error) { var raw [8]byte if _, err := rand.Read(raw[:]); err != nil { - return blob.NewError(blob.KindOther, probeOperation, "", err) + return blob.NewError(blob.KindOther, probeOperation, "", err), nil } key := probePrefix + hex.EncodeToString(raw[:]) objectKey := s.objectKey(key) + // Cleanup is scheduled before the first create: a PUT that returns an ambiguous error may + // still have written the probe object. + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), probeCleanupTimeout) + defer cancel() + _, err := api.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}, noRetry) + if err != nil { + cleanupErr = blob.NewError(blob.KindOther, probeOperation, key, fmt.Errorf("cleanup probe object: %w", err)) + } + }() + return s.probeCreate(ctx, api, key, objectKey), nil +} + +// probeCreate creates the probe object twice and checks the second create is rejected. +func (s *Store) probeCreate(ctx context.Context, api API, key, objectKey string) error { create := func() error { _, err := api.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.cfg.bucket), @@ -86,14 +111,6 @@ func (s *Store) runProbe(ctx context.Context, api API) (resultErr error) { if err := create(); err != nil { return err } - defer func() { - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), probeCleanupTimeout) - defer cancel() - _, err := api.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}, noRetry) - if err != nil { - resultErr = blob.NewError(blob.KindOther, probeOperation, key, fmt.Errorf("cleanup probe object: %w", err)) - } - }() err := create() if blob.IsPreconditionFailed(err) { return s.probeIfMatch(ctx, api, key, objectKey) diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go index 669fc903e07..a7eb8bb2453 100644 --- a/pkg/artifact/blob/s3/conditional_test.go +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "io" "net/http" "testing" @@ -227,6 +228,50 @@ func TestAmbiguousCompletionLeavesTheUploadForReconciliation(t *testing.T) { assert.Equal(t, data, object.Data) } +func TestUnconditionalCompletionLostResponseIsAmbiguousAndNotRetried(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCompleteMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 9) + + _, err := s.Put(context.Background(), "big", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "a lost completion response is not a definite failure: %v", err) + assert.Len(t, fake.RequestsFor(s3test.OpCompleteMultipartUpload), 1, "completion is never retried by the SDK") + assert.Empty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload), "the upload is left for reconciliation") + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) +} + +// failingReader serves data, then fails with err. +type failingReader struct { + data []byte + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +func TestSourceUnexpectedEOFAbortsMultipartPublication(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + body := &failingReader{data: patterned(smallPart*2 + 9), err: io.ErrUnexpectedEOF} + + _, err := s.Put(context.Background(), "big", body, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, io.ErrUnexpectedEOF, "the source failure is not mistaken for end of input") + _, ok := fake.Object("big") + assert.False(t, ok, "a truncated source never becomes an object") + assert.Empty(t, fake.RequestsFor(s3test.OpCompleteMultipartUpload)) + assert.NotEmpty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload)) +} + func TestServiceCodesMapToKinds(t *testing.T) { cases := []struct { name string @@ -271,6 +316,34 @@ func TestProbeCleanupFailureIsReturnedAndNotCached(t *testing.T) { assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 6) } +func TestProbeCleansUpWhenFirstPutResponseIsLost(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + err := s.ProbeConditionalWrites(context.Background()) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Empty(t, fake.Keys(), "the probe object committed by the lost response is removed") +} + +func TestProbeCleanupFailureKeepsUnsupportedVerdictAndIsNotCached(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + err := s.ProbeConditionalWrites(context.Background()) + require.ErrorContains(t, err, "cleanup probe object") + assert.True(t, blob.IsConditionalUnsupported(err), "the probe verdict stays visible: %v", err) + + fake.ClearFaults() + err = s.ProbeConditionalWrites(context.Background()) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err)) + assert.NotContains(t, err.Error(), "cleanup probe object", "the verdict was not cached with the failed cleanup") + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 4, "the probe ran again after the failed cleanup") +} + type ignoresIfMatch struct { API put, delete bool diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go index 56a53a2ff92..f09d96cefa2 100644 --- a/pkg/artifact/blob/s3/multipart.go +++ b/pkg/artifact/blob/s3/multipart.go @@ -187,16 +187,20 @@ func acquireBuffer(ctx context.Context, free chan []byte, partSize int64) ([]byt // readPart fills buf from body, starting with a carried byte when one was already consumed. It // returns the bytes in buf; zero means the body ended exactly on the previous boundary. func readPart(body io.Reader, buf, carry []byte) (int, error) { - offset := copy(buf, carry) - n, err := io.ReadFull(body, buf[offset:]) - switch { - case err == nil: - return offset + n, nil - case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): - return offset + n, nil - default: - return 0, err + filled := copy(buf, carry) + for filled < len(buf) { + n, err := body.Read(buf[filled:]) + filled += n + if errors.Is(err, io.EOF) { + return filled, nil + } + if err != nil { + // Any other error, including a source-returned io.ErrUnexpectedEOF, is a read failure + // and must abort publication. + return 0, err + } } + return filled, nil } // uploadPart sends one part and returns its ETag. @@ -215,7 +219,9 @@ func (m *multipart) uploadPart(ctx context.Context, number int32, data []byte) ( return out.ETag, nil } -// complete assembles the parts into the object under the write condition. +// complete assembles the parts into the object under the write condition. SDK retries are always +// disabled, conditional or not: a retried completion whose first attempt already committed would +// meet a consumed upload ID and look like a definite failure. func (m *multipart) complete(ctx context.Context, parts []types.CompletedPart, total int64) (*blob.ObjectInfo, error) { cond := m.plan.opts.Condition in := &s3.CompleteMultipartUploadInput{ @@ -225,7 +231,7 @@ func (m *multipart) complete(ctx context.Context, parts []types.CompletedPart, t MultipartUpload: &types.CompletedMultipartUpload{Parts: parts}, } in.IfNoneMatch, in.IfMatch = conditionHeaders(cond) - out, err := m.plan.api.CompleteMultipartUpload(ctx, in, append(retryOptions(cond.IsConditional()), skipOptionalChecksums)...) + out, err := m.plan.api.CompleteMultipartUpload(ctx, in, noRetry, skipOptionalChecksums) if err != nil { classified := classify(m.plan.request(true), err) if blob.IsAmbiguous(classified) { diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go index d214ea4f5bf..ed77f6a2e57 100644 --- a/pkg/artifact/blob/s3/options.go +++ b/pkg/artifact/blob/s3/options.go @@ -29,7 +29,9 @@ const ( ConditionalAuto ConditionalMode = iota // ConditionalProbe is like Auto, but first proves once per store that the endpoint enforces - // If-None-Match and If-Match on writes and deletes (see [Store.ProbeConditionalWrites]). Use it for endpoints that + // If-None-Match and If-Match on single-object PutObject and DeleteObject requests (see + // [Store.ProbeConditionalWrites]). The probe does not exercise CompleteMultipartUpload, so + // enforcement on multipart completion is trusted, not verified. Use it for endpoints that // might silently ignore the headers. ConditionalProbe diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index 5a185fe753b..526ae60d30f 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -65,8 +65,8 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) ( if err := blob.ValidateTags(tags); err != nil { return nil, err } - if opts.Condition.Kind == blob.ConditionIfMatch && opts.Condition.ETag == "" { - return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + if err := validateCondition(key, opts.Condition); err != nil { + return nil, err } if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { return nil, err @@ -88,6 +88,22 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) ( }, nil } +// validateCondition rejects conditions the store cannot translate into request headers. Unknown +// kinds must fail before any write: sending them unconditionally would silently drop the guard. +func validateCondition(key string, cond blob.Condition) error { + switch cond.Kind { + case blob.ConditionNone, blob.ConditionIfAbsent: + return nil + case blob.ConditionIfMatch: + if cond.ETag == "" { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + } + return nil + default: + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: unknown condition kind %d", errUtils.ErrBlobInvalid, cond.Kind)) + } +} + // effectivePartSize grows the part size so a known-size body fits within the S3 part limit. func effectivePartSize(size, partSize int64) int64 { if size <= 0 { diff --git a/pkg/artifact/blob/s3/put_test.go b/pkg/artifact/blob/s3/put_test.go index 0abef912bb9..3f5fe0bb6f3 100644 --- a/pkg/artifact/blob/s3/put_test.go +++ b/pkg/artifact/blob/s3/put_test.go @@ -260,3 +260,21 @@ func (r *errAfterReader) Read(p []byte) (int, error) { r.data = r.data[n:] return n, nil } + +func TestInvalidConditionsAreRejectedBeforeAnyRequest(t *testing.T) { + cases := map[string]blob.Condition{ + "unknown kind": {Kind: blob.ConditionKind(99)}, + "negative kind": {Kind: blob.ConditionKind(-1)}, + "if-match no etag": {Kind: blob.ConditionIfMatch}, + "unknown with etag": {Kind: blob.ConditionKind(7), ETag: `"abc"`}, + } + for name, cond := range cases { + t.Run(name, func(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, Condition: cond}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests(), "an invalid condition never reaches the endpoint") + }) + } +} diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 38e95f37f87..0bff11d6e07 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -366,6 +366,7 @@ func TestDiscardBodiesKeepsSizesAndETags(t *testing.T) { func TestEmulatorFromEnvSkipsWhenUnset(t *testing.T) { t.Setenv(EmulatorEndpointEnv, "") t.Setenv(EmulatorBucketEnv, "") + t.Setenv(EmulatorRegionEnv, "") result := t.Run("inner", func(t *testing.T) { EmulatorFromEnv(t) t.Error("EmulatorFromEnv must skip when the environment is unset") From f6aeb2ddde01ee9a62e93d9d3d3dd71998f9e2d6 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:09:47 -0600 Subject: [PATCH 22/43] fix(artifact): tighten blob tag, key, prefix, and staging handling Count S3 tag limits in UTF-16 code units, reject reserved temp and attribute names in every key segment, honor WithLiteralBackslashes when validating list prefixes, and prune empty staging directories after a failed local write. ListAll tests now use the generated MockStore. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/blob/blob_test.go | 59 ++++++++++++++++----------- pkg/artifact/blob/local/list.go | 15 +++++-- pkg/artifact/blob/local/put.go | 9 +++- pkg/artifact/blob/local/store.go | 17 ++++++-- pkg/artifact/blob/local/store_test.go | 41 ++++++++++++++++++- pkg/artifact/blob/tags.go | 22 ++++++++-- 6 files changed, 128 insertions(+), 35 deletions(-) diff --git a/pkg/artifact/blob/blob_test.go b/pkg/artifact/blob/blob_test.go index 10384840652..e6790371145 100644 --- a/pkg/artifact/blob/blob_test.go +++ b/pkg/artifact/blob/blob_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" ) @@ -137,17 +138,28 @@ func TestCapabilityChecksFailClearly(t *testing.T) { } func TestListAllFollowsCursorsAndMergesPrefixes(t *testing.T) { - store := &listOnlyStore{pages: []ListPage{ + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + pages := []ListPage{ {Objects: []ObjectInfo{{Key: "a"}, {Key: "b"}}, CommonPrefixes: []string{"p/"}, NextCursor: "c1"}, {Objects: []ObjectInfo{{Key: "c"}}, NextCursor: "c2"}, {Objects: []ObjectInfo{{Key: "d"}}, CommonPrefixes: []string{"q/"}}, - }} + } + var calls []ListOptions + store.EXPECT().List(gomock.Any(), gomock.Any()).Times(len(pages)). + DoAndReturn(func(_ context.Context, opts ListOptions) (*ListPage, error) { + calls = append(calls, opts) + page := pages[len(calls)-1] + return &page, nil + }) + objects, prefixes, err := ListAll(context.Background(), store, ListOptions{Prefix: "x/", Limit: 2}) require.NoError(t, err) assert.Equal(t, []string{"a", "b", "c", "d"}, []string{objects[0].Key, objects[1].Key, objects[2].Key, objects[3].Key}) assert.Equal(t, []string{"p/", "q/"}, prefixes) - assert.Equal(t, []string{"", "c1", "c2"}, []string{store.calls[0].Cursor, store.calls[1].Cursor, store.calls[2].Cursor}) - for _, call := range store.calls { + require.Len(t, calls, 3) + assert.Equal(t, []string{"", "c1", "c2"}, []string{calls[0].Cursor, calls[1].Cursor, calls[2].Cursor}) + for _, call := range calls { assert.Equal(t, "x/", call.Prefix) assert.Equal(t, 2, call.Limit) } @@ -155,31 +167,32 @@ func TestListAllFollowsCursorsAndMergesPrefixes(t *testing.T) { func TestListAllStopsOnError(t *testing.T) { boom := errors.New("boom") - store := &listOnlyStore{err: boom} + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + store.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, boom) _, _, err := ListAll(context.Background(), store, ListOptions{}) require.ErrorIs(t, err, boom) } -// listOnlyStore implements only List; other methods come from the generated mock and would fail -// the test if called. -type listOnlyStore struct { - MockStore - pages []ListPage - calls []ListOptions - err error -} - -func (s *listOnlyStore) List(_ context.Context, opts ListOptions) (*ListPage, error) { - s.calls = append(s.calls, opts) - if s.err != nil { - return nil, s.err - } - page := s.pages[len(s.calls)-1] - return &page, nil -} - func TestValidateTagsRejectsInvalidUTF8(t *testing.T) { for _, tags := range []map[string]string{{"\xff": "value"}, {"key": "\xff"}} { require.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid) } } + +func TestValidateTagsCountsUTF16CodeUnits(t *testing.T) { + const emoji = "\U0001F600" // One rune, two UTF-16 code units. + keyAtLimit := strings.Repeat(emoji, MaxTagKeyChars/2) + valueAtLimit := strings.Repeat(emoji, MaxTagValueChars/2) + + require.NoError(t, ValidateTags(map[string]string{keyAtLimit: valueAtLimit}), "exactly at both limits") + for name, tags := range map[string]map[string]string{ + "key one unit over": {keyAtLimit + "a": "v"}, + "key one rune over": {keyAtLimit + emoji: "v"}, + "value one unit over": {"k": valueAtLimit + "a"}, + "value one rune over": {"k": valueAtLimit + emoji}, + "rune count under cap": {strings.Repeat(emoji, MaxTagKeyChars/2+1): "v"}, + } { + require.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid, name) + } +} diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go index 1150e6f1ffa..05b8eb33bc3 100644 --- a/pkg/artifact/blob/local/list.go +++ b/pkg/artifact/blob/local/list.go @@ -31,7 +31,7 @@ func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage if err := checkContext(ctx, blob.OpList, opts.Prefix); err != nil { return nil, err } - if err := validatePrefix(opts.Prefix); err != nil { + if err := s.validatePrefix(opts.Prefix); err != nil { return nil, err } var entries []entry @@ -175,14 +175,21 @@ func rollUp(key string, opts blob.ListOptions) (string, bool) { return opts.Prefix + rest[:idx+len(opts.Delimiter)], true } -// validatePrefix rejects listing prefixes that could leave the root. -func validatePrefix(prefix string) error { +// literalBackslashes reports whether backslashes are ordinary characters in keys: the store opted +// in with WithLiteralBackslashes and the platform does not treat them as path separators. +func (s *Store) literalBackslashes() bool { + return s.allowBackslash && os.PathSeparator != '\\' +} + +// validatePrefix rejects listing prefixes that could leave the root. It applies the same +// platform-aware backslash rule as key validation. +func (s *Store) validatePrefix(prefix string) error { for _, segment := range strings.Split(prefix, "/") { if segment == ".." { return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix contains a relative segment", errUtils.ErrBlobInvalid)) } } - if strings.ContainsRune(prefix, '\\') || strings.ContainsRune(prefix, 0) || strings.HasPrefix(prefix, "/") { + if (!s.literalBackslashes() && strings.ContainsRune(prefix, '\\')) || strings.ContainsRune(prefix, 0) || strings.HasPrefix(prefix, "/") { return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix must be a relative slash path", errUtils.ErrBlobInvalid)) } return nil diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index 41ae86b1032..13eb71c524b 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -240,15 +240,22 @@ func (s *Store) statCommitted(root *os.Root, key, target string, pending *staged return &blob.ObjectInfo{Key: key, Size: pending.size, ETag: pending.etag, LastModified: info.ModTime()}, nil } -// discard removes any temporary files that were not committed. +// discard removes any temporary files that were not committed, then prunes the staging +// directories that this put created and left empty. Non-empty directories are kept. func (p *staged) discard(root *os.Root) { if p.preserve { return } + var dirs []string if p.tempPath != "" { _ = root.Remove(p.tempPath) + dirs = append(dirs, filepath.Dir(p.tempPath)) } if p.attrsTemp != "" { _ = root.Remove(p.attrsTemp) + dirs = append(dirs, filepath.Dir(p.attrsTemp)) + } + for _, dir := range dirs { + rootRemoveEmptyDirs(root, dir) } } diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index d2809362dbf..fb41bd5e27a 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -85,19 +85,30 @@ func (s *Store) Capabilities() blob.Capabilities { // validate rejects keys that are unsafe on a filesystem or collide with the store's own files. func (s *Store) validate(op, key string) error { validationKey := key - if s.allowBackslash && os.PathSeparator != '\\' { + if s.literalBackslashes() { validationKey = strings.ReplaceAll(key, "\\", "_") } if err := blob.ValidateKey(validationKey); err != nil { return err } - last := key[strings.LastIndexByte(key, '/')+1:] - if strings.HasPrefix(last, tempPrefix) || strings.HasSuffix(last, attrsSuffix) || key == lockName { + if key == lockName || hasReservedSegment(key) { return blob.NewError(blob.KindInvalid, op, key, fmt.Errorf("%w: key uses a reserved name", errUtils.ErrBlobInvalid)) } return nil } +// hasReservedSegment reports whether any slash-separated segment of key looks like one of the +// store's own temporary or attribute files. A reserved directory name would otherwise hide the +// objects under it from listings, which skip those names at every level. +func hasReservedSegment(key string) bool { + for _, segment := range strings.Split(key, "/") { + if strings.HasPrefix(segment, tempPrefix) || strings.HasSuffix(segment, attrsSuffix) { + return true + } + } + return false +} + // checkContext fails fast when ctx is already done. func checkContext(ctx context.Context, op, key string) error { if err := ctx.Err(); err != nil { diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index e97601dffe6..df18aee9d50 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -84,7 +84,7 @@ func TestAttributeFileHiddenAndRemoved(t *testing.T) { func TestReservedAndUnsafeKeysRejected(t *testing.T) { store := newStore(t) - for _, key := range []string{"../escape", "a/../../b", "/abs", "a//b", "a/", ".", "d/" + tempPrefix + "x", "d/x" + attrsSuffix, lockName, "a\\b"} { + for _, key := range []string{"../escape", "a/../../b", "/abs", "a//b", "a/", ".", "d/" + tempPrefix + "x", "d/x" + attrsSuffix, tempPrefix + "dir/child", "a/" + tempPrefix + "dir/child", "x" + attrsSuffix + "/child", lockName, "a\\b"} { _, err := store.Put(context.Background(), key, bytes.NewReader(nil), blob.PutOptions{Size: 0}) require.Error(t, err, key) assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) @@ -278,3 +278,42 @@ func TestPutDirectoryConflictPreservesExistingSidecar(t *testing.T) { require.NoError(t, err) assert.Equal(t, original, got) } + +func TestFailedPutPrunesEmptyStagingDirectories(t *testing.T) { + for name, opts := range map[string][]Option{"managed": nil, "plain": {WithPlainFileCompatibility()}} { + t.Run(name, func(t *testing.T) { + store := newStore(t, opts...) + _, err := store.Put(context.Background(), "keep/sibling", bytes.NewReader([]byte("ok")), blob.PutOptions{Size: 2}) + require.NoError(t, err) + + _, err = store.Put(context.Background(), "keep/new/deep/object", bytes.NewReader([]byte("abc")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a body longer than the declared size fails") + + assert.NoDirExists(t, filepath.Join(store.Root(), "keep", "new"), "staging directories created by the failed put are pruned") + assert.DirExists(t, filepath.Join(store.Root(), "keep"), "a non-empty parent is left intact") + assert.FileExists(t, filepath.Join(store.Root(), "keep", "sibling")) + }) + } +} + +func TestListPrefixHonorsLiteralBackslashes(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("backslash is a path separator on Windows") + } + strict := newStore(t) + _, err := strict.List(context.Background(), blob.ListOptions{Prefix: `dir\`}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "backslashes stay rejected unless the store opts in") + + literal := newStore(t, WithLiteralBackslashes()) + _, err = literal.Put(context.Background(), `dir\file`, bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + page, err := literal.List(context.Background(), blob.ListOptions{Prefix: `dir\`}) + require.NoError(t, err) + require.Len(t, page.Objects, 1) + assert.Equal(t, `dir\file`, page.Objects[0].Key) + + for _, prefix := range []string{"/abs", "a\x00b", "../x"} { + _, err = literal.List(context.Background(), blob.ListOptions{Prefix: prefix}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "%q stays rejected", prefix) + } +} diff --git a/pkg/artifact/blob/tags.go b/pkg/artifact/blob/tags.go index 662dd223ead..169aed4183a 100644 --- a/pkg/artifact/blob/tags.go +++ b/pkg/artifact/blob/tags.go @@ -16,6 +16,9 @@ const ( MaxTagValueChars = 256 ) +// supplementaryPlane is the first rune that needs a UTF-16 surrogate pair. +const supplementaryPlane = 0x10000 + // MergeTags returns defaults overlaid with overrides. It returns nil when both are empty and never // aliases either input. func MergeTags(defaults, overrides map[string]string) map[string]string { @@ -31,7 +34,7 @@ func MergeTags(defaults, overrides map[string]string) map[string]string { } // ValidateTags checks the S3 object tag limits: at most [MaxTags] tags, non-empty keys of at most -// [MaxTagKeyChars] characters, and values of at most [MaxTagValueChars] characters. +// [MaxTagKeyChars] UTF-16 code units, and values of at most [MaxTagValueChars] UTF-16 code units. func ValidateTags(tags map[string]string) error { defer perf.Track(nil, "blob.ValidateTags")() @@ -39,16 +42,29 @@ func ValidateTags(tags map[string]string) error { return tagError(fmt.Sprintf("%d tags exceed the limit of %d", len(tags), MaxTags)) } for key, value := range tags { - if key == "" || !utf8.ValidString(key) || utf8.RuneCountInString(key) > MaxTagKeyChars { + if key == "" || !utf8.ValidString(key) || utf16Len(key) > MaxTagKeyChars { return tagError(fmt.Sprintf("tag key %q must be 1-%d characters", key, MaxTagKeyChars)) } - if !utf8.ValidString(value) || utf8.RuneCountInString(value) > MaxTagValueChars { + if !utf8.ValidString(value) || utf16Len(value) > MaxTagValueChars { return tagError(fmt.Sprintf("tag %q value exceeds %d characters", key, MaxTagValueChars)) } } return nil } +// utf16Len counts UTF-16 code units, which is how S3 measures tag lengths: a supplementary +// character counts as two. +func utf16Len(s string) int { + n := 0 + for _, r := range s { + n++ + if r >= supplementaryPlane { + n++ + } + } + return n +} + func tagError(reason string) error { return NewError(KindInvalid, OpValidate, "", fmt.Errorf("%w: %s", errUtils.ErrBlobInvalid, reason)) } From 0e5051f28929124898a37e9d23b12a8fa9b11675 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:10:20 -0600 Subject: [PATCH 23/43] fix(artifact): harden layout identifiers and make location rendering hermetic Hash operation IDs that start with the reserved sha256- prefix, validate Prior.RecordDigest format, and remove randInt, ago, and zone-aware date helpers from key_pattern. Verifier tests use the generated MockVerifier and MockSigner. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/layout/layout_test.go | 31 ++++++++++++ pkg/artifact/layout/location.go | 7 ++- pkg/artifact/layout/paths.go | 6 ++- pkg/artifact/layout/records.go | 3 ++ pkg/artifact/local/verifier_test.go | 74 +++++++++++++++-------------- 5 files changed, 82 insertions(+), 39 deletions(-) diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go index a41cca3a4a5..b40020e783a 100644 --- a/pkg/artifact/layout/layout_test.go +++ b/pkg/artifact/layout/layout_test.go @@ -45,6 +45,12 @@ func TestPaths(t *testing.T) { assert.NotEqual(t, LockPath("a"), LockPath("b")) } +func TestOperationFileNameCannotCollideWithHashedNames(t *testing.T) { + hashed := OperationFileName("op/with/slash") + require.True(t, strings.HasPrefix(hashed, "sha256-")) + assert.NotEqual(t, hashed, OperationFileName(hashed), "a literal id equal to another id's hashed name is hashed itself") +} + func TestOperationFileName(t *testing.T) { tests := []struct { name string @@ -58,6 +64,7 @@ func TestOperationFileName(t *testing.T) { {"traversal", "../x", false}, {"overlong", strings.Repeat("a", 200), false}, {"empty", "", false}, + {"reserved hash prefix", "sha256-deadbeef", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -129,6 +136,23 @@ func TestRenderLocation(t *testing.T) { } } +func TestRenderLocation_RejectsRandomAndClockDependentFunctions(t *testing.T) { + in := &PatternInput{Name: "app", Version: "1", Kind: "x", CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)} + for _, name := range []string{ + "now", "ago", "env", "randAlphaNum", "randAlpha", "randNumeric", "randAscii", "randInt", "uuidv4", + "shuffle", "dateInZone", "date_in_zone", "htmlDate", "htmlDateInZone", + } { + t.Run(name, func(t *testing.T) { + _, err := RenderLocation("{{ "+name+" }}", in, artifact.PackagingNone) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "%s must not be defined in key_pattern", name) + assert.Contains(t, err.Error(), "not defined") + }) + } + got, err := RenderLocation("{{ .Name }}/{{ date \"2006\" .CreatedAt }}", in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, "app/2026", got, "the deterministic UTC date function remains available") +} + func TestRenderLocation_DateIsUTCAndStable(t *testing.T) { // 23:30 at UTC-5 is already the next day in UTC; the captured instant decides, not the zone. zone := time.FixedZone("UTC-5", -5*60*60) @@ -373,6 +397,13 @@ func TestOperation_JournalRoundTrip(t *testing.T) { broken.Prior = &Prior{} _, err = broken.Marshal() require.Error(t, err) + for _, malformed := range []digest.Digest{"x", "sha256:short", digest.Digest("md5:" + digestA.Encoded()), digest.Digest(strings.Repeat("g", 71))} { + broken = sampleOperation() + broken.Prior = &Prior{RecordDigest: malformed, Location: "2026/10/29/team/app/1.0.0"} + _, err = broken.Marshal() + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "malformed prior digest %q", malformed) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + } } func TestEvidence_EnvelopeBindsFileNameToContent(t *testing.T) { diff --git a/pkg/artifact/layout/location.go b/pkg/artifact/layout/location.go index 8236ff92c68..a516d1769e6 100644 --- a/pkg/artifact/layout/location.go +++ b/pkg/artifact/layout/location.go @@ -30,10 +30,13 @@ const deleteRune = 0x7f // depend only on the captured publication inputs: not on the clock, the process environment, the // network, or randomness. var nonHermeticFunctions = []string{ - "now", "env", "expandenv", "getHostByName", "uuidv4", "randAlphaNum", "randAlpha", "randNumeric", - "randAscii", "randBytes", "shuffle", "genPrivateKey", "genCA", "genCAWithKey", "genSelfSignedCert", + "now", "ago", "env", "expandenv", "getHostByName", "uuidv4", "randAlphaNum", "randAlpha", "randNumeric", + "randAscii", "randBytes", "randInt", "shuffle", "genPrivateKey", "genCA", "genCAWithKey", "genSelfSignedCert", "genSelfSignedCertWithKey", "genSignedCert", "genSignedCertWithKey", "buildCustomCert", "derivePassword", "htpasswd", "bcrypt", "encryptAES", "decryptAES", + // Sprig's zone-aware date helpers fall back to the current time for unsupported input and + // format in the host's local zone. The "date" function is replaced by a UTC variant instead. + "dateInZone", "date_in_zone", "htmlDate", "htmlDateInZone", } // PatternInput holds the values a key_pattern may reference: .Name, .Version, .Kind, and diff --git a/pkg/artifact/layout/paths.go b/pkg/artifact/layout/paths.go index 13a58bfa20a..4095c4a15d4 100644 --- a/pkg/artifact/layout/paths.go +++ b/pkg/artifact/layout/paths.go @@ -99,11 +99,13 @@ func RepresentationPath(key string) string { // OperationFileName returns the file stem for an operation identifier. Safe identifiers are used // verbatim; anything else (slashes, spaces, overlong values) maps to a hash so that no identifier -// can address a path outside the operations directory. +// can address a path outside the operations directory. The hashed prefix is reserved: an +// identifier that starts with it is hashed too, so a literal identifier can never collide with +// the hashed file name of another identifier. func OperationFileName(operationID string) string { defer perf.Track(nil, "layout.OperationFileName")() - if len(operationID) <= maxSafeOperationID && safeOperationID.MatchString(operationID) && !windowsDeviceName(operationID) { + if len(operationID) <= maxSafeOperationID && !strings.HasPrefix(operationID, hashedOperationPrefix) && safeOperationID.MatchString(operationID) && !windowsDeviceName(operationID) { return operationID } sum := sha256.Sum256([]byte(operationID)) diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go index f7f35be1bd5..6e1eed54282 100644 --- a/pkg/artifact/layout/records.go +++ b/pkg/artifact/layout/records.go @@ -393,6 +393,9 @@ func validatePrior(prior *Prior) error { if prior.RecordDigest == "" { return invalid(kindOperation, "prior needs a record_digest") } + if err := artifact.ValidateDigest(prior.RecordDigest); err != nil { + return invalid(kindOperation, "prior record_digest is malformed: %v", err) + } if prior.Location != "" { if _, err := CleanLocation(prior.Location); err != nil { return invalid(kindOperation, "prior location %q is not a valid relative location", prior.Location) diff --git a/pkg/artifact/local/verifier_test.go b/pkg/artifact/local/verifier_test.go index a3c92bd1840..124126b83e9 100644 --- a/pkg/artifact/local/verifier_test.go +++ b/pkg/artifact/local/verifier_test.go @@ -10,36 +10,39 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/artifact/layout" ) -// stubVerifier returns a canned verifier outcome. -type stubVerifier struct { - result *artifact.VerificationResult - err error - seen int -} - -func (*stubVerifier) Formats() []artifact.EvidenceFormat { - return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle} -} - -func (s *stubVerifier) Verify(_ context.Context, _ artifact.SignatureSubject, evidence []artifact.EvidenceBlob, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { - s.seen = len(evidence) - return s.result, s.err -} +// verifierProbe records how many evidence blobs reached the verifier. +type verifierProbe struct{ seen int } -type fixedSigner struct{ data []byte } - -func (*fixedSigner) Formats() []artifact.EvidenceFormat { - return []artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle} +// mockVerifier returns a generated verifier mock with a canned outcome. Its probe records the +// evidence count of the latest Verify call. +func mockVerifier(t *testing.T, result *artifact.VerificationResult, err error) (*artifact.MockVerifier, *verifierProbe) { + t.Helper() + probe := &verifierProbe{} + verifier := artifact.NewMockVerifier(gomock.NewController(t)) + verifier.EXPECT().Formats().AnyTimes().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}) + verifier.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes(). + DoAndReturn(func(_ context.Context, _ artifact.SignatureSubject, evidence []artifact.EvidenceBlob, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + probe.seen = len(evidence) + return result, err + }) + return verifier, probe } -func (f *fixedSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { - return &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: f.data}, nil +// mockSigner returns a generated signer mock that always signs data. +func mockSigner(t *testing.T, data []byte) *artifact.MockSigner { + t.Helper() + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().AnyTimes().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}) + signer.EXPECT().Sign(gomock.Any(), gomock.Any()).AnyTimes(). + Return(&artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: data}, nil) + return signer } func signedWith(t *testing.T, verifier artifact.Verifier) (*Repository, artifact.SignatureSubject) { @@ -47,7 +50,7 @@ func signedWith(t *testing.T, verifier artifact.Verifier) (*Repository, artifact repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(verifier)}}) publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) subject := recordSubject(t, repo) - _, err := repo.Sign(context.Background(), subject, &fixedSigner{data: []byte(`{"bundle":1}`)}, artifact.SignOptions{}) + _, err := repo.Sign(context.Background(), subject, mockSigner(t, []byte(`{"bundle":1}`)), artifact.SignOptions{}) require.NoError(t, err) return repo, subject } @@ -73,7 +76,7 @@ func TestVerify_TranslatesVerifierOutcomes(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - stub := &stubVerifier{result: tt.result, err: tt.err} + stub, probe := mockVerifier(t, tt.result, tt.err) repo, subject := signedWith(t, stub) result, err := repo.Verify(context.Background(), subject, trustedPolicy) if tt.wantErr != nil { @@ -85,16 +88,16 @@ func TestVerify_TranslatesVerifierOutcomes(t *testing.T) { assert.Equal(t, tt.wantStatus, result.Status) assert.Equal(t, subject, result.Subject) assert.NotEmpty(t, result.PolicyFingerprint, "the applied policy is always fingerprinted") - assert.Equal(t, 1, stub.seen) + assert.Equal(t, 1, probe.seen) }) } } func TestVerify_ReportsDiagnosticsAlongsideTheVerifier(t *testing.T) { - stub := &stubVerifier{result: &artifact.VerificationResult{ + stub, _ := mockVerifier(t, &artifact.VerificationResult{ Status: artifact.VerificationVerified, PolicyFingerprint: "verifier-fingerprint", Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityInfo, Code: "from-verifier", Message: "ok"}}, - }} + }, nil) repo, subject := signedWith(t, stub) dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) require.NoError(t, os.WriteFile(filepath.Join(dir, "note.txt"), []byte("x"), 0o644)) @@ -113,7 +116,7 @@ func TestVerify_ReportsDiagnosticsAlongsideTheVerifier(t *testing.T) { } func TestVerify_ReadsAllValidEvidence(t *testing.T) { - stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) repo, subject := signedWith(t, stub) for index := range maxMalformedEvidence + 5 { evidence, err := layout.NewEvidence(subject, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte(fmt.Sprintf("bundle-%d", index))}) @@ -125,22 +128,23 @@ func TestVerify_ReadsAllValidEvidence(t *testing.T) { } result, err := repo.Verify(context.Background(), subject, trustedPolicy) require.NoError(t, err) - assert.Equal(t, maxMalformedEvidence+6, stub.seen, "all valid evidence reaches the verifier") + assert.Equal(t, maxMalformedEvidence+6, probe.seen, "all valid evidence reaches the verifier") assert.Equal(t, artifact.VerificationVerified, result.Status) } func TestSign_RejectsOversizeEvidence(t *testing.T) { - repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(&stubVerifier{})}}) + verifier, _ := mockVerifier(t, nil, nil) + repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(verifier)}}) publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) subject := recordSubject(t, repo) - _, err := repo.Sign(context.Background(), subject, &fixedSigner{data: make([]byte, artifact.MaxEvidenceSize+1)}, artifact.SignOptions{}) + _, err := repo.Sign(context.Background(), subject, mockSigner(t, make([]byte, artifact.MaxEvidenceSize+1)), artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) - _, err = repo.Sign(context.Background(), subject, &fixedSigner{data: nil}, artifact.SignOptions{}) + _, err = repo.Sign(context.Background(), subject, mockSigner(t, nil), artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "empty evidence is invalid") } func TestVerifyRejectsUnreadableEvidenceDirectory(t *testing.T) { - stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) repo, subject := signedWith(t, stub) dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) require.NoError(t, os.RemoveAll(dir)) @@ -148,11 +152,11 @@ func TestVerifyRejectsUnreadableEvidenceDirectory(t *testing.T) { result, err := repo.Verify(context.Background(), subject, trustedPolicy) require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) require.Contains(t, result.Diagnostics[0].Code, "evidence-read-failed") - assert.Zero(t, stub.seen) + assert.Zero(t, probe.seen) } func TestVerifyFailsClosedOnTooManyMalformedEvidenceFiles(t *testing.T) { - stub := &stubVerifier{result: &artifact.VerificationResult{Status: artifact.VerificationVerified}} + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) repo, subject := signedWith(t, stub) dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) for i := range maxMalformedEvidence + 1 { @@ -161,5 +165,5 @@ func TestVerifyFailsClosedOnTooManyMalformedEvidenceFiles(t *testing.T) { _, err := repo.Verify(context.Background(), subject, trustedPolicy) require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) require.ErrorIs(t, err, errTooManyMalformedEvidence) - assert.Zero(t, stub.seen) + assert.Zero(t, probe.seen) } From 86a9354b3dfe773b446c65499c4db22a2506410d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 08:51:47 -0600 Subject: [PATCH 24/43] fix(artifact): refuse an unusable signer before local signing runs The local repository called the signer before it checked whether it could store any of the signer's evidence formats. It now refuses such a signer up front with Capabilities.CheckSignerFormats, matching the conformance contract. The signature binding tests use generated signer mocks instead of hand-written doubles. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/local/evidence.go | 4 ++++ pkg/artifact/local/signature_test.go | 25 +++++++++---------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index 7c2ede000bd..d65065adcd6 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -143,6 +143,10 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject if signer == nil { return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) } + caps := r.Capabilities() + if err := caps.CheckSignerFormats(r.alias, signer); err != nil { + return nil, err + } t, err := r.openTree(false) if err != nil { return nil, r.missingRoot(err, artifact.Ref{Name: "(subject)", Digest: subject.Digest}) diff --git a/pkg/artifact/local/signature_test.go b/pkg/artifact/local/signature_test.go index d74f425b004..f2ab4f75f20 100644 --- a/pkg/artifact/local/signature_test.go +++ b/pkg/artifact/local/signature_test.go @@ -9,6 +9,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -196,29 +197,21 @@ func TestSignature_SubjectBinding(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) }) t.Run("format the repository cannot verify", func(t *testing.T) { - _, err := repo.Sign(context.Background(), subject, &foreignSigner{}, artifact.SignOptions{}) + // Sign has no expectation: an unusable signer must be refused before it runs. + foreign := artifact.NewMockSigner(gomock.NewController(t)) + foreign.EXPECT().Formats().Return([]artifact.EvidenceFormat{"other"}).AnyTimes() + _, err := repo.Sign(context.Background(), subject, foreign, artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) }) t.Run("signer produces nothing", func(t *testing.T) { - _, err := repo.Sign(context.Background(), subject, &nilSigner{}, artifact.SignOptions{}) + empty := artifact.NewMockSigner(gomock.NewController(t)) + empty.EXPECT().Formats().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}).AnyTimes() + empty.EXPECT().Sign(gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := repo.Sign(context.Background(), subject, empty, artifact.SignOptions{}) require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) }) } -type foreignSigner struct{} - -func (*foreignSigner) Formats() []artifact.EvidenceFormat { return []artifact.EvidenceFormat{"other"} } - -func (*foreignSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { - return &artifact.EvidencePayload{Format: "other", MediaType: "x", Data: []byte("x")}, nil -} - -type nilSigner struct{ foreignSigner } - -func (*nilSigner) Sign(context.Context, artifact.SignatureSubject) (*artifact.EvidencePayload, error) { - return nil, nil -} - func TestSignature_WithoutVerifierIsACapabilityError(t *testing.T) { repo := newRepo(t, dirSpec()) publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) From ff26d9b1b7c739a978b2cba58202e8c97fec28f9 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 13 Nov 2026 12:02:34 -0600 Subject: [PATCH 25/43] refactor(artifact): keep local signing within lint limits Code scanning flagged Repository.Sign at cyclomatic complexity 11 after the signer format check was added. The subject, signer and format checks move into checkSignRequest. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- pkg/artifact/local/evidence.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index d65065adcd6..5f5c873fb7a 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -137,14 +137,7 @@ func (r *Repository) checkSubject(operation string, subject artifact.SignatureSu func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { defer perf.Track(nil, "local.Repository.Sign")() - if err := r.checkSubject("sign", subject); err != nil { - return nil, err - } - if signer == nil { - return nil, fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) - } - caps := r.Capabilities() - if err := caps.CheckSignerFormats(r.alias, signer); err != nil { + if err := r.checkSignRequest(subject, signer); err != nil { return nil, err } t, err := r.openTree(false) @@ -173,6 +166,19 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject return []artifact.SignatureEvidence{stored}, nil } +// checkSignRequest refuses an unsupported subject, a missing signer, and a signer whose evidence +// formats the repository cannot store, before anything is opened or signed. +func (r *Repository) checkSignRequest(subject artifact.SignatureSubject, signer artifact.Signer) error { + if err := r.checkSubject("sign", subject); err != nil { + return err + } + if signer == nil { + return fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) + } + caps := r.Capabilities() + return caps.CheckSignerFormats(r.alias, signer) +} + // storeEvidence validates and writes one evidence payload. func (r *Repository) storeEvidence(t *tree, subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (artifact.SignatureEvidence, error) { if caps := r.Capabilities(); !caps.HasEvidenceFormat(payload.Format) { From 9b25ff8c717a655861196e1c1f2d903127726eb7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:08:49 -0600 Subject: [PATCH 26/43] fix(artifact/blob): give unclassified blob failures a shared sentinel KindOther errors carried only the OS or SDK cause. They now match ErrBlobOperationFailed, so every classified blob error has a static sentinel while the underlying cause stays in the chain. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 4 ++++ pkg/artifact/blob/blob_test.go | 5 +++-- pkg/artifact/blob/errors.go | 26 ++++++++++---------------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index caba6f75f31..74d0f5b8f22 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2164,6 +2164,10 @@ var ( // The generic blob transport fails closed instead of falling back to ambient credentials. ErrBlobIdentityUnresolved = errors.New("blob identity could not be resolved") + // ErrBlobOperationFailed is the sentinel for blob failures without a more specific class, such as + // local I/O errors or an SDK failure with no mapped meaning. The underlying cause stays in the chain. + ErrBlobOperationFailed = errors.New("blob operation failed") + // ErrBlobInvalidChunkTerminator is returned by the S3 test endpoint when an aws-chunked request // body has a chunk that is not followed by the required CRLF terminator. ErrBlobInvalidChunkTerminator = errors.New("invalid aws-chunked chunk terminator") diff --git a/pkg/artifact/blob/blob_test.go b/pkg/artifact/blob/blob_test.go index e6790371145..24350ab6a46 100644 --- a/pkg/artifact/blob/blob_test.go +++ b/pkg/artifact/blob/blob_test.go @@ -21,6 +21,7 @@ func TestErrorKindsMatchSentinelsAndCauses(t *testing.T) { sentinel error check func(error) bool }{ + {KindOther, errUtils.ErrBlobOperationFailed, func(err error) bool { return errors.Is(err, errUtils.ErrBlobOperationFailed) }}, {KindNotFound, errUtils.ErrBlobNotFound, IsNotFound}, {KindPrecondition, errUtils.ErrBlobPreconditionFailed, IsPreconditionFailed}, {KindUnsupported, errUtils.ErrBlobConditionalUnsupported, IsConditionalUnsupported}, @@ -43,8 +44,8 @@ func TestErrorMessagesDoNotDuplicateTheSentinel(t *testing.T) { err := NewError(KindNotFound, "get", "k", fmt.Errorf("%w: gone", errUtils.ErrBlobNotFound)) assert.Equal(t, 1, strings.Count(err.Error(), errUtils.ErrBlobNotFound.Error())) assert.Equal(t, `blob get "k": blob object not found`, NewError(KindNotFound, "get", "k", nil).Error()) - assert.Equal(t, `blob list: failed`, NewError(KindOther, "list", "", nil).Error()) - assert.Equal(t, `blob list: boom`, NewError(KindOther, "list", "", errors.New("boom")).Error()) + assert.Equal(t, `blob list: blob operation failed`, NewError(KindOther, "list", "", nil).Error()) + assert.Equal(t, `blob list: blob operation failed: boom`, NewError(KindOther, "list", "", errors.New("boom")).Error()) } func TestOnlyAmbiguousErrorsAreTemporary(t *testing.T) { diff --git a/pkg/artifact/blob/errors.go b/pkg/artifact/blob/errors.go index 35f6b4ff0f5..81d24eaed1e 100644 --- a/pkg/artifact/blob/errors.go +++ b/pkg/artifact/blob/errors.go @@ -24,7 +24,8 @@ type Kind int // Error kinds. const ( - // KindOther is any failure without a more specific class. Only the cause is reported. + // KindOther is any failure without a more specific class. It matches ErrBlobOperationFailed + // and keeps the cause visible. KindOther Kind = iota // KindNotFound means the addressed object does not exist. @@ -46,7 +47,8 @@ const ( KindIdentity ) -// sentinel returns the static error for the kind, or nil for KindOther. +// sentinel returns the static error for the kind. KindOther maps to ErrBlobOperationFailed so every +// classified error carries a repository sentinel. func (k Kind) sentinel() error { switch k { case KindNotFound: @@ -62,7 +64,7 @@ func (k Kind) sentinel() error { case KindIdentity: return errUtils.ErrBlobIdentityUnresolved default: - return nil + return errUtils.ErrBlobOperationFailed } } @@ -82,7 +84,7 @@ type Error struct { Err error } -// NewError returns a classified error. A nil err is allowed for kinds that have a sentinel. +// NewError returns a classified error. A nil err is allowed because every kind has a sentinel. func NewError(kind Kind, op, key string, err error) *Error { defer perf.Track(nil, "blob.NewError")() @@ -99,14 +101,9 @@ func (e *Error) Error() string { } sentinel := e.Kind.sentinel() switch { - case sentinel == nil && e.Err == nil: - return subject + ": failed" - case sentinel == nil: - return fmt.Sprintf("%s: %v", subject, e.Err) - case e.Err == nil, errors.Is(e.Err, sentinel): - if e.Err == nil { - return fmt.Sprintf("%s: %v", subject, sentinel) - } + case e.Err == nil: + return fmt.Sprintf("%s: %v", subject, sentinel) + case errors.Is(e.Err, sentinel): return fmt.Sprintf("%s: %v", subject, e.Err) default: return fmt.Sprintf("%s: %v: %v", subject, sentinel, e.Err) @@ -117,10 +114,7 @@ func (e *Error) Error() string { func (e *Error) Unwrap() []error { defer perf.Track(nil, "blob.Error.Unwrap")() - var chain []error - if sentinel := e.Kind.sentinel(); sentinel != nil { - chain = append(chain, sentinel) - } + chain := []error{e.Kind.sentinel()} if e.Err != nil { chain = append(chain, e.Err) } From 5fa24870edda8dc5755d5c1b89f17eedf092ce0a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:13:30 -0600 Subject: [PATCH 27/43] fix(artifact/blob/local): prefix deletes, case-folded names, cancelable hashing, plain-file attributes - Deleting a key that is only a prefix of other objects succeeds and leaves the directory and its children in place. - Reserved-name checks fold case, so .ATMOS-BLOB-TMP-TRANSACTION cannot alias the transaction marker on case-insensitive volumes. - ETag hashing observes context cancellation between reads. - Plain-file stores keep no attribute files: a.tar.atmos-blob.json is an ordinary object there, attribute-bearing puts are rejected, and Stat/Get accept SkipETag to avoid hashing the whole object. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/delete.go | 23 +++-- pkg/artifact/blob/local/etag.go | 14 +-- pkg/artifact/blob/local/get.go | 40 ++++++-- pkg/artifact/blob/local/list.go | 9 +- pkg/artifact/blob/local/put.go | 38 +++---- pkg/artifact/blob/local/store.go | 15 +-- pkg/artifact/blob/local/store_test.go | 139 +++++++++++++++++++++++++- pkg/artifact/blob/store.go | 8 ++ 8 files changed, 233 insertions(+), 53 deletions(-) diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go index c5b8c192a7c..d609ac53d71 100644 --- a/pkg/artifact/blob/local/delete.go +++ b/pkg/artifact/blob/local/delete.go @@ -34,19 +34,28 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) return wrapIO(blob.OpDelete, key, err) } defer root.Close() - return deleteFromRoot(root, key, opts.IfMatch) + return s.deleteFromRoot(ctx, root, key, opts.IfMatch) } // deleteFromRoot removes the object and its attributes after an optional ETag check, then prunes -// directories the removal left empty. -func deleteFromRoot(root *os.Root, key, ifMatch string) error { +// directories the removal left empty. A key that is only a prefix of other objects names no object, +// so deleting it succeeds and leaves the directory and its children untouched. +func (s *Store) deleteFromRoot(ctx context.Context, root *os.Root, key, ifMatch string) error { target := filepath.FromSlash(key) if ifMatch != "" { - if err := checkDeleteETag(root, key, target, ifMatch); err != nil { + if err := checkDeleteETag(ctx, root, key, target, ifMatch); err != nil { return err } } - for _, path := range []string{target, attrsPath(target)} { + if info, err := root.Stat(target); err == nil && info.IsDir() { + return nil + } + // Plain-file stores keep no attribute files, and a name that looks like one is another object. + paths := []string{target} + if !s.plainFiles { + paths = append(paths, attrsPath(target)) + } + for _, path := range paths { if err := root.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return wrapIO(blob.OpDelete, key, err) } @@ -56,10 +65,10 @@ func deleteFromRoot(root *os.Root, key, ifMatch string) error { } // checkDeleteETag verifies the if-match token of a conditional delete. Callers hold the lock. -func checkDeleteETag(root *os.Root, key, target, want string) error { +func checkDeleteETag(ctx context.Context, root *os.Root, key, target, want string) error { state, err := inspect(root, target) if err != nil { return wrapIO(blob.OpDelete, key, err) } - return matchETag(root, blob.OpDelete, key, state, want) + return matchETag(ctx, root, objectRef{op: blob.OpDelete, key: key}, state, want) } diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go index 69a5631a4e4..a79d664800f 100644 --- a/pkg/artifact/blob/local/etag.go +++ b/pkg/artifact/blob/local/etag.go @@ -1,26 +1,28 @@ package local import ( + "context" "crypto/sha256" "encoding/hex" "io" "os" ) -// etagOf returns the content-based ETag of the file at path. -func etagOf(root *os.Root, path string) (string, error) { +// etagOf returns the content-based ETag of the file at path. Hashing stops when ctx is done. +func etagOf(ctx context.Context, root *os.Root, path string) (string, error) { file, err := rootOpenFile(root, path) if err != nil { return "", err } defer file.Close() - return etagOpen(file) + return etagOpen(ctx, file) } -// etagOpen hashes the opened file and restores its position for the caller. -func etagOpen(file *os.File) (string, error) { +// etagOpen hashes the opened file and restores its position for the caller. Cancellation is +// observed between reads, so a canceled caller releases the root lock instead of finishing the scan. +func etagOpen(ctx context.Context, file *os.File) (string, error) { hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { + if _, err := io.Copy(hash, &ctxReader{ctx: ctx, r: file}); err != nil { return "", err } if _, err := file.Seek(0, io.SeekStart); err != nil { diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go index fe28bf61b21..8636f922573 100644 --- a/pkg/artifact/blob/local/get.go +++ b/pkg/artifact/blob/local/get.go @@ -52,7 +52,7 @@ func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blo if err != nil { return nil, wrapIO(blob.OpGet, key, err) } - info, err := s.describe(root, blob.OpGet, key, file, false) + info, err := s.describe(ctx, root, describeTarget{op: blob.OpGet, key: key, file: file}, describeOptions{skipETag: opts.SkipETag}) if err != nil { _ = file.Close() return nil, err @@ -109,11 +109,27 @@ func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*b return nil, wrapIO(blob.OpStat, key, err) } defer file.Close() - return s.describe(root, blob.OpStat, key, file, opts.IncludeTags) + return s.describe(ctx, root, describeTarget{op: blob.OpStat, key: key, file: file}, describeOptions{withTags: opts.IncludeTags, skipETag: opts.SkipETag}) +} + +// describeTarget is the open file a description is built for. +type describeTarget struct { + op string + key string + file *os.File +} + +// describeOptions select the optional parts of a description. +type describeOptions struct { + withTags bool + + // skipETag leaves the ETag empty so a metadata check does not hash the whole object. + skipETag bool } // describe builds ObjectInfo for an open regular file. A directory is not an object. -func (s *Store) describe(root *os.Root, op, key string, file *os.File, withTags bool) (*blob.ObjectInfo, error) { +func (s *Store) describe(ctx context.Context, root *os.Root, target describeTarget, opts describeOptions) (*blob.ObjectInfo, error) { + op, key, file := target.op, target.key, target.file stat, err := file.Stat() if err != nil { return nil, wrapIO(op, key, err) @@ -121,16 +137,20 @@ func (s *Store) describe(root *os.Root, op, key string, file *os.File, withTags if !stat.Mode().IsRegular() { return nil, blob.NewError(blob.KindNotFound, op, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) } - etag, err := etagOpen(file) - if err != nil { - return nil, wrapIO(op, key, err) + var etag string + if !opts.skipETag { + if etag, err = etagOpen(ctx, file); err != nil { + return nil, wrapIO(op, key, err) + } } - a, err := readAttrs(root, filepath.FromSlash(key)) - if err != nil { - return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("read attributes: %w", err)) + var a attrs + if !s.plainFiles { + if a, err = readAttrs(root, filepath.FromSlash(key)); err != nil { + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("read attributes: %w", err)) + } } info := &blob.ObjectInfo{Key: key, Size: stat.Size(), ETag: etag, ContentType: a.ContentType, LastModified: stat.ModTime(), Metadata: a.Metadata} - if withTags { + if opts.withTags { info.Tags = a.Tags } return info, nil diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go index 05b8eb33bc3..12a765e75dd 100644 --- a/pkg/artifact/blob/local/list.go +++ b/pkg/artifact/blob/local/list.go @@ -108,7 +108,7 @@ func ignoreMissing(err error) error { // objectAt returns the object stored at a walked path, or false for directories, bookkeeping // files, keys outside the prefix, and files that vanished. func (s *Store) objectAt(name string, d fs.DirEntry, prefix string) (entry, bool, error) { - if d.IsDir() || reserved(name, d.Name()) { + if d.IsDir() || s.reserved(name, d.Name()) { return entry{}, false, nil } key := name @@ -122,10 +122,11 @@ func (s *Store) objectAt(name string, d fs.DirEntry, prefix string) (entry, bool return entry{key: key, info: info}, info.Mode().IsRegular(), nil } -// reserved reports whether a file is one of the store's own bookkeeping files. -func reserved(name, base string) bool { +// reserved reports whether a file is one of the store's own bookkeeping files. Plain-file stores +// write no attribute files, so an attribute-looking name there is an object. +func (s *Store) reserved(name, base string) bool { return strings.HasPrefix(base, tempPrefix) || - strings.HasSuffix(base, attrsSuffix) || + (!s.plainFiles && strings.HasSuffix(base, attrsSuffix)) || (base == lockName && path.Dir(name) == ".") } diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index 13eb71c524b..69ffb826cd6 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -54,7 +54,7 @@ func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.P } defer release() - if err := s.checkCondition(root, key, target, opts.Condition); err != nil { + if err := s.checkCondition(ctx, root, key, target, opts.Condition); err != nil { return nil, err } if err := s.commit(root, key, target, pending, fromPut(opts).empty()); err != nil { @@ -76,6 +76,9 @@ func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOption if opts.Condition.Kind == blob.ConditionIfMatch && opts.Condition.ETag == "" { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) } + if s.plainFiles && !fromPut(opts).empty() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: plain-file stores cannot store content type, metadata, or tags", errUtils.ErrBlobInvalid)) + } return checkContext(ctx, blob.OpPut, key) } @@ -141,7 +144,7 @@ func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (i } // checkCondition evaluates the put condition against the current object. Callers hold the lock. -func (s *Store) checkCondition(root *os.Root, key, target string, cond blob.Condition) error { +func (s *Store) checkCondition(ctx context.Context, root *os.Root, key, target string, cond blob.Condition) error { if cond.Kind == blob.ConditionNone { return nil } @@ -158,7 +161,7 @@ func (s *Store) checkCondition(root *os.Root, key, target string, cond blob.Cond return preconditionFailed(blob.OpPut, key, "object already exists") } case blob.ConditionIfMatch: - return matchETag(root, blob.OpPut, key, state, cond.ETag) + return matchETag(ctx, root, objectRef{op: blob.OpPut, key: key}, state, cond.ETag) } return nil } @@ -179,13 +182,20 @@ func inspect(root *os.Root, path string) (fileState, error) { return fileState{path: path, info: info, exists: err == nil && info.Mode().IsRegular()}, nil } +// objectRef names the object an operation addresses, for error reporting. +type objectRef struct { + op string + key string +} + // matchETag fails with NotFound when the object is missing and with a precondition failure when // its ETag differs from want. -func matchETag(root *os.Root, op, key string, state fileState, want string) error { +func matchETag(ctx context.Context, root *os.Root, ref objectRef, state fileState, want string) error { + op, key := ref.op, ref.key if !state.exists { return blob.NewError(blob.KindNotFound, op, key, nil) } - current, err := etagOf(root, state.path) + current, err := etagOf(ctx, root, state.path) if err != nil { return wrapIO(op, key, err) } @@ -208,22 +218,12 @@ func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAtt if !s.plainFiles { return s.commitManaged(root, key, pending, noAttrs, rootRenameReplace) } - return commitPlain(root, key, target, pending, noAttrs) + return commitPlain(root, key, target, pending) } -// commitPlain installs a staged object in a plain-files store: attributes first, then the object. -func commitPlain(root *os.Root, key, target string, pending *staged, noAttrs bool) error { - switch { - case pending.attrsTemp != "": - if err := rootRenameReplace(root, pending.attrsTemp, attrsPath(target)); err != nil { - return wrapIO(blob.OpPut, key, err) - } - pending.attrsTemp = "" - case noAttrs: - if err := root.Remove(attrsPath(target)); err != nil && !errors.Is(err, os.ErrNotExist) { - return wrapIO(blob.OpPut, key, err) - } - } +// commitPlain installs a staged object in a plain-files store. It never touches attribute paths: +// a name such as "a.tar.atmos-blob.json" is an independent legacy object there. +func commitPlain(root *os.Root, key, target string, pending *staged) error { if err := rootRenameReplace(root, pending.tempPath, target); err != nil { return wrapIO(blob.OpPut, key, err) } diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index fb41bd5e27a..039d43ce7c4 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -74,8 +74,8 @@ func (s *Store) Capabilities() blob.Capabilities { ConditionalPut: true, ConditionalDelete: true, Multipart: true, - Tags: true, - Metadata: true, + Tags: !s.plainFiles, + Metadata: !s.plainFiles, Range: true, Listing: true, ListDelimiter: true, @@ -91,7 +91,7 @@ func (s *Store) validate(op, key string) error { if err := blob.ValidateKey(validationKey); err != nil { return err } - if key == lockName || hasReservedSegment(key) { + if strings.EqualFold(key, lockName) || s.hasReservedSegment(key) { return blob.NewError(blob.KindInvalid, op, key, fmt.Errorf("%w: key uses a reserved name", errUtils.ErrBlobInvalid)) } return nil @@ -99,10 +99,13 @@ func (s *Store) validate(op, key string) error { // hasReservedSegment reports whether any slash-separated segment of key looks like one of the // store's own temporary or attribute files. A reserved directory name would otherwise hide the -// objects under it from listings, which skip those names at every level. -func hasReservedSegment(key string) bool { +// objects under it from listings, which skip those names at every level. The comparison folds +// case: on case-insensitive volumes a differently cased name addresses the same file. Plain-file +// stores keep no attribute files, so for them an attribute-looking name is an ordinary object. +func (s *Store) hasReservedSegment(key string) bool { for _, segment := range strings.Split(key, "/") { - if strings.HasPrefix(segment, tempPrefix) || strings.HasSuffix(segment, attrsSuffix) { + folded := strings.ToLower(segment) + if strings.HasPrefix(folded, tempPrefix) || (!s.plainFiles && strings.HasSuffix(folded, attrsSuffix)) { return true } } diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index df18aee9d50..0c0ae948379 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -181,7 +181,7 @@ func TestDescribeHashesOpenedFile(t *testing.T) { root, err := store.openRoot(false) require.NoError(t, err) defer root.Close() - info, err := store.describe(root, blob.OpGet, "k", file, false) + info, err := store.describe(context.Background(), root, describeTarget{op: blob.OpGet, key: "k", file: file}, describeOptions{}) require.NoError(t, err) sum := sha256.Sum256([]byte("old")) assert.Equal(t, contentETag(sum[:]), info.ETag) @@ -317,3 +317,140 @@ func TestListPrefixHonorsLiteralBackslashes(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "%q stays rejected", prefix) } } + +func TestReservedNamesFoldCase(t *testing.T) { + store := newStore(t) + for _, key := range []string{ + ".ATMOS-BLOB-TMP-TRANSACTION", + "d/.Atmos-Blob-Tmp-x", + "d/x.ATMOS-BLOB.JSON", + ".ATMOS-BLOB.LOCK", + "Mixed/.atmos-blob-tmp-dir/child", + } { + _, err := store.Put(context.Background(), key, bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } + assert.NoDirExists(t, filepath.Join(store.Root(), "d"), "a rejected key stages nothing") +} + +func TestDeletePrefixOnlyDirectoryIsMissingObject(t *testing.T) { + for _, tc := range []struct { + name string + opts []Option + }{ + {name: "managed"}, + {name: "plain", opts: []Option{WithPlainFileCompatibility()}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newStore(t, tc.opts...) + ctx := context.Background() + _, err := store.Put(ctx, "prefix/child", bytes.NewReader([]byte("child")), blob.PutOptions{Size: 5}) + require.NoError(t, err) + + require.NoError(t, store.Delete(ctx, "prefix", blob.DeleteOptions{}), "an unconditional delete of a missing object succeeds") + + info, err := store.Stat(ctx, "prefix/child", blob.StatOptions{}) + require.NoError(t, err, "the children stay in place") + assert.Equal(t, int64(5), info.Size) + assert.DirExists(t, filepath.Join(store.Root(), "prefix")) + + err = store.Delete(ctx, "prefix", blob.DeleteOptions{IfMatch: `"any"`}) + require.ErrorIs(t, err, errUtils.ErrBlobNotFound, "a conditional delete of a prefix reports the object as missing") + assert.FileExists(t, filepath.Join(store.Root(), "prefix", "child")) + }) + } +} + +func TestETagHashingObservesCancellation(t *testing.T) { + path := filepath.Join(t.TempDir(), "big") + content := bytes.Repeat([]byte("x"), 1<<20) + require.NoError(t, os.WriteFile(path, content, 0o644)) + file, err := openFile(path) + require.NoError(t, err) + defer file.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = etagOpen(ctx, file) + require.ErrorIs(t, err, context.Canceled, "hashing stops instead of scanning the whole file") + + etag, err := etagOpen(context.Background(), file) + require.NoError(t, err) + sum := sha256.Sum256(content) + assert.Equal(t, contentETag(sum[:]), etag, "an active context hashes the full content") +} + +func TestPlainFilesTreatAttributeLookingNamesAsObjects(t *testing.T) { + store := newStore(t, WithPlainFileCompatibility()) + ctx := context.Background() + const object, lookalike = "a.tar", "a.tar" + attrsSuffix + put := func(key, body string) { + t.Helper() + _, err := store.Put(ctx, key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body))}) + require.NoError(t, err, key) + } + read := func(key string) string { + t.Helper() + obj, err := store.Get(ctx, key, blob.GetOptions{}) + require.NoError(t, err, key) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + return string(data) + } + + put(object, "tar-v1") + put(lookalike, "not json at all") + assert.Equal(t, "tar-v1", read(object), "an unrelated object beside the data is not parsed as attributes") + + put(object, "tar-v2") + assert.Equal(t, "tar-v2", read(object)) + assert.Equal(t, "not json at all", read(lookalike), "updating a.tar leaves the independent artifact untouched") + + page, err := store.List(ctx, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, page.Objects, 2, "both names are listed as objects") + assert.Equal(t, object, page.Objects[0].Key) + assert.Equal(t, lookalike, page.Objects[1].Key) + + require.NoError(t, store.Delete(ctx, object, blob.DeleteOptions{})) + assert.Equal(t, "not json at all", read(lookalike), "deleting a.tar leaves the independent artifact untouched") + + _, err = store.Put(ctx, "tagged", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, Tags: map[string]string{"a": "b"}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "attributes cannot be stored without a sidecar") + assert.False(t, store.Capabilities().Tags) + assert.False(t, store.Capabilities().Metadata) +} + +func TestSkipETagAvoidsHashingTheObject(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "k", bytes.NewReader([]byte("body")), blob.PutOptions{Size: 4}) + require.NoError(t, err) + + tests := []struct { + name string + skip bool + wantETag bool + }{ + {name: "default computes the content ETag", wantETag: true}, + {name: "skip leaves the ETag empty", skip: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stat, err := store.Stat(ctx, "k", blob.StatOptions{SkipETag: tc.skip}) + require.NoError(t, err) + assert.Equal(t, int64(4), stat.Size) + assert.Equal(t, tc.wantETag, stat.ETag != "") + + obj, err := store.Get(ctx, "k", blob.GetOptions{SkipETag: tc.skip}) + require.NoError(t, err) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + assert.Equal(t, "body", string(data), "the body is complete either way") + assert.Equal(t, tc.wantETag, obj.ETag != "") + }) + } +} diff --git a/pkg/artifact/blob/store.go b/pkg/artifact/blob/store.go index 953ecb9b7e7..24b457a061b 100644 --- a/pkg/artifact/blob/store.go +++ b/pkg/artifact/blob/store.go @@ -116,6 +116,10 @@ type GetOptions struct { // Length is the number of bytes to read from Offset. Zero or negative reads to the end. Length int64 + + // SkipETag tells the store the caller does not need the ETag, so a store that derives it from + // the content may leave it empty instead of reading the whole object first. + SkipETag bool } // IsRange reports whether the read is a partial read. @@ -129,6 +133,10 @@ func (o GetOptions) IsRange() bool { type StatOptions struct { // IncludeTags also loads the object tags, which may cost an extra request. IncludeTags bool + + // SkipETag tells the store the caller does not need the ETag, so a store that derives it from + // the content may leave it empty instead of reading the whole object first. + SkipETag bool } // DeleteOptions describe one delete. From babb0b5b9f226968b1aa6cfd6ca490b9cc0b1a26 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:14:19 -0600 Subject: [PATCH 28/43] fix(ci/artifact/local): keep attribute-looking artifacts independent and skip hashing Legacy artifacts such as a.tar and a.tar.atmos-blob.json no longer shadow or delete each other, and Exists, GetMetadata, and Download no longer hash the whole artifact. Co-Authored-By: Claude Sonnet 5.5 --- pkg/ci/artifact/local/store.go | 8 ++-- .../artifact/local/transport_parity_test.go | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index e3efe45afc4..0047b3a36a7 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -146,7 +146,7 @@ func (s *Store) Download(ctx context.Context, name string) (io.ReadCloser, *arti } key := s.key(name) - object, err := transport.Get(ctx, key, blob.GetOptions{}) + object, err := transport.Get(ctx, key, blob.GetOptions{SkipETag: true}) if err != nil { if blob.IsNotFound(err) { if s.isDirectory(name) { @@ -266,7 +266,7 @@ func (s *Store) Exists(ctx context.Context, name string) (bool, error) { return false, fmt.Errorf("failed to check if artifact %s exists: %w", name, err) } - if _, err := transport.Stat(ctx, s.key(name), blob.StatOptions{}); err != nil { + if _, err := transport.Stat(ctx, s.key(name), blob.StatOptions{SkipETag: true}); err != nil { if blob.IsNotFound(err) { return false, nil } @@ -289,7 +289,7 @@ func (s *Store) GetMetadata(ctx context.Context, name string) (*artifact.Metadat key := s.key(name) // Check if the artifact file exists. - info, err := transport.Stat(ctx, key, blob.StatOptions{}) + info, err := transport.Stat(ctx, key, blob.StatOptions{SkipETag: true}) if err != nil { if blob.IsNotFound(err) { if s.isDirectory(name) { @@ -328,7 +328,7 @@ func (s *Store) validateName(name string) error { // loadMetadata loads metadata from a JSON sidecar object. A missing sidecar yields nil metadata. func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, metadataKey string) (*artifact.Metadata, error) { - object, err := transport.Get(ctx, metadataKey, blob.GetOptions{}) + object, err := transport.Get(ctx, metadataKey, blob.GetOptions{SkipETag: true}) if err != nil { if blob.IsNotFound(err) { return nil, nil diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go index 524d1b12bc3..fca85b69506 100644 --- a/pkg/ci/artifact/local/transport_parity_test.go +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -206,3 +206,49 @@ func TestLegacyLiteralBackslashName(t *testing.T) { assert.Equal(t, name, listed[0].Name) require.NoError(t, store.Delete(ctx, name)) } + +// TestAttributeLookingArtifactNamesStayIndependent covers legacy stores that hold both "a.tar" and +// "a.tar.atmos-blob.json": the transport keeps no attribute files in this layout, so neither +// artifact may shadow, parse, or delete the other. +func TestAttributeLookingArtifactNamesStayIndependent(t *testing.T) { + const object, lookalike = "a.tar", "a.tar.atmos-blob.json" + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + upload := func(name, body string) { + t.Helper() + require.NoError(t, store.Upload(ctx, name, strings.NewReader(body), int64(len(body)), nil), name) + } + download := func(name string) string { + t.Helper() + reader, _, err := store.Download(ctx, name) + require.NoError(t, err, name) + defer reader.Close() + data, err := io.ReadAll(reader) + require.NoError(t, err) + return string(data) + } + + upload(object, "tar-v1") + upload(lookalike, "not an attribute document") + upload(object, "tar-v2") + + assert.Equal(t, "tar-v2", download(object), "a.tar downloads although a.tar.atmos-blob.json exists beside it") + assert.Equal(t, "not an attribute document", download(lookalike), "updating a.tar does not delete the second artifact") + + listed, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + names := make([]string, 0, len(listed)) + for _, info := range listed { + names = append(names, info.Name) + } + assert.ElementsMatch(t, []string{object, lookalike}, names, "both legacy artifacts are listed") + + exists, err := store.Exists(ctx, lookalike) + require.NoError(t, err) + assert.True(t, exists) + + require.NoError(t, store.Delete(ctx, object)) + assert.Equal(t, "not an attribute document", download(lookalike), "deleting a.tar does not delete the second artifact") + assert.Equal(t, []string{lookalike, lookalike + ".metadata.json"}, dirEntries(t, root)) +} From b55a6edc5a67a2fd73a2d947594a055e1392287d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:19:37 -0600 Subject: [PATCH 29/43] fix(artifact/blob/s3): honor cancellation, keep worker errors, share sentinels, mock the API - The conditional-write probe wait gives up when its caller's context ends instead of blocking behind an in-flight probe. - Put checks the context before reading the body and reads through a context-aware reader, so a canceled put never blocks on an idle source. - A multipart worker's service error is returned instead of the producer's bare cancellation when the group canceled it. - The aws-chunked decoder and tag parser in s3test wrap their failures with shared sentinels; the tagging-denial sentinel moves to errors/. - read_version_test uses a mockgen-generated API mock instead of a manual test double. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 11 + pkg/artifact/blob/s3/api.go | 2 + pkg/artifact/blob/s3/conditional.go | 25 ++- pkg/artifact/blob/s3/context_test.go | 132 +++++++++++ pkg/artifact/blob/s3/fallback.go | 9 +- pkg/artifact/blob/s3/mock_api_test.go | 242 +++++++++++++++++++++ pkg/artifact/blob/s3/multipart.go | 24 +- pkg/artifact/blob/s3/put.go | 6 + pkg/artifact/blob/s3/read_version_test.go | 98 ++++----- pkg/artifact/blob/s3/s3test/chunked.go | 11 +- pkg/artifact/blob/s3/s3test/objects.go | 10 +- pkg/artifact/blob/s3/s3test/server_test.go | 50 ++++- pkg/artifact/blob/s3/store.go | 7 +- 13 files changed, 542 insertions(+), 85 deletions(-) create mode 100644 pkg/artifact/blob/s3/context_test.go create mode 100644 pkg/artifact/blob/s3/mock_api_test.go diff --git a/errors/errors.go b/errors/errors.go index 74d0f5b8f22..02e68084f34 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2168,6 +2168,17 @@ var ( // local I/O errors or an SDK failure with no mapped meaning. The underlying cause stays in the chain. ErrBlobOperationFailed = errors.New("blob operation failed") + // ErrBlobTaggingDenied is returned by tests that simulate a version-specific object tagging denial. + ErrBlobTaggingDenied = errors.New("blob tagging denied") + + // ErrBlobInvalidChunk is returned by the S3 test endpoint when an aws-chunked request body cannot + // be decoded: an unreadable or non-hex chunk size, or a chunk cut short. + ErrBlobInvalidChunk = errors.New("invalid aws-chunked body") + + // ErrBlobTooManyTags is returned by the S3 test endpoint when a request carries more object tags + // than S3 allows. + ErrBlobTooManyTags = errors.New("object tags cannot be greater than 10") + // ErrBlobInvalidChunkTerminator is returned by the S3 test endpoint when an aws-chunked request // body has a chunk that is not followed by the required CRLF terminator. ErrBlobInvalidChunkTerminator = errors.New("invalid aws-chunked chunk terminator") diff --git a/pkg/artifact/blob/s3/api.go b/pkg/artifact/blob/s3/api.go index 60d7be14ce3..2b39868ed8b 100644 --- a/pkg/artifact/blob/s3/api.go +++ b/pkg/artifact/blob/s3/api.go @@ -7,6 +7,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) +//go:generate go run go.uber.org/mock/mockgen -package s3 -destination mock_api_test.go github.com/cloudposse/atmos/pkg/artifact/blob/s3 API + // API is the subset of the S3 client the store uses. *s3.Client satisfies it; tests may supply a // recording or faulting implementation. type API interface { diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go index b45e1086eb6..258613df711 100644 --- a/pkg/artifact/blob/s3/conditional.go +++ b/pkg/artifact/blob/s3/conditional.go @@ -54,10 +54,18 @@ func unsupportedConditionalFrom(key string, probeErr error) error { func (s *Store) ProbeConditionalWrites(ctx context.Context) error { defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")() - s.probeMu.Lock() - defer s.probeMu.Unlock() - if s.probed { - return s.probeError + if verdict, cached := s.cachedProbe(); cached { + return verdict + } + select { + case s.probeSlot <- struct{}{}: + defer func() { <-s.probeSlot }() + case <-ctx.Done(): + return blob.NewError(blob.KindOther, probeOperation, "", ctx.Err()) + } + // Another operation may have finished the probe while this one waited for the slot. + if verdict, cached := s.cachedProbe(); cached { + return verdict } api, err := s.client(ctx) if err != nil { @@ -70,11 +78,20 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error { return errors.Join(verdict, cleanupErr) } if verdict == nil || blob.IsConditionalUnsupported(verdict) { + s.probeMu.Lock() s.probed, s.probeError = true, verdict + s.probeMu.Unlock() } return verdict } +// cachedProbe returns the cached probe verdict, when a definite one exists. +func (s *Store) cachedProbe() (verdict error, cached bool) { + s.probeMu.Lock() + defer s.probeMu.Unlock() + return s.probeError, s.probed +} + // runProbe performs the double create and removes the probe object. The verdict is the probe // result; cleanupErr reports a failure to delete the probe object. func (s *Store) runProbe(ctx context.Context, api API) (verdict, cleanupErr error) { diff --git a/pkg/artifact/blob/s3/context_test.go b/pkg/artifact/blob/s3/context_test.go new file mode 100644 index 00000000000..3b2fad54b3d --- /dev/null +++ b/pkg/artifact/blob/s3/context_test.go @@ -0,0 +1,132 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestProbeWaiterStopsWhenItsContextEnds(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) + started := make(chan struct{}) + release := make(chan struct{}) + errEndpointDown := errors.New("endpoint down") + api.EXPECT(). + PutObject(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + close(started) + <-release + return nil, errEndpointDown + }). + Times(1) + api.EXPECT().DeleteObject(gomock.Any(), gomock.Any(), gomock.Any()).Return(&s3.DeleteObjectOutput{}, nil).Times(1) + store, err := NewFromClient(api, "bucket", WithConditionalMode(ConditionalProbe)) + require.NoError(t, err) + + first := make(chan error, 1) + go func() { first <- store.ProbeConditionalWrites(context.Background()) }() + <-started + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + waited := make(chan error, 1) + go func() { waited <- store.ProbeConditionalWrites(ctx) }() + select { + case err := <-waited: + require.ErrorIs(t, err, context.DeadlineExceeded, "the waiter gives up at its own deadline while the first probe is in flight") + require.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + case <-time.After(5 * time.Second): + t.Fatal("the waiting probe stayed blocked after its context ended") + } + + close(release) + require.ErrorIs(t, <-first, errEndpointDown, "the first probe still finishes and reports its own failure") +} + +func TestPutChecksContextBeforeReadingTheBody(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) // Any SDK call fails the test: nothing may be sent. + store, err := NewFromClient(api, "bucket", WithPartSize(smallPart)) + require.NoError(t, err) + reader, writer := io.Pipe() // Nothing is ever written, so reading would block forever. + defer writer.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { + _, putErr := store.Put(ctx, "k", reader, blob.PutOptions{Size: blob.UnknownSize}) + done <- putErr + }() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + case <-time.After(5 * time.Second): + t.Fatal("put blocked reading the body although its context was already canceled") + } +} + +func TestContextReaderStopsConsumingAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader := &ctxReader{ctx: ctx, reader: bytes.NewReader([]byte("abcdef"))} + buf := make([]byte, 3) + n, err := reader.Read(buf) + require.NoError(t, err) + assert.Equal(t, "abc", string(buf[:n])) + + cancel() + n, err = reader.Read(buf) + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, n) +} + +func TestMergeUploadErrorsKeepsTheWorkerFailure(t *testing.T) { + errService := errors.New("service rejected the part") + errRead := errors.New("body read failed") + live := context.Background() + canceled, cancel := context.WithCancel(context.Background()) + cancel() + + tests := []struct { + name string + parent context.Context + produceErr error + waitErr error + want error + }{ + {name: "no failure", parent: live}, + {name: "worker failure surfaces when the producer was canceled by the group", parent: live, produceErr: context.Canceled, waitErr: errService, want: errService}, + {name: "caller cancellation stays the reported error", parent: canceled, produceErr: context.Canceled, waitErr: errService, want: context.Canceled}, + {name: "unrelated producer error is kept", parent: live, produceErr: errRead, waitErr: context.Canceled, want: errRead}, + {name: "worker failure alone is reported", parent: live, waitErr: errService, want: errService}, + {name: "producer failure alone is reported", parent: live, produceErr: errRead, want: errRead}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := mergeUploadErrors(tc.parent, tc.produceErr, tc.waitErr) + if tc.want == nil { + assert.NoError(t, got) + return + } + assert.ErrorIs(t, got, tc.want) + }) + } +} + +func TestUnclassifiedFailuresCarryTheSentinel(t *testing.T) { + cause := errors.New("disk full") + err := asBlobError("put", "k", cause) + require.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + assert.ErrorIs(t, err, cause, "the original cause stays visible") +} diff --git a/pkg/artifact/blob/s3/fallback.go b/pkg/artifact/blob/s3/fallback.go index e0d3afb15b9..340161ea5a2 100644 --- a/pkg/artifact/blob/s3/fallback.go +++ b/pkg/artifact/blob/s3/fallback.go @@ -24,7 +24,7 @@ func (s *Store) putFallback(ctx context.Context, plan *putPlan, first, carry []b _ = file.Close() _ = os.Remove(file.Name()) }() - size, err := io.Copy(file, &fallbackReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}) + size, err := io.Copy(file, &ctxReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}) if err != nil { return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("spool upload: %w", err)) } @@ -43,13 +43,14 @@ func (s *Store) putFallback(ctx context.Context, plan *putPlan, first, carry []b return &blob.ObjectInfo{Key: plan.key, Size: size, ETag: aws.ToString(out.ETag)}, nil } -type fallbackReader struct { +// ctxReader fails reads once its context is done, so a canceled operation stops consuming its body. +type ctxReader struct { ctx context.Context reader io.Reader } -func (r *fallbackReader) Read(p []byte) (int, error) { - defer perf.Track(nil, "s3.fallbackReader.Read")() +func (r *ctxReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "s3.ctxReader.Read")() if err := r.ctx.Err(); err != nil { return 0, err diff --git a/pkg/artifact/blob/s3/mock_api_test.go b/pkg/artifact/blob/s3/mock_api_test.go new file mode 100644 index 00000000000..2141a6811a4 --- /dev/null +++ b/pkg/artifact/blob/s3/mock_api_test.go @@ -0,0 +1,242 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/blob/s3 (interfaces: API) +// +// Generated by this command: +// +// mockgen -package s3 -destination mock_api_test.go github.com/cloudposse/atmos/pkg/artifact/blob/s3 API +// + +// Package s3 is a generated GoMock package. +package s3 + +import ( + context "context" + reflect "reflect" + + s3 "github.com/aws/aws-sdk-go-v2/service/s3" + gomock "go.uber.org/mock/gomock" +) + +// MockAPI is a mock of API interface. +type MockAPI struct { + ctrl *gomock.Controller + recorder *MockAPIMockRecorder + isgomock struct{} +} + +// MockAPIMockRecorder is the mock recorder for MockAPI. +type MockAPIMockRecorder struct { + mock *MockAPI +} + +// NewMockAPI creates a new mock instance. +func NewMockAPI(ctrl *gomock.Controller) *MockAPI { + mock := &MockAPI{ctrl: ctrl} + mock.recorder = &MockAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAPI) EXPECT() *MockAPIMockRecorder { + return m.recorder +} + +// AbortMultipartUpload mocks base method. +func (m *MockAPI) AbortMultipartUpload(ctx context.Context, in *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AbortMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.AbortMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AbortMultipartUpload indicates an expected call of AbortMultipartUpload. +func (mr *MockAPIMockRecorder) AbortMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AbortMultipartUpload", reflect.TypeOf((*MockAPI)(nil).AbortMultipartUpload), varargs...) +} + +// CompleteMultipartUpload mocks base method. +func (m *MockAPI) CompleteMultipartUpload(ctx context.Context, in *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CompleteMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.CompleteMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. +func (mr *MockAPIMockRecorder) CompleteMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockAPI)(nil).CompleteMultipartUpload), varargs...) +} + +// CreateMultipartUpload mocks base method. +func (m *MockAPI) CreateMultipartUpload(ctx context.Context, in *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CreateMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.CreateMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateMultipartUpload indicates an expected call of CreateMultipartUpload. +func (mr *MockAPIMockRecorder) CreateMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateMultipartUpload", reflect.TypeOf((*MockAPI)(nil).CreateMultipartUpload), varargs...) +} + +// DeleteObject mocks base method. +func (m *MockAPI) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteObject", varargs...) + ret0, _ := ret[0].(*s3.DeleteObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteObject indicates an expected call of DeleteObject. +func (mr *MockAPIMockRecorder) DeleteObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObject", reflect.TypeOf((*MockAPI)(nil).DeleteObject), varargs...) +} + +// GetObject mocks base method. +func (m *MockAPI) GetObject(ctx context.Context, in *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetObject", varargs...) + ret0, _ := ret[0].(*s3.GetObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObject indicates an expected call of GetObject. +func (mr *MockAPIMockRecorder) GetObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockAPI)(nil).GetObject), varargs...) +} + +// GetObjectTagging mocks base method. +func (m *MockAPI) GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetObjectTagging", varargs...) + ret0, _ := ret[0].(*s3.GetObjectTaggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObjectTagging indicates an expected call of GetObjectTagging. +func (mr *MockAPIMockRecorder) GetObjectTagging(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObjectTagging", reflect.TypeOf((*MockAPI)(nil).GetObjectTagging), varargs...) +} + +// HeadObject mocks base method. +func (m *MockAPI) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "HeadObject", varargs...) + ret0, _ := ret[0].(*s3.HeadObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HeadObject indicates an expected call of HeadObject. +func (mr *MockAPIMockRecorder) HeadObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*MockAPI)(nil).HeadObject), varargs...) +} + +// ListObjectsV2 mocks base method. +func (m *MockAPI) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListObjectsV2", varargs...) + ret0, _ := ret[0].(*s3.ListObjectsV2Output) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObjectsV2 indicates an expected call of ListObjectsV2. +func (mr *MockAPIMockRecorder) ListObjectsV2(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjectsV2", reflect.TypeOf((*MockAPI)(nil).ListObjectsV2), varargs...) +} + +// PutObject mocks base method. +func (m *MockAPI) PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PutObject", varargs...) + ret0, _ := ret[0].(*s3.PutObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutObject indicates an expected call of PutObject. +func (mr *MockAPIMockRecorder) PutObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockAPI)(nil).PutObject), varargs...) +} + +// UploadPart mocks base method. +func (m *MockAPI) UploadPart(ctx context.Context, in *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UploadPart", varargs...) + ret0, _ := ret[0].(*s3.UploadPartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UploadPart indicates an expected call of UploadPart. +func (mr *MockAPIMockRecorder) UploadPart(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadPart", reflect.TypeOf((*MockAPI)(nil).UploadPart), varargs...) +} diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go index f09d96cefa2..f83fce94d19 100644 --- a/pkg/artifact/blob/s3/multipart.go +++ b/pkg/artifact/blob/s3/multipart.go @@ -76,8 +76,8 @@ func (r *partResult) add(number int32, etag *string) { // uploadParts reads the body part by part and uploads them through a bounded pool. A free-list of // buffers caps memory at one buffer per worker. -func (m *multipart) uploadParts(ctx context.Context, first, carry []byte, body io.Reader) ([]types.CompletedPart, int64, error) { - ctx, cancel := context.WithCancel(ctx) +func (m *multipart) uploadParts(parent context.Context, first, carry []byte, body io.Reader) ([]types.CompletedPart, int64, error) { + ctx, cancel := context.WithCancel(parent) defer cancel() group, gctx := errgroup.WithContext(ctx) @@ -104,11 +104,8 @@ func (m *multipart) uploadParts(ctx context.Context, first, carry []byte, body i cancel() } waitErr := group.Wait() - if produceErr != nil { - return nil, 0, produceErr - } - if waitErr != nil { - return nil, 0, waitErr + if err := mergeUploadErrors(parent, produceErr, waitErr); err != nil { + return nil, 0, err } sort.Slice(result.parts, func(i, j int) bool { return aws.ToInt32(result.parts[i].PartNumber) < aws.ToInt32(result.parts[j].PartNumber) @@ -116,6 +113,19 @@ func (m *multipart) uploadParts(ctx context.Context, first, carry []byte, body i return result.parts, total, nil } +// mergeUploadErrors picks the error that explains a failed upload. When a worker fails, the group +// cancels the producer's context, so the producer reports a bare cancellation that hides the +// worker's service error. That cancellation is not the caller's own when parent is still live. +func mergeUploadErrors(parent context.Context, produceErr, waitErr error) error { + if produceErr != nil && waitErr != nil && errors.Is(produceErr, context.Canceled) && parent.Err() == nil { + return waitErr + } + if produceErr != nil { + return produceErr + } + return waitErr +} + // partFeed hands parts to the worker pool. Its free list holds one buffer slot per worker. type partFeed struct { free chan []byte diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index 526ae60d30f..38f8c278a69 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -43,6 +43,12 @@ func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.P if err != nil { return nil, err } + // A canceled context must stop the put before it consumes any of the body: a source such as an + // idle pipe would otherwise block here before an SDK call could observe the cancellation. + if err := ctx.Err(); err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, key, err) + } + body = &ctxReader{ctx: ctx, reader: body} first, carry, err := readFirstPart(body, plan.partSize, opts.Size) if err != nil { return nil, blob.NewError(blob.KindOther, blob.OpPut, key, fmt.Errorf("read body: %w", err)) diff --git a/pkg/artifact/blob/s3/read_version_test.go b/pkg/artifact/blob/s3/read_version_test.go index 83e33248c12..9027f4f47da 100644 --- a/pkg/artifact/blob/s3/read_version_test.go +++ b/pkg/artifact/blob/s3/read_version_test.go @@ -2,48 +2,37 @@ package s3 import ( "context" - "errors" "testing" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact/blob" - "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" ) -// errTaggingDenied stands in for a version-specific tagging denial. -var errTaggingDenied = errors.New("tagging denied") +const versionedBucket = "versioned-bucket" -// versionedAPI wraps a real API, reporting a fixed HeadObject version ID and recording the -// GetObjectTagging inputs. A non-nil taggingErr fails every tagging request. -type versionedAPI struct { - API - versionID *string - taggingErr error - tagging []*s3.GetObjectTaggingInput +// expectHead makes the mock report a fixed object with the given HeadObject version ID. +func expectHead(api *MockAPI, versionID *string) { + api.EXPECT(). + HeadObject(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, in *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return &s3.HeadObjectOutput{ContentLength: aws.Int64(1), ETag: aws.String(`"etag"`), VersionId: versionID}, nil + }). + Times(1) } -func (a *versionedAPI) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { - out, err := a.API.HeadObject(ctx, in, optFns...) - if err != nil { - return nil, err - } - out.VersionId = a.versionID - return out, nil -} - -func (a *versionedAPI) GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { - a.tagging = append(a.tagging, in) - if a.taggingErr != nil { - return nil, a.taggingErr - } - // The fake endpoint is unversioned, so serve the current tags for any requested version. - plain := *in - plain.VersionId = nil - return a.API.GetObjectTagging(ctx, &plain, optFns...) +// newVersionedStore returns a store over the mocked API. +func newVersionedStore(t *testing.T, api API) *Store { + t.Helper() + s, err := NewFromClient(api, versionedBucket, WithPartSize(smallPart)) + require.NoError(t, err) + return s } func TestStatPinsTagReadToTheHeadedVersion(t *testing.T) { @@ -59,39 +48,44 @@ func TestStatPinsTagReadToTheHeadedVersion(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - fake := newFake(t) - api := &versionedAPI{API: fake.Client(), versionID: tc.versionID} - s := newVersionedStore(t, fake, api) - putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) + api := NewMockAPI(gomock.NewController(t)) + expectHead(api, tc.versionID) + var tagging []*s3.GetObjectTaggingInput + api.EXPECT(). + GetObjectTagging(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, in *s3.GetObjectTaggingInput, _ ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + tagging = append(tagging, in) + return &s3.GetObjectTaggingOutput{TagSet: []types.Tag{{Key: aws.String("a"), Value: aws.String("b")}}}, nil + }). + Times(1) + s := newVersionedStore(t, api) info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) require.NoError(t, err) assert.Equal(t, map[string]string{"a": "b"}, info.Tags) - require.Len(t, api.tagging, 1) - assert.Equal(t, aws.ToString(tc.want), aws.ToString(api.tagging[0].VersionId)) - assert.Equal(t, tc.want == nil, api.tagging[0].VersionId == nil) + require.Len(t, tagging, 1) + assert.Equal(t, aws.ToString(tc.want), aws.ToString(tagging[0].VersionId)) + assert.Equal(t, tc.want == nil, tagging[0].VersionId == nil) + assert.Equal(t, versionedBucket, aws.ToString(tagging[0].Bucket)) }) } } func TestStatDoesNotFallBackWhenVersionedTagReadFails(t *testing.T) { - fake := newFake(t) - api := &versionedAPI{API: fake.Client(), versionID: aws.String("v1")} - s := newVersionedStore(t, fake, api) - putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) - api.taggingErr = errTaggingDenied + api := NewMockAPI(gomock.NewController(t)) + expectHead(api, aws.String("v1")) + // A denied version-specific read must not retry against the current version, so exactly one + // tagging call is allowed and it must carry the headed version. + api.EXPECT(). + GetObjectTagging(gomock.Any(), gomock.Cond(func(x any) bool { + in, ok := x.(*s3.GetObjectTaggingInput) + return ok && aws.ToString(in.VersionId) == "v1" + })). + Return(nil, errUtils.ErrBlobTaggingDenied). + Times(1) + s := newVersionedStore(t, api) info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) - require.ErrorIs(t, err, errTaggingDenied) + require.ErrorIs(t, err, errUtils.ErrBlobTaggingDenied) assert.Nil(t, info) - require.Len(t, api.tagging, 1, "a denied version-specific read must not retry against the current version") - assert.Equal(t, "v1", aws.ToString(api.tagging[0].VersionId)) -} - -// newVersionedStore returns a store whose requests go through the wrapping API. -func newVersionedStore(t *testing.T, fake *s3test.Server, api *versionedAPI) *Store { - t.Helper() - s, err := NewFromClient(api, fake.Bucket(), WithPartSize(smallPart)) - require.NoError(t, err) - return s } diff --git a/pkg/artifact/blob/s3/s3test/chunked.go b/pkg/artifact/blob/s3/s3test/chunked.go index 560cab7343f..7ab31e1155f 100644 --- a/pkg/artifact/blob/s3/s3test/chunked.go +++ b/pkg/artifact/blob/s3/s3test/chunked.go @@ -3,6 +3,7 @@ package s3test import ( "bufio" "bytes" + "fmt" "io" "net/http" "strconv" @@ -18,7 +19,7 @@ const hexBase = 16 func readBody(r *http.Request) ([]byte, error) { raw, err := io.ReadAll(r.Body) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: read request body: %w", errUtils.ErrBlobOperationFailed, err) } if !isChunked(r) { return raw, nil @@ -39,22 +40,22 @@ func decodeAWSChunked(raw []byte) ([]byte, error) { for { line, err := reader.ReadString('\n') if err != nil { - return nil, err + return nil, fmt.Errorf("%w: read chunk header: %w", errUtils.ErrBlobInvalidChunk, err) } sizeText, _, _ := strings.Cut(strings.TrimSpace(line), ";") size, err := strconv.ParseInt(sizeText, hexBase, bitSize64) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: parse chunk size %q: %w", errUtils.ErrBlobInvalidChunk, sizeText, err) } if size == 0 { return out.Bytes(), nil } if _, err := io.CopyN(&out, reader, size); err != nil { - return nil, err + return nil, fmt.Errorf("%w: read chunk data: %w", errUtils.ErrBlobInvalidChunk, err) } var terminator [2]byte if _, err := io.ReadFull(reader, terminator[:]); err != nil { - return nil, err + return nil, fmt.Errorf("%w: read chunk terminator: %w", errUtils.ErrBlobInvalidChunk, err) } if terminator != [2]byte{'\r', '\n'} { return nil, errUtils.ErrBlobInvalidChunkTerminator diff --git a/pkg/artifact/blob/s3/s3test/objects.go b/pkg/artifact/blob/s3/s3test/objects.go index a4c630356bb..a41674327b7 100644 --- a/pkg/artifact/blob/s3/s3test/objects.go +++ b/pkg/artifact/blob/s3/s3test/objects.go @@ -2,12 +2,13 @@ package s3test import ( "encoding/xml" - "errors" "fmt" "net/http" "net/url" "strconv" "strings" + + errUtils "github.com/cloudposse/atmos/errors" ) // Number parsing parameters and the S3 tag limit. @@ -24,9 +25,6 @@ type objectAttrs struct { tags map[string]string } -// errTooManyTags reports a tag set above the S3 limit. -var errTooManyTags = errors.New("object tags cannot be greater than 10") - // attributes extracts content type, user metadata, and tags from request headers. func attributes(r *http.Request) (objectAttrs, error) { attrs := objectAttrs{contentType: r.Header.Get("Content-Type")} @@ -45,10 +43,10 @@ func attributes(r *http.Request) (objectAttrs, error) { } parsed, err := url.ParseQuery(raw) if err != nil { - return objectAttrs{}, err + return objectAttrs{}, fmt.Errorf("%w: parse object tags: %w", errUtils.ErrBlobInvalid, err) } if len(parsed) > maxTagCount { - return objectAttrs{}, errTooManyTags + return objectAttrs{}, errUtils.ErrBlobTooManyTags } attrs.tags = make(map[string]string, len(parsed)) for key, values := range parsed { diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 0bff11d6e07..0360c578f70 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "testing" "time" @@ -344,12 +345,28 @@ func TestDecodeAWSChunked(t *testing.T) { got, err := decodeAWSChunked(raw) require.NoError(t, err) assert.Equal(t, "hello world", string(got)) - _, err = decodeAWSChunked([]byte("zz\r\n")) - require.Error(t, err) - _, err = decodeAWSChunked([]byte("5\r\nhelloXX0\r\n")) - require.ErrorIs(t, err, errUtils.ErrBlobInvalidChunkTerminator) - _, err = decodeAWSChunked([]byte("5\r\nhel")) - require.Error(t, err) + + malformed := []struct { + name string + raw string + want error + cause error + }{ + {name: "missing header newline", raw: "5", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "non-hex chunk size", raw: "zz\r\n", want: errUtils.ErrBlobInvalidChunk, cause: strconv.ErrSyntax}, + {name: "chunk cut short", raw: "5\r\nhel", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "terminator missing", raw: "5\r\nhello", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "wrong terminator", raw: "5\r\nhelloXX0\r\n", want: errUtils.ErrBlobInvalidChunkTerminator}, + } + for _, tc := range malformed { + t.Run(tc.name, func(t *testing.T) { + _, err := decodeAWSChunked([]byte(tc.raw)) + require.ErrorIs(t, err, tc.want) + if tc.cause != nil { + require.ErrorIs(t, err, tc.cause, "the decoder keeps the underlying cause visible") + } + }) + } } func TestDiscardBodiesKeepsSizesAndETags(t *testing.T) { @@ -401,3 +418,24 @@ func TestRangeEndMaxIntDoesNotOverflow(t *testing.T) { require.True(t, ok) assert.Equal(t, int64(10), got.end) } + +func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) { + tests := []struct { + name string + tagging string + want error + }{ + {name: "malformed query", tagging: "a=%zz", want: errUtils.ErrBlobInvalid}, + {name: "too many tags", tagging: "a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11", want: errUtils.ErrBlobTooManyTags}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, "http://example.invalid/k", strings.NewReader("x")) + require.NoError(t, err) + req.Header.Set("X-Amz-Tagging", tc.tagging) + + _, err = attributes(req) + require.ErrorIs(t, err, tc.want) + }) + } +} diff --git a/pkg/artifact/blob/s3/store.go b/pkg/artifact/blob/s3/store.go index 543d82f8765..994392b71b4 100644 --- a/pkg/artifact/blob/s3/store.go +++ b/pkg/artifact/blob/s3/store.go @@ -20,6 +20,11 @@ type Store struct { mu sync.Mutex api API + // probeSlot admits one conditional-write probe at a time. A buffered channel instead of a mutex + // lets a waiting operation give up when its context ends. + probeSlot chan struct{} + + // probeMu guards the cached verdict below. probeMu sync.Mutex probed bool probeError error @@ -36,7 +41,7 @@ func New(bucket string, opts ...Option) (*Store, error) { for _, opt := range opts { opt(&cfg) } - return &Store{cfg: cfg, api: cfg.api}, nil + return &Store{cfg: cfg, api: cfg.api, probeSlot: make(chan struct{}, 1)}, nil } // NewFromClient returns a Store over an existing client. From 6ea33bf4287b2190b4f1664108a8d9fd7ecdf655 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:49:38 -0600 Subject: [PATCH 30/43] fix(artifact/layout): validate representation descriptors A stored representation whose payload or document descriptor carried a malformed digest or a negative size passed validation and reached stream verification, where the digest's Algorithm call panics. Both lists are now checked through the same record-validation path as the other fields. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/layout/layout_test.go | 44 ++++++++++++++++++++++++++++++ pkg/artifact/layout/records.go | 28 ++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go index b40020e783a..e86a73a53df 100644 --- a/pkg/artifact/layout/layout_test.go +++ b/pkg/artifact/layout/layout_test.go @@ -363,6 +363,50 @@ func TestRepresentation_RejectsUnsafeStoredLocation(t *testing.T) { require.Error(t, err) } +func TestRepresentation_RejectsMalformedDescriptors(t *testing.T) { + tests := []struct { + name string + mutate func(rep *Representation) + }{ + {name: "payload digest that is not a digest", mutate: func(rep *Representation) { rep.Payload[0].Digest = "x" }}, + {name: "payload digest with an unsupported algorithm", mutate: func(rep *Representation) { + rep.Payload[0].Digest = digest.SHA512.FromString("b") + }}, + {name: "payload digest empty", mutate: func(rep *Representation) { rep.Payload[0].Digest = "" }}, + {name: "payload size negative", mutate: func(rep *Representation) { rep.Payload[0].Size = -1 }}, + {name: "document digest malformed", mutate: func(rep *Representation) { + rep.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: "sha256:zz", Size: 2}} + }}, + {name: "document size negative", mutate: func(rep *Representation) { + rep.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: digestC, Size: -5}} + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rep := sampleRepresentation() + tc.mutate(&rep) + require.NotPanics(t, func() { _ = rep.Validate() }) + require.ErrorIs(t, rep.Validate(), errUtils.ErrArtifactRecordInvalid) + + _, err := rep.Marshal() + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a malformed descriptor never reaches storage") + }) + } + + valid := sampleRepresentation() + valid.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: digestC, Size: 0}} + require.NoError(t, valid.Validate(), "well-formed descriptors, including an empty document, stay valid") + + // A stored record with a hand-edited digest fails decoding instead of reaching stream verification. + good := sampleRepresentation() + raw, err := good.Marshal() + require.NoError(t, err) + tampered := strings.Replace(string(raw), digestB.String(), "x", 1) + require.NotEqual(t, string(raw), tampered) + _, err = DecodeRepresentation([]byte(tampered)) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) +} + func sampleOperation() Operation { return Operation{ SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op/1", Name: "team/app", Version: "1.0.0", diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go index 6e1eed54282..eb0d38fb82c 100644 --- a/pkg/artifact/layout/records.go +++ b/pkg/artifact/layout/records.go @@ -236,10 +236,36 @@ func (r *Representation) Validate() error { if _, err := CleanLocation(r.Location); err != nil || r.Location != cleanedOrSelf(r.Location) { return invalid(kindRepresentation, "location %q is not a canonical relative location", r.Location) } + if err := r.validateLists(); err != nil { + return err + } + return validateOptionalDigests(r.RepresentationDigest) +} + +// validateLists checks that both blob descriptor lists are present and well formed. +func (r *Representation) validateLists() error { if r.Payload == nil || r.Documents == nil { return invalid(kindRepresentation, "payload and documents lists are required (use empty lists)") } - return validateOptionalDigests(r.RepresentationDigest) + if err := validateDescriptors("payload", r.Payload); err != nil { + return err + } + return validateDescriptors("documents", r.Documents) +} + +// validateDescriptors checks that every blob descriptor carries a sha256 digest and a size that +// stream verification can enforce. A malformed digest must be rejected here: later reads hand it +// straight to the verifying reader, which has no error path for an unparsable algorithm. +func validateDescriptors(list string, descriptors []ocispec.Descriptor) error { + for index := range descriptors { + if err := artifact.ValidateDigest(descriptors[index].Digest); err != nil { + return invalid(kindRepresentation, "%s[%d] has an invalid digest: %v", list, index, err) + } + if descriptors[index].Size < 0 { + return invalid(kindRepresentation, "%s[%d] has a negative size %d", list, index, descriptors[index].Size) + } + } + return nil } // validateNameVersion checks the name and version of a document. From f25a9c1fcd5459a891e121a7f51ad6ff29b01851 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 13:50:25 -0600 Subject: [PATCH 31/43] fix(artifact/local): lock evidence writes, filter exports by subject, share sentinel - Sign stores evidence under the reference lock after re-checking that the Record is still current, so a Delete that ran while the signer worked can no longer be followed by evidence for a deleted Record. - ReadEvidence exports only envelopes bound to the pinned Record, matching what Verify uses, so a foreign envelope cannot block a valid signature at the destination. - Exhausting the malformed-evidence budget returns ErrArtifactEvidenceMalformed. - The lock-owner test keeps stdin open until the helper is killed and waited on, so it checks OS cleanup rather than the helper's normal release path. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 4 + pkg/artifact/local/evidence.go | 71 +++++++++++------- pkg/artifact/local/lock_test.go | 10 ++- pkg/artifact/local/signature_test.go | 107 +++++++++++++++++++++++++++ pkg/artifact/local/verifier_test.go | 2 +- 5 files changed, 163 insertions(+), 31 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 02e68084f34..f2774adc264 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2138,6 +2138,10 @@ var ( // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The // underlying I/O error is wrapped alongside it for diagnostics. ErrArtifactStreamFailed = errors.New("artifact stream failed") + + // ErrArtifactEvidenceMalformed is returned when a subject's evidence directory holds more malformed + // entries than the repository is willing to read, so valid evidence may be hidden behind them. + ErrArtifactEvidenceMalformed = errors.New("too many malformed evidence files") ) // Blob transport sentinels. They classify outcomes of the transport-only object I/O in diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go index 5f5c873fb7a..0ce11ee6117 100644 --- a/pkg/artifact/local/evidence.go +++ b/pkg/artifact/local/evidence.go @@ -19,8 +19,6 @@ import ( // maxMalformedEvidence bounds diagnostics and reads for invalid evidence entries. const maxMalformedEvidence = 64 -var errTooManyMalformedEvidence = errors.New("too many malformed evidence files") - // loadedEvidence is one stored evidence file that parsed and matched its file name. type loadedEvidence struct { evidence *layout.Evidence @@ -56,7 +54,7 @@ func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact } if ignored > maxMalformedEvidence { diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many invalid evidence entries; remaining entries were not read"}) - return nil, diags, errTooManyMalformedEvidence + return nil, diags, fmt.Errorf("%w: more than %d invalid entries for %s", errUtils.ErrArtifactEvidenceMalformed, maxMalformedEvidence, subject) } } if errors.Is(readErr, io.EOF) { @@ -134,7 +132,7 @@ func (r *Repository) checkSubject(operation string, subject artifact.SignatureSu // and stores it as a detached file named by the evidence digest. Evidence lives beside the Record, // so signing never changes the Record, the reference, or the payload. A signature added after // publication covers the Record that describes the archive, not the original archive bytes. -func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, _ artifact.SignOptions) ([]artifact.SignatureEvidence, error) { +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, opts artifact.SignOptions) ([]artifact.SignatureEvidence, error) { defer perf.Track(nil, "local.Repository.Sign")() if err := r.checkSignRequest(subject, signer); err != nil { @@ -159,13 +157,41 @@ func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject if payload == nil { return nil, fmt.Errorf("%w: the signer returned no evidence", errUtils.ErrArtifactVerificationFailed) } - stored, err := r.storeEvidence(t, subject, payload) + // The signer may have waited arbitrarily long, so a concurrent Delete can have removed the Record + // since the first check. Store the evidence under the reference lock, against a re-checked Record. + var stored artifact.SignatureEvidence + err = r.withEvidenceLock(ctx, t, subject.Digest, opts.Lock, func() error { + var storeErr error + stored, storeErr = r.storeEvidence(t, subject, payload) + return storeErr + }) if err != nil { return nil, err } return []artifact.SignatureEvidence{stored}, nil } +// withEvidenceLock runs fn while holding the reference lock of the Record, after confirming the +// Record is still current. A matching caller lock is reused; otherwise the lock is acquired and +// released here, so evidence is never written for a Record a concurrent Delete removed. +func (r *Repository) withEvidenceLock(ctx context.Context, t *tree, recordDigest digest.Digest, lock artifact.LockHandle, fn func() error) (err error) { + held, err := r.lockEvidenceReference(ctx, t, recordDigest, lock) + if err != nil { + return err + } + if held != nil { + defer func() { + if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { + err = releaseErr + } + }() + } + if _, err := r.requireCurrentRecord(t, recordDigest); err != nil { + return err + } + return fn() +} + // checkSignRequest refuses an unsupported subject, a missing signer, and a signer whose evidence // formats the repository cannot store, before anything is opened or signed. func (r *Repository) checkSignRequest(subject artifact.SignatureSubject, signer artifact.Signer) error { @@ -300,11 +326,11 @@ func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedRe if err != nil { return nil, err } - blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) - for index := range loaded { - blobs = append(blobs, loaded[index].evidence.Blob(loaded[index].locator)) - } - return blobs, nil + // Evidence bound to another subject can share the directory. Verification ignores it, so the + // export must too: the destination rejects a foreign envelope and would stop before importing + // the valid signatures that follow it. + var ignored []artifact.Diagnostic + return evidenceBlobs(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: pinned.RecordDigest}, loaded, &ignored), nil } // WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this @@ -321,21 +347,12 @@ func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedR return nil, r.missingRoot(err, pinned.Ref) } defer t.Close() - held, err := r.lockEvidenceReference(ctx, t, pinned, lock) - if err != nil { - return nil, err - } - if held != nil { - defer func() { - if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { - err = releaseErr - } - }() - } - if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { - return nil, err - } - return r.writeEvidenceLocked(t, pinned, evidence) + err = r.withEvidenceLock(ctx, t, pinned.RecordDigest, lock, func() error { + var writeErr error + stored, writeErr = r.writeEvidenceLocked(t, pinned, evidence) + return writeErr + }) + return stored, err } // writeEvidenceLocked validates and stores each evidence blob under the reference lock. @@ -363,8 +380,8 @@ func (r *Repository) writeEvidenceLocked(t *tree, pinned *artifact.PinnedReferen } // lockEvidenceReference reuses a matching caller lock or acquires the reference lock. -func (r *Repository) lockEvidenceReference(ctx context.Context, t *tree, pinned *artifact.PinnedReference, lock artifact.LockHandle) (*fileLock, error) { - record, _, err := t.loadRecord(pinned.RecordDigest) +func (r *Repository) lockEvidenceReference(ctx context.Context, t *tree, recordDigest digest.Digest, lock artifact.LockHandle) (*fileLock, error) { + record, _, err := t.loadRecord(recordDigest) if err != nil { return nil, err } diff --git a/pkg/artifact/local/lock_test.go b/pkg/artifact/local/lock_test.go index 472d1c10cae..f72798b2bb1 100644 --- a/pkg/artifact/local/lock_test.go +++ b/pkg/artifact/local/lock_test.go @@ -362,11 +362,15 @@ func TestLock_ReleasesWhenItsOwnerProcessExits(t *testing.T) { _, err = repo.Lock(ctx, lockRequest(resource)) require.ErrorIs(t, err, context.DeadlineExceeded, "the helper process holds the lock") - // Kill the owner without letting it release: the OS must free the lock. - _ = stdin.Close() + // Kill the owner while its stdin is still open, so it cannot leave its read loop and release + // the lock on the normal path: the OS alone must free it. Stdin closes only after the wait. require.NoError(t, cmd.Process.Kill()) - _ = cmd.Wait() + waitErr := cmd.Wait() exited = true + _ = stdin.Close() + require.Error(t, waitErr, "the owner was terminated rather than exiting on its own") + require.NotNil(t, cmd.ProcessState) + assert.False(t, cmd.ProcessState.Success(), "the owner did not exit cleanly through its release path") acquire, cancelAcquire := context.WithTimeout(context.Background(), 10*time.Second) defer cancelAcquire() diff --git a/pkg/artifact/local/signature_test.go b/pkg/artifact/local/signature_test.go index f2ab4f75f20..47d0a91ac18 100644 --- a/pkg/artifact/local/signature_test.go +++ b/pkg/artifact/local/signature_test.go @@ -2,9 +2,11 @@ package local import ( "context" + "fmt" "os" "path/filepath" "testing" + "time" "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" @@ -293,3 +295,108 @@ func TestSignature_EvidenceFollowsTheRecordLifecycle(t *testing.T) { require.NoError(t, err) assert.NoDirExists(t, currentDir, "delete removes the evidence the artifact exclusively owns") } + +func TestSignature_ReadEvidenceExportsOnlyThePinnedSubject(t *testing.T) { + source := newRepo(t, signedSpec(dirSpec())) + publish(t, source, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, source, defaultRef()) + subject := recordSubject(t, source) + signer := newSigner() + signed, err := source.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + // A valid envelope for another subject sits in the same directory, as the verification test + // above explicitly permits. Exporting it would make the destination reject the whole batch. + dir := filepath.Join(source.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + foreign, err := layout.NewEvidence(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("foreign subject")}, + &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte("{}")}) + require.NoError(t, err) + raw, err := foreign.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, foreign.Digest().Encoded()+".json"), raw, 0o644)) + require.Len(t, listFiles(t, dir), 2, "both envelopes are on disk") + + blobs, err := source.ReadEvidence(context.Background(), &pinned) + require.NoError(t, err) + require.Len(t, blobs, 1, "only evidence bound to the pinned Record is exported") + assert.Equal(t, signed[0].Descriptor.Digest, blobs[0].Evidence.Descriptor.Digest) + assert.Equal(t, subject.Digest, blobs[0].Evidence.Subject.Digest) + + destination := newRepo(t, signedSpec(dirSpec())) + publish(t, destination, &prepareSpec{}, artifact.PublishOptions{}) + destPinned := resolve(t, destination, defaultRef()) + stored, err := destination.WriteEvidence(context.Background(), &destPinned, blobs, nil) + require.NoError(t, err, "the destination accepts the exported batch") + require.Len(t, stored, 1) + verified, err := destination.Verify(context.Background(), recordSubject(t, destination), signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) +} + +func TestSignature_ReadEvidenceReportsExhaustedMalformedEntries(t *testing.T) { + repo, subject := signedWith(t, &artifacttest.FakeVerifier{}) + pinned := resolve(t, repo, defaultRef()) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + for i := range maxMalformedEvidence + 1 { + require.NoError(t, os.WriteFile(filepath.Join(dir, fmt.Sprintf("bad-%03d.json", i)), []byte("bad"), 0o644)) + } + + blobs, err := repo.ReadEvidence(context.Background(), &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactEvidenceMalformed) + assert.Empty(t, blobs) +} + +// A signer may wait long enough for the Record to be deleted. Signing must then fail instead of +// recreating the evidence directory for a Record that no longer exists. +func TestSignature_SignDoesNotResurrectEvidenceForADeletedRecord(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo) + + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + _, deleteErr := repo.Delete(ctx, pinned, artifact.DeleteOptions{}) + require.NoError(t, deleteErr, "the Record is deleted while the signer is working") + return &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte(`{"late":true}`)}, nil + }) + + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.Error(t, err, "signing a deleted Record must not report success") + assert.Empty(t, evidence) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))), + "no evidence directory is recreated for the deleted Record") +} + +func TestSignature_SignStoresEvidenceUnderTheReferenceLock(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + resource := testName + ":" + testVersion + + handle, err := repo.Lock(context.Background(), lockRequest(resource)) + require.NoError(t, err) + + t.Run("another writer holds the lock, so storing waits and honors the deadline", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, signErr := repo.Sign(ctx, subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, signErr, context.DeadlineExceeded) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest)))) + }) + t.Run("a matching caller lock is reused", func(t *testing.T) { + evidence, signErr := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{Lock: handle}) + require.NoError(t, signErr) + require.Len(t, evidence, 1) + }) + require.NoError(t, handle.Release(context.Background())) + + t.Run("the lock is released after signing", func(t *testing.T) { + _, signErr := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{}) + require.NoError(t, signErr) + reacquired, lockErr := repo.Lock(context.Background(), lockRequest(resource)) + require.NoError(t, lockErr) + require.NoError(t, reacquired.Release(context.Background())) + }) +} diff --git a/pkg/artifact/local/verifier_test.go b/pkg/artifact/local/verifier_test.go index 124126b83e9..0e5c618a01d 100644 --- a/pkg/artifact/local/verifier_test.go +++ b/pkg/artifact/local/verifier_test.go @@ -164,6 +164,6 @@ func TestVerifyFailsClosedOnTooManyMalformedEvidenceFiles(t *testing.T) { } _, err := repo.Verify(context.Background(), subject, trustedPolicy) require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) - require.ErrorIs(t, err, errTooManyMalformedEvidence) + require.ErrorIs(t, err, errUtils.ErrArtifactEvidenceMalformed) assert.Zero(t, probe.seen) } From cfa49de76e04fb3cd039b07a5c2efde12f72ac0a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:10:09 -0600 Subject: [PATCH 32/43] fix(artifact/blob/local): refuse conditional operations a plain-file store cannot make atomic A plain-file store leaves unconditional puts and deletes unlocked for legacy writers, so an if-match put could be overwritten between its check and its commit. Such a store now refuses conditional puts and deletes and no longer advertises them, unless WithLockAllWrites makes every mutation take the same root lock. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/delete.go | 3 + pkg/artifact/blob/local/doc.go | 4 +- pkg/artifact/blob/local/options.go | 7 +- .../blob/local/plain_conditional_test.go | 129 ++++++++++++++++++ pkg/artifact/blob/local/put.go | 3 + pkg/artifact/blob/local/store.go | 13 +- pkg/artifact/blob/local/store_test.go | 2 +- 7 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/blob/local/plain_conditional_test.go diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go index d609ac53d71..ebb8ac293b2 100644 --- a/pkg/artifact/blob/local/delete.go +++ b/pkg/artifact/blob/local/delete.go @@ -17,6 +17,9 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) if err := s.validate(blob.OpDelete, key); err != nil { return err } + if err := s.Capabilities().CheckDelete(storeName, key, opts); err != nil { + return err + } if err := checkContext(ctx, blob.OpDelete, key); err != nil { return err } diff --git a/pkg/artifact/blob/local/doc.go b/pkg/artifact/blob/local/doc.go index 0f28761b014..a6472bab9c9 100644 --- a/pkg/artifact/blob/local/doc.go +++ b/pkg/artifact/blob/local/doc.go @@ -10,7 +10,9 @@ // protects content and attributes across interrupted commits; recovery runs before the next managed // operation. Direct filesystem edits and mixing [WithPlainFileCompatibility] with managed stores // on one root are outside this coordination. Plain-file mode preserves the legacy CI layout and -// unlocked unconditional operations. The journal recovers interrupted processes, not power loss. +// unlocked unconditional operations, so it refuses conditional puts and deletes (and does not +// advertise them) unless [WithLockAllWrites] makes every mutation honor the same lock. The journal +// recovers interrupted processes, not power loss. // // A local object cannot share a path with a key prefix: "a" and "a/b" cannot both exist. ETags are // opaque content-based SHA-256 tokens. Identical contents have the same token, including after an diff --git a/pkg/artifact/blob/local/options.go b/pkg/artifact/blob/local/options.go index 3d8f298f08c..a8969be86d6 100644 --- a/pkg/artifact/blob/local/options.go +++ b/pkg/artifact/blob/local/options.go @@ -30,7 +30,9 @@ func WithFilePerm(perm os.FileMode) Option { } // WithLockAllWrites serializes unconditional writes and deletes with conditional ones. Managed -// stores already do this; it can also be used with plain-file compatibility. +// stores already do this. A plain-file store enforces if-absent and if-match conditions only with +// this option, because only then does every mutation honor the lock that makes a condition check +// and its commit atomic; without it the plain-file store refuses conditional operations. func WithLockAllWrites() Option { defer perf.Track(nil, "blob.local.WithLockAllWrites")() @@ -38,7 +40,8 @@ func WithLockAllWrites() Option { } // WithPlainFileCompatibility preserves the legacy CI layout and unlocked unconditional operations. -// Do not mix this mode with managed stores on the same root when consistency is required. +// Because unconditional operations stay unlocked, the store refuses conditional puts and deletes +// and does not advertise them unless [WithLockAllWrites] is also set. Do not mix this mode with managed stores on the same root when consistency is required. func WithPlainFileCompatibility() Option { defer perf.Track(nil, "local.WithPlainFileCompatibility")() diff --git a/pkg/artifact/blob/local/plain_conditional_test.go b/pkg/artifact/blob/local/plain_conditional_test.go new file mode 100644 index 00000000000..cccf824d982 --- /dev/null +++ b/pkg/artifact/blob/local/plain_conditional_test.go @@ -0,0 +1,129 @@ +package local + +import ( + "bytes" + "context" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func putBody(t *testing.T, s *Store, key, body string, cond blob.Condition) error { + t.Helper() + _, err := s.Put(context.Background(), key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body)), Condition: cond}) + return err +} + +func readBody(t *testing.T, s *Store, key string) string { + t.Helper() + obj, err := s.Get(context.Background(), key, blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + return string(data) +} + +// A plain-file store leaves unconditional writes unlocked for legacy writers, so it cannot make a +// conditional write atomic against them. It must say so instead of advertising and accepting +// conditions it cannot enforce. +func TestPlainFileStoreWithoutLockAllRefusesConditionalOperations(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility()) + require.NoError(t, putBody(t, s, "k", "A", blob.Condition{})) + info, err := s.Stat(context.Background(), "k", blob.StatOptions{}) + require.NoError(t, err) + + caps := s.Capabilities() + assert.False(t, caps.ConditionalPut, "conditional writes are not advertised") + assert.False(t, caps.ConditionalDelete, "conditional deletes are not advertised") + + tests := []struct { + name string + run func() error + }{ + {name: "if-absent put", run: func() error { return putBody(t, s, "new", "B", blob.IfAbsent()) }}, + {name: "if-match put", run: func() error { return putBody(t, s, "k", "B", blob.IfMatch(info.ETag)) }}, + {name: "if-match delete", run: func() error { + return s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: info.ETag}) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.run() + require.ErrorIs(t, err, errUtils.ErrBlobConditionalUnsupported) + assert.True(t, blob.IsConditionalUnsupported(err)) + }) + } + assert.Equal(t, "A", readBody(t, s, "k"), "a refused conditional operation changes nothing") + _, err = s.Stat(context.Background(), "new", blob.StatOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobNotFound) + assert.NoFileExists(t, s.Root()+"/"+lockName, "refusing takes no lock") + + require.NoError(t, putBody(t, s, "k", "C", blob.Condition{}), "unconditional writes keep working") + require.NoError(t, s.Delete(context.Background(), "k", blob.DeleteOptions{}), "unconditional deletes keep working") +} + +// With WithLockAllWrites a plain-file store serializes every mutation, so a conditional operation +// is atomic against unconditional ones and the capabilities are honest. +func TestPlainFileStoreWithLockAllEnforcesConditionsAgainstUnconditionalWrites(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility(), WithLockAllWrites()) + assert.True(t, s.Capabilities().ConditionalPut) + assert.True(t, s.Capabilities().ConditionalDelete) + + t.Run("unconditional mutations wait for a held root lock", func(t *testing.T) { + require.NoError(t, putBody(t, s, "held", "A", blob.Condition{})) + release, err := s.guard(context.Background(), blob.OpPut, "held", true) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, putErr := s.Put(ctx, "held", bytes.NewReader([]byte("C")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, putErr, context.DeadlineExceeded, "an unconditional put cannot slip between a check and a commit") + ctx2, cancel2 := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel2() + deleteErr := s.Delete(ctx2, "held", blob.DeleteOptions{}) + require.ErrorIs(t, deleteErr, context.DeadlineExceeded, "an unconditional delete cannot slip in either") + + release() + assert.Equal(t, "A", readBody(t, s, "held"), "nothing was written while the lock was held") + require.NoError(t, putBody(t, s, "held", "C", blob.Condition{})) + }) + + t.Run("an if-match put never overwrites a competing unconditional put", func(t *testing.T) { + for round := 0; round < 50; round++ { + key := "race" + require.NoError(t, putBody(t, s, key, "A", blob.Condition{})) + info, err := s.Stat(context.Background(), key, blob.StatOptions{}) + require.NoError(t, err) + + var wg sync.WaitGroup + start := make(chan struct{}) + var conditionalErr error + wg.Add(2) + go func() { + defer wg.Done() + <-start + conditionalErr = putBody(t, s, key, "B", blob.IfMatch(info.ETag)) + }() + go func() { + defer wg.Done() + <-start + assert.NoError(t, putBody(t, s, key, "C", blob.Condition{})) + }() + close(start) + wg.Wait() + + if conditionalErr != nil { + require.ErrorIs(t, conditionalErr, errUtils.ErrBlobPreconditionFailed) + } + assert.Equal(t, "C", readBody(t, s, key), "round %d: the conditional write either ran before C or failed", round) + } + }) +} diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index 69ffb826cd6..91d7d5d7be0 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -67,6 +67,9 @@ func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOption if err := s.validate(blob.OpPut, key); err != nil { return err } + if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { + return err + } if opts.Size < blob.UnknownSize || opts.Size == math.MaxInt64 { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid or unverifiable size %d", errUtils.ErrBlobInvalid, opts.Size)) } diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go index 039d43ce7c4..42765395e01 100644 --- a/pkg/artifact/blob/local/store.go +++ b/pkg/artifact/blob/local/store.go @@ -70,9 +70,12 @@ func (s *Store) Root() string { func (s *Store) Capabilities() blob.Capabilities { defer perf.Track(nil, "local.Store.Capabilities")() + // A plain-file store leaves unconditional mutations unlocked for legacy writers, so it can only + // make a condition atomic when WithLockAllWrites makes every mutation honor the same lock. + conditional := s.enforcesConditions() return blob.Capabilities{ - ConditionalPut: true, - ConditionalDelete: true, + ConditionalPut: conditional, + ConditionalDelete: conditional, Multipart: true, Tags: !s.plainFiles, Metadata: !s.plainFiles, @@ -82,6 +85,12 @@ func (s *Store) Capabilities() blob.Capabilities { } } +// enforcesConditions reports whether every mutation takes the root lock, which is what makes an +// if-absent or if-match check and its commit atomic against competing writers of this store. +func (s *Store) enforcesConditions() bool { + return !s.plainFiles || s.lockAll +} + // validate rejects keys that are unsafe on a filesystem or collide with the store's own files. func (s *Store) validate(op, key string) error { validationKey := key diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go index 0c0ae948379..545527c1b2b 100644 --- a/pkg/artifact/blob/local/store_test.go +++ b/pkg/artifact/blob/local/store_test.go @@ -340,7 +340,7 @@ func TestDeletePrefixOnlyDirectoryIsMissingObject(t *testing.T) { opts []Option }{ {name: "managed"}, - {name: "plain", opts: []Option{WithPlainFileCompatibility()}}, + {name: "plain", opts: []Option{WithPlainFileCompatibility(), WithLockAllWrites()}}, } { t.Run(tc.name, func(t *testing.T) { store := newStore(t, tc.opts...) From 5bef473d51ac76553cc9e042c585ca7ebaebc41f Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 16:10:30 -0600 Subject: [PATCH 33/43] fix(artifact/service): hold the reference lock while signing Service.Sign locked only subject: and handed that handle to the repository, but a local repository requires the writer lock to cover the reference :, so every signing request against a local repository failed with a conflict. Signing now takes the subject lock and the Record's reference lock together, in the canonical order delete uses, and passes the reference handle. A concurrent delete, through the service or directly on the repository, waits for the signature to be stored. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/service/local_sign_test.go | 199 ++++++++++++++++++++++++ pkg/artifact/service/sign.go | 22 ++- 2 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 pkg/artifact/service/local_sign_test.go diff --git a/pkg/artifact/service/local_sign_test.go b/pkg/artifact/service/local_sign_test.go new file mode 100644 index 00000000000..1032eeb101d --- /dev/null +++ b/pkg/artifact/service/local_sign_test.go @@ -0,0 +1,199 @@ +package service_test + +import ( + "context" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guard for the schema fields the local repository config uses. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": ""}} + +// gatedSigner signs like [artifacttest.FakeSigner] but announces when it starts and waits to be +// released, so a test can run another operation while a signature is in flight. +type gatedSigner struct { + artifacttest.FakeSigner + started chan struct{} + release chan struct{} + once sync.Once +} + +func newGatedSigner() *gatedSigner { + return &gatedSigner{ + FakeSigner: artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"}, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +func (g *gatedSigner) Sign(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + g.once.Do(func() { close(g.started) }) + select { + case <-g.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + return g.FakeSigner.Sign(ctx, subject) +} + +// localSignEnv is a service over a real local directory repository. +type localSignEnv struct { + svc *service.Service + repo *local.Repository + published *service.PublishOutcome + digestRef artifact.Ref +} + +func newLocalSignEnv(t *testing.T, name string, signer artifact.Signer) *localSignEnv { + t.Helper() + cfg := &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": filepath.Join(t.TempDir(), "repo")}, + } + repo, err := local.NewDir("dst", cfg, local.WithVerifier(&artifacttest.FakeVerifier{}), local.WithOwner("local-test")) + require.NoError(t, err) + static := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{}} + svc, err := service.New(static, nil, service.WithSigner(signer), service.WithOwner("service-test"), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + src := (&env{t: t}).source("hello") + published, err := svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: src, Name: name, Version: "1.0.0"}) + require.NoError(t, err) + return &localSignEnv{svc: svc, repo: repo, published: published, digestRef: artifact.Ref{Name: name, Digest: published.Operation.RecordDigest}} +} + +func (e *localSignEnv) sign(ctx context.Context) (*artifact.SignResult, error) { + return e.svc.Sign(ctx, artifact.SignRequest{Repository: "dst", Ref: e.digestRef}) +} + +// Signing through the service must work against a real local repository: the lock the service +// holds has to be one the repository accepts. Names cover a reference that sorts before and after +// the "subject:" lock resource, because the service takes its locks in canonical order. +func TestSign_ThroughTheServiceStoresEvidenceInALocalRepository(t *testing.T) { + tests := []struct { + name string + ref string + }{ + {name: "reference sorts after the subject lock", ref: "tools/example"}, + {name: "reference sorts before the subject lock", ref: "a/example"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newLocalSignEnv(t, tt.ref, releaseSigner) + + result, err := e.sign(context.Background()) + require.NoError(t, err) + require.Len(t, result.Evidence, 1) + assert.Equal(t, artifact.SubjectRecord, result.Subject.Kind) + assert.Equal(t, e.published.Operation.RecordDigest, result.Subject.Digest) + assert.Equal(t, artifact.LockScopeHost, result.Operation.LockScope) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: artifact.Ref{Name: tt.ref, Version: "1.0.0"}}) + require.NoError(t, err) + require.Len(t, inspected.Inspection.Evidence, 1) + assert.Equal(t, result.Evidence[0].Descriptor.Digest, inspected.Inspection.Evidence[0].Descriptor.Digest) + + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: e.digestRef}) + require.Error(t, err, "no trust configuration is set, so verification reports unchecked rather than green") + assert.Equal(t, artifact.VerificationUnchecked, verified.Verification.Status) + + // Every lock the service took was released: a second signature and a delete both proceed. + _, err = e.sign(context.Background()) + require.NoError(t, err) + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: tt.ref, Version: "1.0.0"}}) + require.NoError(t, err) + }) + } +} + +// While a signer is working, a concurrent delete through the service must wait for the signature +// to be stored instead of removing the Record underneath it, and it must not deadlock with the +// signing locks whichever way the reference sorts against the subject lock. +func TestSign_ThroughTheServiceExcludesAConcurrentDelete(t *testing.T) { + for _, name := range []string{"tools/example", "a/example"} { + t.Run(name, func(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, name, signer) + + signed := make(chan error, 1) + go func() { + _, err := e.sign(context.Background()) + signed <- err + }() + <-signer.started + + deleted := make(chan error, 1) + go func() { + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: name, Version: "1.0.0"}}) + deleted <- err + }() + + select { + case err := <-deleted: + t.Fatalf("delete finished while the signer was still working: %v", err) + case <-time.After(200 * time.Millisecond): + } + + close(signer.release) + require.NoError(t, waitFor(t, signed), "signing completes once the signer returns") + require.NoError(t, waitFor(t, deleted), "the delete runs after the signature is stored") + + _, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: artifact.Ref{Name: name, Version: "1.0.0"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// A writer that bypasses the service and deletes through the repository takes the repository's own +// reference lock, which signing through the service must also hold. +func TestSign_ThroughTheServiceExcludesADirectRepositoryDelete(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, "tools/example", signer) + + signed := make(chan error, 1) + go func() { + _, err := e.sign(context.Background()) + signed <- err + }() + <-signer.started + + pinned, err := e.repo.Resolve(context.Background(), artifact.Ref{Name: "tools/example", Version: "1.0.0"}) + require.NoError(t, err) + deleted := make(chan error, 1) + go func() { + _, deleteErr := e.repo.Delete(context.Background(), pinned, artifact.DeleteOptions{}) + deleted <- deleteErr + }() + + select { + case err := <-deleted: + t.Fatalf("a direct delete finished while the signer was still working: %v", err) + case <-time.After(200 * time.Millisecond): + } + close(signer.release) + require.NoError(t, waitFor(t, signed)) + require.NoError(t, waitFor(t, deleted)) +} + +func waitFor(t *testing.T, ch <-chan error) error { + t.Helper() + select { + case err := <-ch: + return err + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for the operation to finish") + return nil + } +} diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go index b126bf60034..afe2b3f483e 100644 --- a/pkg/artifact/service/sign.go +++ b/pkg/artifact/service/sign.go @@ -40,8 +40,8 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig if err := checkSigningCapability(request.Repository, &info, s.signer, subject); err != nil { return err } - resource := subjectLockResource(request.Ref.Name) - locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: resource}}) + targets, writerResource := signLockTargets(request, repo, &info, inspection) + locks, err := s.acquireLocks(ctx, op, opSign, targets) if err != nil { return err } @@ -50,7 +50,7 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig if err := verifyLockedSubject(ctx, repo, request.Ref, &pinned); err != nil { return err } - evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, resource)}) + evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, writerResource)}) if err != nil { return fmt.Errorf("sign %s in repository %q: %w", request.Ref, request.Repository, err) } @@ -61,6 +61,22 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig return nil } +// signLockTargets returns the locks signing holds and the resource whose handle is passed to the +// repository as the writer lock. Signing takes the same two locks as a delete: the subject lock, +// which serializes every operation on the artifact name, and the reference lock of the Record, +// which publish, delete and a repository's own internal locking all honor. They are acquired in +// canonical order by [Service.acquireLocks], so signing and deleting never deadlock. A subject +// without a Record (an OCI root) has no reference lock to hold and uses the subject lock alone. +func signLockTargets(request *artifact.SignRequest, repo artifact.Repository, info *RepositoryInfo, inspection *artifact.Inspection) (targets []lockTarget, writerResource string) { + subjectResource := subjectLockResource(request.Ref.Name) + targets = []lockTarget{{alias: request.Repository, repo: repo, info: info, resource: subjectResource}} + if inspection.Record == nil { + return targets, subjectResource + } + writerResource = artifact.Ref{Name: inspection.Record.Name, Version: inspection.Record.Version}.String() + return append(targets, lockTarget{alias: request.Repository, repo: repo, info: info, resource: writerResource}), writerResource +} + func verifyLockedSubject(ctx context.Context, repo artifact.Repository, ref artifact.Ref, pinned *artifact.PinnedReference) error { current, err := repo.Resolve(ctx, ref) if err != nil { From a283b44a2739b08d04075f4627a0cab221e9a0d3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:46:21 -0600 Subject: [PATCH 34/43] test(artifact/blob/local): release held guards and roots when a test aborts, bound commit waits A failed assertion left the root lock held for the next subtest, and the commit-ordering test waited without a bound for a commit that could already have failed. Guards now release through a once-only wrapper registered with the test cleanup, root handles close on cleanup, and every wait on another goroutine has a timeout and reports the real error. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/hold_test.go | 72 +++++++++++++++++++ .../blob/local/plain_conditional_test.go | 10 ++- pkg/artifact/blob/local/transaction_test.go | 28 +++++--- 3 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 pkg/artifact/blob/local/hold_test.go diff --git a/pkg/artifact/blob/local/hold_test.go b/pkg/artifact/blob/local/hold_test.go new file mode 100644 index 00000000000..01425da3953 --- /dev/null +++ b/pkg/artifact/blob/local/hold_test.go @@ -0,0 +1,72 @@ +package local + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// waitTimeout bounds every wait on another goroutine so a broken product path fails the test instead of hanging it. +const waitTimeout = 10 * time.Second + +// holdGuard takes the put guard for a test and returns a release that is safe to call any number +// of times. The guard is also released when the test ends, so an assertion that aborts the test can +// never leave the root lock held for the next subtest. +func holdGuard(t *testing.T, s *Store, key string, conditional bool) func() { + t.Helper() + release, err := s.guard(context.Background(), blob.OpPut, key, conditional) + require.NoError(t, err) + return onceRelease(t, release) +} + +// onceRelease wraps a release function that is not idempotent so it runs once, at the first explicit +// call or at test cleanup, whichever comes first. +func onceRelease(t *testing.T, release func()) func() { + t.Helper() + var once sync.Once + wrapped := func() { once.Do(release) } + t.Cleanup(wrapped) + return wrapped +} + +// closeOnCleanup closes a handle when the test ends, ignoring the error of a second close. +func closeOnCleanup(t *testing.T, root *os.Root) { + t.Helper() + t.Cleanup(func() { _ = root.Close() }) +} + +// receive waits for one value from ch and fails the test, instead of hanging it, when nothing arrives. +func receive[T any](t *testing.T, ch <-chan T, what string) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(waitTimeout): + t.Fatalf("timed out waiting for %s", what) + var zero T + return zero + } +} + +func TestHoldGuardReleasesWhenTheTestEnds(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility(), WithLockAllWrites()) + + t.Run("a subtest that never releases", func(t *testing.T) { + explicit := holdGuard(t, s, "held", true) + explicit() + explicit() // A second release is a no-op rather than a double unlock. + _ = holdGuard(t, s, "held", true) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + release, err := s.guard(ctx, blob.OpPut, "held", true) + require.NoError(t, err, "the lock a finished subtest still held is free for the next one") + release() +} diff --git a/pkg/artifact/blob/local/plain_conditional_test.go b/pkg/artifact/blob/local/plain_conditional_test.go index cccf824d982..41eaa9efbf8 100644 --- a/pkg/artifact/blob/local/plain_conditional_test.go +++ b/pkg/artifact/blob/local/plain_conditional_test.go @@ -79,8 +79,7 @@ func TestPlainFileStoreWithLockAllEnforcesConditionsAgainstUnconditionalWrites(t t.Run("unconditional mutations wait for a held root lock", func(t *testing.T) { require.NoError(t, putBody(t, s, "held", "A", blob.Condition{})) - release, err := s.guard(context.Background(), blob.OpPut, "held", true) - require.NoError(t, err) + release := holdGuard(t, s, "held", true) ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) defer cancel() @@ -118,7 +117,12 @@ func TestPlainFileStoreWithLockAllEnforcesConditionsAgainstUnconditionalWrites(t assert.NoError(t, putBody(t, s, key, "C", blob.Condition{})) }() close(start) - wg.Wait() + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + receive(t, finished, "both writers to finish") if conditionalErr != nil { require.ErrorIs(t, conditionalErr, errUtils.ErrBlobPreconditionFailed) diff --git a/pkg/artifact/blob/local/transaction_test.go b/pkg/artifact/blob/local/transaction_test.go index 8a6001037b6..e8503cff904 100644 --- a/pkg/artifact/blob/local/transaction_test.go +++ b/pkg/artifact/blob/local/transaction_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "sync" "testing" "time" @@ -28,9 +29,13 @@ func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) require.NoError(t, err) defer pending.discard(root) - release, err := s.guard(ctx, blob.OpPut, "key", false) - require.NoError(t, err) + release := holdGuard(t, s, "key", false) between, proceed := make(chan struct{}), make(chan struct{}) + // The commit stays parked on proceed until the test releases it. Release it on every exit so an + // assertion that aborts the test never strands the commit goroutine holding the root lock. + var proceedOnce sync.Once + letProceed := func() { proceedOnce.Do(func() { close(proceed) }) } + t.Cleanup(letProceed) commitDone := make(chan error, 1) go func() { defer release() @@ -45,7 +50,13 @@ func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { return nil }) }() - <-between + select { + case <-between: + case err := <-commitDone: + t.Fatalf("the commit ended before its attribute rename: %v", err) + case <-time.After(waitTimeout): + t.Fatal("timed out waiting for the commit to reach its attribute rename") + } type result struct { body string info *blob.ObjectInfo @@ -76,10 +87,10 @@ func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { early = true case <-time.After(100 * time.Millisecond): } - close(proceed) - require.NoError(t, <-commitDone) + letProceed() + require.NoError(t, receive(t, commitDone, "the commit to finish")) require.False(t, early) - got, stat := <-getDone, <-statDone + got, stat := receive(t, getDone, "Get to finish"), receive(t, statDone, "Stat to finish") require.NoError(t, got.err) require.NoError(t, stat.err) assert.Equal(t, "new", got.body) @@ -98,8 +109,7 @@ func TestManagedCommitFailureRestoresOldAttributes(t *testing.T) { pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) require.NoError(t, err) defer pending.discard(root) - release, err := s.guard(ctx, blob.OpPut, "key", false) - require.NoError(t, err) + release := holdGuard(t, s, "key", false) failure := errors.New("injected data rename failure") err = s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { if to == "key" { @@ -128,6 +138,7 @@ func TestManagedRecoveryAfterInterruptedCommit(t *testing.T) { require.NoError(t, err) root, err := s.openRoot(false) require.NoError(t, err) + closeOnCleanup(t, root) pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) require.NoError(t, err) oldAttrs, err := rootReadFile(root, attrsPath("key")) @@ -165,6 +176,7 @@ func TestManagedRecoveryRemovesNewAttributesWhenOldObjectHadNone(t *testing.T) { require.NoError(t, err) root, err := s.openRoot(false) require.NoError(t, err) + closeOnCleanup(t, root) pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) require.NoError(t, err) marker, err := json.Marshal(transaction{Target: "key", DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp}) From 33b84ee6b916cf38c8854c69c31c9607ccc076d9 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:46:44 -0600 Subject: [PATCH 35/43] test(artifact/service): bound the wait for the signer and free gates on every exit The signing tests waited on the signer's start without a bound, so a Sign that failed before calling the signer, such as an inspection or lock error, hung the test and hid the real error. They now select on the signer start, the Sign result and a timeout, and cancel the signing and free the signer when the test ends. The list-concurrency gate opens on cleanup and its result wait is bounded, and a lock taken for another owner is released when the test ends. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/service/consume_test.go | 15 ++- pkg/artifact/service/local_sign_test.go | 129 ++++++++++++++++++++---- pkg/artifact/service/mutate_test.go | 1 + 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index fa77e9db59c..8ac8fa8e1e0 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -379,8 +379,12 @@ type listGate struct { peak atomic.Int32 entered chan struct{} release chan struct{} + freeing sync.Once } +// open lets every blocked listing continue. It is safe to call any number of times. +func (g *listGate) open() { g.freeing.Do(func() { close(g.release) }) } + // gatedList returns a generated repository that blocks every List on the gate before delegating. func gatedList(t *testing.T, base *artifacttest.Repository, gate *listGate) artifact.Repository { return delegatingRepository(t, base, repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { @@ -403,6 +407,7 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { seedList(t, e, alias, map[string]string{"app/x:1": alias}, nil) } gate := &listGate{entered: make(chan struct{}, 3), release: make(chan struct{})} + t.Cleanup(gate.open) // A failed assertion must not leave a listing blocked on the gate. for _, alias := range []string{"a", "b", "c"} { e.static.Repositories[alias] = gatedList(t, e.repos[alias], gate) } @@ -422,15 +427,19 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { select { case <-gate.entered: case <-time.After(2 * time.Second): - close(gate.release) t.Fatal("listing did not start") } select { case <-gate.entered: case <-time.After(200 * time.Millisecond): } - close(gate.release) - outcome := <-done + gate.open() + var outcome listOutcome + select { + case outcome = <-done: + case <-time.After(10 * time.Second): + t.Fatal("the listing did not finish after the gate opened") + } require.NoError(t, outcome.err) assert.Equal(t, 3, outcome.count) assert.Equal(t, int32(1), gate.peak.Load()) diff --git a/pkg/artifact/service/local_sign_test.go b/pkg/artifact/service/local_sign_test.go index 1032eeb101d..9d1b17c0e0a 100644 --- a/pkg/artifact/service/local_sign_test.go +++ b/pkg/artifact/service/local_sign_test.go @@ -2,6 +2,8 @@ package service_test import ( "context" + "errors" + "fmt" "path/filepath" "sync" "testing" @@ -28,6 +30,7 @@ type gatedSigner struct { started chan struct{} release chan struct{} once sync.Once + freeing sync.Once } func newGatedSigner() *gatedSigner { @@ -38,6 +41,9 @@ func newGatedSigner() *gatedSigner { } } +// free lets the signer finish. It is safe to call any number of times. +func (g *gatedSigner) free() { g.freeing.Do(func() { close(g.release) }) } + func (g *gatedSigner) Sign(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { g.once.Do(func() { close(g.started) }) select { @@ -78,6 +84,52 @@ func (e *localSignEnv) sign(ctx context.Context) (*artifact.SignResult, error) { return e.svc.Sign(ctx, artifact.SignRequest{Repository: "dst", Ref: e.digestRef}) } +// signerEntryTimeout bounds how long a test waits for the signer to start or an operation to finish. +const signerEntryTimeout = 10 * time.Second + +// awaitSignerEntry waits until the signer has started. When signing ends first, for example because +// the service refused it before calling the signer, it returns that real result instead of waiting +// for a signer that will never run. A signer that never starts within the timeout is an error too. +func awaitSignerEntry(started <-chan struct{}, signed <-chan error, timeout time.Duration) error { + select { + case <-started: + return nil + case err := <-signed: + if err == nil { + return errSignedWithoutTheSigner + } + return fmt.Errorf("signing ended before the signer started: %w", err) + case <-time.After(timeout): + return errSignerNeverStarted + } +} + +var ( + errSignedWithoutTheSigner = errors.New("signing finished without calling the signer") + errSignerNeverStarted = errors.New("the signer did not start in time") +) + +// startSigning signs through the service in the background and waits for the signer to be working. +// Its context is cancelled and the signer freed when the test ends, so a failed assertion cannot +// leave the signing goroutine holding the reference locks. +func startSigning(t *testing.T, e *localSignEnv, signer *gatedSigner) <-chan error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + signer.free() + }) + signed := make(chan error, 1) + go func() { + _, err := e.sign(ctx) + signed <- err + }() + if err := awaitSignerEntry(signer.started, signed, signerEntryTimeout); err != nil { + t.Fatal(err) + } + return signed +} + // Signing through the service must work against a real local repository: the lock the service // holds has to be one the repository accepts. Names cover a reference that sorts before and after // the "subject:" lock resource, because the service takes its locks in canonical order. @@ -126,13 +178,7 @@ func TestSign_ThroughTheServiceExcludesAConcurrentDelete(t *testing.T) { t.Run(name, func(t *testing.T) { signer := newGatedSigner() e := newLocalSignEnv(t, name, signer) - - signed := make(chan error, 1) - go func() { - _, err := e.sign(context.Background()) - signed <- err - }() - <-signer.started + signed := startSigning(t, e, signer) deleted := make(chan error, 1) go func() { @@ -146,7 +192,7 @@ func TestSign_ThroughTheServiceExcludesAConcurrentDelete(t *testing.T) { case <-time.After(200 * time.Millisecond): } - close(signer.release) + signer.free() require.NoError(t, waitFor(t, signed), "signing completes once the signer returns") require.NoError(t, waitFor(t, deleted), "the delete runs after the signature is stored") @@ -161,13 +207,7 @@ func TestSign_ThroughTheServiceExcludesAConcurrentDelete(t *testing.T) { func TestSign_ThroughTheServiceExcludesADirectRepositoryDelete(t *testing.T) { signer := newGatedSigner() e := newLocalSignEnv(t, "tools/example", signer) - - signed := make(chan error, 1) - go func() { - _, err := e.sign(context.Background()) - signed <- err - }() - <-signer.started + signed := startSigning(t, e, signer) pinned, err := e.repo.Resolve(context.Background(), artifact.Ref{Name: "tools/example", Version: "1.0.0"}) require.NoError(t, err) @@ -182,7 +222,7 @@ func TestSign_ThroughTheServiceExcludesADirectRepositoryDelete(t *testing.T) { t.Fatalf("a direct delete finished while the signer was still working: %v", err) case <-time.After(200 * time.Millisecond): } - close(signer.release) + signer.free() require.NoError(t, waitFor(t, signed)) require.NoError(t, waitFor(t, deleted)) } @@ -192,8 +232,63 @@ func waitFor(t *testing.T, ch <-chan error) error { select { case err := <-ch: return err - case <-time.After(10 * time.Second): + case <-time.After(signerEntryTimeout): t.Fatal("timed out waiting for the operation to finish") return nil } } + +// A signing that ends before it reaches the signer must surface its own error. Waiting only on the +// signer's start would hang forever, because the signer is never called. +func TestAwaitSignerEntry(t *testing.T) { + boom := errors.New("boom") + closed := make(chan struct{}) + close(closed) + + tests := []struct { + name string + started <-chan struct{} + signed func() <-chan error + wantErr error + }{ + {name: "signer started", started: closed, signed: func() <-chan error { return make(chan error) }}, + {name: "signing failed first", started: make(chan struct{}), signed: func() <-chan error { + ch := make(chan error, 1) + ch <- boom + return ch + }, wantErr: boom}, + {name: "signing returned without the signer", started: make(chan struct{}), signed: func() <-chan error { + ch := make(chan error, 1) + ch <- nil + return ch + }, wantErr: errSignedWithoutTheSigner}, + {name: "nothing happens", started: make(chan struct{}), signed: func() <-chan error { return make(chan error) }, wantErr: errSignerNeverStarted}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := awaitSignerEntry(tt.started, tt.signed(), 50*time.Millisecond) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.wantErr) + }) + } +} + +// Signing through the service with a context that is already cancelled fails before the signer +// runs. The test helper must report that failure rather than wait for the signer to start. +func TestStartSigning_ReportsAnEarlyServiceError(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, "tools/example", signer) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + signed := make(chan error, 1) + go func() { + _, err := e.sign(ctx) + signed <- err + }() + err := awaitSignerEntry(signer.started, signed, signerEntryTimeout) + require.ErrorIs(t, err, context.Canceled, "the real service error is reported") +} diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go index 07436eac227..797abce9b8d 100644 --- a/pkg/artifact/service/mutate_test.go +++ b/pkg/artifact/service/mutate_test.go @@ -145,6 +145,7 @@ func TestLocks_HeldByAnotherOwnerBlocksUntilCanceledAndLeaksNothing(t *testing.T e := newEnv(t, envConfig{}) other, err := e.repos["dst"].Lock(context.Background(), artifact.LockRequest{Resource: exampleRef.String(), Owner: "someone-else"}) require.NoError(t, err) + t.Cleanup(func() { _ = other.Release(context.Background()) }) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) defer cancel() From ed5bcf79506a91a3785990bed2da405b3483f905 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:47:03 -0600 Subject: [PATCH 36/43] test(artifact/local): release held locks and handles when a test aborts Locks taken with repo.Lock and the tree and document handles opened in tests were released only on the success path, so a failed assertion left them held. They are now released on test cleanup through holdLock, and a second release after an explicit one is ignored. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/local/delete_test.go | 1 + pkg/artifact/local/hold_test.go | 36 ++++++++++++++++++++++++ pkg/artifact/local/lock_test.go | 42 ++++++++++------------------ pkg/artifact/local/roundtrip_test.go | 1 + pkg/artifact/local/signature_test.go | 3 +- 5 files changed, 54 insertions(+), 29 deletions(-) create mode 100644 pkg/artifact/local/hold_test.go diff --git a/pkg/artifact/local/delete_test.go b/pkg/artifact/local/delete_test.go index 250ff908e65..b67d76eb0b5 100644 --- a/pkg/artifact/local/delete_test.go +++ b/pkg/artifact/local/delete_test.go @@ -137,6 +137,7 @@ func TestStoredLocationOutsideTheRootIsRejected(t *testing.T) { tr, err := repo.openTree(false) require.NoError(t, err) + t.Cleanup(func() { _ = tr.Close() }) rep, err := tr.loadRepresentation(key) require.NoError(t, err) require.NoError(t, tr.Close()) diff --git a/pkg/artifact/local/hold_test.go b/pkg/artifact/local/hold_test.go new file mode 100644 index 00000000000..1e10cba80d1 --- /dev/null +++ b/pkg/artifact/local/hold_test.go @@ -0,0 +1,36 @@ +package local + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// holdLock takes a repository lock for a test. The lock is also released when the test ends, so an +// assertion that aborts the test never leaves the resource held. Releasing it explicitly first is +// fine: the cleanup's second release only reports that the handle is gone, and that is ignored. +func holdLock(t *testing.T, repo *Repository, request artifact.LockRequest) artifact.LockHandle { + t.Helper() + handle, err := repo.Lock(context.Background(), request) + require.NoError(t, err) + t.Cleanup(func() { _ = handle.Release(context.Background()) }) + return handle +} + +func TestHoldLockReleasesWhenTheTestEnds(t *testing.T) { + repo := newRepo(t, dirSpec()) + + t.Run("a subtest that never releases", func(t *testing.T) { + _ = holdLock(t, repo, lockRequest("r")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + next, err := repo.Lock(ctx, lockRequest("r")) + require.NoError(t, err, "the lock a finished subtest still held is free for the next one") + require.NoError(t, next.Release(context.Background())) +} diff --git a/pkg/artifact/local/lock_test.go b/pkg/artifact/local/lock_test.go index f72798b2bb1..2e1193e33a9 100644 --- a/pkg/artifact/local/lock_test.go +++ b/pkg/artifact/local/lock_test.go @@ -102,8 +102,7 @@ func lockRequest(resource string) artifact.LockRequest { func TestLock_HandleContract(t *testing.T) { repo := newRepo(t, dirSpec()) - handle, err := repo.Lock(context.Background(), lockRequest("team/app:1.0.0")) - require.NoError(t, err) + handle := holdLock(t, repo, lockRequest("team/app:1.0.0")) assert.Equal(t, "team/app:1.0.0", handle.Resource()) assert.Equal(t, "test-owner", handle.Owner()) assert.Equal(t, artifact.LockScopeHost, handle.Scope()) @@ -117,8 +116,7 @@ func TestLock_HandleContract(t *testing.T) { require.ErrorIs(t, handle.Release(context.Background()), errUtils.ErrArtifactConflict, "a handle cannot be released twice") assert.FileExists(t, lockFile(repo, "team/app:1.0.0")) - other, err := repo.Lock(context.Background(), artifact.LockRequest{Resource: "x"}) - require.NoError(t, err) + other := holdLock(t, repo, artifact.LockRequest{Resource: "x"}) assert.NotEmpty(t, other.Owner(), "an empty owner defaults to the repository owner") require.NoError(t, other.Release(context.Background())) } @@ -147,8 +145,7 @@ func TestLock_RejectsUnsupportedGuarantees(t *testing.T) { func TestLock_ReleaseChecksTheOwnershipToken(t *testing.T) { repo := newRepo(t, dirSpec()) - handle, err := repo.Lock(context.Background(), lockRequest("r")) - require.NoError(t, err) + handle := holdLock(t, repo, lockRequest("r")) real, ok := handle.(*fileLock) require.True(t, ok) @@ -157,7 +154,7 @@ func TestLock_ReleaseChecksTheOwnershipToken(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - _, err = repo.Lock(ctx, lockRequest("r")) + _, err := repo.Lock(ctx, lockRequest("r")) require.ErrorIs(t, err, context.DeadlineExceeded, "the real owner still holds the lock") require.NoError(t, handle.Release(context.Background())) } @@ -203,8 +200,7 @@ func TestLock_CompetingHandlesAreMutuallyExclusive(t *testing.T) { func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { repo := newRepo(t, dirSpec()) - holder, err := repo.Lock(context.Background(), lockRequest("r")) - require.NoError(t, err) + holder := holdLock(t, repo, lockRequest("r")) var wg sync.WaitGroup for range 5 { @@ -278,8 +274,7 @@ func TestTwoWritersToOneReference(t *testing.T) { // existing holder; with a caller-held lock it proceeds without deadlocking on itself. func TestPublishHonorsTheReferenceLock(t *testing.T) { repo := newRepo(t, dirSpec()) - holder, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) - require.NoError(t, err) + holder := holdLock(t, repo, lockRequest(testName+":"+testVersion)) prepared := prepare(t, repo, &prepareSpec{}) done := make(chan error, 1) @@ -301,20 +296,17 @@ func TestPublishHonorsTheReferenceLock(t *testing.T) { } second := newRepo(t, dirSpec()) - held, err := second.Lock(context.Background(), lockRequest("team/other:1.0.0")) - require.NoError(t, err) - t.Cleanup(func() { _ = held.Release(context.Background()) }) - _, err = second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "team/other"}), artifact.PublishOptions{Lock: held}) + held := holdLock(t, second, lockRequest("team/other:1.0.0")) + _, err := second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "team/other"}), artifact.PublishOptions{Lock: held}) require.NoError(t, err, "a service-held lock is honored, not re-acquired") } func TestPublishCancellationWhileWaitingLeaksNothing(t *testing.T) { repo := newRepo(t, dirSpec()) - holder, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) - require.NoError(t, err) + holder := holdLock(t, repo, lockRequest(testName+":"+testVersion)) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() - _, err = repo.Publish(ctx, prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + _, err := repo.Publish(ctx, prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) require.ErrorIs(t, err, context.DeadlineExceeded) require.NoError(t, holder.Release(context.Background())) _, err = repo.Resolve(context.Background(), defaultRef()) @@ -325,8 +317,7 @@ func TestPublishCancellationWhileWaitingLeaksNothing(t *testing.T) { func TestLock_ExcludesOtherProcesses(t *testing.T) { repo := newRepo(t, dirSpec()) resource := "team/app:1.0.0" - handle, err := repo.Lock(context.Background(), lockRequest(resource)) - require.NoError(t, err) + handle := holdLock(t, repo, lockRequest(resource)) assert.Equal(t, helperExitBusy, exitCode(t, helperCommand(t, "try", repo.root, resource)), "another process cannot take a held lock") other := resource + "-other" @@ -385,10 +376,8 @@ func lockFile(repo *Repository, resource string) string { func TestPublishRejectsLockForAnotherReference(t *testing.T) { repo := newRepo(t, dirSpec()) - held, err := repo.Lock(context.Background(), lockRequest("other:1.0.0")) - require.NoError(t, err) - defer held.Release(context.Background()) - _, err = repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{Lock: held}) + held := holdLock(t, repo, lockRequest("other:1.0.0")) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{Lock: held}) require.ErrorIs(t, err, errUtils.ErrArtifactConflict) } @@ -396,11 +385,10 @@ func TestWriteEvidenceHonorsReferenceLock(t *testing.T) { repo := newRepo(t, dirSpec()) publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) pinned := resolve(t, repo, defaultRef()) - held, err := repo.Lock(context.Background(), lockRequest(testName+":"+testVersion)) - require.NoError(t, err) + held := holdLock(t, repo, lockRequest(testName+":"+testVersion)) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() - _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + _, err := repo.WriteEvidence(ctx, &pinned, nil, nil) require.ErrorIs(t, err, context.DeadlineExceeded) _, err = repo.WriteEvidence(context.Background(), &pinned, nil, held) require.NoError(t, err) diff --git a/pkg/artifact/local/roundtrip_test.go b/pkg/artifact/local/roundtrip_test.go index 4b2b632d62c..f8b566418ab 100644 --- a/pkg/artifact/local/roundtrip_test.go +++ b/pkg/artifact/local/roundtrip_test.go @@ -193,6 +193,7 @@ func TestDocumentsAreReadWithoutThePayload(t *testing.T) { stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) require.NoError(t, err) + t.Cleanup(func() { _ = stream.Reader.Close() }) data, err := io.ReadAll(stream.Reader) require.NoError(t, err) require.NoError(t, stream.Reader.Close()) diff --git a/pkg/artifact/local/signature_test.go b/pkg/artifact/local/signature_test.go index 47d0a91ac18..c56d7976c61 100644 --- a/pkg/artifact/local/signature_test.go +++ b/pkg/artifact/local/signature_test.go @@ -375,8 +375,7 @@ func TestSignature_SignStoresEvidenceUnderTheReferenceLock(t *testing.T) { subject := recordSubject(t, repo) resource := testName + ":" + testVersion - handle, err := repo.Lock(context.Background(), lockRequest(resource)) - require.NoError(t, err) + handle := holdLock(t, repo, lockRequest(resource)) t.Run("another writer holds the lock, so storing waits and honors the deadline", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) From e8b7a4b7c4ec6132ad1f6d1b18742c4553f17845 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 17:47:35 -0600 Subject: [PATCH 37/43] test(artifact): bound S3 probe waits and close readers when a test aborts The S3 probe test waited on the mocked endpoint without a bound and left it parked when an assertion failed, and the emulator request handlers blocked behind an unread channel on a retry. Readers and files opened by the S3, CI-store, compat, produce, snapshot and archive tests are now closed on test cleanup as well as explicitly. Co-Authored-By: Claude Sonnet 5.5 --- pkg/archive/deterministic_test.go | 1 + pkg/artifact/blob/s3/client_test.go | 12 ++++++++-- pkg/artifact/blob/s3/context_test.go | 23 ++++++++++++++++--- pkg/artifact/compat/compat_test.go | 1 + pkg/artifact/produce/produce_test.go | 1 + pkg/artifact/snapshot/snapshot_test.go | 1 + .../artifact/local/transport_parity_test.go | 5 +++- pkg/ci/artifact/s3/endpoint_request_test.go | 10 ++++++-- pkg/ci/artifact/s3/transport_parity_test.go | 2 ++ 9 files changed, 48 insertions(+), 8 deletions(-) diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go index 618517f99a6..668aee0a87b 100644 --- a/pkg/archive/deterministic_test.go +++ b/pkg/archive/deterministic_test.go @@ -104,6 +104,7 @@ func TestWriteDeterministic_ZipEntries(t *testing.T) { assert.NotZero(t, byName["link"].Mode()&fs.ModeSymlink) rc, err := byName["link"].Open() require.NoError(t, err) + t.Cleanup(func() { _ = rc.Close() }) target, err := io.ReadAll(rc) require.NoError(t, err) require.NoError(t, rc.Close()) diff --git a/pkg/artifact/blob/s3/client_test.go b/pkg/artifact/blob/s3/client_test.go index 2bc5531fc88..8074d3a8586 100644 --- a/pkg/artifact/blob/s3/client_test.go +++ b/pkg/artifact/blob/s3/client_test.go @@ -82,7 +82,10 @@ func TestLegacyClientKeepsAmbientFallback(t *testing.T) { func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) { requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests <- r.Clone(r.Context()) + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -100,7 +103,12 @@ func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) { _, err = s.Stat(context.Background(), "templates/stack.yaml", blob.StatOptions{}) require.NoError(t, err) - request := <-requests + var request *http.Request + select { + case request = <-requests: + default: + t.Fatal("the emulator endpoint received no request") + } assert.Equal(t, endpoint.Host, request.Host, "path-style: the bucket is not in the host name") assert.Equal(t, "/artifacts/templates/stack.yaml", request.URL.Path) } diff --git a/pkg/artifact/blob/s3/context_test.go b/pkg/artifact/blob/s3/context_test.go index 3b2fad54b3d..4305daf46ee 100644 --- a/pkg/artifact/blob/s3/context_test.go +++ b/pkg/artifact/blob/s3/context_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "sync" "testing" "time" @@ -21,6 +22,11 @@ func TestProbeWaiterStopsWhenItsContextEnds(t *testing.T) { api := NewMockAPI(gomock.NewController(t)) started := make(chan struct{}) release := make(chan struct{}) + // The mocked PutObject parks on release. Free it on every exit so a failed assertion never + // strands the probe goroutine. + var freeOnce sync.Once + free := func() { freeOnce.Do(func() { close(release) }) } + t.Cleanup(free) errEndpointDown := errors.New("endpoint down") api.EXPECT(). PutObject(gomock.Any(), gomock.Any(), gomock.Any()). @@ -36,7 +42,13 @@ func TestProbeWaiterStopsWhenItsContextEnds(t *testing.T) { first := make(chan error, 1) go func() { first <- store.ProbeConditionalWrites(context.Background()) }() - <-started + select { + case <-started: + case err := <-first: + t.Fatalf("the first probe ended before it reached the endpoint: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("the first probe never reached the endpoint") + } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() @@ -50,8 +62,13 @@ func TestProbeWaiterStopsWhenItsContextEnds(t *testing.T) { t.Fatal("the waiting probe stayed blocked after its context ended") } - close(release) - require.ErrorIs(t, <-first, errEndpointDown, "the first probe still finishes and reports its own failure") + free() + select { + case err := <-first: + require.ErrorIs(t, err, errEndpointDown, "the first probe still finishes and reports its own failure") + case <-time.After(5 * time.Second): + t.Fatal("the first probe never finished after the endpoint was released") + } } func TestPutChecksContextBeforeReadingTheBody(t *testing.T) { diff --git a/pkg/artifact/compat/compat_test.go b/pkg/artifact/compat/compat_test.go index ec81d680c89..e8323e7d821 100644 --- a/pkg/artifact/compat/compat_test.go +++ b/pkg/artifact/compat/compat_test.go @@ -121,6 +121,7 @@ func TestLegacyCIFixtureStillReadsThroughCurrentStore(t *testing.T) { require.Len(t, files, 2) bodies := map[string]string{} for _, file := range files { + t.Cleanup(func() { _ = file.Data.Close() }) body, readErr := io.ReadAll(file.Data) require.NoError(t, readErr) require.NoError(t, file.Data.Close()) diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 709701757fd..c71d332d9f4 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -51,6 +51,7 @@ func readBlob(t *testing.T, blob *artifact.StagedBlob) []byte { t.Helper() reader, err := blob.Open(context.Background()) require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) data, err := io.ReadAll(reader) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 6ebf58bef8c..0016d6413f0 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -69,6 +69,7 @@ func TestCapture_FilePackageInventory(t *testing.T) { reader, err := snap.Open("nested/deep/file.txt") require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) data, err := io.ReadAll(reader) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go index fca85b69506..d6bc601b67b 100644 --- a/pkg/ci/artifact/local/transport_parity_test.go +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -87,7 +87,9 @@ func TestBundledStoreTarBytesAndChecksumSurviveTheTransport(t *testing.T) { assert.Equal(t, "plan.tfplan", got[0].Name) assert.Equal(t, "lock.hcl", got[1].Name) for index, want := range []string{"plan-bytes", "lock"} { - data, readErr := io.ReadAll(got[index].Data) + body := got[index].Data + t.Cleanup(func() { _ = body.Close() }) + data, readErr := io.ReadAll(body) require.NoError(t, readErr) require.NoError(t, got[index].Data.Close()) assert.Equal(t, want, string(data)) @@ -196,6 +198,7 @@ func TestLegacyLiteralBackslashName(t *testing.T) { assert.FileExists(t, filepath.Join(root, name)) reader, _, err := store.Download(ctx, name) require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) data, err := io.ReadAll(reader) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/pkg/ci/artifact/s3/endpoint_request_test.go b/pkg/ci/artifact/s3/endpoint_request_test.go index 4cc13f31987..8bd73858e0e 100644 --- a/pkg/ci/artifact/s3/endpoint_request_test.go +++ b/pkg/ci/artifact/s3/endpoint_request_test.go @@ -18,7 +18,10 @@ import ( func TestIdentityEndpointUsesBucketPath(t *testing.T) { requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests <- r.Clone(r.Context()) + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -44,7 +47,10 @@ func TestIdentityEndpointUsesBucketPath(t *testing.T) { func TestAdapterIdentityResolverRoutesToEndpoint(t *testing.T) { requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests <- r.Clone(r.Context()) + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } w.WriteHeader(http.StatusOK) })) defer server.Close() diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go index 28c4ee46e57..d6f56cf6edb 100644 --- a/pkg/ci/artifact/s3/transport_parity_test.go +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -79,6 +79,7 @@ func TestAdapterRoundTripWithoutPrefix(t *testing.T) { reader, got, err := store.Download(ctx, "k.tar") require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) var buf bytes.Buffer _, err = buf.ReadFrom(reader) require.NoError(t, err) @@ -161,6 +162,7 @@ func TestAdapterBundledStoreChecksumAndTarSurviveTheTransport(t *testing.T) { require.NoError(t, err) require.Len(t, got, 1) assert.Equal(t, "plan.tfplan", got[0].Name) + t.Cleanup(func() { _ = got[0].Data.Close() }) data, err := io.ReadAll(got[0].Data) require.NoError(t, err) require.NoError(t, got[0].Data.Close()) From 737aced46f7ab73f01143c5f83970e47b74694ad Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 21:25:19 -0600 Subject: [PATCH 38/43] test(artifacts): bound the guard acquisition in holdGuard holdGuard waited on context.Background(), so a release that failed to free the root lock hung the test before its timed acquisition and reported nothing. Bound the wait with waitTimeout and fail with a message naming the key. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/hold_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/blob/local/hold_test.go b/pkg/artifact/blob/local/hold_test.go index 01425da3953..44c99a635ef 100644 --- a/pkg/artifact/blob/local/hold_test.go +++ b/pkg/artifact/blob/local/hold_test.go @@ -20,8 +20,12 @@ const waitTimeout = 10 * time.Second // never leave the root lock held for the next subtest. func holdGuard(t *testing.T, s *Store, key string, conditional bool) func() { t.Helper() - release, err := s.guard(context.Background(), blob.OpPut, key, conditional) - require.NoError(t, err) + // Bound the acquisition so a release that failed to free the lock fails the test with a clear + // message instead of hanging it. The lock does not depend on the context once it is held. + ctx, cancel := context.WithTimeout(context.Background(), waitTimeout) + defer cancel() + release, err := s.guard(ctx, blob.OpPut, key, conditional) + require.NoError(t, err, "timed out or failed acquiring the put guard for %q; a previous release did not free the root lock", key) return onceRelease(t, release) } From c44c1d9010e13984a23cdac1d2f77c3e054bc6e0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 21:25:56 -0600 Subject: [PATCH 39/43] test(artifacts): join the commit before closing its root TestManagedReadWaitsForBothCommitRenames closed the root and discarded staging in defers that ran before the cleanup that opens the commit gate, so a commit still parked on the gate could continue with a closed root. Tear down in order instead: open the gate, wait for the commit goroutine with a bound, then discard staging and close the root. If the commit never finishes, the test fails and leaves the root open rather than closing it under the commit. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/transaction_test.go | 28 ++++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pkg/artifact/blob/local/transaction_test.go b/pkg/artifact/blob/local/transaction_test.go index e8503cff904..2afe16ecfac 100644 --- a/pkg/artifact/blob/local/transaction_test.go +++ b/pkg/artifact/blob/local/transaction_test.go @@ -25,19 +25,27 @@ func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { require.NoError(t, err) root, err := s.openRoot(false) require.NoError(t, err) - defer root.Close() pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) require.NoError(t, err) - defer pending.discard(root) + // commitJoined reports whether the commit goroutine ended. The root and staging area are torn + // down only after it did, so a commit that is still running never sees a closed root. + commitJoined := true + t.Cleanup(func() { + if !commitJoined { + return + } + pending.discard(root) + _ = root.Close() + }) release := holdGuard(t, s, "key", false) between, proceed := make(chan struct{}), make(chan struct{}) // The commit stays parked on proceed until the test releases it. Release it on every exit so an // assertion that aborts the test never strands the commit goroutine holding the root lock. var proceedOnce sync.Once letProceed := func() { proceedOnce.Do(func() { close(proceed) }) } - t.Cleanup(letProceed) - commitDone := make(chan error, 1) + commitDone, commitFinished := make(chan error, 1), make(chan struct{}) go func() { + defer close(commitFinished) defer release() commitDone <- s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { if err := rootRenameReplace(r, from, to); err != nil { @@ -50,6 +58,18 @@ func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { return nil }) }() + // Registered after the goroutine starts, so it runs before the root teardown above: open the + // gate, then wait for the commit to finish within a bound. + commitJoined = false + t.Cleanup(func() { + letProceed() + select { + case <-commitFinished: + commitJoined = true + case <-time.After(waitTimeout): + t.Errorf("timed out waiting for the commit goroutine to finish; leaving its root open") + } + }) select { case <-between: case err := <-commitDone: From c9281221706653f661c8b93cb42651bb820e7940 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:29:44 -0600 Subject: [PATCH 40/43] fix(artifacts): clear code-scanning lint findings in the CI stores Track the public Name and SetAuthContext methods with perf.Track, keep the by-value Query on List because the Backend interface requires it, pass the Query to queryToPrefix by pointer, and name the repeated wrap format. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- pkg/ci/artifact/local/store.go | 4 +++- pkg/ci/artifact/s3/store.go | 23 ++++++++++++++--------- pkg/ci/artifact/s3/store_test.go | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index 0047b3a36a7..a6d7989ec25 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -67,6 +67,8 @@ func NewStore(opts artifact.StoreOptions) (artifact.Backend, error) { // Name returns the store type name. func (s *Store) Name() string { + defer perf.Track(nil, "artifact.local.Name")() + return storeName } @@ -198,7 +200,7 @@ func (s *Store) Delete(ctx context.Context, name string) error { // List lists artifacts matching the given query. // Finds metadata sidecar files with a corresponding artifact file (not dir), // applies Query filters, and sorts newest first. -func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { +func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { //nolint:gocritic // Query is passed by value to satisfy the Backend interface. defer perf.Track(nil, "artifact.local.List")() transport, err := s.transport() diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index 9e096f489f1..f044a6b9c18 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -27,6 +27,9 @@ const ( // Metadata is stored as a JSON file alongside the artifact. metadataSuffix = ".metadata.json" + + // The wrap format joins a static sentinel error with the underlying cause so both stay in the chain. + wrapFormat = "%w: %w" ) // Store implements the artifact.Backend interface using AWS S3. @@ -96,6 +99,8 @@ func NewStore(opts artifact.StoreOptions) (artifact.Backend, error) { // registry honors this contract by calling SetAuthContext synchronously // before returning the backend. func (s *Store) SetAuthContext(resolver artifact.AuthContextResolver, identityName string) { + defer perf.Track(nil, "s3.Store.SetAuthContext")() + s.authResolver = resolver if identityName != "" { s.identityName = identityName @@ -200,7 +205,7 @@ func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int transport, err := s.transport(ctx) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrArtifactUploadFailed, err) + return fmt.Errorf(wrapFormat, errUtils.ErrArtifactUploadFailed, err) } fullKey := s.fullKey(key) @@ -238,7 +243,7 @@ func (s *Store) Download(ctx context.Context, key string) (io.ReadCloser, *artif transport, err := s.transport(ctx) if err != nil { - return nil, nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactDownloadFailed, err) + return nil, nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactDownloadFailed, err) } fullKey := s.fullKey(key) @@ -264,7 +269,7 @@ func (s *Store) Delete(ctx context.Context, key string) error { transport, err := s.transport(ctx) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) + return fmt.Errorf(wrapFormat, errUtils.ErrArtifactDeleteFailed, err) } fullKey := s.fullKey(key) @@ -281,16 +286,16 @@ func (s *Store) Delete(ctx context.Context, key string) error { } // List lists artifacts matching the given query. -func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { +func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { //nolint:gocritic // Query is passed by value to satisfy the Backend interface. defer perf.Track(nil, "s3.List")() transport, err := s.transport(ctx) if err != nil { - return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) + return nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactListFailed, err) } // Convert query to prefix-based S3 listing. - prefix := s.queryToPrefix(query) + prefix := s.queryToPrefix(&query) fullPrefix := s.fullKey(prefix) objects, _, err := blob.ListAll(ctx, transport, blob.ListOptions{Prefix: fullPrefix}) @@ -333,7 +338,7 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) { transport, err := s.transport(ctx) if err != nil { - return false, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) + return false, fmt.Errorf(wrapFormat, errUtils.ErrArtifactListFailed, err) } fullKey := s.fullKey(key) @@ -354,7 +359,7 @@ func (s *Store) GetMetadata(ctx context.Context, key string) (*artifact.Metadata transport, err := s.transport(ctx) if err != nil { - return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactMetadataFailed, err) + return nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactMetadataFailed, err) } fullKey := s.fullKey(key) @@ -397,7 +402,7 @@ func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, artifact } // queryToPrefix converts an artifact.Query to an S3 prefix string. -func (s *Store) queryToPrefix(query artifact.Query) string { +func (s *Store) queryToPrefix(query *artifact.Query) string { if query.All { return "" } diff --git a/pkg/ci/artifact/s3/store_test.go b/pkg/ci/artifact/s3/store_test.go index 6541a6c316a..2a58c34804e 100644 --- a/pkg/ci/artifact/s3/store_test.go +++ b/pkg/ci/artifact/s3/store_test.go @@ -502,7 +502,7 @@ func TestStore_QueryToPrefix(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := store.queryToPrefix(tt.query) + result := store.queryToPrefix(&tt.query) assert.Equal(t, tt.expected, result) }) } From 6951aa0ec4df4567638f7af61e0715bf07973c52 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:21:28 -0600 Subject: [PATCH 41/43] fix(artifact): keep the destination's file mode when an upload replaces an object The shared local transport staged every object with the store's default mode and renamed it over the destination, so overwriting an artifact whose mode had been restricted by hand (for example 0600) widened it to 0644. The previous CI adapter truncated in place and kept the mode. Stage the new file with the destination's permission bits, set before any content is written, in both the plain-file and managed layouts. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/local/put.go | 37 ++++++++++++ pkg/artifact/blob/local/put_mode_test.go | 71 ++++++++++++++++++++++++ pkg/ci/artifact/local/store_test.go | 53 ++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 pkg/artifact/blob/local/put_mode_test.go diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go index 91d7d5d7be0..929dc1cd9e1 100644 --- a/pkg/artifact/blob/local/put.go +++ b/pkg/artifact/blob/local/put.go @@ -93,6 +93,11 @@ func (s *Store) stage(ctx context.Context, root *os.Root, key string, body io.Re return nil, s.classifyCreate(root, key, target, err) } pending := &staged{tempPath: name} + if err := keepDestinationMode(root, file, target); err != nil { + _ = file.Close() + pending.discard(root) + return nil, wrapIO(blob.OpPut, key, err) + } hash := sha256.New() written, copyErr := copyBody(ctx, io.MultiWriter(file, hash), body, opts.Size) closeErr := file.Close() @@ -109,6 +114,38 @@ func (s *Store) stage(ctx context.Context, root *os.Root, key string, body io.Re return pending, nil } +// keepDestinationMode gives the staged file the permission bits of the regular file it will +// replace, so an overwrite does not widen (or narrow) a mode that was set by hand. It runs before +// any content is written, so restricted data is never exposed under the default mode. A missing +// or non-regular destination keeps the store's default mode. +func keepDestinationMode(root *os.Root, file *os.File, target string) error { + want, ok := destinationMode(root, target) + if !ok { + return nil + } + created, err := file.Stat() + if err != nil { + return err + } + if created.Mode().Perm() == want { + return nil + } + if err := file.Chmod(want); err != nil && !errors.Is(err, errors.ErrUnsupported) { + return err + } + return nil +} + +// destinationMode returns the permission bits of the regular file at target. It reports false +// when there is no such file; a destination that cannot be inspected fails at commit instead. +func destinationMode(root *os.Root, target string) (os.FileMode, bool) { + info, err := root.Stat(target) + if err != nil || !info.Mode().IsRegular() { + return 0, false + } + return info.Mode().Perm(), true +} + // stageAttrs writes the attribute file to a temporary path when any attribute is set. func (s *Store) stageAttrs(root *os.Root, pending *staged, target string, a attrs) error { if a.empty() { diff --git a/pkg/artifact/blob/local/put_mode_test.go b/pkg/artifact/blob/local/put_mode_test.go new file mode 100644 index 00000000000..c16bf85c339 --- /dev/null +++ b/pkg/artifact/blob/local/put_mode_test.go @@ -0,0 +1,71 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "testing/iotest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// Replacing an object keeps the mode of the file on disk, in both layouts, and never exposes the +// new content under a wider mode. +func TestPutReplaceKeepsDestinationMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + + layouts := []struct { + name string + opts []Option + }{ + {name: "managed"}, + {name: "plain files", opts: []Option{WithPlainFileCompatibility()}}, + } + modes := []os.FileMode{0o600, 0o644, 0o664} + for _, layout := range layouts { + for _, mode := range modes { + t.Run(layout.name+"/"+mode.String(), func(t *testing.T) { + s := newStore(t, layout.opts...) + require.NoError(t, putBody(t, s, "dir/obj", "old", blob.Condition{})) + path := filepath.Join(s.root, "dir", "obj") + require.NoError(t, os.Chmod(path, mode)) + + require.NoError(t, putBody(t, s, "dir/obj", "new contents", blob.Condition{})) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, mode, info.Mode().Perm()) + assert.Equal(t, "new contents", readBody(t, s, "dir/obj")) + }) + } + } +} + +// A destination that does not exist yet gets the store's default mode, and a failed put leaves the +// existing destination untouched. +func TestPutNewObjectUsesStoreMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + s := newStore(t, WithPlainFileCompatibility()) + require.NoError(t, putBody(t, s, "fresh", "data", blob.Condition{})) + + info, err := os.Stat(filepath.Join(s.root, "fresh")) + require.NoError(t, err) + assert.Zero(t, info.Mode().Perm()&^DefaultFilePerm) + + require.NoError(t, os.Chmod(filepath.Join(s.root, "fresh"), 0o600)) + _, err = s.Put(context.Background(), "fresh", iotest.ErrReader(os.ErrDeadlineExceeded), blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + info, err = os.Stat(filepath.Join(s.root, "fresh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + assert.Equal(t, "data", readBody(t, s, "fresh")) +} diff --git a/pkg/ci/artifact/local/store_test.go b/pkg/ci/artifact/local/store_test.go index 71fcc8f34d1..d762b4a1c3e 100644 --- a/pkg/ci/artifact/local/store_test.go +++ b/pkg/ci/artifact/local/store_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -541,3 +542,55 @@ func TestIntegration_FullLifecycle(t *testing.T) { _, err = os.Stat(filepath.Join(tmpDir, name+metadataSuffix)) assert.True(t, os.IsNotExist(err)) } + +// Replacing an artifact keeps the permission bits of the file already on disk. The previous +// adapter truncated the file in place, so an artifact restricted by hand stayed restricted. +func TestUploadReplaceKeepsDestinationMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + + tests := []struct { + name string + existing os.FileMode + want os.FileMode + }{ + {name: "restricted artifact stays restricted", existing: 0o600, want: 0o600}, + {name: "readable artifact stays readable", existing: 0o644, want: 0o644}, + {name: "group-writable artifact stays group-writable", existing: 0o664, want: 0o664}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + store := &Store{basePath: dir} + path := filepath.Join(dir, "plan.tar") + require.NoError(t, os.WriteFile(path, []byte("old"), tt.existing)) + require.NoError(t, os.Chmod(path, tt.existing)) + + require.NoError(t, store.Upload(context.Background(), "plan.tar", strings.NewReader("new contents"), -1, nil)) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, tt.want, info.Mode().Perm()) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new contents", string(data)) + }) + } +} + +// A new artifact has no destination to inherit from and keeps the default mode. +func TestUploadNewArtifactUsesDefaultMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + dir := t.TempDir() + store := &Store{basePath: dir} + + require.NoError(t, store.Upload(context.Background(), "fresh.tar", strings.NewReader("data"), -1, nil)) + + info, err := os.Stat(filepath.Join(dir, "fresh.tar")) + require.NoError(t, err) + assert.NotZero(t, info.Mode().Perm()&0o600) + assert.Zero(t, info.Mode().Perm()&^os.FileMode(0o644), "default mode must not exceed 0644 before umask") +} From 69ede65d4690615dae79c68924ad7257c8ead0cb Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 09:24:34 -0600 Subject: [PATCH 42/43] fix(artifact): keep CI S3 uploads on one PutObject so write-only roles still work The CI S3 adapter switched artifacts above 8 MiB to a multipart upload. The previous adapter issued a single PutObject, which needs only s3:PutObject and kms:GenerateDataKey on an SSE-KMS bucket. Multipart needs kms:Decrypt as well as s3:AbortMultipartUpload for cleanup, so an upload that worked before could start failing with AccessDenied. Add WithSinglePutOnly to the shared transport: a body that fits one part is sent from memory, a larger one is spooled to a temporary file (memory stays bounded at one part) and sent with one PutObject, and a body over the 5 GiB single-request limit fails with a clear error instead of being rejected after the transfer. The CI adapter uses it, so it issues only PutObject for uploads. Tests run against a fake S3 that refuses every operation except PutObject. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/s3/fallback.go | 7 +- pkg/artifact/blob/s3/options.go | 16 ++ pkg/artifact/blob/s3/put.go | 20 ++ pkg/artifact/blob/s3/put_single_only_test.go | 186 ++++++++++++++++++ pkg/ci/artifact/s3/store.go | 7 +- pkg/ci/artifact/s3/transport_parity_test.go | 32 +-- pkg/ci/artifact/s3/upload_permissions_test.go | 96 +++++++++ 7 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 pkg/artifact/blob/s3/put_single_only_test.go create mode 100644 pkg/ci/artifact/s3/upload_permissions_test.go diff --git a/pkg/artifact/blob/s3/fallback.go b/pkg/artifact/blob/s3/fallback.go index 340161ea5a2..317f255d1de 100644 --- a/pkg/artifact/blob/s3/fallback.go +++ b/pkg/artifact/blob/s3/fallback.go @@ -24,10 +24,15 @@ func (s *Store) putFallback(ctx context.Context, plan *putPlan, first, carry []b _ = file.Close() _ = os.Remove(file.Name()) }() - size, err := io.Copy(file, &ctxReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}) + // One byte past the limit is enough to prove the body is too large without spooling all of it. + source := io.LimitReader(&ctxReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}, s.singlePutLimit()+1) + size, err := io.Copy(file, source) if err != nil { return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("spool upload: %w", err)) } + if size > s.singlePutLimit() { + return nil, tooLargeForSinglePut(plan.key, size, s.singlePutLimit()) + } if plan.opts.Size >= 0 && size != plan.opts.Size { return nil, sizeMismatch(plan.key, size, plan.opts.Size) } diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go index ed77f6a2e57..fb3247b5c7f 100644 --- a/pkg/artifact/blob/s3/options.go +++ b/pkg/artifact/blob/s3/options.go @@ -49,6 +49,8 @@ type settings struct { conditional ConditionalMode opaqueKeys bool singleFallback bool + singleOnly bool + maxSinglePut int64 client ClientConfig api API } @@ -185,3 +187,17 @@ func WithSinglePutFallback() Option { return func(s *settings) { s.singleFallback = true } } + +// MaxSinglePutSize is the largest object S3 accepts in one PutObject request. +const MaxSinglePutSize int64 = 5 << 30 + +// WithSinglePutOnly never uses a multipart upload. A body that fits one part goes out from memory; +// a larger body is spooled to a temporary file and sent with one PutObject, so memory stays bounded +// at one part. Use it when the caller's credentials may be limited to PutObject: on an SSE-KMS +// bucket, multipart operations also need kms:Decrypt, which a write-only role lacks. Bodies above +// [MaxSinglePutSize] fail with a clear error instead of being sent to be rejected. +func WithSinglePutOnly() Option { + defer perf.Track(nil, "blob.s3.WithSinglePutOnly")() + + return func(s *settings) { s.singleOnly = true } +} diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go index 38f8c278a69..dd3355cf3be 100644 --- a/pkg/artifact/blob/s3/put.go +++ b/pkg/artifact/blob/s3/put.go @@ -56,6 +56,9 @@ func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.P if len(carry) == 0 { return s.putSingle(ctx, plan, first) } + if s.cfg.singleOnly { + return s.putFallback(ctx, plan, first, carry, body) + } return s.putMultipart(ctx, plan, first, carry, body) } @@ -67,6 +70,9 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) ( if opts.Size < blob.UnknownSize { return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid size %d", errUtils.ErrBlobInvalid, opts.Size)) } + if s.cfg.singleOnly && opts.Size > s.singlePutLimit() { + return nil, tooLargeForSinglePut(key, opts.Size, s.singlePutLimit()) + } tags := blob.MergeTags(s.cfg.tags, opts.Tags) if err := blob.ValidateTags(tags); err != nil { return nil, err @@ -210,3 +216,17 @@ func encodeTags(tags map[string]string) string { func sizeMismatch(key string, got, declared int64) error { return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: body is %d bytes, declared %d", errUtils.ErrBlobInvalid, got, declared)) } + +// tooLargeForSinglePut reports a body that one PutObject cannot carry when multipart is off. +func tooLargeForSinglePut(key string, size, limit int64) error { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, + fmt.Errorf("%w: %d bytes exceeds the %d-byte single-request limit and multipart uploads are disabled", errUtils.ErrBlobInvalid, size, limit)) +} + +// singlePutLimit is the largest body one PutObject may carry. +func (s *Store) singlePutLimit() int64 { + if s.cfg.maxSinglePut > 0 { + return s.cfg.maxSinglePut + } + return MaxSinglePutSize +} diff --git a/pkg/artifact/blob/s3/put_single_only_test.go b/pkg/artifact/blob/s3/put_single_only_test.go new file mode 100644 index 00000000000..87fa1878ec5 --- /dev/null +++ b/pkg/artifact/blob/s3/put_single_only_test.go @@ -0,0 +1,186 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// isolateTempDir points the process temp directory at an empty directory and returns it, so a test +// can prove that the upload spool is removed. +func isolateTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"TMPDIR", "TMP", "TEMP"} { + t.Setenv(name, dir) + } + return dir +} + +func requireEmptyDir(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, "the upload spool must be removed") +} + +// With multipart off, every body goes out as one PutObject, whatever its size and whether or not +// its length is declared. No multipart or read-back request is ever issued, so a write-only role +// is enough. +func TestSinglePutOnlyNeverUsesMultipartOrReadsBack(t *testing.T) { + tests := []struct { + name string + size int + // declared reports whether the caller declares the length. + declared bool + }{ + {name: "empty", size: 0, declared: true}, + {name: "below one part", size: smallPart - 1, declared: true}, + {name: "exactly one part", size: smallPart, declared: true}, + {name: "one byte past one part", size: smallPart + 1, declared: true}, + {name: "many parts", size: smallPart*5 + 17, declared: true}, + {name: "many parts of unknown length", size: smallPart*5 + 17}, + {name: "one byte past one part of unknown length", size: smallPart + 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + for _, op := range []string{ + s3test.OpGetObject, s3test.OpHeadObject, s3test.OpGetObjectTagging, s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, s3test.OpCompleteMultipartUpload, s3test.OpAbortMultipartUpload, + } { + fake.Inject(s3test.Fault{Op: op, Times: -1, Action: s3test.FailStatus, Status: http.StatusForbidden, Code: "AccessDenied"}) + } + s := newTestStore(t, fake, WithSinglePutOnly()) + data := patterned(tt.size) + opts := blob.PutOptions{Size: blob.UnknownSize} + if tt.declared { + opts.Size = int64(len(data)) + } + + info, err := s.Put(context.Background(), "obj", bytes.NewReader(data), opts) + + require.NoError(t, err) + assert.Equal(t, int64(len(data)), info.Size) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data), "stored %d bytes, want %d", len(stored.Data), len(data)) + requests := fake.Requests() + require.Len(t, requests, 1) + assert.Equal(t, s3test.OpPutObject, requests[0].Op) + requireEmptyDir(t, tmp) + }) + } +} + +// A declared size that cannot fit one request is refused before the body is read or any request +// is sent, with an error that names the cause. +func TestSinglePutOnlyRejectsDeclaredSizeAboveTheLimit(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithSinglePutOnly()) + reader := &countingReader{size: MaxSinglePutSize + 1} + + _, err := s.Put(context.Background(), "obj", reader, blob.PutOptions{Size: MaxSinglePutSize + 1}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Contains(t, err.Error(), "multipart uploads are disabled") + assert.Zero(t, reader.read.Load(), "the body is not consumed") + assert.Empty(t, fake.Requests()) +} + +// A stream of unknown length that outgrows one request stops at the limit, sends nothing, and +// removes its spool. +func TestSinglePutOnlyStopsAnUnknownLengthStreamAtTheLimit(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + s := newTestStore(t, fake, WithSinglePutOnly()) + s.cfg.maxSinglePut = smallPart * 3 + data := patterned(smallPart*3 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) + requireEmptyDir(t, tmp) + + // A body exactly at the limit is still accepted. + exact := patterned(smallPart * 3) + _, err = s.Put(context.Background(), "obj", bytes.NewReader(exact), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(exact, stored.Data)) + requireEmptyDir(t, tmp) +} + +// A refused PutObject is reported as the upload's failure; no fallback hides it, and the spool is +// removed. +func TestSinglePutOnlyReportsPutObjectDenied(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Times: -1, Action: s3test.FailStatus, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithSinglePutOnly()) + data := patterned(smallPart*2 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.Error(t, err) + assert.Empty(t, fake.Keys()) + assert.Empty(t, fake.RequestsFor(s3test.OpCreateMultipartUpload)) + requireEmptyDir(t, tmp) +} + +// The fallback option still turns an unimplemented multipart upload into one PutObject, and it +// refuses to spool without bound. +func TestSinglePutFallbackWhenMultipartUnsupported(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + data := patterned(smallPart*2 + 3) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.NoError(t, err) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data)) + assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestSinglePutFallbackDoesNotMaskOtherMultipartErrors(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + data := patterned(smallPart*2 + 3) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.Error(t, err) + assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) +} + +func TestSinglePutFallbackStopsAtTheLimit(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + s.cfg.maxSinglePut = smallPart * 2 + data := patterned(smallPart*2 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) + requireEmptyDir(t, tmp) +} diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index f044a6b9c18..c3f61e937c5 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -188,13 +188,16 @@ func (s *Store) fullKey(key string) string { // transport returns the shared S3 transport over the lazily initialized client. Keys are passed // through untouched (fullKey already applied the legacy prefix), so the transport is created -// without a prefix and with opaque-key validation to keep historical key patterns valid. +// without a prefix and with opaque-key validation to keep historical key patterns valid. Uploads use +// one PutObject at any size, as the previous adapter did, because multipart uploads need extra +// permissions (kms:Decrypt on SSE-KMS buckets, s3:AbortMultipartUpload) that a write-only CI role +// may not hold. func (s *Store) transport(ctx context.Context) (*blobs3.Store, error) { if err := s.ensureClient(ctx); err != nil { return nil, err } s.transportOnce.Do(func() { - s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys(), blobs3.WithSinglePutFallback()) + s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys(), blobs3.WithSinglePutOnly()) }) return s.transportStore, s.transportErr } diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go index d6f56cf6edb..678f9dc6733 100644 --- a/pkg/ci/artifact/s3/transport_parity_test.go +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "strings" "testing" "time" @@ -169,7 +168,9 @@ func TestAdapterBundledStoreChecksumAndTarSurviveTheTransport(t *testing.T) { assert.Equal(t, "plan-bytes", string(data)) } -func TestAdapterLargeUploadStreamsAsMultipartWithLegacyClient(t *testing.T) { +// The adapter sends one PutObject at any size, as the previous adapter did, so a role limited to +// PutObject keeps working. Memory stays bounded because the body is spooled, not buffered. +func TestAdapterLargeUploadIsOnePutObjectWithLegacyClient(t *testing.T) { store, fake := newFakeStore(t, "") body := bytes.Repeat([]byte("0123456789abcdef"), (9<<20)/16) require.NoError(t, store.Upload(context.Background(), "big.tar", bytes.NewReader(body), int64(len(body)), nil)) @@ -178,11 +179,10 @@ func TestAdapterLargeUploadStreamsAsMultipartWithLegacyClient(t *testing.T) { require.True(t, ok) assert.Equal(t, len(body), len(stored.Data)) assert.True(t, bytes.Equal(body, stored.Data)) - assert.Equal(t, 1, fake.CompletedUploads()) + assert.Equal(t, 0, fake.CompletedUploads()) assert.Empty(t, fake.OpenUploads()) - for _, req := range fake.RequestsFor(s3test.OpUploadPart) { - assert.Empty(t, req.Header.Get("X-Amz-Checksum-Crc32"), "multipart requests carry no optional checksums") - } + require.Len(t, fake.Requests(), 1) + assert.Equal(t, s3test.OpPutObject, fake.Requests()[0].Op) } func TestAdapterWrapsTransportFailuresInLegacySentinels(t *testing.T) { @@ -213,23 +213,3 @@ func TestAdapterWrapsTransportFailuresInLegacySentinels(t *testing.T) { fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: 403, Code: "AccessDenied", Times: -1}) require.ErrorIs(t, store.Delete(ctx, "k.tar"), errUtils.ErrArtifactDeleteFailed) } - -func TestAdapterFallsBackWhenMultipartUnsupported(t *testing.T) { - store, fake := newFakeStore(t, "") - fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) - body := bytes.Repeat([]byte("x"), 9<<20) - require.NoError(t, store.Upload(context.Background(), "big.tar", bytes.NewBuffer(body), int64(len(body)), nil)) - object, ok := fake.Object("big.tar") - require.True(t, ok) - assert.Equal(t, body, object.Data) - assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) - assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) -} - -func TestAdapterDoesNotFallbackForOtherMultipartErrors(t *testing.T) { - store, fake := newFakeStore(t, "") - fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusForbidden, Code: "AccessDenied"}) - body := bytes.Repeat([]byte("x"), 9<<20) - require.Error(t, store.Upload(context.Background(), "big.tar", bytes.NewReader(body), int64(len(body)), nil)) - assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) -} diff --git a/pkg/ci/artifact/s3/upload_permissions_test.go b/pkg/ci/artifact/s3/upload_permissions_test.go new file mode 100644 index 00000000000..f6e06746c61 --- /dev/null +++ b/pkg/ci/artifact/s3/upload_permissions_test.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +const ( + partSize = 8 << 20 + denied = 403 +) + +// denyEverythingButPutObject models a role that may only write objects. On an SSE-KMS bucket such +// a role holds s3:PutObject and kms:GenerateDataKey but not kms:Decrypt, so every operation that +// reads an object back, and every multipart operation, is refused. +func denyEverythingButPutObject(server *s3test.Server) { + for _, op := range []string{ + s3test.OpGetObject, + s3test.OpHeadObject, + s3test.OpGetObjectTagging, + s3test.OpListObjectsV2, + s3test.OpDeleteObject, + s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, + s3test.OpCompleteMultipartUpload, + s3test.OpAbortMultipartUpload, + } { + server.Inject(s3test.Fault{Op: op, Times: -1, Action: s3test.FailStatus, Status: denied, Code: "AccessDenied"}) + } +} + +func patterned(n int) []byte { + data := make([]byte, n) + for i := range data { + data[i] = byte(i*7 + i/251) + } + return data +} + +// An upload that worked with the previous single-PutObject adapter must keep working for a role +// that can only put objects, whatever the artifact size, including sizes past the multipart +// threshold. The adapter may only issue PutObject requests. +func TestUploadNeedsOnlyPutObjectPermission(t *testing.T) { + tests := []struct { + name string + size int + }{ + {name: "empty", size: 0}, + {name: "small", size: 1 << 10}, + {name: "exactly one part", size: partSize}, + {name: "one byte past one part", size: partSize + 1}, + {name: "multiple parts", size: partSize*2 + 123}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s3test.NewServer(t) + denyEverythingButPutObject(server) + store := &Store{client: server.Client(), bucket: server.Bucket(), prefix: "planfiles"} + data := patterned(tt.size) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc"} + + err := store.Upload(context.Background(), "dev/vpc/abc.tfplan", bytes.NewReader(data), int64(len(data)), metadata) + + require.NoError(t, err) + stored, ok := server.Object("planfiles/dev/vpc/abc.tfplan") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data), "stored %d bytes, want %d", len(stored.Data), len(data)) + _, ok = server.Object("planfiles/dev/vpc/abc.tfplan.metadata.json") + assert.True(t, ok, "the metadata sidecar is uploaded") + for _, request := range server.Requests() { + assert.Equal(t, s3test.OpPutObject, request.Op, "request %d", request.Seq) + assert.Empty(t, request.Query.Get("uploadId"), "no multipart request") + } + assert.Empty(t, server.OpenUploads()) + }) + } +} + +// Refusing an upload for lack of PutObject permission is reported, not hidden by a fallback. +func TestUploadReportsPutObjectDenied(t *testing.T) { + server := s3test.NewServer(t) + server.Inject(s3test.Fault{Op: s3test.OpPutObject, Times: -1, Action: s3test.FailStatus, Status: denied, Code: "AccessDenied"}) + store := &Store{client: server.Client(), bucket: server.Bucket()} + + err := store.Upload(context.Background(), "k", bytes.NewReader(patterned(partSize+1)), -1, nil) + + require.Error(t, err) + assert.Empty(t, server.Keys()) +} From 992587d85d5ef9a31285e85408353dc6f3ef5d60 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 18:32:22 -0600 Subject: [PATCH 43/43] fix(artifact): preserve sentinel ordering from historical integration --- errors/errors.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index f2774adc264..7a989a94a6e 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2131,6 +2131,10 @@ var ( // reconciled: the completed reference could not be proven to hold, or to lack, the expected content. ErrArtifactPublicationAmbiguous = errors.New("artifact publication outcome is ambiguous") + // ErrArtifactEvidenceMalformed is returned when a subject's evidence directory holds more malformed + // entries than the repository is willing to read, so valid evidence may be hidden behind them. + ErrArtifactEvidenceMalformed = errors.New("too many malformed evidence files") + // ErrSymlinkLoop is returned when resolving a path follows more symlinks than the bound allows, which // means the links form a loop. ErrSymlinkLoop = errors.New("symlink loop") @@ -2138,10 +2142,6 @@ var ( // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The // underlying I/O error is wrapped alongside it for diagnostics. ErrArtifactStreamFailed = errors.New("artifact stream failed") - - // ErrArtifactEvidenceMalformed is returned when a subject's evidence directory holds more malformed - // entries than the repository is willing to read, so valid evidence may be hidden behind them. - ErrArtifactEvidenceMalformed = errors.New("too many malformed evidence files") ) // Blob transport sentinels. They classify outcomes of the transport-only object I/O in