Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
942fc52
feat(artifacts): add aws/s3 artifact repository (A05b)
osterman Oct 31, 2026
47e6954
fix(artifacts): settle an S3 lock claim when the context ends mid-req…
osterman Oct 31, 2026
ae6de69
feat(artifacts): add OCI registry and layout repositories with sigsto…
osterman Oct 31, 2026
ef0a81c
fix: avoid overflow in S3 artifact pagination
osterman Nov 11, 2026
736af73
fix: harden artifact backends and evidence handling
osterman Nov 11, 2026
718d12b
fix: retain oversized evidence diagnostics
osterman Nov 11, 2026
4cab69c
test: cover S3 replacement cleanup recovery
osterman Nov 11, 2026
45b1957
fix: recover interrupted S3 deletions and refresh identity
osterman Nov 11, 2026
5079f7b
fix: report unresolved S3 claim creation
osterman Nov 11, 2026
ba1a9d5
fix: fence uncertain S3 lock claims
osterman Nov 11, 2026
3814ae6
fix(artifacts): coordinate delegated S3 mutations
osterman Nov 12, 2026
ca5fa81
fix(artifact): identify uncertain S3 lock attempts for recovery
osterman Nov 12, 2026
6d0c9a9
fix(artifact): close remaining OCI review findings
osterman Nov 13, 2026
8331b70
refactor(artifact): keep OCI publish and attachment checks within com…
osterman Nov 13, 2026
c7bc62d
fix(artifact): refuse an unusable signer before S3 signing runs
osterman Nov 13, 2026
4bb1b46
fix(terminal): size the query scan buffer from one length
osterman Nov 13, 2026
4d8e4a9
refactor(artifact): keep delete lock planning within lint limits
osterman Nov 13, 2026
cbaf2c7
fix(artifact): never echo endpoint credentials in validation errors
osterman Nov 14, 2026
0fb2438
test(artifact): clear ambient AWS profiles in the S3 test isolation h…
osterman Nov 14, 2026
c7b572d
fix(artifact): reject a canceled context before claiming an S3 lock
osterman Nov 14, 2026
802df01
fix(artifact): exclude evidence bound to another Record from S3 exports
osterman Nov 14, 2026
b831cad
fix(oci): reject an empty reference before filtering the layout index
osterman Nov 14, 2026
23c58b1
fix(artifact): carry abandoned S3 payloads in the journal until a ret…
osterman Nov 14, 2026
279e93f
fix(artifact): bind the S3 conditional-write verdict to the endpoint …
osterman Nov 14, 2026
3791054
fix(oci): treat an oversized referrer as malformed evidence
osterman Nov 14, 2026
bc9f286
fix(oci): build the copy result from the copied digest
osterman Nov 14, 2026
7f26395
fix(oci): recheck sibling tags right before a manifest-scoped delete
osterman Nov 14, 2026
c5c4e5b
fix(oci): keep pinned layout reads working after reopening the reposi…
osterman Nov 14, 2026
f6744c7
fix(oci): propagate operational failures during digest scans
osterman Nov 14, 2026
4ca1f9b
fix(oci): serialize layout writes across processes with a layout-wide…
osterman Nov 14, 2026
e5ad760
fix(artifact): redact registry userinfo in OCI option validation errors
osterman Nov 14, 2026
7379365
fix(oci): treat an integrity-failed referrer as malformed evidence
osterman Nov 14, 2026
b11efb2
fix(oci): preserve stat errors when checking whether a layout exists
osterman Nov 14, 2026
917b115
fix(artifact): redact credentials in scheme-relative registry values
osterman Nov 14, 2026
21ebc8e
fix(s3): stop polling when the configured lock wait expires
osterman Nov 14, 2026
65cb293
fix(s3): keep claim-release errors beside a failed mutation
osterman Nov 14, 2026
6800689
fix(s3): include abandoned payload locations in the reservation claim
osterman Nov 14, 2026
135e20d
fix(oci): count one root per digest when selecting a layout root
osterman Nov 14, 2026
913eacd
fix(oci): check the Record identity before ImportLayout writes the re…
osterman Nov 14, 2026
4077b93
fix(oci): serialize the registry copy's read-decide-write sequence
osterman Nov 14, 2026
1c5d5a2
fix(oci): reject malformed descriptor digests before reading them
osterman Nov 14, 2026
05adcd6
fix(oci): validate an import's destination and package before publishing
osterman Nov 14, 2026
1bafd61
fix(s3): keep every location a publication journal owes cleanup on cl…
osterman Nov 15, 2026
51f51bd
fix(s3): check every key a publication writes against the keys its re…
osterman Nov 15, 2026
a84664d
test(artifacts): outlast the first claim attempt in the lock wait test
osterman Nov 15, 2026
1875575
fix(artifact/s3): keep the recovered payload location until the repla…
osterman Nov 15, 2026
a8179ed
test(artifact/s3): bound the newcomer race by the retirement, not an …
osterman Nov 15, 2026
cc79d63
fix(artifact/oci): bind the source Record to its reference before a n…
osterman Nov 15, 2026
19141d6
fix(artifact/oci): classify only the root media types the copy transp…
osterman Nov 15, 2026
0bf5c4d
test(artifact): isolate every S3 config test from the machine's share…
osterman Nov 15, 2026
98d9507
fix(s3): keep the held-claim conflict when the context ends between l…
osterman Nov 15, 2026
4455fa9
fix(artifact): address the CodeRabbit findings on the S3 and OCI repo…
osterman Nov 16, 2026
e953f75
fix(artifact): preserve remote backend merge resolutions after rebase
osterman Nov 17, 2026
a6eb222
Merge verified L4 review fixes into artifacts backends remote
osterman Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions pkg/artifact/blob/s3/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
Expand All @@ -18,10 +20,12 @@ import (

errUtils "github.com/cloudposse/atmos/errors"
"github.com/cloudposse/atmos/pkg/artifact/blob"
"github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test"
"github.com/cloudposse/atmos/pkg/store"
)

func TestIdentityIsResolvedLazily(t *testing.T) {
s3test.IsolateAWSConfig(t)
resolver := &fakeResolver{err: errors.New("sso expired")}
s, err := New("bucket", WithIdentity("deploy", resolver), WithRegion("us-east-1"))
require.NoError(t, err, "constructing the store must not touch credentials")
Expand All @@ -33,6 +37,7 @@ func TestIdentityIsResolvedLazily(t *testing.T) {
}

func TestUnresolvableIdentityFailsClosed(t *testing.T) {
s3test.IsolateAWSConfig(t)
t.Setenv("AWS_ACCESS_KEY_ID", "ambient-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret")
t.Setenv("AWS_REGION", "us-east-1")
Expand Down Expand Up @@ -66,6 +71,7 @@ func TestUnresolvableIdentityFailsClosed(t *testing.T) {
}

func TestLegacyClientKeepsAmbientFallback(t *testing.T) {
s3test.IsolateAWSConfig(t)
t.Setenv("AWS_REGION", "us-east-1")
t.Setenv("AWS_ACCESS_KEY_ID", "k")
t.Setenv("AWS_SECRET_ACCESS_KEY", "s")
Expand All @@ -80,6 +86,7 @@ func TestLegacyClientKeepsAmbientFallback(t *testing.T) {
}

func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) {
s3test.IsolateAWSConfig(t)
requests := make(chan *http.Request, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
Expand Down Expand Up @@ -184,6 +191,7 @@ func TestBuildClientOptFnsMatchesLegacyEndpointBehavior(t *testing.T) {
}

func TestHTTPClientInjectionReachesTheSDK(t *testing.T) {
s3test.IsolateAWSConfig(t)
fake := newFake(t)
var used bool
client := httpClientFunc(func(r *http.Request) (*http.Response, error) {
Expand All @@ -204,3 +212,35 @@ func TestHTTPClientInjectionReachesTheSDK(t *testing.T) {
type httpClientFunc func(*http.Request) (*http.Response, error)

func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { return f(r) }

func TestNamedIdentityReloadsRotatedCredentials(t *testing.T) {
isolated := s3test.IsolateAWSConfig(t)
headers := make(chan string, 2)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
headers <- r.Header.Get("Authorization")
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
creds := filepath.Join(t.TempDir(), "credentials")
write := func(key string) {
require.NoError(t, os.WriteFile(creds, []byte("[default]\naws_access_key_id = "+key+"\naws_secret_access_key = secret\n"), 0o600))
}
write("first-key")
resolver := &fakeResolver{result: &store.AWSAuthConfig{
CredentialsFile: creds,
// The shared config is named too, so the developer's own default profile is never merged in.
ConfigFile: filepath.Join(isolated, "config"),
Profile: "default",
Region: "us-east-1",
}}
s, err := New("artifacts", WithIdentity("deploy", resolver), WithEndpoint(server.URL))
require.NoError(t, err)
_, err = s.Stat(context.Background(), "file", blob.StatOptions{})
require.NoError(t, err)
assert.Contains(t, <-headers, "Credential=first-key/")
write("second-key")
_, err = s.Stat(context.Background(), "file", blob.StatOptions{})
require.NoError(t, err)
assert.Contains(t, <-headers, "Credential=second-key/")
assert.Equal(t, int32(2), resolver.calls.Load())
}
61 changes: 45 additions & 16 deletions pkg/artifact/blob/s3/conditional.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,16 @@ const probeOperation = "probe"

const probeCleanupTimeout = 5 * time.Second

// ensureConditional makes sure a conditional operation is safe to send. In probe mode it first
// proves, once per store, that the endpoint enforces If-None-Match and If-Match; an endpoint that silently
// ignores the header would otherwise turn create-if-absent into an unconditional overwrite.
func (s *Store) ensureConditional(ctx context.Context, key string, cond blob.Condition) error {
// ensureConditional makes sure a conditional operation is safe to send through api. In probe mode
// it first proves, once per endpoint, that the endpoint enforces If-None-Match and If-Match; an
// endpoint that silently ignores the header would otherwise turn create-if-absent into an
// unconditional overwrite. The verdict covers the endpoint of api, the client the operation is
// about to use, so a named identity that now resolves to another endpoint is probed again.
func (s *Store) ensureConditional(ctx context.Context, api API, key string, cond blob.Condition) error {
if !cond.IsConditional() || s.cfg.conditional != ConditionalProbe {
return nil
}
if err := s.ProbeConditionalWrites(ctx); err != nil {
if err := s.probeConditional(ctx, api); err != nil {
return unsupportedConditionalFrom(key, err)
}
return nil
Expand All @@ -48,13 +50,26 @@ func unsupportedConditionalFrom(key string, probeErr error) error {
// object twice: the second create must be rejected. It then checks that stale If-Match validators
// are rejected on PutObject and DeleteObject. The guarantee covers those single-object requests
// only; conditional enforcement on CompleteMultipartUpload is not probed. A definite answer is
// cached for the life of the store; a transient failure is returned without being cached. It
// cached for the endpoint it probed; a different endpoint, which a named identity can resolve to
// at any time, is probed again. A transient failure is returned without being cached. It
// returns an error matching ErrBlobConditionalUnsupported when the endpoint ignores or rejects the
// condition.
func (s *Store) ProbeConditionalWrites(ctx context.Context) error {
defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")()

if verdict, cached := s.cachedProbe(); cached {
api, err := s.client(ctx)
if err != nil {
return err
}
return s.probeConditional(ctx, api)
}

// probeConditional probes the endpoint of api unless its verdict is already cached. One probe runs
// at a time; a waiting operation gives up when its context ends instead of blocking on the probe
// in flight.
func (s *Store) probeConditional(ctx context.Context, api API) error {
endpoint := endpointOf(api)
if verdict, cached := s.cachedProbe(endpoint); cached {
return verdict
}
select {
Expand All @@ -64,13 +79,9 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error {
return blob.NewError(blob.KindOther, probeOperation, "", ctx.Err())
}
// Another operation may have finished the probe while this one waited for the slot.
if verdict, cached := s.cachedProbe(); cached {
if verdict, cached := s.cachedProbe(endpoint); cached {
return verdict
}
api, err := s.client(ctx)
if err != nil {
return err
}
verdict, cleanupErr := s.runProbe(ctx, api)
if cleanupErr != nil {
// A leftover probe object is a transient failure: neither verdict is cached, and both
Expand All @@ -79,17 +90,35 @@ func (s *Store) ProbeConditionalWrites(ctx context.Context) error {
}
if verdict == nil || blob.IsConditionalUnsupported(verdict) {
s.probeMu.Lock()
s.probed, s.probeError = true, verdict
s.probed, s.probedEndpoint, s.probeError = true, endpoint, verdict
s.probeMu.Unlock()
}
return verdict
}

// cachedProbe returns the cached probe verdict, when a definite one exists.
func (s *Store) cachedProbe() (verdict error, cached bool) {
// cachedProbe returns the cached probe verdict, when a definite one exists for endpoint.
func (s *Store) cachedProbe(endpoint string) (verdict error, cached bool) {
s.probeMu.Lock()
defer s.probeMu.Unlock()
return s.probeError, s.probed
if !s.probed || s.probedEndpoint != endpoint {
return nil, false
}
return s.probeError, true
}

// endpointOf names the endpoint a client targets: its base endpoint override, or its region for
// the standard AWS endpoint. A client that does not expose its options, such as a test double or
// an injected wrapper, has one fixed endpoint for the life of the store.
func endpointOf(api API) string {
reporter, ok := api.(interface{ Options() s3.Options })
if !ok {
return ""
}
options := reporter.Options()
if options.BaseEndpoint != nil && *options.BaseEndpoint != "" {
return *options.BaseEndpoint
}
return "region:" + options.Region
}

// runProbe performs the double create and removes the probe object. The verdict is the probe
Expand Down
53 changes: 53 additions & 0 deletions pkg/artifact/blob/s3/conditional_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"github.com/cloudposse/atmos/pkg/artifact/blob"
"github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test"
"github.com/cloudposse/atmos/pkg/store"
)

func putIfAbsent(s *Store, key, body string) (*blob.ObjectInfo, error) {
Expand Down Expand Up @@ -381,3 +382,55 @@ func TestProbeRejectsIgnoredIfMatch(t *testing.T) {
})
}
}

// A probe verdict belongs to the endpoint it probed. A named identity is re-resolved on every
// operation, so its endpoint can change during the life of the store; a verdict from the old
// endpoint must never vouch for the new one.
func TestProbeVerdictIsBoundToTheEndpointItProbed(t *testing.T) {
s3test.IsolateAWSConfig(t)
t.Setenv("AWS_ACCESS_KEY_ID", "test")
t.Setenv("AWS_SECRET_ACCESS_KEY", "test")
enforcing := newFake(t)
ignoring := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore))
ignoring.Seed("lock", []byte("owner-a"), nil)

resolver := &fakeResolver{result: &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: enforcing.Endpoint()}}
s, err := New(enforcing.Bucket(), WithIdentity("deploy", resolver), WithConditionalMode(ConditionalProbe))
require.NoError(t, err)

_, err = putIfAbsent(s, "first", "a")
require.NoError(t, err, "the enforcing endpoint passes its probe")
probes := len(enforcing.RequestsFor(s3test.OpPutObject))
_, err = putIfAbsent(s, "second", "a")
require.NoError(t, err)
assert.Len(t, enforcing.RequestsFor(s3test.OpPutObject), probes+1, "an unchanged endpoint keeps its cached verdict: no second probe")

resolver.result = &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: ignoring.Endpoint()}
_, err = putIfAbsent(s, "lock", "owner-b")
require.Error(t, err)
assert.True(t, blob.IsConditionalUnsupported(err), "the new endpoint is probed, not trusted: %v", err)
object, ok := ignoring.Object("lock")
require.True(t, ok)
assert.Equal(t, []byte("owner-a"), object.Data, "the existing object on the ignoring endpoint was never overwritten")

before := len(ignoring.Requests())
_, err = putIfAbsent(s, "lock", "owner-c")
assert.True(t, blob.IsConditionalUnsupported(err))
assert.Len(t, ignoring.Requests(), before, "a definite verdict for the new endpoint is cached for it")

// Switching back is a different endpoint again; the earlier verdict for it does not apply blindly.
resolver.result = &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: enforcing.Endpoint()}
_, err = putIfAbsent(s, "third", "a")
require.NoError(t, err)
}

// A transient probe failure is never cached, whatever the endpoint.
func TestProbeTransientFailureIsNotCachedAcrossEndpoints(t *testing.T) {
fake := newFake(t)
fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusServiceUnavailable, Code: "SlowDown", Times: 1})
s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe))
err := s.ProbeConditionalWrites(context.Background())
require.Error(t, err)
assert.False(t, blob.IsConditionalUnsupported(err))
require.NoError(t, s.ProbeConditionalWrites(context.Background()), "the transient failure was not cached")
}
10 changes: 5 additions & 5 deletions pkg/artifact/blob/s3/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions)
return err
}
conditional := opts.IfMatch != ""
if conditional {
if err := s.ensureConditional(ctx, key, blob.IfMatch(opts.IfMatch)); err != nil {
return err
}
}
api, err := s.client(ctx)
if err != nil {
return err
}
if conditional {
if err := s.ensureConditional(ctx, api, key, blob.IfMatch(opts.IfMatch)); err != nil {
return err
}
}
in := &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), IfMatch: optionalString(opts.IfMatch)}
if _, err := api.DeleteObject(ctx, in, retryOptions(conditional)...); err != nil {
return classify(request{op: blob.OpDelete, key: key, mutating: true, conditional: conditional}, err)
Expand Down
6 changes: 3 additions & 3 deletions pkg/artifact/blob/s3/put.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,13 @@ func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) (
if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil {
return nil, err
}
if err := s.ensureConditional(ctx, key, opts.Condition); err != nil {
return nil, err
}
api, err := s.client(ctx)
if err != nil {
return nil, err
}
if err := s.ensureConditional(ctx, api, key, opts.Condition); err != nil {
return nil, err
}
return &putPlan{
api: api,
key: key,
Expand Down
59 changes: 59 additions & 0 deletions pkg/artifact/blob/s3/s3test/isolate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package s3test

import (
"os"
"path/filepath"
"testing"

"github.com/cloudposse/atmos/pkg/perf"
)

// ambientAWSVariables are the variables through which the developer's or the runner's AWS setup
// reaches the SDK's default chain: the profile selection, the web identity and container credential
// sources, and the endpoint, FIPS, and dual-stack overrides. Static credentials and the region are
// left to each test, which sets the ones it needs.
var ambientAWSVariables = []string{
"AWS_PROFILE",
"AWS_DEFAULT_PROFILE",
"AWS_SESSION_TOKEN",
"AWS_ROLE_ARN",
"AWS_ROLE_SESSION_NAME",
"AWS_WEB_IDENTITY_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
"AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_AUTHORIZATION_TOKEN",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_ENDPOINT_URL",
"AWS_ENDPOINT_URL_S3",
"AWS_USE_FIPS_ENDPOINT",
"AWS_USE_DUALSTACK_ENDPOINT",
}

// isolatedFileMode is the permission of the empty shared files, which hold no secret.
const isolatedFileMode = 0o600

// IsolateAWSConfig keeps a test that builds an AWS SDK configuration away from the AWS setup of the
// machine it runs on. The shared config and credentials files point at empty temporary files, the
// variables that select a profile or another credential source are cleared, and instance metadata is
// disabled. A developer's default profile with a role_arn would otherwise be merged into the test's
// configuration and select AssumeRole, which reaches the real STS service. It returns the directory
// holding the files, so a test can write the profile it wants into them.
func IsolateAWSConfig(t testing.TB) string {
defer perf.Track(nil, "s3test.IsolateAWSConfig")()

t.Helper()
dir := t.TempDir()
config, credentials := filepath.Join(dir, "config"), filepath.Join(dir, "credentials")
for _, path := range []string{config, credentials} {
if err := os.WriteFile(path, nil, isolatedFileMode); err != nil {
t.Fatalf("create an empty AWS file: %v", err)
}
}
for _, name := range ambientAWSVariables {
t.Setenv(name, "")
}
t.Setenv("AWS_CONFIG_FILE", config)
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", credentials)
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
return dir
}
Loading
Loading