diff --git a/.github/scripts/govulncheck-monitor.sh b/.github/scripts/govulncheck-monitor.sh new file mode 100644 index 00000000000..7826010a7dd --- /dev/null +++ b/.github/scripts/govulncheck-monitor.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Run the scanner unchanged while recording resource pressure for slow CI jobs. +set -u + +monitor() { + local sleeper="" + trap 'kill "$sleeper" 2>/dev/null || :; exit 0' TERM INT + while :; do + { + echo "--- govulncheck resources $(date -u +%FT%TZ) ---" + free -m || : + for resource in cpu memory io; do + if [[ -r "/proc/pressure/$resource" ]]; then + echo "$resource pressure:" + cat "/proc/pressure/$resource" || : + fi + done + # Show the largest processes and the scanner even if it is not among them. + ps -eo pid,ppid,stat,pcpu,rss,vsz,comm --sort=-rss | head -n 26 || : + for pid in $(pgrep -x govulncheck || :); do + echo "govulncheck pid $pid:" + awk '/^(State|VmRSS|VmSwap):/ { print }' "/proc/$pid/status" 2>/dev/null || : + done + } >&2 + sleep 60 & + sleeper=$! + wait "$sleeper" || : + sleeper="" + done +} + +monitor & +monitor_pid=$! +cleanup() { + kill "$monitor_pid" 2>/dev/null || : + wait "$monitor_pid" 2>/dev/null || : +} +trap cleanup EXIT + +"$@" diff --git a/.github/scripts/govulncheck-monitor.test.cjs b/.github/scripts/govulncheck-monitor.test.cjs new file mode 100644 index 00000000000..bbc431af514 --- /dev/null +++ b/.github/scripts/govulncheck-monitor.test.cjs @@ -0,0 +1,20 @@ +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +// Run the script by its name from its own directory, so no absolute path reaches the command line. +const script = 'govulncheck-monitor.sh'; +const options = { skip: process.platform === 'win32' }; + +for (const status of [0, 23]) { + test(`preserves scanner output and exit status ${status}, and stops the monitor`, options, () => { + const result = spawnSync('bash', [script, 'bash', '-c', + `sleep 0.2; printf '%s' '{"version":"2.1.0"}'; printf 'scanner diagnostic\\n' >&2; exit ${status}`], + { cwd: __dirname, encoding: 'utf8', timeout: 5000 }); + assert.ifError(result.error); + assert.equal(result.status, status); + assert.equal(result.stdout, '{"version":"2.1.0"}'); + assert.match(result.stderr, /govulncheck resources/); + assert.match(result.stderr, /scanner diagnostic/); + }); +} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ccd026c124c..7d46037adc5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -201,10 +201,16 @@ jobs: # (govulncheck's SSA construction parallelizes via goroutines, which # -p doesn't govern) while nearly tripling run time. GOMEMLIMIT: "32GiB" + # Controlled scanner-only comparison against the GOGC=100 baseline. + # GC traces report live heap and collection cost on scanner stderr. + ATMOS_VULNCHECK_GOGC: "50" + ATMOS_VULNCHECK_GCTRACE: "1" # The target also removes exact duplicate stacks from the report: the # SARIF schema requires them to be unique and the upload rejects the # report otherwise. See internal/ci/vulncheck. - run: go tool mage ci:vulncheck govulncheck.sarif + # Diagnostics run separately on stderr; the scanner and SARIF output + # are unchanged. They help distinguish pressure from a stalled scan. + run: bash .github/scripts/govulncheck-monitor.sh go tool mage ci:vulncheck govulncheck.sarif - name: Upload SARIF file # Setup failures produce no report; preserve the original failure without diff --git a/.golangci.yml b/.golangci.yml index a820f830cd8..334edfaed82 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -104,6 +104,10 @@ linters: # pkg/ci/artifact/s3 is the S3-backed CI artifact store; it legitimately # depends on the AWS SDK, same rationale as pkg/ci/planfile/s3 above. - "!**/pkg/ci/artifact/s3/**" + # pkg/artifact/blob/s3 is the shared AWS SDK object transport behind the artifact S3 + # repository and the CI artifact S3 adapter (including its in-process S3 test fake). + # The provider-neutral contract is pkg/artifact/blob, which imports no SDK. + - "!**/pkg/artifact/blob/s3/**" # The S3 downloader signs source requests with request-scoped AWS credentials. # Only this cloud-specific leaf imports the SDK; the parent downloader stays generic. - "!**/pkg/downloader/s3/**" diff --git a/errors/errors.go b/errors/errors.go index f7cd45d8f15..7a989a94a6e 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2131,6 +2131,10 @@ var ( // reconciled: the completed reference could not be proven to hold, or to lack, the expected content. ErrArtifactPublicationAmbiguous = errors.New("artifact publication outcome is ambiguous") + // ErrArtifactEvidenceMalformed is returned when a subject's evidence directory holds more malformed + // entries than the repository is willing to read, so valid evidence may be hidden behind them. + ErrArtifactEvidenceMalformed = errors.New("too many malformed evidence files") + // ErrSymlinkLoop is returned when resolving a path follows more symlinks than the bound allows, which // means the links form a loop. ErrSymlinkLoop = errors.New("symlink loop") @@ -2140,6 +2144,50 @@ var ( ErrArtifactStreamFailed = errors.New("artifact stream failed") ) +// Blob transport sentinels. They classify outcomes of the transport-only object I/O in +// pkg/artifact/blob so callers can branch with errors.Is without depending on a backend SDK. +var ( + // ErrBlobNotFound is returned when the addressed object does not exist. + ErrBlobNotFound = errors.New("blob object not found") + + // ErrBlobPreconditionFailed is returned when a conditional write or delete (if-absent or if-match) lost its race. + ErrBlobPreconditionFailed = errors.New("blob precondition failed") + + // ErrBlobConditionalUnsupported is returned when the endpoint cannot enforce a required condition. + // The operation is rejected instead of silently overwriting. + ErrBlobConditionalUnsupported = errors.New("blob endpoint cannot enforce the required condition") + + // ErrBlobOutcomeUnknown is returned when a mutation was sent but its result was not observed + // (timeout, connection reset, server error). Callers must reconcile by reading the object. + ErrBlobOutcomeUnknown = errors.New("blob mutation outcome is unknown") + + // ErrBlobInvalid is returned for malformed keys, tags, options, sizes, or ranges. + ErrBlobInvalid = errors.New("invalid blob request") + + // ErrBlobIdentityUnresolved is returned when a configured identity cannot be resolved to credentials. + // The generic blob transport fails closed instead of falling back to ambient credentials. + ErrBlobIdentityUnresolved = errors.New("blob identity could not be resolved") + + // ErrBlobOperationFailed is the sentinel for blob failures without a more specific class, such as + // local I/O errors or an SDK failure with no mapped meaning. The underlying cause stays in the chain. + ErrBlobOperationFailed = errors.New("blob operation failed") + + // ErrBlobTaggingDenied is returned by tests that simulate a version-specific object tagging denial. + ErrBlobTaggingDenied = errors.New("blob tagging denied") + + // ErrBlobInvalidChunk is returned by the S3 test endpoint when an aws-chunked request body cannot + // be decoded: an unreadable or non-hex chunk size, or a chunk cut short. + ErrBlobInvalidChunk = errors.New("invalid aws-chunked body") + + // ErrBlobTooManyTags is returned by the S3 test endpoint when a request carries more object tags + // than S3 allows. + ErrBlobTooManyTags = errors.New("object tags cannot be greater than 10") + + // ErrBlobInvalidChunkTerminator is returned by the S3 test endpoint when an aws-chunked request + // body has a chunk that is not followed by the required CRLF terminator. + ErrBlobInvalidChunkTerminator = errors.New("invalid aws-chunked chunk terminator") +) + // ExitCodeError is a typed error that preserves subcommand exit codes. // This allows the root command to exit with the same code as the subcommand. // When Code is 0, it indicates successful completion that should exit cleanly without printing errors. diff --git a/internal/ci/vulncheck/scan.go b/internal/ci/vulncheck/scan.go index 90a8cfc739c..4853e09d4d3 100644 --- a/internal/ci/vulncheck/scan.go +++ b/internal/ci/vulncheck/scan.go @@ -74,6 +74,19 @@ func scan(ctx context.Context, params *Params, run commandRunner) error { func runCommand(ctx context.Context, stdout, stderr io.Writer, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) // #nosec G204 -- fixed tool name, mage-target-controlled args. + // These optional CI experiment settings affect only the scanner child, not + // Mage, compilation, or other Go processes in the job. + cmd.Env = cmd.Environ() + if gc, ok := os.LookupEnv("ATMOS_VULNCHECK_GOGC"); ok && gc != "" { + cmd.Env = append(cmd.Env, "GOGC="+gc) + } + if trace, ok := os.LookupEnv("ATMOS_VULNCHECK_GCTRACE"); ok && trace == "1" { + debug, _ := os.LookupEnv("GODEBUG") + if debug != "" { + debug += "," + } + cmd.Env = append(cmd.Env, "GODEBUG="+debug+"gctrace=1") + } cmd.Stdout = stdout cmd.Stderr = stderr return cmd.Run() diff --git a/internal/ci/vulncheck/scan_test.go b/internal/ci/vulncheck/scan_test.go index 07ee609f68c..a90a70428bd 100644 --- a/internal/ci/vulncheck/scan_test.go +++ b/internal/ci/vulncheck/scan_test.go @@ -176,6 +176,25 @@ func TestRunCommand_MissingBinary(t *testing.T) { require.Error(t, err) } +func TestRunCommand_ScannerEnvironment(t *testing.T) { + if os.Getenv("VULNCHECK_ENV_HELPER") == "1" { + _, _ = io.WriteString(os.Stdout, os.Getenv("GOGC")+"|"+os.Getenv("GODEBUG")) + return + } + t.Setenv("VULNCHECK_ENV_HELPER", "1") + t.Setenv("GOGC", "100") + t.Setenv("GODEBUG", "invalidptr=1") + t.Setenv("ATMOS_VULNCHECK_GOGC", "50") + t.Setenv("ATMOS_VULNCHECK_GCTRACE", "1") + exe, err := os.Executable() + require.NoError(t, err) + var stdout, stderr bytes.Buffer + require.NoError(t, runCommand(context.Background(), &stdout, &stderr, exe, "-test.run=^TestRunCommand_ScannerEnvironment$")) + assert.Contains(t, stdout.String(), "50|invalidptr=1,gctrace=1") + assert.Equal(t, "100", os.Getenv("GOGC")) + assert.Equal(t, "invalidptr=1", os.Getenv("GODEBUG")) +} + func TestScan_ReportsMissingGovulncheck(t *testing.T) { // With nothing on PATH the real runner cannot find the tool, and Scan must // say so instead of writing an empty report. diff --git a/pkg/archive/deterministic_test.go b/pkg/archive/deterministic_test.go index 618517f99a6..668aee0a87b 100644 --- a/pkg/archive/deterministic_test.go +++ b/pkg/archive/deterministic_test.go @@ -104,6 +104,7 @@ func TestWriteDeterministic_ZipEntries(t *testing.T) { assert.NotZero(t, byName["link"].Mode()&fs.ModeSymlink) rc, err := byName["link"].Open() require.NoError(t, err) + t.Cleanup(func() { _ = rc.Close() }) target, err := io.ReadAll(rc) require.NoError(t, err) require.NoError(t, rc.Close()) diff --git a/pkg/artifact/artifacttest/verifier.go b/pkg/artifact/artifacttest/verifier.go new file mode 100644 index 00000000000..ddaec659442 --- /dev/null +++ b/pkg/artifact/artifacttest/verifier.go @@ -0,0 +1,75 @@ +package artifacttest + +import ( + "context" + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// FakeVerifier is an [artifact.Verifier] for the evidence [FakeSigner] produces. Repositories that +// delegate cryptography to an injected verifier (the local repositories) use it in conformance +// tests, so those tests exercise the real storage and policy flow without Sigstore. It is a +// harness: it checks subject binding and the trust set, nothing cryptographic. +type FakeVerifier struct{} + +var _ artifact.Verifier = (*FakeVerifier)(nil) + +// Formats implements [artifact.Verifier]. +func (*FakeVerifier) Formats() []artifact.EvidenceFormat { + defer perf.Track(nil, "artifacttest.FakeVerifier.Formats")() + + return []artifact.EvidenceFormat{FakeEvidenceFormat} +} + +// Verify implements [artifact.Verifier]. Every trust set must be satisfied by at least one +// evidence item bound to the subject. Malformed, unsupported, and unrelated evidence is reported +// and ignored; it can never override matching evidence. +func (*FakeVerifier) Verify(_ context.Context, subject artifact.SignatureSubject, evidence []artifact.EvidenceBlob, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "artifacttest.FakeVerifier.Verify")() + + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed} + if fingerprint, err := policy.Fingerprint(); err == nil { + result.PolicyFingerprint = fingerprint + } + if !policy.HasTrustConstraints() { + result.Status = artifact.VerificationUnchecked + return result, nil + } + var claims []artifact.SignerClaim + for index := range evidence { + claim, ok := authenticateFake(subject, &evidence[index], result) + if ok { + claims = append(claims, claim) + result.EvidenceDigests = append(result.EvidenceDigests, evidence[index].Evidence.Descriptor.Digest) + } + } + if !satisfies(policy, claims) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-permitted-signature", Message: "no valid permitted signature satisfies the trust set"}) + return result, fmt.Errorf("%w: no valid permitted signature for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + result.Claims = claims + result.Status = artifact.VerificationVerified + return result, nil +} + +func authenticateFake(subject artifact.SignatureSubject, held *artifact.EvidenceBlob, result *artifact.VerificationResult) (artifact.SignerClaim, bool) { + skip := func(code, message string) (artifact.SignerClaim, bool) { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: code, Message: message}) + return artifact.SignerClaim{}, false + } + if held.Evidence.Format != FakeEvidenceFormat { + return skip("unsupported-evidence", "evidence in an unsupported format was ignored") + } + var parsed fakeEvidence + if err := json.Unmarshal(held.Data, &parsed); err != nil { + return skip("malformed-evidence", "malformed evidence was ignored") + } + if parsed.Subject != subject.Digest || held.Evidence.Subject.Digest != subject.Digest { + return skip("subject-mismatch", "evidence bound to a different subject was ignored") + } + return artifact.SignerClaim{Identity: parsed.Identity, Issuer: parsed.Issuer}, true +} diff --git a/pkg/artifact/blob/blob_test.go b/pkg/artifact/blob/blob_test.go new file mode 100644 index 00000000000..24350ab6a46 --- /dev/null +++ b/pkg/artifact/blob/blob_test.go @@ -0,0 +1,199 @@ +package blob + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestErrorKindsMatchSentinelsAndCauses(t *testing.T) { + cause := errors.New("sdk detail") + cases := []struct { + kind Kind + sentinel error + check func(error) bool + }{ + {KindOther, errUtils.ErrBlobOperationFailed, func(err error) bool { return errors.Is(err, errUtils.ErrBlobOperationFailed) }}, + {KindNotFound, errUtils.ErrBlobNotFound, IsNotFound}, + {KindPrecondition, errUtils.ErrBlobPreconditionFailed, IsPreconditionFailed}, + {KindUnsupported, errUtils.ErrBlobConditionalUnsupported, IsConditionalUnsupported}, + {KindAmbiguous, errUtils.ErrBlobOutcomeUnknown, IsAmbiguous}, + {KindInvalid, errUtils.ErrBlobInvalid, func(err error) bool { return errors.Is(err, errUtils.ErrBlobInvalid) }}, + {KindIdentity, errUtils.ErrBlobIdentityUnresolved, func(err error) bool { return errors.Is(err, errUtils.ErrBlobIdentityUnresolved) }}, + } + for _, tc := range cases { + err := fmt.Errorf("wrapped: %w", NewError(tc.kind, "put", "k", cause)) + assert.ErrorIs(t, err, tc.sentinel) + assert.ErrorIs(t, err, cause, "the cause stays visible") + assert.True(t, tc.check(err)) + assert.Contains(t, err.Error(), `blob put "k"`) + assert.Contains(t, err.Error(), "sdk detail") + } + assert.False(t, IsNotFound(NewError(KindPrecondition, "put", "k", nil))) +} + +func TestErrorMessagesDoNotDuplicateTheSentinel(t *testing.T) { + err := NewError(KindNotFound, "get", "k", fmt.Errorf("%w: gone", errUtils.ErrBlobNotFound)) + assert.Equal(t, 1, strings.Count(err.Error(), errUtils.ErrBlobNotFound.Error())) + assert.Equal(t, `blob get "k": blob object not found`, NewError(KindNotFound, "get", "k", nil).Error()) + assert.Equal(t, `blob list: blob operation failed`, NewError(KindOther, "list", "", nil).Error()) + assert.Equal(t, `blob list: blob operation failed: boom`, NewError(KindOther, "list", "", errors.New("boom")).Error()) +} + +func TestOnlyAmbiguousErrorsAreTemporary(t *testing.T) { + for kind := KindOther; kind <= KindIdentity; kind++ { + assert.Equal(t, kind == KindAmbiguous, NewError(kind, "put", "k", nil).Temporary(), "kind %d", kind) + } + var temporary interface{ Temporary() bool } + require.True(t, errors.As(NewError(KindAmbiguous, "put", "k", nil), &temporary)) +} + +func TestValidateKey(t *testing.T) { + valid := []string{"a", "a/b", "dir/file.tar", "x/y.z/w", strings.Repeat("a", MaxKeyBytes)} + for _, key := range valid { + assert.NoError(t, ValidateKey(key), key) + } + invalid := []string{"", "/a", "a/", "a//b", "a/./b", "a/../b", "..", ".", "a\\b", "a\x00b", "\xff", strings.Repeat("a", MaxKeyBytes+1)} + for _, key := range invalid { + err := ValidateKey(key) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } +} + +func TestValidateOpaqueKeyAllowsLegacyPatterns(t *testing.T) { + for _, key := range []string{"a//b", "/leading", "trailing/", "a/../b", "a\\b"} { + assert.NoError(t, ValidateOpaqueKey(key), key) + } + for _, key := range []string{"", "a\x00b", strings.Repeat("a", MaxKeyBytes+1)} { + assert.Error(t, ValidateOpaqueKey(key), key) + } +} + +func TestMergeTagsNeverAliasesInputs(t *testing.T) { + defaults := map[string]string{"a": "1", "b": "2"} + overrides := map[string]string{"b": "3", "c": "4"} + merged := MergeTags(defaults, overrides) + assert.Equal(t, map[string]string{"a": "1", "b": "3", "c": "4"}, merged) + + merged["a"] = "mutated" + assert.Equal(t, "1", defaults["a"], "result to source isolation") + defaults["z"] = "late" + assert.NotContains(t, merged, "z", "source to result isolation") + assert.Equal(t, "2", map[string]string{"a": "1", "b": "2"}["b"]) + assert.Nil(t, MergeTags(nil, nil)) + assert.Nil(t, CloneMap(nil)) + clone := CloneMap(overrides) + clone["c"] = "x" + assert.Equal(t, "4", overrides["c"]) +} + +func TestValidateTags(t *testing.T) { + assert.NoError(t, ValidateTags(nil)) + assert.NoError(t, ValidateTags(map[string]string{"k": strings.Repeat("v", MaxTagValueChars)})) + + many := map[string]string{} + for i := range MaxTags + 1 { + many[fmt.Sprint("k", i)] = "v" + } + for name, tags := range map[string]map[string]string{ + "too many": many, + "empty key": {"": "v"}, + "long key": {strings.Repeat("k", MaxTagKeyChars+1): "v"}, + "long val": {"k": strings.Repeat("v", MaxTagValueChars+1)}, + } { + assert.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid, name) + } +} + +func TestConditionHelpers(t *testing.T) { + assert.False(t, Condition{}.IsConditional()) + assert.True(t, IfAbsent().IsConditional()) + assert.Equal(t, Condition{Kind: ConditionIfMatch, ETag: `"x"`}, IfMatch(`"x"`)) + assert.False(t, GetOptions{}.IsRange()) + assert.True(t, GetOptions{Offset: 1}.IsRange()) + assert.True(t, GetOptions{Length: 1}.IsRange()) +} + +func TestCapabilityChecksFailClearly(t *testing.T) { + none := Capabilities{} + assert.NoError(t, none.CheckPut("s", "k", Condition{})) + err := none.CheckPut("s", "k", IfAbsent()) + assert.True(t, IsConditionalUnsupported(err)) + assert.Contains(t, err.Error(), "s store cannot enforce conditional writes") + + assert.NoError(t, none.CheckDelete("s", "k", DeleteOptions{})) + assert.True(t, IsConditionalUnsupported(none.CheckDelete("s", "k", DeleteOptions{IfMatch: "x"}))) + + full := Capabilities{ConditionalPut: true, ConditionalDelete: true} + assert.NoError(t, full.CheckPut("s", "k", IfMatch("x"))) + assert.NoError(t, full.CheckDelete("s", "k", DeleteOptions{IfMatch: "x"})) +} + +func TestListAllFollowsCursorsAndMergesPrefixes(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + pages := []ListPage{ + {Objects: []ObjectInfo{{Key: "a"}, {Key: "b"}}, CommonPrefixes: []string{"p/"}, NextCursor: "c1"}, + {Objects: []ObjectInfo{{Key: "c"}}, NextCursor: "c2"}, + {Objects: []ObjectInfo{{Key: "d"}}, CommonPrefixes: []string{"q/"}}, + } + var calls []ListOptions + store.EXPECT().List(gomock.Any(), gomock.Any()).Times(len(pages)). + DoAndReturn(func(_ context.Context, opts ListOptions) (*ListPage, error) { + calls = append(calls, opts) + page := pages[len(calls)-1] + return &page, nil + }) + + objects, prefixes, err := ListAll(context.Background(), store, ListOptions{Prefix: "x/", Limit: 2}) + require.NoError(t, err) + assert.Equal(t, []string{"a", "b", "c", "d"}, []string{objects[0].Key, objects[1].Key, objects[2].Key, objects[3].Key}) + assert.Equal(t, []string{"p/", "q/"}, prefixes) + require.Len(t, calls, 3) + assert.Equal(t, []string{"", "c1", "c2"}, []string{calls[0].Cursor, calls[1].Cursor, calls[2].Cursor}) + for _, call := range calls { + assert.Equal(t, "x/", call.Prefix) + assert.Equal(t, 2, call.Limit) + } +} + +func TestListAllStopsOnError(t *testing.T) { + boom := errors.New("boom") + ctrl := gomock.NewController(t) + store := NewMockStore(ctrl) + store.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, boom) + _, _, err := ListAll(context.Background(), store, ListOptions{}) + require.ErrorIs(t, err, boom) +} + +func TestValidateTagsRejectsInvalidUTF8(t *testing.T) { + for _, tags := range []map[string]string{{"\xff": "value"}, {"key": "\xff"}} { + require.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid) + } +} + +func TestValidateTagsCountsUTF16CodeUnits(t *testing.T) { + const emoji = "\U0001F600" // One rune, two UTF-16 code units. + keyAtLimit := strings.Repeat(emoji, MaxTagKeyChars/2) + valueAtLimit := strings.Repeat(emoji, MaxTagValueChars/2) + + require.NoError(t, ValidateTags(map[string]string{keyAtLimit: valueAtLimit}), "exactly at both limits") + for name, tags := range map[string]map[string]string{ + "key one unit over": {keyAtLimit + "a": "v"}, + "key one rune over": {keyAtLimit + emoji: "v"}, + "value one unit over": {"k": valueAtLimit + "a"}, + "value one rune over": {"k": valueAtLimit + emoji}, + "rune count under cap": {strings.Repeat(emoji, MaxTagKeyChars/2+1): "v"}, + } { + require.ErrorIs(t, ValidateTags(tags), errUtils.ErrBlobInvalid, name) + } +} diff --git a/pkg/artifact/blob/blobtest/cases_basic.go b/pkg/artifact/blob/blobtest/cases_basic.go new file mode 100644 index 00000000000..602d8f8ca48 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_basic.go @@ -0,0 +1,245 @@ +package blobtest + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func (c *conformance) putGetRoundTrip(t *testing.T) { + store := c.newStore(t) + data := []byte("hello blob") + + info := put(t, store, keyNested, data) + assert.Equal(t, keyNested, info.Key) + assert.Equal(t, int64(len(data)), info.Size) + assert.NotEmpty(t, info.ETag) + + assert.Equal(t, data, read(t, store, keyNested)) + stat, err := store.Stat(context.Background(), keyNested, blob.StatOptions{}) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), stat.Size) + assert.Equal(t, info.ETag, stat.ETag, "Put and Stat must report the same concurrency token") +} + +func (c *conformance) zeroLength(t *testing.T) { + store := c.newStore(t) + info := put(t, store, "empty", nil) + assert.Zero(t, info.Size) + assert.NotEmpty(t, info.ETag) + + assert.Empty(t, read(t, store, "empty")) + stat, err := store.Stat(context.Background(), "empty", blob.StatOptions{}) + require.NoError(t, err) + assert.Zero(t, stat.Size) + + // A zero-length body of unknown size is also a valid object. + _, err = store.Put(context.Background(), "empty-unknown", bytes.NewReader(nil), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Empty(t, read(t, store, "empty-unknown")) +} + +func (c *conformance) missing(t *testing.T) { + store := c.newStore(t) + ctx := context.Background() + + _, err := store.Get(ctx, "nope", blob.GetOptions{}) + assert.True(t, blob.IsNotFound(err), "Get: %v", err) + _, err = store.Stat(ctx, "nope", blob.StatOptions{}) + assert.True(t, blob.IsNotFound(err), "Stat: %v", err) + assert.NoError(t, store.Delete(ctx, "nope", blob.DeleteOptions{}), "unconditional delete of a missing object succeeds") +} + +func (c *conformance) overwrite(t *testing.T) { + store := c.newStore(t) + first := put(t, store, keyObject, []byte("one")) + second := put(t, store, keyObject, []byte("two!")) + assert.NotEqual(t, first.ETag, second.ETag, "different content must yield a different token") + assert.Equal(t, []byte("two!"), read(t, store, keyObject)) + + require.NoError(t, store.Delete(context.Background(), keyObject, blob.DeleteOptions{})) + requireAbsent(t, store, keyObject) +} + +func (c *conformance) invalidKeys(t *testing.T) { + store := c.newStore(t) + _, err := store.Put(context.Background(), "", bytes.NewReader([]byte(bodyX)), blob.PutOptions{Size: 1}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = store.Get(context.Background(), "", blob.GetOptions{}) + require.Error(t, err) + _, err = store.Stat(context.Background(), "", blob.StatOptions{}) + require.Error(t, err) +} + +func (c *conformance) attributes(t *testing.T) { + store := c.newStore(t) + caps := store.Capabilities() + ctx := context.Background() + opts := blob.PutOptions{ + Size: 3, + ContentType: "application/json", + Metadata: map[string]string{"owner": "atmos"}, + Tags: map[string]string{"team": "platform", "env": "dev"}, + } + _, err := store.Put(ctx, keyAttrs, bytes.NewReader([]byte("{ }")), opts) + require.NoError(t, err) + + stat, err := store.Stat(ctx, keyAttrs, blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + if caps.Metadata { + assert.Equal(t, "application/json", stat.ContentType) + assert.Equal(t, "atmos", stat.Metadata["owner"]) + } + if caps.Tags { + assert.Equal(t, opts.Tags, stat.Tags) + } + + // Rewriting without attributes must not keep the old ones. + put(t, store, keyAttrs, []byte("{ }")) + stat, err = store.Stat(ctx, keyAttrs, blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Empty(t, stat.Tags) + assert.Empty(t, stat.Metadata) + if caps.Metadata { + assert.NotEqual(t, "application/json", stat.ContentType) + } + + tooMany := map[string]string{} + for _, key := range []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} { + tooMany[key] = "v" + } + _, err = store.Put(ctx, "bad-tags", bytes.NewReader(nil), blob.PutOptions{Size: 0, Tags: tooMany}) + require.Error(t, err, "more than the S3 limit of 10 tags must be rejected on every backend") + requireAbsent(t, store, "bad-tags") +} + +func (c *conformance) ranged(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().Range { + t.Skipf("store does not advertise range reads") + } + put(t, store, "r", []byte("0123456789")) + for _, tc := range []struct { + name string + opts blob.GetOptions + want string + }{ + {"middle", blob.GetOptions{Offset: 2, Length: 4}, "2345"}, + {"tail", blob.GetOptions{Offset: tailOffset}, "789"}, + {"clamped", blob.GetOptions{Offset: clampedOffset, Length: listPageGuard}, "89"}, + } { + t.Run(tc.name, func(t *testing.T) { + object, err := store.Get(context.Background(), "r", tc.opts) + require.NoError(t, err) + defer object.Body.Close() + data, err := io.ReadAll(object.Body) + require.NoError(t, err) + assert.Equal(t, tc.want, string(data)) + assert.Equal(t, int64(10), object.Size, "Size is the full object size") + }) + } +} + +func (c *conformance) sizeMismatch(t *testing.T) { + store := c.newStore(t) + for name, body := range map[string]string{"short": "ab", "long": "abcdef"} { + _, err := store.Put(context.Background(), name, bytes.NewReader([]byte(body)), blob.PutOptions{Size: 4}) + require.Error(t, err, name) + requireAbsent(t, store, name) + } +} + +// errSourceBroke is the failure the broken-source case injects. +var errSourceBroke = errors.New("source broke") + +// mockFailingReader yields data and then fails. +type mockFailingReader struct { + data []byte + err error +} + +func (f *mockFailingReader) Read(p []byte) (int, error) { + if len(f.data) == 0 { + return 0, f.err + } + n := copy(p, f.data) + f.data = f.data[n:] + return n, nil +} + +func (c *conformance) failedBody(t *testing.T) { + store := c.newStore(t) + boom := errSourceBroke + _, err := store.Put(context.Background(), "partial", &mockFailingReader{data: bytes.Repeat([]byte(bodyX), sourceChunk), err: boom}, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, boom) + requireAbsent(t, store, "partial") +} + +func (c *conformance) canceled(t *testing.T) { + store := c.newStore(t) + ctx, cancel := context.WithCancel(context.Background()) + body := io.MultiReader(bytes.NewReader(bytes.Repeat([]byte(bodyX), sourceChunk)), &mockCancelingReader{cancel: cancel}) + _, err := store.Put(ctx, "canceled", body, blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + requireAbsent(t, store, "canceled") +} + +// mockCancelingReader cancels its context on first read and then reports the cancellation. +type mockCancelingReader struct{ cancel context.CancelFunc } + +func (r *mockCancelingReader) Read([]byte) (int, error) { + r.cancel() + return 0, context.Canceled +} + +func (c *conformance) largeStream(t *testing.T) { + store := c.newStore(t) + const size = 3<<20 + 123 + unit := []byte("0123456789abcdef") + build := func() io.Reader { + return io.LimitReader(&mockRepeatReader{unit: unit}, size) + } + want := sha256.New() + _, err := io.Copy(want, build()) + require.NoError(t, err) + + for name, declared := range map[string]int64{"known-size": size, "unknown-size": blob.UnknownSize} { + info, err := store.Put(context.Background(), name, build(), blob.PutOptions{Size: declared}) + require.NoError(t, err, name) + assert.Equal(t, int64(size), info.Size, name) + + object, err := store.Get(context.Background(), name, blob.GetOptions{}) + require.NoError(t, err, name) + got := sha256.New() + n, err := io.Copy(got, object.Body) + require.NoError(t, err, name) + require.NoError(t, object.Body.Close()) + assert.Equal(t, int64(size), n, name) + assert.Equal(t, want.Sum(nil), got.Sum(nil), name) + } +} + +// mockRepeatReader yields unit forever. +type mockRepeatReader struct { + unit []byte + off int +} + +func (r *mockRepeatReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = r.unit[(r.off+i)%len(r.unit)] + } + r.off = (r.off + len(p)) % len(r.unit) + return len(p), nil +} diff --git a/pkg/artifact/blob/blobtest/cases_conditional.go b/pkg/artifact/blob/blobtest/cases_conditional.go new file mode 100644 index 00000000000..099f8c0e631 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_conditional.go @@ -0,0 +1,187 @@ +package blobtest + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// putWith stores data under a condition and returns the result. +func putWith(store blob.Store, key string, data []byte, cond blob.Condition) (*blob.ObjectInfo, error) { + return store.Put(context.Background(), key, bytes.NewReader(data), blob.PutOptions{Size: int64(len(data)), Condition: cond}) +} + +func (c *conformance) ifAbsent(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + _, err := putWith(store, keyLock, []byte("a"), blob.IfAbsent()) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "a store without conditional writes must say so: %v", err) + requireAbsent(t, store, keyLock) + return + } + first, err := putWith(store, keyLock, []byte("owner-a"), blob.IfAbsent()) + require.NoError(t, err) + assert.NotEmpty(t, first.ETag) + + _, err = putWith(store, keyLock, []byte("owner-b"), blob.IfAbsent()) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "second create-if-absent must lose: %v", err) + assert.False(t, blob.IsAmbiguous(err)) + assert.Equal(t, []byte("owner-a"), read(t, store, keyLock), "a lost race must not change the object") + + require.NoError(t, store.Delete(context.Background(), keyLock, blob.DeleteOptions{})) + _, err = putWith(store, keyLock, []byte("owner-c"), blob.IfAbsent()) + require.NoError(t, err, "create-if-absent succeeds again after delete") +} + +func (c *conformance) ifMatch(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + first := put(t, store, keyRef, []byte("v1")) + + second, err := putWith(store, keyRef, []byte("v2"), blob.IfMatch(first.ETag)) + require.NoError(t, err) + assert.NotEqual(t, first.ETag, second.ETag) + + _, err = putWith(store, keyRef, []byte("v3"), blob.IfMatch(first.ETag)) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "stale token must lose: %v", err) + assert.Equal(t, []byte("v2"), read(t, store, keyRef)) + + _, err = putWith(store, "ghost", []byte(bodyX), blob.IfMatch(`"abc"`)) + require.Error(t, err) + assert.True(t, blob.IsNotFound(err), "if-match on a missing object: %v", err) + requireAbsent(t, store, "ghost") + + _, err = putWith(store, keyRef, []byte(bodyX), blob.Condition{Kind: blob.ConditionIfMatch}) + require.Error(t, err, "if-match without a token is invalid") +} + +func (c *conformance) conditionalDelete(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalDelete { + put(t, store, keyObject, []byte(bodyX)) + err := store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: `"abc"`}) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Equal(t, []byte(bodyX), read(t, store, keyObject)) + return + } + info := put(t, store, keyObject, []byte(bodyX)) + + err := store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: `"stale"`}) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + assert.Equal(t, []byte(bodyX), read(t, store, keyObject), "a rejected delete must keep the object") + + require.NoError(t, store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: info.ETag})) + requireAbsent(t, store, keyObject) + + err = store.Delete(context.Background(), keyObject, blob.DeleteOptions{IfMatch: info.ETag}) + assert.True(t, blob.IsNotFound(err), "conditional delete of a missing object: %v", err) +} + +func (c *conformance) concurrentIfAbsent(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + const writers = 12 + var ( + wg sync.WaitGroup + winners atomic.Int32 + losers atomic.Int32 + start = make(chan struct{}) + ) + for i := range writers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := putWith(store, "claim", []byte("owner-"+strconv.Itoa(i)), blob.IfAbsent()) + switch { + case err == nil: + winners.Add(1) + case blob.IsPreconditionFailed(err): + losers.Add(1) + default: + t.Errorf("writer %d: unexpected error %v", i, err) + } + }() + } + close(start) + wg.Wait() + assert.Equal(t, int32(1), winners.Load(), "exactly one creator wins") + assert.Equal(t, int32(writers-1), losers.Load()) +} + +func (c *conformance) concurrentSwap(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ConditionalPut { + t.Skipf("store does not advertise conditional writes") + } + put(t, store, keyCounter, []byte("0")) + const workers, increments = 6, 5 + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for range increments { + incrementWithRetry(t, store) + } + }() + } + wg.Wait() + assert.Equal(t, strconv.Itoa(workers*increments), string(read(t, store, keyCounter)), "every successful swap must be reflected") +} + +// incrementWithRetry performs a read-modify-write loop guarded by If-Match. +func incrementWithRetry(t *testing.T, store blob.Store) { + t.Helper() + for range maxRetries { + stat, err := store.Stat(context.Background(), keyCounter, blob.StatOptions{}) + if err != nil { + t.Errorf("stat: %v", err) + return + } + object, err := store.Get(context.Background(), keyCounter, blob.GetOptions{}) + if err != nil { + t.Errorf("get: %v", err) + return + } + raw, readErr := io.ReadAll(object.Body) + _ = object.Body.Close() + if readErr != nil { + t.Errorf("read counter: %v", readErr) + return + } + current, err := strconv.Atoi(string(raw)) + if err != nil { + continue + } + next := []byte(fmt.Sprint(current + 1)) + _, err = putWith(store, keyCounter, next, blob.IfMatch(stat.ETag)) + if err == nil { + return + } + if !blob.IsPreconditionFailed(err) { + t.Errorf("swap: %v", err) + return + } + } + t.Errorf("swap did not converge") +} diff --git a/pkg/artifact/blob/blobtest/cases_list.go b/pkg/artifact/blob/blobtest/cases_list.go new file mode 100644 index 00000000000..5187a254041 --- /dev/null +++ b/pkg/artifact/blob/blobtest/cases_list.go @@ -0,0 +1,105 @@ +package blobtest + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func (c *conformance) listPrefix(t *testing.T) { + store := c.newStore(t) + for _, key := range []string{"a/2", "a/1", "b/1", "a/sub/3", "ab"} { + put(t, store, key, []byte(key)) + } + objects, prefixes, err := blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "a/"}) + require.NoError(t, err) + assert.Empty(t, prefixes) + assert.Equal(t, []string{"a/1", "a/2", "a/sub/3"}, keysOf(objects), "lexicographic order, prefix-scoped") + for _, object := range objects { + assert.Equal(t, int64(len(object.Key)), object.Size) + } + + objects, _, err = blob.ListAll(context.Background(), store, blob.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, []string{"a/1", "a/2", "a/sub/3", "ab", "b/1"}, keysOf(objects)) + + objects, _, err = blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "zzz/"}) + require.NoError(t, err) + assert.Empty(t, objects) +} + +func (c *conformance) listPagination(t *testing.T) { + store := c.newStore(t) + const total = listKeyCount + want := make([]string, 0, total) + for i := range total { + key := fmt.Sprintf("page/%04d", i) + want = append(want, key) + put(t, store, key, nil) + } + slices.Sort(want) + + var ( + got []string + cursor string + pages int + ) + for { + page, err := store.List(context.Background(), blob.ListOptions{Prefix: "page/", Cursor: cursor, Limit: listPageSize}) + require.NoError(t, err) + assert.LessOrEqual(t, len(page.Objects), listPageSize, "a page never exceeds the limit") + got = append(got, keysOf(page.Objects)...) + pages++ + require.Less(t, pages, listPageGuard, "pagination must terminate") + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + } + assert.Equal(t, want, got, "every key exactly once, in order") + assert.GreaterOrEqual(t, pages, listMinPages, "53 keys at 10 per page need at least 6 pages") +} + +func (c *conformance) listDelimiter(t *testing.T) { + store := c.newStore(t) + if !store.Capabilities().ListDelimiter { + t.Skipf("store does not advertise delimiter listing") + } + for _, key := range []string{"root.txt", "x/1", "x/2", "x/deep/3", "y/1", "z"} { + put(t, store, key, nil) + } + objects, prefixes, err := blob.ListAll(context.Background(), store, blob.ListOptions{Delimiter: "/"}) + require.NoError(t, err) + assert.Equal(t, []string{"root.txt", "z"}, keysOf(objects)) + assert.Equal(t, []string{"x/", "y/"}, prefixes) + + // Paging must not split or repeat a rolled-up prefix. + var ( + gotObjects []string + gotPrefixes []string + cursor string + ) + for range delimiterPages { + page, err := store.List(context.Background(), blob.ListOptions{Delimiter: "/", Cursor: cursor, Limit: 1}) + require.NoError(t, err) + gotObjects = append(gotObjects, keysOf(page.Objects)...) + gotPrefixes = append(gotPrefixes, page.CommonPrefixes...) + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + } + assert.Equal(t, []string{"root.txt", "z"}, gotObjects) + assert.Equal(t, []string{"x/", "y/"}, gotPrefixes) + + objects, prefixes, err = blob.ListAll(context.Background(), store, blob.ListOptions{Prefix: "x/", Delimiter: "/"}) + require.NoError(t, err) + assert.Equal(t, []string{"x/1", "x/2"}, keysOf(objects)) + assert.Equal(t, []string{"x/deep/"}, prefixes) +} diff --git a/pkg/artifact/blob/blobtest/conformance.go b/pkg/artifact/blob/blobtest/conformance.go new file mode 100644 index 00000000000..f06da218ac9 --- /dev/null +++ b/pkg/artifact/blob/blobtest/conformance.go @@ -0,0 +1,115 @@ +package blobtest + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Keys and sizes the cases share. +const ( + keyObject = "k" + keyNested = "dir/one.txt" + keyAttrs = "attrs.json" + keyLock = "lock" + keyRef = "ref" + keyCounter = "counter" + bodyX = "x" + + sourceChunk = 1024 + maxRetries = 500 + listPageSize = 10 + listPageGuard = 100 + listKeyCount = 53 + listMinPages = 6 + delimiterPages = 20 + tailOffset = 7 + clampedOffset = 8 +) + +// Factory returns a fresh, empty store for one subtest. It must register cleanup with t.Cleanup +// and must not share state with stores returned by earlier calls. +type Factory func(t *testing.T) blob.Store + +// RunStoreConformance runs the transport contract against stores built by factory. +func RunStoreConformance(t *testing.T, factory Factory) { + defer perf.Track(nil, "blobtest.RunStoreConformance")() + + t.Helper() + require.NotNil(t, factory, "a store factory is required") + suite := &conformance{factory: factory} + t.Run("PutGetRoundTrip", suite.putGetRoundTrip) + t.Run("ZeroLengthObject", suite.zeroLength) + t.Run("MissingObjects", suite.missing) + t.Run("OverwriteChangesETag", suite.overwrite) + t.Run("RejectsInvalidKeys", suite.invalidKeys) + t.Run("AttributesRoundTrip", suite.attributes) + t.Run("RangedRead", suite.ranged) + t.Run("SizeMismatchLeavesNothing", suite.sizeMismatch) + t.Run("FailedBodyLeavesNothing", suite.failedBody) + t.Run("CanceledPutLeavesNothing", suite.canceled) + t.Run("LargeStreamedBody", suite.largeStream) + t.Run("ConditionalIfAbsent", suite.ifAbsent) + t.Run("ConditionalIfMatch", suite.ifMatch) + t.Run("ConditionalDelete", suite.conditionalDelete) + t.Run("ConcurrentCreateIfAbsentHasOneWinner", suite.concurrentIfAbsent) + t.Run("ConcurrentCompareAndSwapHasNoLostUpdate", suite.concurrentSwap) + t.Run("ListPrefixAndOrder", suite.listPrefix) + t.Run("ListPaginationVisitsEveryKeyOnce", suite.listPagination) + t.Run("ListDelimiterRollsUpPrefixes", suite.listDelimiter) +} + +// conformance holds the factory shared by the cases. +type conformance struct { + factory Factory +} + +// newStore builds a store for one subtest. +func (c *conformance) newStore(t *testing.T) blob.Store { + t.Helper() + store := c.factory(t) + require.NotNil(t, store) + return store +} + +// put stores data unconditionally and fails the test on error. +func put(t *testing.T, store blob.Store, key string, data []byte) *blob.ObjectInfo { + t.Helper() + info, err := store.Put(context.Background(), key, bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.NoError(t, err) + require.NotNil(t, info) + return info +} + +// read returns the full content of key. +func read(t *testing.T, store blob.Store, key string) []byte { + t.Helper() + object, err := store.Get(context.Background(), key, blob.GetOptions{}) + require.NoError(t, err) + defer object.Body.Close() + data, err := io.ReadAll(object.Body) + require.NoError(t, err) + return data +} + +// requireAbsent asserts the key does not exist. +func requireAbsent(t *testing.T, store blob.Store, key string) { + t.Helper() + _, err := store.Stat(context.Background(), key, blob.StatOptions{}) + require.True(t, blob.IsNotFound(err), "expected %q to be absent, got %v", key, err) +} + +// keysOf collects the keys of listed objects. +func keysOf(objects []blob.ObjectInfo) []string { + keys := make([]string, 0, len(objects)) + for _, object := range objects { + keys = append(keys, object.Key) + } + return keys +} diff --git a/pkg/artifact/blob/blobtest/doc.go b/pkg/artifact/blob/blobtest/doc.go new file mode 100644 index 00000000000..2d8ed32e6ac --- /dev/null +++ b/pkg/artifact/blob/blobtest/doc.go @@ -0,0 +1,4 @@ +// Package blobtest is the shared conformance suite for [blob.Store] implementations. Every backend +// runs [RunStoreConformance] from its own tests; unsupported behavior is checked for clear errors where the suite can assert them; other +// cases absent from [blob.Capabilities] are skipped. +package blobtest diff --git a/pkg/artifact/blob/boundary_test.go b/pkg/artifact/blob/boundary_test.go new file mode 100644 index 00000000000..79802d3f037 --- /dev/null +++ b/pkg/artifact/blob/boundary_test.go @@ -0,0 +1,45 @@ +package blob + +import ( + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestTransportHasNoCIOrCommandDependencies keeps the dependency direction one way: CI adapters +// import the blob transport, never the reverse. The blob packages (including every backend and the +// S3 fake) must not reach for CI, command, hook, plugin, runner, or exec packages. +func TestTransportHasNoCIOrCommandDependencies(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + } + checked := 0 + require.NoError(t, filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + parsed, parseErr := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + require.NoError(t, parseErr) + checked++ + for _, spec := range parsed.Imports { + imported, unquoteErr := strconv.Unquote(spec.Path.Value) + require.NoError(t, unquoteErr) + for _, prefix := range forbidden { + require.Falsef(t, imported == prefix || strings.HasPrefix(imported, prefix+"/"), "%s imports forbidden package %s", path, imported) + } + } + return nil + })) + require.Greater(t, checked, 20, "the boundary check must walk the backends and test helpers") +} diff --git a/pkg/artifact/blob/capabilities.go b/pkg/artifact/blob/capabilities.go new file mode 100644 index 00000000000..143c04de211 --- /dev/null +++ b/pkg/artifact/blob/capabilities.go @@ -0,0 +1,62 @@ +package blob + +import ( + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Capabilities describes what a store can enforce. It is computed without network access. +type Capabilities struct { + // ConditionalPut reports that if-absent and if-match writes are enforced. + ConditionalPut bool + + // ConditionalDelete reports that if-match deletes are enforced. + ConditionalDelete bool + + // Multipart reports that bodies larger than one buffer stream without being held in memory. + Multipart bool + + // Tags reports that object tags are stored and returned. + Tags bool + + // Metadata reports that user metadata and content type are stored and returned. + Metadata bool + + // Range reports that partial reads are supported. + Range bool + + // Listing reports that prefix listing with pagination is supported. + Listing bool + + // ListDelimiter reports that listing rolls keys up into common prefixes. + ListDelimiter bool +} + +// SupportsPut reports whether the store can enforce the put condition. +func (c Capabilities) SupportsPut(cond Condition) bool { + defer perf.Track(nil, "blob.Capabilities.SupportsPut")() + + return !cond.IsConditional() || c.ConditionalPut +} + +// CheckPut returns a clear ErrBlobConditionalUnsupported error when the store cannot enforce cond. +func (c Capabilities) CheckPut(store, key string, cond Condition) error { + defer perf.Track(nil, "blob.Capabilities.CheckPut")() + + if c.SupportsPut(cond) { + return nil + } + return NewError(KindUnsupported, OpPut, key, fmt.Errorf("%w: %s store cannot enforce conditional writes", errUtils.ErrBlobConditionalUnsupported, store)) +} + +// CheckDelete returns a clear ErrBlobConditionalUnsupported error when the store cannot enforce an if-match delete. +func (c Capabilities) CheckDelete(store, key string, opts DeleteOptions) error { + defer perf.Track(nil, "blob.Capabilities.CheckDelete")() + + if opts.IfMatch == "" || c.ConditionalDelete { + return nil + } + return NewError(KindUnsupported, OpDelete, key, fmt.Errorf("%w: %s store cannot enforce conditional deletes", errUtils.ErrBlobConditionalUnsupported, store)) +} diff --git a/pkg/artifact/blob/doc.go b/pkg/artifact/blob/doc.go new file mode 100644 index 00000000000..7011e5c482d --- /dev/null +++ b/pkg/artifact/blob/doc.go @@ -0,0 +1,21 @@ +// Package blob is the transport-only object I/O layer shared by artifact repositories and the +// legacy CI artifact adapters. +// +// A [Store] reads and writes opaque byte streams addressed by slash-separated keys. It knows +// nothing about artifact records, CI metadata, signatures, or key layouts: callers own those and +// pass fully formed keys. The package defines streaming put/get, head, conditional write and +// delete (if-absent and if-match), prefix listing with pagination, object tags, user metadata, +// capability reporting, and a small set of classified errors. +// +// Conditional operations carry an ETag. ETags are opaque concurrency tokens and are never payload +// checksums. A store that cannot enforce a requested condition fails that operation with +// [errors.ErrBlobConditionalUnsupported] instead of silently overwriting. A mutation whose result +// was not observed (for example a connection reset after the request was sent) fails with an error +// for which [IsAmbiguous] is true and whose Temporary method reports true; callers reconcile by +// reading the object back. +// +// Implementations live in subpackages: blob/local (filesystem) and blob/s3 (AWS SDK v2). The +// blobtest subpackage holds the shared conformance suite every implementation must pass. +// +// This package must not import pkg/ci or any CI types. Dependencies point from CI adapters to blob. +package blob diff --git a/pkg/artifact/blob/errors.go b/pkg/artifact/blob/errors.go new file mode 100644 index 00000000000..81d24eaed1e --- /dev/null +++ b/pkg/artifact/blob/errors.go @@ -0,0 +1,158 @@ +package blob + +import ( + "errors" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Operation names carried by [Error.Op] and used by every backend. +const ( + OpPut = "put" + OpGet = "get" + OpStat = "stat" + OpDelete = "delete" + OpList = "list" + OpValidate = "validate" + OpNew = "new" +) + +// Kind classifies a blob error independently of the backend that produced it. +type Kind int + +// Error kinds. +const ( + // KindOther is any failure without a more specific class. It matches ErrBlobOperationFailed + // and keeps the cause visible. + KindOther Kind = iota + + // KindNotFound means the addressed object does not exist. + KindNotFound + + // KindPrecondition means a conditional write or delete lost its race. + KindPrecondition + + // KindUnsupported means the endpoint cannot enforce a required condition. + KindUnsupported + + // KindAmbiguous means a mutation was sent but its result was not observed. + KindAmbiguous + + // KindInvalid means the request itself was malformed. + KindInvalid + + // KindIdentity means a configured identity could not be resolved to credentials. + KindIdentity +) + +// sentinel returns the static error for the kind. KindOther maps to ErrBlobOperationFailed so every +// classified error carries a repository sentinel. +func (k Kind) sentinel() error { + switch k { + case KindNotFound: + return errUtils.ErrBlobNotFound + case KindPrecondition: + return errUtils.ErrBlobPreconditionFailed + case KindUnsupported: + return errUtils.ErrBlobConditionalUnsupported + case KindAmbiguous: + return errUtils.ErrBlobOutcomeUnknown + case KindInvalid: + return errUtils.ErrBlobInvalid + case KindIdentity: + return errUtils.ErrBlobIdentityUnresolved + default: + return errUtils.ErrBlobOperationFailed + } +} + +// Error is a classified blob failure. It matches both the kind sentinel and the cause under +// errors.Is, so context cancellation and SDK errors stay visible. +type Error struct { + // Kind is the failure class. + Kind Kind + + // Op is the transport operation, for example "put" or "stat". + Op string + + // Key is the object key, when the operation addressed one. + Key string + + // Err is the underlying cause. + Err error +} + +// NewError returns a classified error. A nil err is allowed because every kind has a sentinel. +func NewError(kind Kind, op, key string, err error) *Error { + defer perf.Track(nil, "blob.NewError")() + + return &Error{Kind: kind, Op: op, Key: key, Err: err} +} + +// Error implements error. +func (e *Error) Error() string { + defer perf.Track(nil, "blob.Error.Error")() + + subject := "blob " + e.Op + if e.Key != "" { + subject = fmt.Sprintf("%s %q", subject, e.Key) + } + sentinel := e.Kind.sentinel() + switch { + case e.Err == nil: + return fmt.Sprintf("%s: %v", subject, sentinel) + case errors.Is(e.Err, sentinel): + return fmt.Sprintf("%s: %v", subject, e.Err) + default: + return fmt.Sprintf("%s: %v: %v", subject, sentinel, e.Err) + } +} + +// Unwrap exposes the kind sentinel and the cause to errors.Is and errors.As. +func (e *Error) Unwrap() []error { + defer perf.Track(nil, "blob.Error.Unwrap")() + + chain := []error{e.Kind.sentinel()} + if e.Err != nil { + chain = append(chain, e.Err) + } + return chain +} + +// Temporary reports an ambiguous outcome: the request may or may not have been applied, so the +// caller must reconcile by reading the object instead of assuming failure. +func (e *Error) Temporary() bool { + defer perf.Track(nil, "blob.Error.Temporary")() + + return e.Kind == KindAmbiguous +} + +// IsNotFound reports whether err means the object does not exist. +func IsNotFound(err error) bool { + defer perf.Track(nil, "blob.IsNotFound")() + + return errors.Is(err, errUtils.ErrBlobNotFound) +} + +// IsPreconditionFailed reports whether err means a conditional write or delete lost its race. +func IsPreconditionFailed(err error) bool { + defer perf.Track(nil, "blob.IsPreconditionFailed")() + + return errors.Is(err, errUtils.ErrBlobPreconditionFailed) +} + +// IsConditionalUnsupported reports whether err means the endpoint cannot enforce a condition. +func IsConditionalUnsupported(err error) bool { + defer perf.Track(nil, "blob.IsConditionalUnsupported")() + + return errors.Is(err, errUtils.ErrBlobConditionalUnsupported) +} + +// IsAmbiguous reports whether err means a mutation's outcome was not observed. +func IsAmbiguous(err error) bool { + defer perf.Track(nil, "blob.IsAmbiguous")() + + return errors.Is(err, errUtils.ErrBlobOutcomeUnknown) +} diff --git a/pkg/artifact/blob/key.go b/pkg/artifact/blob/key.go new file mode 100644 index 00000000000..8f5f12d28bf --- /dev/null +++ b/pkg/artifact/blob/key.go @@ -0,0 +1,62 @@ +package blob + +import ( + "fmt" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// MaxKeyBytes is the longest key any store accepts. It matches the S3 object key limit. +const MaxKeyBytes = 1024 + +// ValidateKey checks that key is a portable relative key: non-empty, valid UTF-8 without NUL or +// backslash, no leading or trailing slash, and no empty, "." or ".." segments. Filesystem stores +// require it; S3 stores apply it by default so a key is valid on every backend. +func ValidateKey(key string) error { + defer perf.Track(nil, "blob.ValidateKey")() + + if err := ValidateOpaqueKey(key); err != nil { + return err + } + if strings.ContainsRune(key, '\\') { + return invalidKey(key, "contains a backslash") + } + if strings.HasPrefix(key, "/") || strings.HasSuffix(key, "/") { + return invalidKey(key, "has a leading or trailing slash") + } + for _, segment := range strings.Split(key, "/") { + switch segment { + case "": + return invalidKey(key, "contains an empty segment") + case ".", "..": + return invalidKey(key, "contains a relative segment") + } + } + return nil +} + +// ValidateOpaqueKey applies only the limits every object store shares: non-empty, at most +// [MaxKeyBytes] bytes, valid UTF-8, and no NUL. It is used for legacy key patterns that are +// opaque strings rather than portable paths. +func ValidateOpaqueKey(key string) error { + defer perf.Track(nil, "blob.ValidateOpaqueKey")() + + switch { + case key == "": + return invalidKey(key, "is empty") + case len(key) > MaxKeyBytes: + return invalidKey(key, "is too long") + case !utf8.ValidString(key): + return invalidKey(key, "is not valid UTF-8") + case strings.ContainsRune(key, 0): + return invalidKey(key, "contains NUL") + } + return nil +} + +func invalidKey(key, reason string) error { + return NewError(KindInvalid, OpValidate, key, fmt.Errorf("%w: key %s", errUtils.ErrBlobInvalid, reason)) +} diff --git a/pkg/artifact/blob/list.go b/pkg/artifact/blob/list.go new file mode 100644 index 00000000000..96780780ec9 --- /dev/null +++ b/pkg/artifact/blob/list.go @@ -0,0 +1,30 @@ +package blob + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// ListAll follows the pagination cursor until the listing is complete and returns every object and +// common prefix. Callers that can stop early should page manually. +func ListAll(ctx context.Context, store Store, opts ListOptions) ([]ObjectInfo, []string, error) { + defer perf.Track(nil, "blob.ListAll")() + + var ( + objects []ObjectInfo + prefixes []string + ) + for { + page, err := store.List(ctx, opts) + if err != nil { + return nil, nil, err + } + objects = append(objects, page.Objects...) + prefixes = append(prefixes, page.CommonPrefixes...) + if page.NextCursor == "" { + return objects, prefixes, nil + } + opts.Cursor = page.NextCursor + } +} diff --git a/pkg/artifact/blob/local/attrs.go b/pkg/artifact/blob/local/attrs.go new file mode 100644 index 00000000000..0e2069a9be4 --- /dev/null +++ b/pkg/artifact/blob/local/attrs.go @@ -0,0 +1,45 @@ +package local + +import ( + "encoding/json" + "errors" + "os" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// attrs is the on-disk attribute file stored beside an object when any attribute is supplied. +type attrs struct { + ContentType string `json:"content_type,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +func (a attrs) empty() bool { + return a.ContentType == "" && len(a.Metadata) == 0 && len(a.Tags) == 0 +} + +func attrsPath(objectPath string) string { + return objectPath + attrsSuffix +} + +// readAttrs loads the attribute file of objectPath; a missing file yields empty attributes. +func readAttrs(root *os.Root, objectPath string) (attrs, error) { + var a attrs + data, err := rootReadFile(root, attrsPath(objectPath)) + if errors.Is(err, os.ErrNotExist) { + return a, nil + } + if err != nil { + return a, err + } + if err := json.Unmarshal(data, &a); err != nil { + return attrs{}, err + } + return a, nil +} + +// fromPut builds the attributes requested by a put. +func fromPut(opts blob.PutOptions) attrs { + return attrs{ContentType: opts.ContentType, Metadata: blob.CloneMap(opts.Metadata), Tags: blob.CloneMap(opts.Tags)} +} diff --git a/pkg/artifact/blob/local/delete.go b/pkg/artifact/blob/local/delete.go new file mode 100644 index 00000000000..ebb8ac293b2 --- /dev/null +++ b/pkg/artifact/blob/local/delete.go @@ -0,0 +1,77 @@ +package local + +import ( + "context" + "errors" + "os" + "path/filepath" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [blob.Store]. +func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) error { + defer perf.Track(nil, "blob.local.Delete")() + + if err := s.validate(blob.OpDelete, key); err != nil { + return err + } + if err := s.Capabilities().CheckDelete(storeName, key, opts); err != nil { + return err + } + if err := checkContext(ctx, blob.OpDelete, key); err != nil { + return err + } + release, err := s.guard(ctx, blob.OpDelete, key, opts.IfMatch != "") + if err != nil { + return err + } + defer release() + + root, err := s.openRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return wrapIO(blob.OpDelete, key, err) + } + defer root.Close() + return s.deleteFromRoot(ctx, root, key, opts.IfMatch) +} + +// deleteFromRoot removes the object and its attributes after an optional ETag check, then prunes +// directories the removal left empty. A key that is only a prefix of other objects names no object, +// so deleting it succeeds and leaves the directory and its children untouched. +func (s *Store) deleteFromRoot(ctx context.Context, root *os.Root, key, ifMatch string) error { + target := filepath.FromSlash(key) + if ifMatch != "" { + if err := checkDeleteETag(ctx, root, key, target, ifMatch); err != nil { + return err + } + } + if info, err := root.Stat(target); err == nil && info.IsDir() { + return nil + } + // Plain-file stores keep no attribute files, and a name that looks like one is another object. + paths := []string{target} + if !s.plainFiles { + paths = append(paths, attrsPath(target)) + } + for _, path := range paths { + if err := root.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return wrapIO(blob.OpDelete, key, err) + } + } + rootRemoveEmptyDirs(root, filepath.Dir(target)) + return nil +} + +// checkDeleteETag verifies the if-match token of a conditional delete. Callers hold the lock. +func checkDeleteETag(ctx context.Context, root *os.Root, key, target, want string) error { + state, err := inspect(root, target) + if err != nil { + return wrapIO(blob.OpDelete, key, err) + } + return matchETag(ctx, root, objectRef{op: blob.OpDelete, key: key}, state, want) +} diff --git a/pkg/artifact/blob/local/doc.go b/pkg/artifact/blob/local/doc.go new file mode 100644 index 00000000000..a6472bab9c9 --- /dev/null +++ b/pkg/artifact/blob/local/doc.go @@ -0,0 +1,21 @@ +// Package local is the filesystem implementation of [blob.Store]. +// +// Objects are plain files under a root directory, so a directory written here is readable with +// ordinary tools. Writes stream to a +// temporary file in the destination directory and are renamed into place, so readers never see a +// partial object. Content type, user metadata, and tags live in an optional ".atmos-blob.json" +// attribute file that is written only when an attribute is supplied. +// +// Managed reads, writes, and deletes hold a per-root process and file lock. A recoverable journal +// protects content and attributes across interrupted commits; recovery runs before the next managed +// operation. Direct filesystem edits and mixing [WithPlainFileCompatibility] with managed stores +// on one root are outside this coordination. Plain-file mode preserves the legacy CI layout and +// unlocked unconditional operations, so it refuses conditional puts and deletes (and does not +// advertise them) unless [WithLockAllWrites] makes every mutation honor the same lock. The journal +// recovers interrupted processes, not power loss. +// +// A local object cannot share a path with a key prefix: "a" and "a/b" cannot both exist. ETags are +// opaque content-based SHA-256 tokens. Identical contents have the same token, including after an +// intervening write. Get and Stat stream the entire file to compute the token, using bounded memory; +// listings do not populate ETag. +package local diff --git a/pkg/artifact/blob/local/errors.go b/pkg/artifact/blob/local/errors.go new file mode 100644 index 00000000000..86fb1249497 --- /dev/null +++ b/pkg/artifact/blob/local/errors.go @@ -0,0 +1,64 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// wrapIO classifies an OS error: a missing path is NotFound, everything else is a plain failure. +func wrapIO(op, key string, err error) error { + if errors.Is(err, os.ErrNotExist) { + return blob.NewError(blob.KindNotFound, op, key, err) + } + return blob.NewError(blob.KindOther, op, key, err) +} + +func preconditionFailed(op, key, reason string) error { + return blob.NewError(blob.KindPrecondition, op, key, fmt.Errorf("%w: %s", errUtils.ErrBlobPreconditionFailed, reason)) +} + +// classifyCopy maps a body copy failure: size mismatches are invalid requests, the rest are I/O. +func classifyCopy(key string, err error) error { + if errors.Is(err, errUtils.ErrBlobInvalid) { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, err) + } + return blob.NewError(blob.KindOther, blob.OpPut, key, err) +} + +// classifyCreate explains why a temporary file could not be created beside target. An existing +// object occupying a parent path is a key conflict, which the filesystem cannot represent. +func (s *Store) classifyCreate(root *os.Root, key, target string, err error) error { + for dir := filepath.Dir(target); dir != "."; dir = filepath.Dir(dir) { + if info, statErr := root.Stat(dir); statErr == nil && !info.IsDir() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: a parent path is an existing object", errUtils.ErrBlobInvalid)) + } + } + return wrapIO(blob.OpPut, key, err) +} + +func firstError(errs ...error) error { + return errors.Join(errs...) +} + +// ctxReader fails reads once its context is done so a canceled put stops promptly. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "local.ctxReader.Read")() + + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/pkg/artifact/blob/local/etag.go b/pkg/artifact/blob/local/etag.go new file mode 100644 index 00000000000..a79d664800f --- /dev/null +++ b/pkg/artifact/blob/local/etag.go @@ -0,0 +1,37 @@ +package local + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" +) + +// etagOf returns the content-based ETag of the file at path. Hashing stops when ctx is done. +func etagOf(ctx context.Context, root *os.Root, path string) (string, error) { + file, err := rootOpenFile(root, path) + if err != nil { + return "", err + } + defer file.Close() + return etagOpen(ctx, file) +} + +// etagOpen hashes the opened file and restores its position for the caller. Cancellation is +// observed between reads, so a canceled caller releases the root lock instead of finishing the scan. +func etagOpen(ctx context.Context, file *os.File) (string, error) { + hash := sha256.New() + if _, err := io.Copy(hash, &ctxReader{ctx: ctx, r: file}); err != nil { + return "", err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return "", err + } + return contentETag(hash.Sum(nil)), nil +} + +// contentETag formats a content hash as an ETag. +func contentETag(sum []byte) string { + return `"` + hex.EncodeToString(sum) + `"` +} diff --git a/pkg/artifact/blob/local/fsutil.go b/pkg/artifact/blob/local/fsutil.go new file mode 100644 index 00000000000..443c1b73f91 --- /dev/null +++ b/pkg/artifact/blob/local/fsutil.go @@ -0,0 +1,69 @@ +package local + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "io" + "os" + "runtime" + "time" +) + +const ( + renameAttempts = 8 + renameBackoff = 5 * time.Millisecond + + // The openAttempts constant bounds the Windows retry for reads that race a concurrent rename-over. + openAttempts = 20 +) + +func mkdirAll(dir string, perm os.FileMode) error { + return os.MkdirAll(dir, perm) +} + +func tempName() (string, error) { + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return tempPrefix + hex.EncodeToString(raw[:]), nil +} + +// openFile opens path for reading. Windows refuses to open a file with a sharing violation while a +// concurrent writer is renaming over it, so it retries briefly there. A missing file is reported +// immediately because absence is a normal answer. +func openFile(path string) (*os.File, error) { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = openAttempts + } + return openFileWith(path, attempts, openRead) +} + +func openFileWith(path string, attempts int, open func(string) (*os.File, error)) (*os.File, error) { + var ( + file *os.File + err error + ) + for i := range attempts { + file, err = open(path) + if err == nil || errors.Is(err, os.ErrNotExist) { + return file, err + } + if i < attempts-1 { + time.Sleep(renameBackoff) + } + } + return nil, err +} + +// readFile returns the contents of path, retrying a transient Windows sharing violation like openFile. +func readFile(path string) ([]byte, error) { + file, err := openFile(path) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(file) +} diff --git a/pkg/artifact/blob/local/fsutil_test.go b/pkg/artifact/blob/local/fsutil_test.go new file mode 100644 index 00000000000..1392533a584 --- /dev/null +++ b/pkg/artifact/blob/local/fsutil_test.go @@ -0,0 +1,60 @@ +package local + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenFile_ReturnsExistingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "object") + require.NoError(t, os.WriteFile(path, []byte("payload"), 0o600)) + + file, err := openFile(path) + require.NoError(t, err) + defer file.Close() + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "payload", string(data)) +} + +func TestOpenFile_MissingFileReportsNotExistWithoutRetrying(t *testing.T) { + file, err := openFile(filepath.Join(t.TempDir(), "absent")) + require.ErrorIs(t, err, os.ErrNotExist) + assert.Nil(t, file) +} + +func TestReadFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "attrs") + require.NoError(t, os.WriteFile(path, []byte(`{"a":1}`), 0o600)) + + data, err := readFile(path) + require.NoError(t, err) + assert.JSONEq(t, `{"a":1}`, string(data)) + + _, err = readFile(filepath.Join(t.TempDir(), "absent")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestOpenFileRetriesTransientFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "object") + require.NoError(t, os.WriteFile(path, []byte("payload"), 0o600)) + attempts := 0 + file, err := openFileWith(path, 2, func(path string) (*os.File, error) { + attempts++ + if attempts == 1 { + return nil, os.ErrPermission + } + return os.Open(path) + }) + require.NoError(t, err) + defer file.Close() + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "payload", string(data)) + assert.Equal(t, 2, attempts) +} diff --git a/pkg/artifact/blob/local/get.go b/pkg/artifact/blob/local/get.go new file mode 100644 index 00000000000..8636f922573 --- /dev/null +++ b/pkg/artifact/blob/local/get.go @@ -0,0 +1,157 @@ +package local + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// rangedBody closes the file while reading only a window of it. +type rangedBody struct { + io.Reader + file *os.File +} + +func (r *rangedBody) Close() error { + defer perf.Track(nil, "local.rangedBody.Close")() + + return r.file.Close() +} + +// Get implements [blob.Store]. +func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blob.Object, error) { + defer perf.Track(nil, "blob.local.Get")() + + if err := s.validate(blob.OpGet, key); err != nil { + return nil, err + } + if opts.Offset < 0 { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: negative offset", errUtils.ErrBlobInvalid)) + } + if err := checkContext(ctx, blob.OpGet, key); err != nil { + return nil, err + } + root, err := s.openRoot(false) + if err != nil { + return nil, wrapIO(blob.OpGet, key, err) + } + defer root.Close() + release, err := s.guard(ctx, blob.OpGet, key, false) + if err != nil { + return nil, err + } + defer release() + target := filepath.FromSlash(key) + file, err := rootOpenFile(root, target) + if err != nil { + return nil, wrapIO(blob.OpGet, key, err) + } + info, err := s.describe(ctx, root, describeTarget{op: blob.OpGet, key: key, file: file}, describeOptions{skipETag: opts.SkipETag}) + if err != nil { + _ = file.Close() + return nil, err + } + body, err := openBody(file, info.Size, opts) + if err != nil { + _ = file.Close() + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, err) + } + return &blob.Object{ObjectInfo: *info, Body: body}, nil +} + +// openBody positions file for a full or ranged read. +func openBody(file *os.File, size int64, opts blob.GetOptions) (io.ReadCloser, error) { + if !opts.IsRange() { + return file, nil + } + if opts.Offset > size { + return nil, fmt.Errorf("%w: offset %d is past the end of a %d byte object", errUtils.ErrBlobInvalid, opts.Offset, size) + } + if _, err := file.Seek(opts.Offset, io.SeekStart); err != nil { + return nil, err + } + var reader io.Reader = file + if opts.Length > 0 { + reader = io.LimitReader(file, opts.Length) + } + return &rangedBody{Reader: reader, file: file}, nil +} + +// Stat implements [blob.Store]. +func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.local.Stat")() + + if err := s.validate(blob.OpStat, key); err != nil { + return nil, err + } + if err := checkContext(ctx, blob.OpStat, key); err != nil { + return nil, err + } + root, err := s.openRoot(false) + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + defer root.Close() + release, err := s.guard(ctx, blob.OpStat, key, false) + if err != nil { + return nil, err + } + defer release() + target := filepath.FromSlash(key) + file, err := rootOpenFile(root, target) + if err != nil { + return nil, wrapIO(blob.OpStat, key, err) + } + defer file.Close() + return s.describe(ctx, root, describeTarget{op: blob.OpStat, key: key, file: file}, describeOptions{withTags: opts.IncludeTags, skipETag: opts.SkipETag}) +} + +// describeTarget is the open file a description is built for. +type describeTarget struct { + op string + key string + file *os.File +} + +// describeOptions select the optional parts of a description. +type describeOptions struct { + withTags bool + + // skipETag leaves the ETag empty so a metadata check does not hash the whole object. + skipETag bool +} + +// describe builds ObjectInfo for an open regular file. A directory is not an object. +func (s *Store) describe(ctx context.Context, root *os.Root, target describeTarget, opts describeOptions) (*blob.ObjectInfo, error) { + op, key, file := target.op, target.key, target.file + stat, err := file.Stat() + if err != nil { + return nil, wrapIO(op, key, err) + } + if !stat.Mode().IsRegular() { + return nil, blob.NewError(blob.KindNotFound, op, key, fmt.Errorf("%w: key is a prefix, not an object", errUtils.ErrBlobNotFound)) + } + var etag string + if !opts.skipETag { + if etag, err = etagOpen(ctx, file); err != nil { + return nil, wrapIO(op, key, err) + } + } + var a attrs + if !s.plainFiles { + if a, err = readAttrs(root, filepath.FromSlash(key)); err != nil { + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("read attributes: %w", err)) + } + } + info := &blob.ObjectInfo{Key: key, Size: stat.Size(), ETag: etag, ContentType: a.ContentType, LastModified: stat.ModTime(), Metadata: a.Metadata} + if opts.withTags { + info.Tags = a.Tags + } + return info, nil +} diff --git a/pkg/artifact/blob/local/hold_test.go b/pkg/artifact/blob/local/hold_test.go new file mode 100644 index 00000000000..44c99a635ef --- /dev/null +++ b/pkg/artifact/blob/local/hold_test.go @@ -0,0 +1,76 @@ +package local + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// waitTimeout bounds every wait on another goroutine so a broken product path fails the test instead of hanging it. +const waitTimeout = 10 * time.Second + +// holdGuard takes the put guard for a test and returns a release that is safe to call any number +// of times. The guard is also released when the test ends, so an assertion that aborts the test can +// never leave the root lock held for the next subtest. +func holdGuard(t *testing.T, s *Store, key string, conditional bool) func() { + t.Helper() + // Bound the acquisition so a release that failed to free the lock fails the test with a clear + // message instead of hanging it. The lock does not depend on the context once it is held. + ctx, cancel := context.WithTimeout(context.Background(), waitTimeout) + defer cancel() + release, err := s.guard(ctx, blob.OpPut, key, conditional) + require.NoError(t, err, "timed out or failed acquiring the put guard for %q; a previous release did not free the root lock", key) + return onceRelease(t, release) +} + +// onceRelease wraps a release function that is not idempotent so it runs once, at the first explicit +// call or at test cleanup, whichever comes first. +func onceRelease(t *testing.T, release func()) func() { + t.Helper() + var once sync.Once + wrapped := func() { once.Do(release) } + t.Cleanup(wrapped) + return wrapped +} + +// closeOnCleanup closes a handle when the test ends, ignoring the error of a second close. +func closeOnCleanup(t *testing.T, root *os.Root) { + t.Helper() + t.Cleanup(func() { _ = root.Close() }) +} + +// receive waits for one value from ch and fails the test, instead of hanging it, when nothing arrives. +func receive[T any](t *testing.T, ch <-chan T, what string) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(waitTimeout): + t.Fatalf("timed out waiting for %s", what) + var zero T + return zero + } +} + +func TestHoldGuardReleasesWhenTheTestEnds(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility(), WithLockAllWrites()) + + t.Run("a subtest that never releases", func(t *testing.T) { + explicit := holdGuard(t, s, "held", true) + explicit() + explicit() // A second release is a no-op rather than a double unlock. + _ = holdGuard(t, s, "held", true) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + release, err := s.guard(ctx, blob.OpPut, "held", true) + require.NoError(t, err, "the lock a finished subtest still held is free for the next one") + release() +} diff --git a/pkg/artifact/blob/local/list.go b/pkg/artifact/blob/local/list.go new file mode 100644 index 00000000000..12a765e75dd --- /dev/null +++ b/pkg/artifact/blob/local/list.go @@ -0,0 +1,197 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "sort" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// defaultPageSize is the entries per page when ListOptions.Limit is zero, matching S3's maximum. +const defaultPageSize = 1000 + +// entry is one object found during a walk. +type entry struct { + key string + info os.FileInfo +} + +// List implements [blob.Store]. +func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage, error) { + defer perf.Track(nil, "blob.local.List")() + + if err := checkContext(ctx, blob.OpList, opts.Prefix); err != nil { + return nil, err + } + if err := s.validatePrefix(opts.Prefix); err != nil { + return nil, err + } + var entries []entry + var snapshotID string + if id, last, ok := parseSnapshotCursor(opts.Cursor); ok { + opts.Cursor = last + snapshotID = id + entries = s.lookupSnapshot(id, opts) + } + if entries == nil { + var err error + entries, err = s.walk(ctx, opts.Prefix) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpList, opts.Prefix, err) + } + snapshotID = "" + } + page := paginate(entries, opts) + if page.NextCursor != "" { + if snapshotID == "" { + snapshotID = s.rememberSnapshot(entries, opts) + } + if snapshotID != "" { + page.NextCursor = snapshotCursor(snapshotID, page.NextCursor) + } + } + return page, nil +} + +// walk returns every object whose key starts with prefix, sorted by key. +func (s *Store) walk(ctx context.Context, prefix string) ([]entry, error) { + root, err := s.openRoot(false) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + defer root.Close() + start := "." + if idx := strings.LastIndexByte(prefix, '/'); idx >= 0 { + start = prefix[:idx] + } + var entries []entry + err = fs.WalkDir(root.FS(), start, func(name string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return ignoreMissing(walkErr) + } + if err := ctx.Err(); err != nil { + return err + } + found, ok, err := s.objectAt(name, d, prefix) + if err != nil || !ok { + return err + } + entries = append(entries, found) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].key < entries[j].key }) + return entries, nil +} + +// ignoreMissing treats a path that vanished during the walk as absent. +func ignoreMissing(err error) error { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +// objectAt returns the object stored at a walked path, or false for directories, bookkeeping +// files, keys outside the prefix, and files that vanished. +func (s *Store) objectAt(name string, d fs.DirEntry, prefix string) (entry, bool, error) { + if d.IsDir() || s.reserved(name, d.Name()) { + return entry{}, false, nil + } + key := name + if !strings.HasPrefix(key, prefix) { + return entry{}, false, nil + } + info, err := d.Info() + if err != nil { + return entry{}, false, ignoreMissing(err) + } + return entry{key: key, info: info}, info.Mode().IsRegular(), nil +} + +// reserved reports whether a file is one of the store's own bookkeeping files. Plain-file stores +// write no attribute files, so an attribute-looking name there is an object. +func (s *Store) reserved(name, base string) bool { + return strings.HasPrefix(base, tempPrefix) || + (!s.plainFiles && strings.HasSuffix(base, attrsSuffix)) || + (base == lockName && path.Dir(name) == ".") +} + +// paginate applies the delimiter roll-up, cursor, and limit to sorted entries. +func paginate(entries []entry, opts blob.ListOptions) *blob.ListPage { + limit := opts.Limit + if limit <= 0 || limit > defaultPageSize { + limit = defaultPageSize + } + page := &blob.ListPage{} + count := 0 + lastToken := "" + for _, e := range entries { + token, rolled := rollUp(e.key, opts) + if opts.Cursor != "" && token <= opts.Cursor { + continue + } + if rolled && token == lastToken { + continue + } + if count == limit { + page.NextCursor = lastToken + return page + } + if rolled { + page.CommonPrefixes = append(page.CommonPrefixes, token) + } else { + page.Objects = append(page.Objects, blob.ObjectInfo{Key: e.key, Size: e.info.Size(), LastModified: e.info.ModTime()}) + } + lastToken = token + count++ + } + return page +} + +// rollUp returns the ordering token of a key: the key itself, or its common prefix when the +// delimiter occurs after the listing prefix. +func rollUp(key string, opts blob.ListOptions) (string, bool) { + if opts.Delimiter == "" { + return key, false + } + rest := key[len(opts.Prefix):] + idx := strings.Index(rest, opts.Delimiter) + if idx < 0 { + return key, false + } + return opts.Prefix + rest[:idx+len(opts.Delimiter)], true +} + +// literalBackslashes reports whether backslashes are ordinary characters in keys: the store opted +// in with WithLiteralBackslashes and the platform does not treat them as path separators. +func (s *Store) literalBackslashes() bool { + return s.allowBackslash && os.PathSeparator != '\\' +} + +// validatePrefix rejects listing prefixes that could leave the root. It applies the same +// platform-aware backslash rule as key validation. +func (s *Store) validatePrefix(prefix string) error { + for _, segment := range strings.Split(prefix, "/") { + if segment == ".." { + return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix contains a relative segment", errUtils.ErrBlobInvalid)) + } + } + if (!s.literalBackslashes() && strings.ContainsRune(prefix, '\\')) || strings.ContainsRune(prefix, 0) || strings.HasPrefix(prefix, "/") { + return blob.NewError(blob.KindInvalid, blob.OpList, prefix, fmt.Errorf("%w: prefix must be a relative slash path", errUtils.ErrBlobInvalid)) + } + return nil +} diff --git a/pkg/artifact/blob/local/lock.go b/pkg/artifact/blob/local/lock.go new file mode 100644 index 00000000000..87f70833e7b --- /dev/null +++ b/pkg/artifact/blob/local/lock.go @@ -0,0 +1,78 @@ +package local + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/gofrs/flock" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +const lockRetry = 10 * time.Millisecond + +// rootMutexes holds one in-process mutex per root so separate Store values over one directory +// serialize with each other without relying on the file lock alone. +var rootMutexes sync.Map + +func rootMutex(root string) chan struct{} { + key := root + if abs, err := filepath.Abs(root); err == nil { + key = abs + } + mutex, _ := rootMutexes.LoadOrStore(key, make(chan struct{}, 1)) + return mutex.(chan struct{}) +} + +// lockRoot takes the per-root lock (in-process mutex, then OS file lock) and returns its release. +func (s *Store) lockRoot(ctx context.Context, op, key string) (func(), error) { + mutex := rootMutex(s.root) + select { + case mutex <- struct{}{}: + case <-ctx.Done(): + return nil, blob.NewError(blob.KindOther, op, key, ctx.Err()) + } + if err := ctx.Err(); err != nil { + <-mutex + return nil, blob.NewError(blob.KindOther, op, key, err) + } + if err := mkdirAll(s.root, s.dirPerm); err != nil { + <-mutex + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("create lock directory: %w", err)) + } + fileLock := flock.New(filepath.Join(s.root, lockName)) + if _, err := fileLock.TryLockContext(ctx, lockRetry); err != nil { + <-mutex + return nil, blob.NewError(blob.KindOther, op, key, fmt.Errorf("acquire root lock: %w", err)) + } + return func() { + _ = fileLock.Unlock() + <-mutex + }, nil +} + +// guard returns the lock release for a write or delete, or a no-op when no lock is needed. +func (s *Store) guard(ctx context.Context, op, key string, conditional bool) (func(), error) { + if s.plainFiles && !conditional && !s.lockAll { + return func() {}, nil + } + release, err := s.lockRoot(ctx, op, key) + if err != nil { + return nil, err + } + if !s.plainFiles { + root, openErr := s.openRoot(false) + if openErr == nil { + openErr = s.recoverCommit(root) + _ = root.Close() + } + if openErr != nil { + release() + return nil, wrapIO(op, key, openErr) + } + } + return release, nil +} diff --git a/pkg/artifact/blob/local/open_other.go b/pkg/artifact/blob/local/open_other.go new file mode 100644 index 00000000000..1932e726a2e --- /dev/null +++ b/pkg/artifact/blob/local/open_other.go @@ -0,0 +1,9 @@ +//go:build !unix + +package local + +import "os" + +const nonblockFlag = 0 + +func openRead(path string) (*os.File, error) { return os.Open(path) } diff --git a/pkg/artifact/blob/local/open_unix.go b/pkg/artifact/blob/local/open_unix.go new file mode 100644 index 00000000000..19a2249c8e4 --- /dev/null +++ b/pkg/artifact/blob/local/open_unix.go @@ -0,0 +1,15 @@ +//go:build unix + +package local + +import ( + "os" + + "golang.org/x/sys/unix" +) + +const nonblockFlag = unix.O_NONBLOCK + +func openRead(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|unix.O_NONBLOCK, 0) +} diff --git a/pkg/artifact/blob/local/open_unix_test.go b/pkg/artifact/blob/local/open_unix_test.go new file mode 100644 index 00000000000..5d8f3f27359 --- /dev/null +++ b/pkg/artifact/blob/local/open_unix_test.go @@ -0,0 +1,24 @@ +//go:build unix + +package local + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestGetAndStatRejectFIFOWithoutBlocking(t *testing.T) { + store := newStore(t) + require.NoError(t, mkdirAll(store.Root(), DefaultDirPerm)) + require.NoError(t, unix.Mkfifo(filepath.Join(store.Root(), "fifo"), 0o600)) + _, err := store.Get(context.Background(), "fifo", blob.GetOptions{}) + require.True(t, blob.IsNotFound(err)) + _, err = store.Stat(context.Background(), "fifo", blob.StatOptions{}) + require.True(t, blob.IsNotFound(err)) +} diff --git a/pkg/artifact/blob/local/options.go b/pkg/artifact/blob/local/options.go new file mode 100644 index 00000000000..a8969be86d6 --- /dev/null +++ b/pkg/artifact/blob/local/options.go @@ -0,0 +1,57 @@ +package local + +import ( + "os" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Default permissions for created directories and files. They match the legacy local CI store. +const ( + DefaultDirPerm os.FileMode = 0o755 + DefaultFilePerm os.FileMode = 0o644 +) + +// Option configures a [Store]. +type Option func(*Store) + +// WithDirPerm sets the permission bits of created directories. +func WithDirPerm(perm os.FileMode) Option { + defer perf.Track(nil, "blob.local.WithDirPerm")() + + return func(s *Store) { s.dirPerm = perm } +} + +// WithFilePerm sets the permission bits of created files. +func WithFilePerm(perm os.FileMode) Option { + defer perf.Track(nil, "blob.local.WithFilePerm")() + + return func(s *Store) { s.filePerm = perm } +} + +// WithLockAllWrites serializes unconditional writes and deletes with conditional ones. Managed +// stores already do this. A plain-file store enforces if-absent and if-match conditions only with +// this option, because only then does every mutation honor the lock that makes a condition check +// and its commit atomic; without it the plain-file store refuses conditional operations. +func WithLockAllWrites() Option { + defer perf.Track(nil, "blob.local.WithLockAllWrites")() + + return func(s *Store) { s.lockAll = true } +} + +// WithPlainFileCompatibility preserves the legacy CI layout and unlocked unconditional operations. +// Because unconditional operations stay unlocked, the store refuses conditional puts and deletes +// and does not advertise them unless [WithLockAllWrites] is also set. Do not mix this mode with managed stores on the same root when consistency is required. +func WithPlainFileCompatibility() Option { + defer perf.Track(nil, "local.WithPlainFileCompatibility")() + + return func(s *Store) { s.plainFiles = true } +} + +// WithLiteralBackslashes permits literal backslashes on filesystems where they are not separators. +// This supports legacy local artifact names; other key validation remains in effect. +func WithLiteralBackslashes() Option { + defer perf.Track(nil, "blob.local.WithLiteralBackslashes")() + + return func(s *Store) { s.allowBackslash = true } +} diff --git a/pkg/artifact/blob/local/plain_conditional_test.go b/pkg/artifact/blob/local/plain_conditional_test.go new file mode 100644 index 00000000000..41eaa9efbf8 --- /dev/null +++ b/pkg/artifact/blob/local/plain_conditional_test.go @@ -0,0 +1,133 @@ +package local + +import ( + "bytes" + "context" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func putBody(t *testing.T, s *Store, key, body string, cond blob.Condition) error { + t.Helper() + _, err := s.Put(context.Background(), key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body)), Condition: cond}) + return err +} + +func readBody(t *testing.T, s *Store, key string) string { + t.Helper() + obj, err := s.Get(context.Background(), key, blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + return string(data) +} + +// A plain-file store leaves unconditional writes unlocked for legacy writers, so it cannot make a +// conditional write atomic against them. It must say so instead of advertising and accepting +// conditions it cannot enforce. +func TestPlainFileStoreWithoutLockAllRefusesConditionalOperations(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility()) + require.NoError(t, putBody(t, s, "k", "A", blob.Condition{})) + info, err := s.Stat(context.Background(), "k", blob.StatOptions{}) + require.NoError(t, err) + + caps := s.Capabilities() + assert.False(t, caps.ConditionalPut, "conditional writes are not advertised") + assert.False(t, caps.ConditionalDelete, "conditional deletes are not advertised") + + tests := []struct { + name string + run func() error + }{ + {name: "if-absent put", run: func() error { return putBody(t, s, "new", "B", blob.IfAbsent()) }}, + {name: "if-match put", run: func() error { return putBody(t, s, "k", "B", blob.IfMatch(info.ETag)) }}, + {name: "if-match delete", run: func() error { + return s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: info.ETag}) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.run() + require.ErrorIs(t, err, errUtils.ErrBlobConditionalUnsupported) + assert.True(t, blob.IsConditionalUnsupported(err)) + }) + } + assert.Equal(t, "A", readBody(t, s, "k"), "a refused conditional operation changes nothing") + _, err = s.Stat(context.Background(), "new", blob.StatOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobNotFound) + assert.NoFileExists(t, s.Root()+"/"+lockName, "refusing takes no lock") + + require.NoError(t, putBody(t, s, "k", "C", blob.Condition{}), "unconditional writes keep working") + require.NoError(t, s.Delete(context.Background(), "k", blob.DeleteOptions{}), "unconditional deletes keep working") +} + +// With WithLockAllWrites a plain-file store serializes every mutation, so a conditional operation +// is atomic against unconditional ones and the capabilities are honest. +func TestPlainFileStoreWithLockAllEnforcesConditionsAgainstUnconditionalWrites(t *testing.T) { + s := newStore(t, WithPlainFileCompatibility(), WithLockAllWrites()) + assert.True(t, s.Capabilities().ConditionalPut) + assert.True(t, s.Capabilities().ConditionalDelete) + + t.Run("unconditional mutations wait for a held root lock", func(t *testing.T) { + require.NoError(t, putBody(t, s, "held", "A", blob.Condition{})) + release := holdGuard(t, s, "held", true) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + _, putErr := s.Put(ctx, "held", bytes.NewReader([]byte("C")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, putErr, context.DeadlineExceeded, "an unconditional put cannot slip between a check and a commit") + ctx2, cancel2 := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel2() + deleteErr := s.Delete(ctx2, "held", blob.DeleteOptions{}) + require.ErrorIs(t, deleteErr, context.DeadlineExceeded, "an unconditional delete cannot slip in either") + + release() + assert.Equal(t, "A", readBody(t, s, "held"), "nothing was written while the lock was held") + require.NoError(t, putBody(t, s, "held", "C", blob.Condition{})) + }) + + t.Run("an if-match put never overwrites a competing unconditional put", func(t *testing.T) { + for round := 0; round < 50; round++ { + key := "race" + require.NoError(t, putBody(t, s, key, "A", blob.Condition{})) + info, err := s.Stat(context.Background(), key, blob.StatOptions{}) + require.NoError(t, err) + + var wg sync.WaitGroup + start := make(chan struct{}) + var conditionalErr error + wg.Add(2) + go func() { + defer wg.Done() + <-start + conditionalErr = putBody(t, s, key, "B", blob.IfMatch(info.ETag)) + }() + go func() { + defer wg.Done() + <-start + assert.NoError(t, putBody(t, s, key, "C", blob.Condition{})) + }() + close(start) + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + receive(t, finished, "both writers to finish") + + if conditionalErr != nil { + require.ErrorIs(t, conditionalErr, errUtils.ErrBlobPreconditionFailed) + } + assert.Equal(t, "C", readBody(t, s, key), "round %d: the conditional write either ran before C or failed", round) + } + }) +} diff --git a/pkg/artifact/blob/local/put.go b/pkg/artifact/blob/local/put.go new file mode 100644 index 00000000000..929dc1cd9e1 --- /dev/null +++ b/pkg/artifact/blob/local/put.go @@ -0,0 +1,301 @@ +package local + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// staged is a fully written temporary object waiting to be renamed into place. +type staged struct { + tempPath string + attrsTemp string + size int64 + etag string + preserve bool +} + +// Put implements [blob.Store]. The body is streamed to a temporary file and renamed into place, +// so a failed or canceled put never leaves a partial object. +func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.local.Put")() + + if body == nil { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: nil body", errUtils.ErrBlobInvalid)) + } + if err := s.validatePut(ctx, key, opts); err != nil { + return nil, err + } + root, err := s.openRoot(true) + if err != nil { + return nil, wrapIO(blob.OpPut, key, err) + } + defer root.Close() + target := filepath.FromSlash(key) + pending, err := s.stage(ctx, root, key, body, opts) + if err != nil { + return nil, err + } + defer pending.discard(root) + + release, err := s.guard(ctx, blob.OpPut, key, opts.Condition.IsConditional()) + if err != nil { + return nil, err + } + defer release() + + if err := s.checkCondition(ctx, root, key, target, opts.Condition); err != nil { + return nil, err + } + if err := s.commit(root, key, target, pending, fromPut(opts).empty()); err != nil { + return nil, err + } + return s.statCommitted(root, key, target, pending) +} + +func (s *Store) validatePut(ctx context.Context, key string, opts blob.PutOptions) error { + if err := s.validate(blob.OpPut, key); err != nil { + return err + } + if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { + return err + } + if opts.Size < blob.UnknownSize || opts.Size == math.MaxInt64 { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid or unverifiable size %d", errUtils.ErrBlobInvalid, opts.Size)) + } + if err := blob.ValidateTags(opts.Tags); err != nil { + return err + } + if opts.Condition.Kind == blob.ConditionIfMatch && opts.Condition.ETag == "" { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + } + if s.plainFiles && !fromPut(opts).empty() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: plain-file stores cannot store content type, metadata, or tags", errUtils.ErrBlobInvalid)) + } + return checkContext(ctx, blob.OpPut, key) +} + +// stage writes the body (and attribute file, when needed) to temporary files next to target. +func (s *Store) stage(ctx context.Context, root *os.Root, key string, body io.Reader, opts blob.PutOptions) (*staged, error) { + target := filepath.FromSlash(key) + file, name, err := rootCreateTemp(root, filepath.Dir(target), s.dirPerm, s.filePerm) + if err != nil { + return nil, s.classifyCreate(root, key, target, err) + } + pending := &staged{tempPath: name} + if err := keepDestinationMode(root, file, target); err != nil { + _ = file.Close() + pending.discard(root) + return nil, wrapIO(blob.OpPut, key, err) + } + hash := sha256.New() + written, copyErr := copyBody(ctx, io.MultiWriter(file, hash), body, opts.Size) + closeErr := file.Close() + if err := firstError(copyErr, closeErr); err != nil { + pending.discard(root) + return nil, classifyCopy(key, err) + } + pending.size = written + pending.etag = contentETag(hash.Sum(nil)) + if err := s.stageAttrs(root, pending, target, fromPut(opts)); err != nil { + pending.discard(root) + return nil, wrapIO(blob.OpPut, key, err) + } + return pending, nil +} + +// keepDestinationMode gives the staged file the permission bits of the regular file it will +// replace, so an overwrite does not widen (or narrow) a mode that was set by hand. It runs before +// any content is written, so restricted data is never exposed under the default mode. A missing +// or non-regular destination keeps the store's default mode. +func keepDestinationMode(root *os.Root, file *os.File, target string) error { + want, ok := destinationMode(root, target) + if !ok { + return nil + } + created, err := file.Stat() + if err != nil { + return err + } + if created.Mode().Perm() == want { + return nil + } + if err := file.Chmod(want); err != nil && !errors.Is(err, errors.ErrUnsupported) { + return err + } + return nil +} + +// destinationMode returns the permission bits of the regular file at target. It reports false +// when there is no such file; a destination that cannot be inspected fails at commit instead. +func destinationMode(root *os.Root, target string) (os.FileMode, bool) { + info, err := root.Stat(target) + if err != nil || !info.Mode().IsRegular() { + return 0, false + } + return info.Mode().Perm(), true +} + +// stageAttrs writes the attribute file to a temporary path when any attribute is set. +func (s *Store) stageAttrs(root *os.Root, pending *staged, target string, a attrs) error { + if a.empty() { + return nil + } + data, err := json.Marshal(a) + if err != nil { + return err + } + file, name, err := rootCreateTemp(root, filepath.Dir(target), s.dirPerm, s.filePerm) + if err != nil { + return err + } + pending.attrsTemp = name + _, writeErr := file.Write(data) + return firstError(writeErr, file.Close()) +} + +// copyBody streams body into dst, enforcing a declared size and honoring cancellation. +func copyBody(ctx context.Context, dst io.Writer, body io.Reader, size int64) (int64, error) { + if size == math.MaxInt64 { + return 0, fmt.Errorf("%w: size %d is unverifiable", errUtils.ErrBlobInvalid, size) + } + reader := io.Reader(&ctxReader{ctx: ctx, r: body}) + if size >= 0 { + reader = io.LimitReader(reader, size+1) + } + written, err := io.Copy(dst, reader) + if err != nil { + return written, err + } + if size >= 0 && written != size { + return written, fmt.Errorf("%w: body is %d bytes, declared %d", errUtils.ErrBlobInvalid, written, size) + } + return written, nil +} + +// checkCondition evaluates the put condition against the current object. Callers hold the lock. +func (s *Store) checkCondition(ctx context.Context, root *os.Root, key, target string, cond blob.Condition) error { + if cond.Kind == blob.ConditionNone { + return nil + } + if cond.Kind != blob.ConditionIfAbsent && cond.Kind != blob.ConditionIfMatch { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: unknown condition %d", errUtils.ErrBlobInvalid, cond.Kind)) + } + state, err := inspect(root, target) + if err != nil { + return wrapIO(blob.OpPut, key, err) + } + switch cond.Kind { + case blob.ConditionIfAbsent: + if state.exists { + return preconditionFailed(blob.OpPut, key, "object already exists") + } + case blob.ConditionIfMatch: + return matchETag(ctx, root, objectRef{op: blob.OpPut, key: key}, state, cond.ETag) + } + return nil +} + +// fileState is the current state of the file backing a key. +type fileState struct { + path string + info os.FileInfo + exists bool +} + +// inspect reports whether a regular file exists at path. A missing path is not an error. +func inspect(root *os.Root, path string) (fileState, error) { + info, err := root.Stat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fileState{}, err + } + return fileState{path: path, info: info, exists: err == nil && info.Mode().IsRegular()}, nil +} + +// objectRef names the object an operation addresses, for error reporting. +type objectRef struct { + op string + key string +} + +// matchETag fails with NotFound when the object is missing and with a precondition failure when +// its ETag differs from want. +func matchETag(ctx context.Context, root *os.Root, ref objectRef, state fileState, want string) error { + op, key := ref.op, ref.key + if !state.exists { + return blob.NewError(blob.KindNotFound, op, key, nil) + } + current, err := etagOf(ctx, root, state.path) + if err != nil { + return wrapIO(op, key, err) + } + if current != want { + return preconditionFailed(op, key, "ETag does not match") + } + return nil +} + +// commit moves the staged files into place: attributes first, so an object is only ever visible +// together with its attributes or with stale ones until the data rename lands. +func (s *Store) commit(root *os.Root, key, target string, pending *staged, noAttrs bool) error { + info, err := root.Stat(target) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return wrapIO(blob.OpPut, key, err) + } + if err == nil && info.IsDir() { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: key is also a prefix of other objects", errUtils.ErrBlobInvalid)) + } + if !s.plainFiles { + return s.commitManaged(root, key, pending, noAttrs, rootRenameReplace) + } + return commitPlain(root, key, target, pending) +} + +// commitPlain installs a staged object in a plain-files store. It never touches attribute paths: +// a name such as "a.tar.atmos-blob.json" is an independent legacy object there. +func commitPlain(root *os.Root, key, target string, pending *staged) error { + if err := rootRenameReplace(root, pending.tempPath, target); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.tempPath = "" + return nil +} + +// statCommitted describes the object that was just written. +func (s *Store) statCommitted(root *os.Root, key, target string, pending *staged) (*blob.ObjectInfo, error) { + info, err := root.Stat(target) + if err != nil { + return nil, wrapIO(blob.OpPut, key, err) + } + return &blob.ObjectInfo{Key: key, Size: pending.size, ETag: pending.etag, LastModified: info.ModTime()}, nil +} + +// discard removes any temporary files that were not committed, then prunes the staging +// directories that this put created and left empty. Non-empty directories are kept. +func (p *staged) discard(root *os.Root) { + if p.preserve { + return + } + var dirs []string + if p.tempPath != "" { + _ = root.Remove(p.tempPath) + dirs = append(dirs, filepath.Dir(p.tempPath)) + } + if p.attrsTemp != "" { + _ = root.Remove(p.attrsTemp) + dirs = append(dirs, filepath.Dir(p.attrsTemp)) + } + for _, dir := range dirs { + rootRemoveEmptyDirs(root, dir) + } +} diff --git a/pkg/artifact/blob/local/put_mode_test.go b/pkg/artifact/blob/local/put_mode_test.go new file mode 100644 index 00000000000..c16bf85c339 --- /dev/null +++ b/pkg/artifact/blob/local/put_mode_test.go @@ -0,0 +1,71 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "testing/iotest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// Replacing an object keeps the mode of the file on disk, in both layouts, and never exposes the +// new content under a wider mode. +func TestPutReplaceKeepsDestinationMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + + layouts := []struct { + name string + opts []Option + }{ + {name: "managed"}, + {name: "plain files", opts: []Option{WithPlainFileCompatibility()}}, + } + modes := []os.FileMode{0o600, 0o644, 0o664} + for _, layout := range layouts { + for _, mode := range modes { + t.Run(layout.name+"/"+mode.String(), func(t *testing.T) { + s := newStore(t, layout.opts...) + require.NoError(t, putBody(t, s, "dir/obj", "old", blob.Condition{})) + path := filepath.Join(s.root, "dir", "obj") + require.NoError(t, os.Chmod(path, mode)) + + require.NoError(t, putBody(t, s, "dir/obj", "new contents", blob.Condition{})) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, mode, info.Mode().Perm()) + assert.Equal(t, "new contents", readBody(t, s, "dir/obj")) + }) + } + } +} + +// A destination that does not exist yet gets the store's default mode, and a failed put leaves the +// existing destination untouched. +func TestPutNewObjectUsesStoreMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + s := newStore(t, WithPlainFileCompatibility()) + require.NoError(t, putBody(t, s, "fresh", "data", blob.Condition{})) + + info, err := os.Stat(filepath.Join(s.root, "fresh")) + require.NoError(t, err) + assert.Zero(t, info.Mode().Perm()&^DefaultFilePerm) + + require.NoError(t, os.Chmod(filepath.Join(s.root, "fresh"), 0o600)) + _, err = s.Put(context.Background(), "fresh", iotest.ErrReader(os.ErrDeadlineExceeded), blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + info, err = os.Stat(filepath.Join(s.root, "fresh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + assert.Equal(t, "data", readBody(t, s, "fresh")) +} diff --git a/pkg/artifact/blob/local/root.go b/pkg/artifact/blob/local/root.go new file mode 100644 index 00000000000..8babc9f5c79 --- /dev/null +++ b/pkg/artifact/blob/local/root.go @@ -0,0 +1,94 @@ +package local + +import ( + "errors" + "io" + "os" + "path/filepath" + "runtime" + "time" +) + +// openRoot confines object operations to the configured directory, including through symlinks. +func (s *Store) openRoot(create bool) (*os.Root, error) { + if create { + if err := os.MkdirAll(s.root, s.dirPerm); err != nil { + return nil, err + } + } + return os.OpenRoot(s.root) +} + +func rootOpenFile(root *os.Root, name string) (*os.File, error) { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = openAttempts + } + return openFileWith(name, attempts, func(name string) (*os.File, error) { + return root.OpenFile(name, os.O_RDONLY|nonblockFlag, 0) + }) +} + +func rootReadFile(root *os.Root, name string) ([]byte, error) { + file, err := rootOpenFile(root, name) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(file) +} + +func rootCreateTemp(root *os.Root, dir string, dirPerm, filePerm os.FileMode) (*os.File, string, error) { + var lastErr error + for range renameAttempts { + if err := root.MkdirAll(dir, dirPerm); err != nil { + return nil, "", err + } + name, err := tempName() + if err != nil { + return nil, "", err + } + name = filepath.Join(dir, name) + file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, filePerm) + if err == nil { + return file, name, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, "", err + } + lastErr = err + } + return nil, "", lastErr +} + +func rootRenameReplace(root *os.Root, src, dst string) error { + attempts := 1 + if runtime.GOOS == "windows" { + attempts = renameAttempts + } + var err error + for i := range attempts { + if err = root.Rename(src, dst); err == nil { + return nil + } + if i < attempts-1 { + time.Sleep(renameBackoff) + } + } + return err +} + +func rootRemoveEmptyDirs(root *os.Root, dir string) { + for dir != "." && dir != string(filepath.Separator) { + file, err := root.Open(dir) + if err != nil { + return + } + _, readErr := file.ReadDir(1) + _ = file.Close() + if !errors.Is(readErr, io.EOF) || root.Remove(dir) != nil { + return + } + dir = filepath.Dir(dir) + } +} diff --git a/pkg/artifact/blob/local/snapshot.go b/pkg/artifact/blob/local/snapshot.go new file mode 100644 index 00000000000..f04203ae65f --- /dev/null +++ b/pkg/artifact/blob/local/snapshot.go @@ -0,0 +1,77 @@ +package local + +import ( + "encoding/base64" + "strings" + "time" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +const ( + snapshotPrefix = "\x00atmos-local:" + snapshotLifetime = 5 * time.Minute + maxSnapshots = 8 +) + +type listSnapshot struct { + entries []entry + prefix, delimiter string + created time.Time +} + +// snapshotCursor stores the last ordering token as well as the cache key, so a cursor remains +// usable if the bounded cache is evicted or a caller uses a new Store. +func snapshotCursor(id, last string) string { + return snapshotPrefix + id + ":" + base64.RawURLEncoding.EncodeToString([]byte(last)) +} + +func parseSnapshotCursor(cursor string) (id, last string, ok bool) { + value, found := strings.CutPrefix(cursor, snapshotPrefix) + if !found { + return "", cursor, false + } + id, encoded, found := strings.Cut(value, ":") + if !found || id == "" { + return "", cursor, false + } + decoded, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return "", cursor, false + } + return id, string(decoded), true +} + +func (s *Store) lookupSnapshot(id string, opts blob.ListOptions) []entry { + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + snapshot, ok := s.snapshots[id] + if !ok || time.Since(snapshot.created) > snapshotLifetime || snapshot.prefix != opts.Prefix || snapshot.delimiter != opts.Delimiter { + return nil + } + return snapshot.entries +} + +func (s *Store) rememberSnapshot(entries []entry, opts blob.ListOptions) string { + id, err := tempName() + if err != nil { + return "" + } + s.snapshotMu.Lock() + defer s.snapshotMu.Unlock() + if s.snapshots == nil { + s.snapshots = make(map[string]listSnapshot) + } + if len(s.snapshots) >= maxSnapshots { + var oldestID string + var oldest time.Time + for key, snapshot := range s.snapshots { + if oldestID == "" || snapshot.created.Before(oldest) { + oldestID, oldest = key, snapshot.created + } + } + delete(s.snapshots, oldestID) + } + s.snapshots[id] = listSnapshot{entries: entries, prefix: opts.Prefix, delimiter: opts.Delimiter, created: time.Now()} + return id +} diff --git a/pkg/artifact/blob/local/store.go b/pkg/artifact/blob/local/store.go new file mode 100644 index 00000000000..42765395e01 --- /dev/null +++ b/pkg/artifact/blob/local/store.go @@ -0,0 +1,132 @@ +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + storeName = "local" + + // Name prefix of in-flight temporary files, which listings never show. + tempPrefix = ".atmos-blob-tmp-" + + // Name suffix of the optional attribute file stored beside an object. + attrsSuffix = ".atmos-blob.json" + + // Name of the per-root lock file. Only conditional operations create it. + lockName = ".atmos-blob.lock" +) + +// Store is a filesystem [blob.Store]. +type Store struct { + root string + dirPerm os.FileMode + filePerm os.FileMode + lockAll bool + plainFiles bool + allowBackslash bool + snapshotMu sync.Mutex + snapshots map[string]listSnapshot +} + +// New returns a Store rooted at root. The directory is created on first write. +func New(root string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.local.New")() + + if root == "" { + return nil, blob.NewError(blob.KindInvalid, "new", "", fmt.Errorf("%w: root directory is required", errUtils.ErrBlobInvalid)) + } + store := &Store{root: filepath.Clean(root), dirPerm: DefaultDirPerm, filePerm: DefaultFilePerm} + for _, opt := range opts { + opt(store) + } + return store, nil +} + +// Name implements [blob.Store]. +func (s *Store) Name() string { + defer perf.Track(nil, "local.Store.Name")() + + return storeName +} + +// Root returns the root directory. +func (s *Store) Root() string { + defer perf.Track(nil, "local.Store.Root")() + + return s.root +} + +// Capabilities implements [blob.Store]. +func (s *Store) Capabilities() blob.Capabilities { + defer perf.Track(nil, "local.Store.Capabilities")() + + // A plain-file store leaves unconditional mutations unlocked for legacy writers, so it can only + // make a condition atomic when WithLockAllWrites makes every mutation honor the same lock. + conditional := s.enforcesConditions() + return blob.Capabilities{ + ConditionalPut: conditional, + ConditionalDelete: conditional, + Multipart: true, + Tags: !s.plainFiles, + Metadata: !s.plainFiles, + Range: true, + Listing: true, + ListDelimiter: true, + } +} + +// enforcesConditions reports whether every mutation takes the root lock, which is what makes an +// if-absent or if-match check and its commit atomic against competing writers of this store. +func (s *Store) enforcesConditions() bool { + return !s.plainFiles || s.lockAll +} + +// validate rejects keys that are unsafe on a filesystem or collide with the store's own files. +func (s *Store) validate(op, key string) error { + validationKey := key + if s.literalBackslashes() { + validationKey = strings.ReplaceAll(key, "\\", "_") + } + if err := blob.ValidateKey(validationKey); err != nil { + return err + } + if strings.EqualFold(key, lockName) || s.hasReservedSegment(key) { + return blob.NewError(blob.KindInvalid, op, key, fmt.Errorf("%w: key uses a reserved name", errUtils.ErrBlobInvalid)) + } + return nil +} + +// hasReservedSegment reports whether any slash-separated segment of key looks like one of the +// store's own temporary or attribute files. A reserved directory name would otherwise hide the +// objects under it from listings, which skip those names at every level. The comparison folds +// case: on case-insensitive volumes a differently cased name addresses the same file. Plain-file +// stores keep no attribute files, so for them an attribute-looking name is an ordinary object. +func (s *Store) hasReservedSegment(key string) bool { + for _, segment := range strings.Split(key, "/") { + folded := strings.ToLower(segment) + if strings.HasPrefix(folded, tempPrefix) || (!s.plainFiles && strings.HasSuffix(folded, attrsSuffix)) { + return true + } + } + return false +} + +// checkContext fails fast when ctx is already done. +func checkContext(ctx context.Context, op, key string) error { + if err := ctx.Err(); err != nil { + return blob.NewError(blob.KindOther, op, key, err) + } + return nil +} + +var _ blob.Store = (*Store)(nil) diff --git a/pkg/artifact/blob/local/store_test.go b/pkg/artifact/blob/local/store_test.go new file mode 100644 index 00000000000..545527c1b2b --- /dev/null +++ b/pkg/artifact/blob/local/store_test.go @@ -0,0 +1,456 @@ +package local + +import ( + "bytes" + "context" + "crypto/sha256" + "io" + "math" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" +) + +func newStore(t *testing.T, opts ...Option) *Store { + t.Helper() + store, err := New(filepath.Join(t.TempDir(), "blobs"), opts...) + require.NoError(t, err) + return store +} + +func TestConformance(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { return newStore(t) }) +} + +func TestConformanceLockAllWrites(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { return newStore(t, WithLockAllWrites()) }) +} + +func TestNewRequiresRoot(t *testing.T) { + _, err := New("") + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestWritesPlainFilesWithoutExtraArtifacts(t *testing.T) { + store := newStore(t, WithPlainFileCompatibility()) + _, err := store.Put(context.Background(), "stack/component/sha.tar", bytes.NewReader([]byte("tar")), blob.PutOptions{Size: 3}) + require.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(store.Root(), "stack", "component", "sha.tar")) + require.NoError(t, err) + assert.Equal(t, []byte("tar"), data) + + entries, err := os.ReadDir(filepath.Join(store.Root(), "stack", "component")) + require.NoError(t, err) + require.Len(t, entries, 1, "no temp, attribute, or lock files remain for a plain unconditional put") + assert.NoFileExists(t, filepath.Join(store.Root(), lockName)) +} + +func TestConditionalPutCreatesLockFileOnly(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader([]byte("a")), blob.PutOptions{Size: 1, Condition: blob.IfAbsent()}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(store.Root(), lockName)) + + objects, _, err := blob.ListAll(context.Background(), store, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, objects, 1, "bookkeeping files are not listed") + assert.Equal(t, "k", objects[0].Key) +} + +func TestAttributeFileHiddenAndRemoved(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "d/k", bytes.NewReader([]byte("a")), blob.PutOptions{Size: 1, Tags: map[string]string{"a": "b"}}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(store.Root(), "d", "k"+attrsSuffix)) + + objects, _, err := blob.ListAll(ctx, store, blob.ListOptions{}) + require.NoError(t, err) + assert.Len(t, objects, 1) + + require.NoError(t, store.Delete(ctx, "d/k", blob.DeleteOptions{})) + assert.NoDirExists(t, filepath.Join(store.Root(), "d"), "empty parent directories are cleaned up") +} + +func TestReservedAndUnsafeKeysRejected(t *testing.T) { + store := newStore(t) + for _, key := range []string{"../escape", "a/../../b", "/abs", "a//b", "a/", ".", "d/" + tempPrefix + "x", "d/x" + attrsSuffix, tempPrefix + "dir/child", "a/" + tempPrefix + "dir/child", "x" + attrsSuffix + "/child", lockName, "a\\b"} { + _, err := store.Put(context.Background(), key, bytes.NewReader(nil), blob.PutOptions{Size: 0}) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } + assert.NoDirExists(t, filepath.Join(filepath.Dir(store.Root()), "escape")) +} + +func TestObjectAndPrefixCannotShareAPath(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "a", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + + _, err = store.Put(ctx, "a/b", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a parent path that is an object") + + _, err = store.Put(ctx, "c/d", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + _, err = store.Put(ctx, "c", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a key that is a prefix of other objects") + + _, err = store.Stat(ctx, "c", blob.StatOptions{}) + assert.True(t, blob.IsNotFound(err), "a prefix is not an object") +} + +func TestLargeObjectContentETag(t *testing.T) { + store := newStore(t) + big := bytes.Repeat([]byte("x"), (4<<20)+1) + info, err := store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big))}) + require.NoError(t, err) + stamp := time.Unix(1700000000, 0) + require.NoError(t, os.Chtimes(filepath.Join(store.Root(), "big"), stamp, stamp)) + stat, err := store.Stat(context.Background(), "big", blob.StatOptions{}) + require.NoError(t, err) + assert.Equal(t, info.ETag, stat.ETag) + big[len(big)-1] = 'y' + _, err = store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big))}) + require.NoError(t, err) + require.NoError(t, os.Chtimes(filepath.Join(store.Root(), "big"), stamp, stamp)) + updated, err := store.Stat(context.Background(), "big", blob.StatOptions{}) + require.NoError(t, err) + assert.NotEqual(t, stat.ETag, updated.ETag) + _, err = store.Put(context.Background(), "big", bytes.NewReader(big), blob.PutOptions{Size: int64(len(big)), Condition: blob.IfMatch(stat.ETag)}) + assert.ErrorIs(t, err, errUtils.ErrBlobPreconditionFailed) +} + +func TestListRejectsEscapingPrefix(t *testing.T) { + store := newStore(t) + _, err := store.List(context.Background(), blob.ListOptions{Prefix: "../"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestGetRejectsNegativeOffsetAndPastEnd(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader([]byte("abc")), blob.PutOptions{Size: 3}) + require.NoError(t, err) + _, err = store.Get(context.Background(), "k", blob.GetOptions{Offset: -1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = store.Get(context.Background(), "k", blob.GetOptions{Offset: 9, Length: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestCanceledContextFailsFast(t *testing.T) { + store := newStore(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := store.Put(ctx, "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, context.Canceled) + _, err = store.Stat(ctx, "k", blob.StatOptions{}) + require.ErrorIs(t, err, context.Canceled) + assert.NoDirExists(t, store.Root(), "nothing is created for a canceled operation") +} + +func TestInvalidSizesAndNULPrefix(t *testing.T) { + store := newStore(t) + for _, size := range []int64{-2, math.MaxInt64} { + _, err := store.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Size: size}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + } + _, err := store.List(context.Background(), blob.ListOptions{Prefix: "x\x00y"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestDescribeHashesOpenedFile(t *testing.T) { + store := newStore(t) + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + path := filepath.Join(store.Root(), "k") + oldPath := filepath.Join(store.Root(), "old") + require.NoError(t, os.WriteFile(oldPath, []byte("old"), 0o644)) + file, err := openFile(oldPath) + require.NoError(t, err) + defer file.Close() + require.NoError(t, os.WriteFile(path, []byte("new"), 0o644)) + root, err := store.openRoot(false) + require.NoError(t, err) + defer root.Close() + info, err := store.describe(context.Background(), root, describeTarget{op: blob.OpGet, key: "k", file: file}, describeOptions{}) + require.NoError(t, err) + sum := sha256.Sum256([]byte("old")) + assert.Equal(t, contentETag(sum[:]), info.ETag) + body, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "old", string(body)) +} + +func TestRootLockWaitRespectsCancellation(t *testing.T) { + store := newStore(t) + release, err := store.lockRoot(context.Background(), blob.OpPut, "k") + require.NoError(t, err) + defer release() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = store.lockRoot(ctx, blob.OpPut, "k") + require.ErrorIs(t, err, context.Canceled) +} + +func TestPutRejectsUnknownCondition(t *testing.T) { + store := newStore(t) + _, err := store.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Condition: blob.Condition{Kind: blob.ConditionKind(999)}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestSymlinksCannotEscapeRoot(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("creating symlinks requires extra privileges on Windows") + } + store := newStore(t) + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + outside := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(outside, "object"), []byte("keep"), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(store.Root(), "team"))) + ctx := context.Background() + _, err := store.Put(ctx, "team/object", bytes.NewReader([]byte("overwrite")), blob.PutOptions{Size: 9}) + require.Error(t, err) + _, err = store.Get(ctx, "team/object", blob.GetOptions{}) + require.Error(t, err) + _, err = store.Stat(ctx, "team/object", blob.StatOptions{}) + require.Error(t, err) + require.Error(t, store.Delete(ctx, "team/object", blob.DeleteOptions{})) + page, listErr := store.List(ctx, blob.ListOptions{Prefix: "team/"}) + if listErr == nil { + assert.Empty(t, page.Objects) + } + data, err := os.ReadFile(filepath.Join(outside, "object")) + require.NoError(t, err) + assert.Equal(t, "keep", string(data)) +} + +func TestListingPagesUseSnapshotAndFreshListingWalksAgain(t *testing.T) { + store := newStore(t) + ctx := context.Background() + for _, key := range []string{"a", "b"} { + _, err := store.Put(ctx, key, bytes.NewReader([]byte(key)), blob.PutOptions{Size: 1}) + require.NoError(t, err) + } + first, err := store.List(ctx, blob.ListOptions{Limit: 1}) + require.NoError(t, err) + require.Len(t, first.Objects, 1) + assert.Equal(t, "a", first.Objects[0].Key) + require.NotEmpty(t, first.NextCursor) + require.NoError(t, store.Delete(ctx, "b", blob.DeleteOptions{})) + _, err = store.Put(ctx, "c", bytes.NewReader([]byte("c")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + second, err := store.List(ctx, blob.ListOptions{Limit: 1, Cursor: first.NextCursor}) + require.NoError(t, err) + require.Len(t, second.Objects, 1) + assert.Equal(t, "b", second.Objects[0].Key, "continuation uses the original walk") + fresh, err := store.List(ctx, blob.ListOptions{Limit: 2}) + require.NoError(t, err) + require.Len(t, fresh.Objects, 2) + assert.Equal(t, "c", fresh.Objects[1].Key, "a fresh listing sees filesystem changes") + other, err := New(store.Root()) + require.NoError(t, err) + resumed, err := other.List(ctx, blob.ListOptions{Limit: 1, Cursor: first.NextCursor}) + require.NoError(t, err) + require.Len(t, resumed.Objects, 1) + assert.Equal(t, "c", resumed.Objects[0].Key, "an evicted snapshot resumes from its ordering token") +} + +func TestPutDirectoryConflictPreservesExistingSidecar(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "prefix/child", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + sidecar := filepath.Join(store.Root(), "prefix"+attrsSuffix) + original := []byte(`{"content_type":"original"}`) + require.NoError(t, os.WriteFile(sidecar, original, 0o644)) + _, err = store.Put(ctx, "prefix", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, ContentType: "new"}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + got, err := os.ReadFile(sidecar) + require.NoError(t, err) + assert.Equal(t, original, got) +} + +func TestFailedPutPrunesEmptyStagingDirectories(t *testing.T) { + for name, opts := range map[string][]Option{"managed": nil, "plain": {WithPlainFileCompatibility()}} { + t.Run(name, func(t *testing.T) { + store := newStore(t, opts...) + _, err := store.Put(context.Background(), "keep/sibling", bytes.NewReader([]byte("ok")), blob.PutOptions{Size: 2}) + require.NoError(t, err) + + _, err = store.Put(context.Background(), "keep/new/deep/object", bytes.NewReader([]byte("abc")), blob.PutOptions{Size: 1}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "a body longer than the declared size fails") + + assert.NoDirExists(t, filepath.Join(store.Root(), "keep", "new"), "staging directories created by the failed put are pruned") + assert.DirExists(t, filepath.Join(store.Root(), "keep"), "a non-empty parent is left intact") + assert.FileExists(t, filepath.Join(store.Root(), "keep", "sibling")) + }) + } +} + +func TestListPrefixHonorsLiteralBackslashes(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("backslash is a path separator on Windows") + } + strict := newStore(t) + _, err := strict.List(context.Background(), blob.ListOptions{Prefix: `dir\`}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "backslashes stay rejected unless the store opts in") + + literal := newStore(t, WithLiteralBackslashes()) + _, err = literal.Put(context.Background(), `dir\file`, bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + page, err := literal.List(context.Background(), blob.ListOptions{Prefix: `dir\`}) + require.NoError(t, err) + require.Len(t, page.Objects, 1) + assert.Equal(t, `dir\file`, page.Objects[0].Key) + + for _, prefix := range []string{"/abs", "a\x00b", "../x"} { + _, err = literal.List(context.Background(), blob.ListOptions{Prefix: prefix}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "%q stays rejected", prefix) + } +} + +func TestReservedNamesFoldCase(t *testing.T) { + store := newStore(t) + for _, key := range []string{ + ".ATMOS-BLOB-TMP-TRANSACTION", + "d/.Atmos-Blob-Tmp-x", + "d/x.ATMOS-BLOB.JSON", + ".ATMOS-BLOB.LOCK", + "Mixed/.atmos-blob-tmp-dir/child", + } { + _, err := store.Put(context.Background(), key, bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.Error(t, err, key) + assert.ErrorIs(t, err, errUtils.ErrBlobInvalid, key) + } + assert.NoDirExists(t, filepath.Join(store.Root(), "d"), "a rejected key stages nothing") +} + +func TestDeletePrefixOnlyDirectoryIsMissingObject(t *testing.T) { + for _, tc := range []struct { + name string + opts []Option + }{ + {name: "managed"}, + {name: "plain", opts: []Option{WithPlainFileCompatibility(), WithLockAllWrites()}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newStore(t, tc.opts...) + ctx := context.Background() + _, err := store.Put(ctx, "prefix/child", bytes.NewReader([]byte("child")), blob.PutOptions{Size: 5}) + require.NoError(t, err) + + require.NoError(t, store.Delete(ctx, "prefix", blob.DeleteOptions{}), "an unconditional delete of a missing object succeeds") + + info, err := store.Stat(ctx, "prefix/child", blob.StatOptions{}) + require.NoError(t, err, "the children stay in place") + assert.Equal(t, int64(5), info.Size) + assert.DirExists(t, filepath.Join(store.Root(), "prefix")) + + err = store.Delete(ctx, "prefix", blob.DeleteOptions{IfMatch: `"any"`}) + require.ErrorIs(t, err, errUtils.ErrBlobNotFound, "a conditional delete of a prefix reports the object as missing") + assert.FileExists(t, filepath.Join(store.Root(), "prefix", "child")) + }) + } +} + +func TestETagHashingObservesCancellation(t *testing.T) { + path := filepath.Join(t.TempDir(), "big") + content := bytes.Repeat([]byte("x"), 1<<20) + require.NoError(t, os.WriteFile(path, content, 0o644)) + file, err := openFile(path) + require.NoError(t, err) + defer file.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = etagOpen(ctx, file) + require.ErrorIs(t, err, context.Canceled, "hashing stops instead of scanning the whole file") + + etag, err := etagOpen(context.Background(), file) + require.NoError(t, err) + sum := sha256.Sum256(content) + assert.Equal(t, contentETag(sum[:]), etag, "an active context hashes the full content") +} + +func TestPlainFilesTreatAttributeLookingNamesAsObjects(t *testing.T) { + store := newStore(t, WithPlainFileCompatibility()) + ctx := context.Background() + const object, lookalike = "a.tar", "a.tar" + attrsSuffix + put := func(key, body string) { + t.Helper() + _, err := store.Put(ctx, key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body))}) + require.NoError(t, err, key) + } + read := func(key string) string { + t.Helper() + obj, err := store.Get(ctx, key, blob.GetOptions{}) + require.NoError(t, err, key) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + return string(data) + } + + put(object, "tar-v1") + put(lookalike, "not json at all") + assert.Equal(t, "tar-v1", read(object), "an unrelated object beside the data is not parsed as attributes") + + put(object, "tar-v2") + assert.Equal(t, "tar-v2", read(object)) + assert.Equal(t, "not json at all", read(lookalike), "updating a.tar leaves the independent artifact untouched") + + page, err := store.List(ctx, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, page.Objects, 2, "both names are listed as objects") + assert.Equal(t, object, page.Objects[0].Key) + assert.Equal(t, lookalike, page.Objects[1].Key) + + require.NoError(t, store.Delete(ctx, object, blob.DeleteOptions{})) + assert.Equal(t, "not json at all", read(lookalike), "deleting a.tar leaves the independent artifact untouched") + + _, err = store.Put(ctx, "tagged", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, Tags: map[string]string{"a": "b"}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "attributes cannot be stored without a sidecar") + assert.False(t, store.Capabilities().Tags) + assert.False(t, store.Capabilities().Metadata) +} + +func TestSkipETagAvoidsHashingTheObject(t *testing.T) { + store := newStore(t) + ctx := context.Background() + _, err := store.Put(ctx, "k", bytes.NewReader([]byte("body")), blob.PutOptions{Size: 4}) + require.NoError(t, err) + + tests := []struct { + name string + skip bool + wantETag bool + }{ + {name: "default computes the content ETag", wantETag: true}, + {name: "skip leaves the ETag empty", skip: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stat, err := store.Stat(ctx, "k", blob.StatOptions{SkipETag: tc.skip}) + require.NoError(t, err) + assert.Equal(t, int64(4), stat.Size) + assert.Equal(t, tc.wantETag, stat.ETag != "") + + obj, err := store.Get(ctx, "k", blob.GetOptions{SkipETag: tc.skip}) + require.NoError(t, err) + defer obj.Body.Close() + data, err := io.ReadAll(obj.Body) + require.NoError(t, err) + assert.Equal(t, "body", string(data), "the body is complete either way") + assert.Equal(t, tc.wantETag, obj.ETag != "") + }) + } +} diff --git a/pkg/artifact/blob/local/transaction.go b/pkg/artifact/blob/local/transaction.go new file mode 100644 index 00000000000..0db9ac2dc5c --- /dev/null +++ b/pkg/artifact/blob/local/transaction.go @@ -0,0 +1,159 @@ +package local + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// A root has at most one transaction because managed operations hold its lock. +// The marker is hidden by the existing temporary-file listing rule. +const transactionName = tempPrefix + "transaction" + +type transaction struct { + Target string `json:"target"` + DataTemp string `json:"data_temp"` + AttrsTemp string `json:"attrs_temp,omitempty"` + HadAttrs bool `json:"had_attrs"` + OldAttrs []byte `json:"old_attrs,omitempty"` +} + +// commitManaged records the old attributes before publishing either replacement. The presence of +// DataTemp distinguishes a failed/incomplete data rename from a completed one during recovery. +func (s *Store) commitManaged(root *os.Root, key string, pending *staged, noAttrs bool, rename func(*os.Root, string, string) error) error { + target := filepath.FromSlash(key) + if err := s.writeTransaction(root, target, pending); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.preserve = true + err := publishTransaction(root, target, pending, noAttrs, rename) + if err != nil { + if recoveryErr := s.recoverCommit(root); recoveryErr != nil { + return wrapIO(blob.OpPut, key, errors.Join(err, recoveryErr)) + } + pending.preserve = false + return wrapIO(blob.OpPut, key, err) + } + if err := s.recoverCommit(root); err != nil { + return wrapIO(blob.OpPut, key, err) + } + pending.preserve = false + pending.tempPath, pending.attrsTemp = "", "" + return nil +} + +func (s *Store) writeTransaction(root *os.Root, target string, pending *staged) error { + if _, err := root.Stat(pending.tempPath); err != nil { + return err + } + old, err := rootReadFile(root, attrsPath(target)) + hadAttrs := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + txn := transaction{Target: target, DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp, HadAttrs: hadAttrs, OldAttrs: old} + data, err := json.Marshal(txn) + if err != nil { + return err + } + file, name, err := rootCreateTemp(root, ".", s.dirPerm, s.filePerm) + if err != nil { + return err + } + defer func() { _ = root.Remove(name) }() + _, writeErr := file.Write(data) + if err := firstError(writeErr, file.Close()); err != nil { + return err + } + return rootRenameReplace(root, name, transactionName) +} + +func publishTransaction(root *os.Root, target string, pending *staged, noAttrs bool, rename func(*os.Root, string, string) error) error { + var err error + if pending.attrsTemp != "" { + err = rename(root, pending.attrsTemp, attrsPath(target)) + } else if noAttrs { + err = root.Remove(attrsPath(target)) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + } + if err == nil { + err = rename(root, pending.tempPath, target) + } + return err +} + +// recoverCommit is idempotent: it restores the old attributes while the data temp still exists, +// and otherwise finishes cleanup after the data rename. Callers hold the root lock. +func (s *Store) recoverCommit(root *os.Root) error { + data, err := rootReadFile(root, transactionName) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var txn transaction + if err := json.Unmarshal(data, &txn); err != nil { + return fmt.Errorf("decode local transaction: %w", err) + } + if err := s.validateTransaction(&txn); err != nil { + return err + } + _, statErr := root.Stat(txn.DataTemp) + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + if statErr == nil { + if err := s.restoreAttrs(root, &txn); err != nil { + return err + } + } + if err := root.Remove(transactionName); err != nil { + return err + } + _ = root.Remove(txn.DataTemp) + s.removeAttrsTemp(root, &txn) + return nil +} + +func (s *Store) validateTransaction(txn *transaction) error { + if err := s.validate(blob.OpPut, filepath.ToSlash(txn.Target)); err != nil || + filepath.Dir(txn.DataTemp) != filepath.Dir(txn.Target) || !strings.HasPrefix(filepath.Base(txn.DataTemp), tempPrefix) { + return fmt.Errorf("%w: invalid local transaction", errUtils.ErrBlobInvalid) + } + return nil +} + +func (s *Store) removeAttrsTemp(root *os.Root, txn *transaction) { + if txn.AttrsTemp != "" && filepath.Dir(txn.AttrsTemp) == filepath.Dir(txn.Target) && strings.HasPrefix(filepath.Base(txn.AttrsTemp), tempPrefix) { + _ = root.Remove(txn.AttrsTemp) + } +} + +func (s *Store) restoreAttrs(root *os.Root, txn *transaction) error { + if !txn.HadAttrs { + err := root.Remove(attrsPath(txn.Target)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + file, name, err := rootCreateTemp(root, filepath.Dir(txn.Target), s.dirPerm, s.filePerm) + if err != nil { + return err + } + defer func() { _ = root.Remove(name) }() + _, writeErr := file.Write(txn.OldAttrs) + if err := firstError(writeErr, file.Close()); err != nil { + return err + } + return rootRenameReplace(root, name, attrsPath(txn.Target)) +} diff --git a/pkg/artifact/blob/local/transaction_test.go b/pkg/artifact/blob/local/transaction_test.go new file mode 100644 index 00000000000..2afe16ecfac --- /dev/null +++ b/pkg/artifact/blob/local/transaction_test.go @@ -0,0 +1,212 @@ +package local + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestManagedReadWaitsForBothCommitRenames(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + // commitJoined reports whether the commit goroutine ended. The root and staging area are torn + // down only after it did, so a commit that is still running never sees a closed root. + commitJoined := true + t.Cleanup(func() { + if !commitJoined { + return + } + pending.discard(root) + _ = root.Close() + }) + release := holdGuard(t, s, "key", false) + between, proceed := make(chan struct{}), make(chan struct{}) + // The commit stays parked on proceed until the test releases it. Release it on every exit so an + // assertion that aborts the test never strands the commit goroutine holding the root lock. + var proceedOnce sync.Once + letProceed := func() { proceedOnce.Do(func() { close(proceed) }) } + commitDone, commitFinished := make(chan error, 1), make(chan struct{}) + go func() { + defer close(commitFinished) + defer release() + commitDone <- s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { + if err := rootRenameReplace(r, from, to); err != nil { + return err + } + if to == attrsPath("key") { + close(between) + <-proceed + } + return nil + }) + }() + // Registered after the goroutine starts, so it runs before the root teardown above: open the + // gate, then wait for the commit to finish within a bound. + commitJoined = false + t.Cleanup(func() { + letProceed() + select { + case <-commitFinished: + commitJoined = true + case <-time.After(waitTimeout): + t.Errorf("timed out waiting for the commit goroutine to finish; leaving its root open") + } + }) + select { + case <-between: + case err := <-commitDone: + t.Fatalf("the commit ended before its attribute rename: %v", err) + case <-time.After(waitTimeout): + t.Fatal("timed out waiting for the commit to reach its attribute rename") + } + type result struct { + body string + info *blob.ObjectInfo + err error + } + getDone, statDone := make(chan result, 1), make(chan result, 1) + go func() { + obj, err := s.Get(ctx, "key", blob.GetOptions{}) + if err != nil { + getDone <- result{err: err} + return + } + body, err := io.ReadAll(obj.Body) + _ = obj.Body.Close() + getDone <- result{body: string(body), info: &obj.ObjectInfo, err: err} + }() + go func() { + info, err := s.Stat(ctx, "key", blob.StatOptions{}) + statDone <- result{info: info, err: err} + }() + early := false + select { + case got := <-getDone: + t.Errorf("Get returned during an incomplete commit: %+v", got) + early = true + case got := <-statDone: + t.Errorf("Stat returned during an incomplete commit: %+v", got) + early = true + case <-time.After(100 * time.Millisecond): + } + letProceed() + require.NoError(t, receive(t, commitDone, "the commit to finish")) + require.False(t, early) + got, stat := receive(t, getDone, "Get to finish"), receive(t, statDone, "Stat to finish") + require.NoError(t, got.err) + require.NoError(t, stat.err) + assert.Equal(t, "new", got.body) + assert.Equal(t, "new", got.info.Metadata["version"]) + assert.Equal(t, "new", stat.info.Metadata["version"]) +} + +func TestManagedCommitFailureRestoresOldAttributes(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + defer root.Close() + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + defer pending.discard(root) + release := holdGuard(t, s, "key", false) + failure := errors.New("injected data rename failure") + err = s.commitManaged(root, "key", pending, false, func(r *os.Root, from, to string) error { + if to == "key" { + return failure + } + return rootRenameReplace(r, from, to) + }) + release() + require.ErrorIs(t, err, failure) + obj, err := s.Get(ctx, "key", blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + body, err := io.ReadAll(obj.Body) + require.NoError(t, err) + assert.Equal(t, "old", string(body)) + assert.Equal(t, "old", obj.Metadata["version"]) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) +} + +func TestManagedRecoveryAfterInterruptedCommit(t *testing.T) { + for _, dataRenamed := range []bool{false, true} { + t.Run(map[bool]string{false: "before data rename", true: "after data rename"}[dataRenamed], func(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "old"}}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + closeOnCleanup(t, root) + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + oldAttrs, err := rootReadFile(root, attrsPath("key")) + require.NoError(t, err) + marker, err := json.Marshal(transaction{Target: "key", DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp, HadAttrs: true, OldAttrs: oldAttrs}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(s.root, transactionName), marker, 0o644)) + require.NoError(t, rootRenameReplace(root, pending.attrsTemp, attrsPath("key"))) + if dataRenamed { + require.NoError(t, rootRenameReplace(root, pending.tempPath, "key")) + } + require.NoError(t, root.Close()) + restarted, err := New(s.root) + require.NoError(t, err) + obj, err := restarted.Get(ctx, "key", blob.GetOptions{}) + require.NoError(t, err) + defer obj.Body.Close() + body, err := io.ReadAll(obj.Body) + require.NoError(t, err) + want := "old" + if dataRenamed { + want = "new" + } + assert.Equal(t, want, string(body)) + assert.Equal(t, want, obj.Metadata["version"]) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) + }) + } +} + +func TestManagedRecoveryRemovesNewAttributesWhenOldObjectHadNone(t *testing.T) { + ctx := context.Background() + s := newStore(t) + _, err := s.Put(ctx, "key", bytes.NewBufferString("old"), blob.PutOptions{Size: 3}) + require.NoError(t, err) + root, err := s.openRoot(false) + require.NoError(t, err) + closeOnCleanup(t, root) + pending, err := s.stage(ctx, root, "key", bytes.NewBufferString("new"), blob.PutOptions{Size: 3, Metadata: map[string]string{"version": "new"}}) + require.NoError(t, err) + marker, err := json.Marshal(transaction{Target: "key", DataTemp: pending.tempPath, AttrsTemp: pending.attrsTemp}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(s.root, transactionName), marker, 0o644)) + require.NoError(t, rootRenameReplace(root, pending.attrsTemp, attrsPath("key"))) + require.NoError(t, root.Close()) + info, err := s.Stat(ctx, "key", blob.StatOptions{}) + require.NoError(t, err) + assert.Empty(t, info.Metadata) + assert.NoFileExists(t, filepath.Join(s.root, attrsPath("key"))) + assert.NoFileExists(t, filepath.Join(s.root, transactionName)) +} diff --git a/pkg/artifact/blob/mock_store.go b/pkg/artifact/blob/mock_store.go new file mode 100644 index 00000000000..fb8a0effbaf --- /dev/null +++ b/pkg/artifact/blob/mock_store.go @@ -0,0 +1,144 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/blob (interfaces: Store) +// +// Generated by this command: +// +// mockgen -package blob -destination pkg/artifact/blob/mock_store.go github.com/cloudposse/atmos/pkg/artifact/blob Store +// + +// Package blob is a generated GoMock package. +package blob + +import ( + context "context" + io "io" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Capabilities mocks base method. +func (m *MockStore) Capabilities() Capabilities { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Capabilities") + ret0, _ := ret[0].(Capabilities) + return ret0 +} + +// Capabilities indicates an expected call of Capabilities. +func (mr *MockStoreMockRecorder) Capabilities() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Capabilities", reflect.TypeOf((*MockStore)(nil).Capabilities)) +} + +// Delete mocks base method. +func (m *MockStore) Delete(ctx context.Context, key string, opts DeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, key, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockStoreMockRecorder) Delete(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), ctx, key, opts) +} + +// Get mocks base method. +func (m *MockStore) Get(ctx context.Context, key string, opts GetOptions) (*Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, key, opts) + ret0, _ := ret[0].(*Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockStoreMockRecorder) Get(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStore)(nil).Get), ctx, key, opts) +} + +// List mocks base method. +func (m *MockStore) List(ctx context.Context, opts ListOptions) (*ListPage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, opts) + ret0, _ := ret[0].(*ListPage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockStoreMockRecorder) List(ctx, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockStore)(nil).List), ctx, opts) +} + +// Name mocks base method. +func (m *MockStore) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockStoreMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockStore)(nil).Name)) +} + +// Put mocks base method. +func (m *MockStore) Put(ctx context.Context, key string, body io.Reader, opts PutOptions) (*ObjectInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Put", ctx, key, body, opts) + ret0, _ := ret[0].(*ObjectInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockStoreMockRecorder) Put(ctx, key, body, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockStore)(nil).Put), ctx, key, body, opts) +} + +// Stat mocks base method. +func (m *MockStore) Stat(ctx context.Context, key string, opts StatOptions) (*ObjectInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stat", ctx, key, opts) + ret0, _ := ret[0].(*ObjectInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stat indicates an expected call of Stat. +func (mr *MockStoreMockRecorder) Stat(ctx, key, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockStore)(nil).Stat), ctx, key, opts) +} diff --git a/pkg/artifact/blob/s3/api.go b/pkg/artifact/blob/s3/api.go new file mode 100644 index 00000000000..2b39868ed8b --- /dev/null +++ b/pkg/artifact/blob/s3/api.go @@ -0,0 +1,43 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +//go:generate go run go.uber.org/mock/mockgen -package s3 -destination mock_api_test.go github.com/cloudposse/atmos/pkg/artifact/blob/s3 API + +// API is the subset of the S3 client the store uses. *s3.Client satisfies it; tests may supply a +// recording or faulting implementation. +type API interface { + PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + GetObject(ctx context.Context, in *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + DeleteObject(ctx context.Context, in *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) + GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) + CreateMultipartUpload(ctx context.Context, in *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) + UploadPart(ctx context.Context, in *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) + CompleteMultipartUpload(ctx context.Context, in *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) + AbortMultipartUpload(ctx context.Context, in *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) +} + +var _ API = (*s3.Client)(nil) + +// sdkOptions is the per-request SDK option type, aliased so helpers need not import the SDK. +type sdkOptions = s3.Options + +// noRetry disables SDK retries for one request. +func noRetry(o *s3.Options) { + o.Retryer = aws.NopRetryer{} +} + +// skipOptionalChecksums keeps the SDK from attaching optional per-part checksums to multipart +// requests. An upload created without a checksum algorithm must also be completed without them, +// whatever defaults the injected client was built with. Artifact digests provide integrity. +func skipOptionalChecksums(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired +} diff --git a/pkg/artifact/blob/s3/classify.go b/pkg/artifact/blob/s3/classify.go new file mode 100644 index 00000000000..1ad3c2d5ed2 --- /dev/null +++ b/pkg/artifact/blob/s3/classify.go @@ -0,0 +1,146 @@ +package s3 + +import ( + "errors" + "fmt" + "net" + "net/http" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// request describes the operation that failed so the classifier can judge ambiguity. +type request struct { + op string + key string + mutating bool + conditional bool +} + +// Error codes S3-compatible endpoints return when they cannot honor a conditional header. +var conditionalRejectCodes = map[string]bool{ + "NotImplemented": true, + "XNotImplemented": true, + "InvalidArgument": true, + "InvalidRequest": true, + "UnsupportedHeader": true, + "MalformedHeader": true, + "HeaderNotSupported": true, +} + +// classify turns an SDK error into a [blob.Error]. Nil stays nil and classified errors pass through. +func classify(req request, err error) error { + if err == nil { + return nil + } + var classified *blob.Error + if errors.As(err, &classified) { + return err + } + status, code := responseDetails(err) + if status == 0 { + return classifyTransport(req, err) + } + return blob.NewError(kindForResponse(req, status, code), req.op, req.key, err) +} + +// responseDetails extracts the HTTP status and S3 error code, or zero when no response arrived. +func responseDetails(err error) (int, string) { + var code string + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + code = apiErr.ErrorCode() + } + var respErr *awshttp.ResponseError + if errors.As(err, &respErr) { + return respErr.HTTPStatusCode(), code + } + return 0, code +} + +// kindForResponse maps a service response to an error kind. +func kindForResponse(req request, status int, code string) blob.Kind { + if kind, ok := definiteKind(status, code); ok { + return kind + } + if req.conditional && rejectsCondition(status, code) { + return blob.KindUnsupported + } + if req.mutating && ambiguousStatus(status) { + return blob.KindAmbiguous + } + return blob.KindOther +} + +// definiteKind classifies responses whose meaning does not depend on the request. +func definiteKind(status int, code string) (blob.Kind, bool) { + switch { + case code == "NoSuchBucket" || code == "NoSuchUpload": + // A missing bucket or upload is a configuration or ordering problem, never a missing object. + return blob.KindOther, true + case isMissingObject(status, code): + return blob.KindNotFound, true + case isLostRace(status, code): + return blob.KindPrecondition, true + case status == http.StatusRequestedRangeNotSatisfiable || code == "InvalidRange": + return blob.KindInvalid, true + default: + return blob.KindOther, false + } +} + +func isMissingObject(status int, code string) bool { + return status == http.StatusNotFound || code == "NoSuchKey" || code == "NotFound" +} + +// isLostRace reports a failed condition, including S3's conflict answer for concurrent conditional writes. +func isLostRace(status int, code string) bool { + return status == http.StatusPreconditionFailed || code == "PreconditionFailed" || + (status == http.StatusConflict && code == "ConditionalRequestConflict") +} + +// rejectsCondition reports a response that says the endpoint cannot honor a conditional header. +func rejectsCondition(status int, code string) bool { + return status == http.StatusNotImplemented || + (status == http.StatusBadRequest && conditionalRejectCodes[code]) +} + +// ambiguousStatus reports server errors after which a mutation may or may not have been applied. +// 501 (not implemented) is a definite rejection. +func ambiguousStatus(status int) bool { + return status >= http.StatusInternalServerError && + status != http.StatusNotImplemented +} + +// classifyTransport judges a failure with no response: a mutation whose request may have reached +// the server is ambiguous, everything else is a plain failure. +func classifyTransport(req request, err error) error { + if req.mutating && requestMayHaveBeenSent(err) { + return blob.NewError(blob.KindAmbiguous, req.op, req.key, err) + } + return blob.NewError(blob.KindOther, req.op, req.key, err) +} + +// requestMayHaveBeenSent distinguishes failures after the request left the client from failures +// before it did (credential, signing, endpoint, and connection-establishment errors). +func requestMayHaveBeenSent(err error) bool { + var send *smithyhttp.RequestSendError + if !errors.As(err, &send) { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + return false + } + return true +} + +// unsupportedConditional builds the clear failure for an endpoint that cannot enforce a condition. +func unsupportedConditional(op, key, reason string) error { + return blob.NewError(blob.KindUnsupported, op, key, fmt.Errorf("%w: %s", errUtils.ErrBlobConditionalUnsupported, reason)) +} diff --git a/pkg/artifact/blob/s3/client.go b/pkg/artifact/blob/s3/client.go new file mode 100644 index 00000000000..01a00805aae --- /dev/null +++ b/pkg/artifact/blob/s3/client.go @@ -0,0 +1,176 @@ +package s3 + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/store" +) + +// AuthResolver resolves an Atmos identity name to AWS credentials configuration. It is the AWS +// slice of store.AuthContextResolver, which satisfies it. +type AuthResolver interface { + ResolveAWSAuthContext(ctx context.Context, identityName string) (*store.AWSAuthConfig, error) +} + +// ClientConfig describes how to build an S3 client. Both the generic [Store] and the legacy CI +// adapter build clients through [NewClient] so credential, identity, and endpoint handling live in +// one place. +type ClientConfig struct { + // Region overrides the region of the identity and the default chain. + Region string + + // Identity is the Atmos identity name. Empty uses the default AWS credential chain. + Identity string + + // Resolver turns Identity into credentials configuration. + Resolver AuthResolver + + // EndpointURL overrides the S3 endpoint (for example an emulator). It wins over the endpoint of + // a resolved identity. + EndpointURL string + + // PathStyle selects path-style addressing. Nil means path-style whenever an endpoint override + // applies, which is what emulators such as Floci require. + PathStyle *bool + + // HTTPClient replaces the SDK HTTP client, for example to trust a private CA. The SDK honors + // AWS_CA_BUNDLE when nil. + HTTPClient aws.HTTPClient + + // Legacy keeps the pre-artifact CI behavior: when an identity is configured but no resolver was + // injected, ambient credentials are used, and the SDK's default checksum behavior is kept. Only + // the legacy CI adapter sets it. The generic store never does, so an unresolvable identity + // fails closed. + Legacy bool +} + +// NewClient builds an S3 client from cfg. With an identity it resolves credentials through the +// resolver; without one it uses the default credential chain. See [ClientConfig.Legacy] for the +// one deliberate difference between the generic and legacy paths. +func NewClient(ctx context.Context, cfg *ClientConfig) (*s3.Client, error) { + defer perf.Track(nil, "blob.s3.NewClient")() + + if cfg.Identity == "" { + return newDefaultClient(ctx, cfg, nil) + } + if cfg.Resolver == nil { + if cfg.Legacy { + return newDefaultClient(ctx, cfg, nil) + } + return nil, identityError(cfg.Identity, errUtils.ErrBlobIdentityUnresolved) + } + auth, err := cfg.Resolver.ResolveAWSAuthContext(ctx, cfg.Identity) + if err != nil { + return nil, identityError(cfg.Identity, err) + } + if auth == nil && !cfg.Legacy { + return nil, identityError(cfg.Identity, errUtils.ErrBlobIdentityUnresolved) + } + loadOpts := BuildAuthConfigOpts(cfg.Region, auth) + awsCfg, err := config.LoadDefaultConfig(ctx, withHTTPClient(loadOpts, cfg.HTTPClient)...) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + } + return s3.NewFromConfig(awsCfg, clientOptFns(cfg, auth)...), nil +} + +// identityError reports an unresolvable identity. It wraps both the configuration-load sentinel the +// legacy callers match and the blob identity sentinel the generic path matches. +func identityError(identity string, cause error) error { + return fmt.Errorf("%w: %w: failed to resolve AWS auth context for identity %q: %w", + errUtils.ErrAWSConfigLoadFailed, errUtils.ErrBlobIdentityUnresolved, identity, cause) +} + +// newDefaultClient builds a client from the default credential chain. +func newDefaultClient(ctx context.Context, cfg *ClientConfig, auth *store.AWSAuthConfig) (*s3.Client, error) { + var loadOpts []func(*config.LoadOptions) error + if cfg.Region != "" { + loadOpts = append(loadOpts, config.WithRegion(cfg.Region)) + } + awsCfg, err := config.LoadDefaultConfig(ctx, withHTTPClient(loadOpts, cfg.HTTPClient)...) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + } + return s3.NewFromConfig(awsCfg, clientOptFns(cfg, auth)...), nil +} + +func withHTTPClient(opts []func(*config.LoadOptions) error, client aws.HTTPClient) []func(*config.LoadOptions) error { + if client == nil { + return opts + } + return append(opts, config.WithHTTPClient(client)) +} + +// clientOptFns assembles the S3 client options for cfg and the resolved identity context. +func clientOptFns(cfg *ClientConfig, auth *store.AWSAuthConfig) []func(*s3.Options) { + var optFns []func(*s3.Options) + endpoint := cfg.EndpointURL + if endpoint == "" && auth != nil { + endpoint = auth.EndpointURL + } + if endpoint != "" { + pathStyle := true + if cfg.PathStyle != nil { + pathStyle = *cfg.PathStyle + } + optFns = append(optFns, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = pathStyle + }) + } else if cfg.PathStyle != nil { + pathStyle := *cfg.PathStyle + optFns = append(optFns, func(o *s3.Options) { o.UsePathStyle = pathStyle }) + } + if !cfg.Legacy { + // Integrity is established by artifact digests. Skipping optional request checksums keeps + // multipart completion and emulator endpoints simple. + optFns = append(optFns, func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) + } + return optFns +} + +// BuildClientOptFns returns the S3 client options an identity's auth context requires: an endpoint +// override (for example an emulator) with path-style addressing. Without it an identity scoped to an +// emulator would still route S3 calls to the real AWS endpoint. +func BuildClientOptFns(auth *store.AWSAuthConfig) []func(*s3.Options) { + defer perf.Track(nil, "blob.s3.BuildClientOptFns")() + + return clientOptFns(&ClientConfig{Legacy: true}, auth) +} + +// BuildAuthConfigOpts returns the AWS SDK config options for a resolved auth context. The +// store-level region wins over the identity's region. +func BuildAuthConfigOpts(region string, auth *store.AWSAuthConfig) []func(*config.LoadOptions) error { + defer perf.Track(nil, "blob.s3.BuildAuthConfigOpts")() + + var cfgOpts []func(*config.LoadOptions) error + if auth == nil { + return cfgOpts + } + if auth.CredentialsFile != "" { + cfgOpts = append(cfgOpts, config.WithSharedCredentialsFiles([]string{auth.CredentialsFile})) + } + if auth.ConfigFile != "" { + cfgOpts = append(cfgOpts, config.WithSharedConfigFiles([]string{auth.ConfigFile})) + } + if auth.Profile != "" { + cfgOpts = append(cfgOpts, config.WithSharedConfigProfile(auth.Profile)) + } + if region == "" { + region = auth.Region + } + if region != "" { + cfgOpts = append(cfgOpts, config.WithRegion(region)) + } + return cfgOpts +} diff --git a/pkg/artifact/blob/s3/client_test.go b/pkg/artifact/blob/s3/client_test.go new file mode 100644 index 00000000000..8074d3a8586 --- /dev/null +++ b/pkg/artifact/blob/s3/client_test.go @@ -0,0 +1,206 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + sdk "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/store" +) + +func TestIdentityIsResolvedLazily(t *testing.T) { + resolver := &fakeResolver{err: errors.New("sso expired")} + s, err := New("bucket", WithIdentity("deploy", resolver), WithRegion("us-east-1")) + require.NoError(t, err, "constructing the store must not touch credentials") + assert.Zero(t, resolver.calls.Load()) + + _, err = s.Stat(context.Background(), "k", blob.StatOptions{}) + require.Error(t, err) + assert.Equal(t, int32(1), resolver.calls.Load()) +} + +func TestUnresolvableIdentityFailsClosed(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "ambient-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret") + t.Setenv("AWS_REGION", "us-east-1") + + cases := map[string]*fakeResolver{ + "resolver error": {err: errors.New("sso expired")}, + "nil auth context": {}, + "no resolver injected": nil, + } + for name, resolver := range cases { + t.Run(name, func(t *testing.T) { + var opt Option + if resolver == nil { + opt = WithIdentity("deploy", nil) + } else { + opt = WithIdentity("deploy", resolver) + } + s, err := New("bucket", opt) + require.NoError(t, err) + for range 2 { + _, err = s.Get(context.Background(), "k", blob.GetOptions{}) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved, "never fall back to ambient credentials") + assert.Contains(t, err.Error(), "deploy", "the error names the identity") + } + if resolver != nil { + assert.Equal(t, int32(2), resolver.calls.Load(), "a failed resolution is retried by the next operation, not cached") + } + }) + } +} + +func TestLegacyClientKeepsAmbientFallback(t *testing.T) { + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_ACCESS_KEY_ID", "k") + t.Setenv("AWS_SECRET_ACCESS_KEY", "s") + + client, err := NewClient(context.Background(), &ClientConfig{Identity: "deploy", Legacy: true}) + require.NoError(t, err, "the legacy CI adapter keeps its historical fallback") + assert.NotNil(t, client) + + _, err = NewClient(context.Background(), &ClientConfig{Identity: "deploy"}) + require.ErrorIs(t, err, errUtils.ErrBlobIdentityUnresolved, "the generic path never does") + require.ErrorIs(t, err, errUtils.ErrAWSConfigLoadFailed) +} + +func TestIdentityEndpointAndPathStyleRouteToTheEmulator(t *testing.T) { + requests := make(chan *http.Request, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + endpoint, err := url.Parse(server.URL) + require.NoError(t, err) + _, port, err := net.SplitHostPort(endpoint.Host) + require.NoError(t, err) + endpoint.Host = net.JoinHostPort("localhost", port) + + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + resolver := &fakeResolver{result: &store.AWSAuthConfig{Region: "us-east-1", EndpointURL: endpoint.String()}} + s, err := New("artifacts", WithIdentity("floci", resolver)) + require.NoError(t, err) + + _, err = s.Stat(context.Background(), "templates/stack.yaml", blob.StatOptions{}) + require.NoError(t, err) + var request *http.Request + select { + case request = <-requests: + default: + t.Fatal("the emulator endpoint received no request") + } + assert.Equal(t, endpoint.Host, request.Host, "path-style: the bucket is not in the host name") + assert.Equal(t, "/artifacts/templates/stack.yaml", request.URL.Path) +} + +func TestExplicitEndpointWinsOverIdentityEndpoint(t *testing.T) { + cfg := &ClientConfig{EndpointURL: "http://explicit:9000"} + var opts sdk.Options + for _, fn := range clientOptFns(cfg, &store.AWSAuthConfig{EndpointURL: "http://identity:4566"}) { + fn(&opts) + } + assert.Equal(t, aws.String("http://explicit:9000"), opts.BaseEndpoint) + assert.True(t, opts.UsePathStyle, "an endpoint override implies path-style addressing") + + pathStyle := false + cfg.PathStyle = &pathStyle + opts = sdk.Options{} + for _, fn := range clientOptFns(cfg, nil) { + fn(&opts) + } + assert.False(t, opts.UsePathStyle, "path-style can be turned off for a custom endpoint") +} + +func TestGenericClientSkipsOptionalChecksumsButLegacyKeepsSDKDefaults(t *testing.T) { + var generic, legacy sdk.Options + for _, fn := range clientOptFns(&ClientConfig{}, nil) { + fn(&generic) + } + for _, fn := range clientOptFns(&ClientConfig{Legacy: true}, nil) { + fn(&legacy) + } + assert.Equal(t, aws.RequestChecksumCalculationWhenRequired, generic.RequestChecksumCalculation) + assert.Equal(t, aws.RequestChecksumCalculation(0), legacy.RequestChecksumCalculation) +} + +func TestBuildAuthConfigOpts(t *testing.T) { + cases := []struct { + name string + region string + auth *store.AWSAuthConfig + want config.LoadOptions + }{ + {"nil", "", nil, config.LoadOptions{}}, + { + "files and profile", "", &store.AWSAuthConfig{CredentialsFile: "/c", ConfigFile: "/f", Profile: "p"}, + config.LoadOptions{SharedCredentialsFiles: []string{"/c"}, SharedConfigFiles: []string{"/f"}, SharedConfigProfile: "p"}, + }, + {"store region wins", "us-east-1", &store.AWSAuthConfig{Region: "us-west-2"}, config.LoadOptions{Region: "us-east-1"}}, + {"identity region fills in", "", &store.AWSAuthConfig{Region: "eu-west-1"}, config.LoadOptions{Region: "eu-west-1"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got config.LoadOptions + for _, opt := range BuildAuthConfigOpts(tc.region, tc.auth) { + require.NoError(t, opt(&got)) + } + assert.Equal(t, tc.want.Region, got.Region) + assert.Equal(t, tc.want.SharedConfigProfile, got.SharedConfigProfile) + assert.Equal(t, tc.want.SharedCredentialsFiles, got.SharedCredentialsFiles) + assert.Equal(t, tc.want.SharedConfigFiles, got.SharedConfigFiles) + }) + } +} + +func TestBuildClientOptFnsMatchesLegacyEndpointBehavior(t *testing.T) { + assert.Empty(t, BuildClientOptFns(nil)) + assert.Empty(t, BuildClientOptFns(&store.AWSAuthConfig{})) + var opts sdk.Options + for _, fn := range BuildClientOptFns(&store.AWSAuthConfig{EndpointURL: "http://localhost:4566"}) { + fn(&opts) + } + assert.Equal(t, aws.String("http://localhost:4566"), opts.BaseEndpoint) + assert.True(t, opts.UsePathStyle) +} + +func TestHTTPClientInjectionReachesTheSDK(t *testing.T) { + fake := newFake(t) + var used bool + client := httpClientFunc(func(r *http.Request) (*http.Response, error) { + used = true + return http.DefaultClient.Do(r) + }) + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + api, err := NewClient(context.Background(), &ClientConfig{Region: "us-east-1", EndpointURL: fake.Endpoint(), HTTPClient: client}) + require.NoError(t, err) + s, err := NewFromClient(api, fake.Bucket()) + require.NoError(t, err) + _, err = s.Put(context.Background(), "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.NoError(t, err) + assert.True(t, used) +} + +type httpClientFunc func(*http.Request) (*http.Response, error) + +func (f httpClientFunc) Do(r *http.Request) (*http.Response, error) { return f(r) } diff --git a/pkg/artifact/blob/s3/conditional.go b/pkg/artifact/blob/s3/conditional.go new file mode 100644 index 00000000000..258613df711 --- /dev/null +++ b/pkg/artifact/blob/s3/conditional.go @@ -0,0 +1,159 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// probePrefix holds the short-lived objects the conditional-write probe creates. +const probePrefix = ".atmos-blob-probe/" + +const probeOperation = "probe" + +const probeCleanupTimeout = 5 * time.Second + +// ensureConditional makes sure a conditional operation is safe to send. In probe mode it first +// proves, once per store, that the endpoint enforces If-None-Match and If-Match; an endpoint that silently +// ignores the header would otherwise turn create-if-absent into an unconditional overwrite. +func (s *Store) ensureConditional(ctx context.Context, key string, cond blob.Condition) error { + if !cond.IsConditional() || s.cfg.conditional != ConditionalProbe { + return nil + } + if err := s.ProbeConditionalWrites(ctx); err != nil { + return unsupportedConditionalFrom(key, err) + } + return nil +} + +// unsupportedConditionalFrom reports a failed probe as the operation's error. +func unsupportedConditionalFrom(key string, probeErr error) error { + if blob.IsConditionalUnsupported(probeErr) { + return blob.NewError(blob.KindUnsupported, blob.OpPut, key, probeErr) + } + return probeErr +} + +// ProbeConditionalWrites verifies that the endpoint enforces If-None-Match by creating a throwaway +// object twice: the second create must be rejected. It then checks that stale If-Match validators +// are rejected on PutObject and DeleteObject. The guarantee covers those single-object requests +// only; conditional enforcement on CompleteMultipartUpload is not probed. A definite answer is +// cached for the life of the store; a transient failure is returned without being cached. It +// returns an error matching ErrBlobConditionalUnsupported when the endpoint ignores or rejects the +// condition. +func (s *Store) ProbeConditionalWrites(ctx context.Context) error { + defer perf.Track(nil, "blob.s3.ProbeConditionalWrites")() + + if verdict, cached := s.cachedProbe(); cached { + return verdict + } + select { + case s.probeSlot <- struct{}{}: + defer func() { <-s.probeSlot }() + case <-ctx.Done(): + return blob.NewError(blob.KindOther, probeOperation, "", ctx.Err()) + } + // Another operation may have finished the probe while this one waited for the slot. + if verdict, cached := s.cachedProbe(); cached { + return verdict + } + api, err := s.client(ctx) + if err != nil { + return err + } + verdict, cleanupErr := s.runProbe(ctx, api) + if cleanupErr != nil { + // A leftover probe object is a transient failure: neither verdict is cached, and both + // errors stay visible to callers. + return errors.Join(verdict, cleanupErr) + } + if verdict == nil || blob.IsConditionalUnsupported(verdict) { + s.probeMu.Lock() + s.probed, s.probeError = true, verdict + s.probeMu.Unlock() + } + return verdict +} + +// cachedProbe returns the cached probe verdict, when a definite one exists. +func (s *Store) cachedProbe() (verdict error, cached bool) { + s.probeMu.Lock() + defer s.probeMu.Unlock() + return s.probeError, s.probed +} + +// runProbe performs the double create and removes the probe object. The verdict is the probe +// result; cleanupErr reports a failure to delete the probe object. +func (s *Store) runProbe(ctx context.Context, api API) (verdict, cleanupErr error) { + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return blob.NewError(blob.KindOther, probeOperation, "", err), nil + } + key := probePrefix + hex.EncodeToString(raw[:]) + objectKey := s.objectKey(key) + // Cleanup is scheduled before the first create: a PUT that returns an ambiguous error may + // still have written the probe object. + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), probeCleanupTimeout) + defer cancel() + _, err := api.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey)}, noRetry) + if err != nil { + cleanupErr = blob.NewError(blob.KindOther, probeOperation, key, fmt.Errorf("cleanup probe object: %w", err)) + } + }() + return s.probeCreate(ctx, api, key, objectKey), nil +} + +// probeCreate creates the probe object twice and checks the second create is rejected. +func (s *Store) probeCreate(ctx context.Context, api API, key, objectKey string) error { + create := func() error { + _, err := api.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(objectKey), + Body: bytes.NewReader([]byte("atmos conditional write probe")), + IfNoneMatch: aws.String("*"), + }, noRetry) + return classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err) + } + if err := create(); err != nil { + return err + } + err := create() + if blob.IsPreconditionFailed(err) { + return s.probeIfMatch(ctx, api, key, objectKey) + } + if err == nil { + return unsupportedConditional(probeOperation, key, "the endpoint accepted a second create-if-absent for the same key, so it ignores If-None-Match") + } + return err +} + +// probeIfMatch verifies that stale validators cannot overwrite or delete the probe object. +func (s *Store) probeIfMatch(ctx context.Context, api API, key, objectKey string) error { + stale := aws.String(`"atmos-probe-nonmatching-etag"`) + _, err := api.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey), Body: bytes.NewReader([]byte("unexpected")), IfMatch: stale}, noRetry) + if err = classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err); !blob.IsPreconditionFailed(err) { + if err == nil { + return unsupportedConditional(probeOperation, key, "the endpoint ignored If-Match on put") + } + return err + } + _, err = api.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(objectKey), IfMatch: stale}, noRetry) + if err = classify(request{op: probeOperation, key: key, mutating: true, conditional: true}, err); !blob.IsPreconditionFailed(err) { + if err == nil { + return unsupportedConditional(probeOperation, key, "the endpoint ignored If-Match on delete") + } + return err + } + return nil +} diff --git a/pkg/artifact/blob/s3/conditional_test.go b/pkg/artifact/blob/s3/conditional_test.go new file mode 100644 index 00000000000..a7eb8bb2453 --- /dev/null +++ b/pkg/artifact/blob/s3/conditional_test.go @@ -0,0 +1,383 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "testing" + + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func putIfAbsent(s *Store, key, body string) (*blob.ObjectInfo, error) { + return s.Put(context.Background(), key, bytes.NewReader([]byte(body)), blob.PutOptions{Size: int64(len(body)), Condition: blob.IfAbsent()}) +} + +func TestConditionalHeadersAreSent(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + + first, err := putIfAbsent(s, "lock", "a") + require.NoError(t, err) + _, err = s.Put(context.Background(), "lock", bytes.NewReader([]byte("b")), blob.PutOptions{Size: 1, Condition: blob.IfMatch(first.ETag)}) + require.NoError(t, err) + + puts := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, puts, 2) + assert.Equal(t, "*", puts[0].Header.Get("If-None-Match")) + assert.Empty(t, puts[0].Header.Get("If-Match")) + assert.Equal(t, first.ETag, puts[1].Header.Get("If-Match")) + assert.Empty(t, puts[1].Header.Get("If-None-Match")) +} + +func TestConditionalDeleteSendsIfMatch(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + info := putBytes(t, s, "k", []byte("x"), blob.PutOptions{}) + + err := s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: `"stale"`}) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + require.NoError(t, s.Delete(context.Background(), "k", blob.DeleteOptions{IfMatch: info.ETag})) + + deletes := fake.RequestsFor(s3test.OpDeleteObject) + require.Len(t, deletes, 2) + assert.Equal(t, `"stale"`, deletes[0].Header.Get("If-Match")) + assert.Equal(t, info.ETag, deletes[1].Header.Get("If-Match")) +} + +func TestEndpointThatCannotEnforceConditionsFailsClearly(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalNotImplemented)) + s := newTestStore(t, fake) + putBytes(t, s, "existing", []byte("keep"), blob.PutOptions{}) + + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "501 maps to a clear unsupported-condition error: %v", err) + assert.False(t, blob.IsAmbiguous(err)) + _, ok := fake.Object("lock") + assert.False(t, ok, "nothing was written") + + _, err = s.Put(context.Background(), "existing", bytes.NewReader([]byte("new")), blob.PutOptions{Size: 3, Condition: blob.IfMatch(`"x"`)}) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + object, _ := fake.Object("existing") + assert.Equal(t, []byte("keep"), object.Data, "the existing object was not overwritten") +} + +func TestEndpointRejectingTheHeaderWithBadRequestIsUnsupported(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusBadRequest, Code: "InvalidArgument", Times: -1}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + + // The same rejection of an unconditional put is a plain failure, not a capability problem. + _, err = s.Put(context.Background(), "plain", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1}) + require.Error(t, err) + assert.False(t, blob.IsConditionalUnsupported(err)) +} + +func TestDisabledModeNeverSendsConditions(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithConditionalMode(ConditionalDisabled)) + + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + err = s.Delete(context.Background(), "lock", blob.DeleteOptions{IfMatch: `"x"`}) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Empty(t, fake.Requests(), "an endpoint known not to enforce conditions is never asked to") + + putBytes(t, s, "plain", []byte("x"), blob.PutOptions{}) + assert.Len(t, fake.Requests(), 1, "unconditional writes still work") +} + +func TestProbeDetectsEndpointThatIgnoresConditions(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + fake.Seed("lock", []byte("owner-a"), nil) + + _, err := putIfAbsent(s, "lock", "owner-b") + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err), "a silently-ignoring endpoint must not be trusted: %v", err) + object, _ := fake.Object("lock") + assert.Equal(t, []byte("owner-a"), object.Data, "the caller's object was never touched") + assert.Len(t, fake.Keys(), 1, "probe objects are removed") + + before := len(fake.Requests()) + _, err = putIfAbsent(s, "lock", "owner-c") + assert.True(t, blob.IsConditionalUnsupported(err)) + assert.Len(t, fake.Requests(), before, "a definite probe verdict is cached") +} + +func TestProbeAcceptsEnforcingEndpointOnce(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + _, err := putIfAbsent(s, "lock", "a") + require.NoError(t, err) + _, err = putIfAbsent(s, "lock2", "a") + require.NoError(t, err) + + puts := fake.RequestsFor(s3test.OpPutObject) + assert.Len(t, puts, 5, "three probe puts plus the two caller puts; the probe runs once") + assert.Equal(t, []string{"lock", "lock2"}, fake.Keys(), "the probe object was cleaned up") + assert.Equal(t, 1, countPrefix(puts, probePrefix)/3, "exactly one probe key was created twice") + + require.NoError(t, s.ProbeConditionalWrites(context.Background())) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 5, "explicit probe reuses the cached verdict") +} + +func countPrefix(requests []s3test.Request, prefix string) int { + count := 0 + for _, req := range requests { + if len(req.Key) >= len(prefix) && req.Key[:len(prefix)] == prefix { + count++ + } + } + return count +} + +func TestResponseDroppedAfterCommitIsAmbiguousAndNotRetried(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + + _, err := putIfAbsent(s, "lock", "mine") + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + var blobErr *blob.Error + require.True(t, errors.As(err, &blobErr)) + assert.True(t, blobErr.Temporary(), "ambiguity is signaled through Temporary()") + assert.False(t, blob.IsPreconditionFailed(err), "an SDK retry must not turn our own commit into a fake conflict") + + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1, "conditional requests are never retried by the SDK") + object, ok := fake.Object("lock") + require.True(t, ok, "the write did commit") + assert.Equal(t, []byte("mine"), object.Data) + + // Reconciliation: reading the object back proves ownership. + stat, statErr := s.Stat(context.Background(), "lock", blob.StatOptions{}) + require.NoError(t, statErr) + assert.Equal(t, object.ETag, stat.ETag) +} + +func TestResponseDroppedBeforeCommitIsAlsoAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropBeforeCommit}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "mine") + assert.True(t, blob.IsAmbiguous(err), "the client cannot tell the two apart: %v", err) + _, ok := fake.Object("lock") + assert.False(t, ok) +} + +func TestServerErrorOnConditionalMutationIsAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusInternalServerError, Code: "InternalError"}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestServiceUnavailableIsAmbiguous(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusServiceUnavailable, Code: "SlowDown", Times: -1}) + s := newTestStore(t, fake) + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "an endpoint may apply a mutation before responding 503") +} + +func TestUnconditionalPutRetriesTransientServerError(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: http.StatusInternalServerError, Code: "InternalError"}) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{}) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 2, "the SDK retried once and succeeded") +} + +func TestConnectionFailureBeforeSendingIsNotAmbiguous(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Close() + _, err := putIfAbsent(s, "lock", "a") + require.Error(t, err) + assert.False(t, blob.IsAmbiguous(err), "the request never reached a server: %v", err) +} + +func TestAmbiguousCompletionLeavesTheUploadForReconciliation(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCompleteMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 9) + + _, err := s.Put(context.Background(), "big", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data)), Condition: blob.IfAbsent()}) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Empty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload), "an upload that may have completed is never aborted") + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) +} + +func TestUnconditionalCompletionLostResponseIsAmbiguousAndNotRetried(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCompleteMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 9) + + _, err := s.Put(context.Background(), "big", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "a lost completion response is not a definite failure: %v", err) + assert.Len(t, fake.RequestsFor(s3test.OpCompleteMultipartUpload), 1, "completion is never retried by the SDK") + assert.Empty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload), "the upload is left for reconciliation") + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) +} + +// failingReader serves data, then fails with err. +type failingReader struct { + data []byte + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +func TestSourceUnexpectedEOFAbortsMultipartPublication(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + body := &failingReader{data: patterned(smallPart*2 + 9), err: io.ErrUnexpectedEOF} + + _, err := s.Put(context.Background(), "big", body, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, io.ErrUnexpectedEOF, "the source failure is not mistaken for end of input") + _, ok := fake.Object("big") + assert.False(t, ok, "a truncated source never becomes an object") + assert.Empty(t, fake.RequestsFor(s3test.OpCompleteMultipartUpload)) + assert.NotEmpty(t, fake.RequestsFor(s3test.OpAbortMultipartUpload)) +} + +func TestServiceCodesMapToKinds(t *testing.T) { + cases := []struct { + name string + op string + status int32 + code string + check func(error) bool + }{ + {"missing key", s3test.OpGetObject, 404, "NoSuchKey", blob.IsNotFound}, + {"conflicting concurrent write", s3test.OpPutObject, 409, "ConditionalRequestConflict", blob.IsPreconditionFailed}, + {"explicit precondition", s3test.OpPutObject, 412, "PreconditionFailed", blob.IsPreconditionFailed}, + {"access denied is plain", s3test.OpGetObject, 403, "AccessDenied", func(err error) bool { + return !blob.IsNotFound(err) && !blob.IsPreconditionFailed(err) && !blob.IsAmbiguous(err) + }}, + {"missing bucket is not a missing object", s3test.OpGetObject, 404, "NoSuchBucket", func(err error) bool { return !blob.IsNotFound(err) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: tc.op, Status: tc.status, Code: tc.code, Times: -1}) + s := newTestStore(t, fake) + var err error + if tc.op == s3test.OpGetObject { + _, err = s.Get(context.Background(), "k", blob.GetOptions{}) + } else { + _, err = putIfAbsent(s, "k", "x") + } + require.Error(t, err) + assert.True(t, tc.check(err), "%v", err) + }) + } +} + +func TestProbeCleanupFailureIsReturnedAndNotCached(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: http.StatusForbidden, Code: "AccessDenied", Skip: 1}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + err := s.ProbeConditionalWrites(context.Background()) + require.ErrorContains(t, err, "cleanup probe object") + fake.ClearFaults() + require.NoError(t, s.ProbeConditionalWrites(context.Background())) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 6) +} + +func TestProbeCleansUpWhenFirstPutResponseIsLost(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + err := s.ProbeConditionalWrites(context.Background()) + require.Error(t, err) + assert.True(t, blob.IsAmbiguous(err), "%v", err) + assert.Empty(t, fake.Keys(), "the probe object committed by the lost response is removed") +} + +func TestProbeCleanupFailureKeepsUnsupportedVerdictAndIsNotCached(t *testing.T) { + fake := newFake(t, s3test.WithConditionalBehavior(s3test.ConditionalIgnore)) + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithConditionalMode(ConditionalProbe)) + + err := s.ProbeConditionalWrites(context.Background()) + require.ErrorContains(t, err, "cleanup probe object") + assert.True(t, blob.IsConditionalUnsupported(err), "the probe verdict stays visible: %v", err) + + fake.ClearFaults() + err = s.ProbeConditionalWrites(context.Background()) + require.Error(t, err) + assert.True(t, blob.IsConditionalUnsupported(err)) + assert.NotContains(t, err.Error(), "cleanup probe object", "the verdict was not cached with the failed cleanup") + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 4, "the probe ran again after the failed cleanup") +} + +type ignoresIfMatch struct { + API + put, delete bool +} + +func (i ignoresIfMatch) PutObject(ctx context.Context, in *awss3.PutObjectInput, opts ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) { + copy := *in + if i.put { + copy.IfMatch = nil + } + return i.API.PutObject(ctx, ©, opts...) +} + +func (i ignoresIfMatch) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput, opts ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { + copy := *in + if i.delete { + copy.IfMatch = nil + } + return i.API.DeleteObject(ctx, ©, opts...) +} + +func TestProbeRejectsIgnoredIfMatch(t *testing.T) { + for _, tc := range []struct { + name string + put, delete bool + }{{"put", true, false}, {"delete", false, true}} { + t.Run(tc.name, func(t *testing.T) { + fake := newFake(t) + api := ignoresIfMatch{API: fake.Client(), put: tc.put, delete: tc.delete} + s, err := NewFromClient(api, fake.Bucket(), WithConditionalMode(ConditionalProbe)) + require.NoError(t, err) + err = s.ProbeConditionalWrites(context.Background()) + assert.True(t, blob.IsConditionalUnsupported(err), "%v", err) + assert.Empty(t, fake.Keys()) + }) + } +} diff --git a/pkg/artifact/blob/s3/context_test.go b/pkg/artifact/blob/s3/context_test.go new file mode 100644 index 00000000000..4305daf46ee --- /dev/null +++ b/pkg/artifact/blob/s3/context_test.go @@ -0,0 +1,149 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +func TestProbeWaiterStopsWhenItsContextEnds(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) + started := make(chan struct{}) + release := make(chan struct{}) + // The mocked PutObject parks on release. Free it on every exit so a failed assertion never + // strands the probe goroutine. + var freeOnce sync.Once + free := func() { freeOnce.Do(func() { close(release) }) } + t.Cleanup(free) + errEndpointDown := errors.New("endpoint down") + api.EXPECT(). + PutObject(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + close(started) + <-release + return nil, errEndpointDown + }). + Times(1) + api.EXPECT().DeleteObject(gomock.Any(), gomock.Any(), gomock.Any()).Return(&s3.DeleteObjectOutput{}, nil).Times(1) + store, err := NewFromClient(api, "bucket", WithConditionalMode(ConditionalProbe)) + require.NoError(t, err) + + first := make(chan error, 1) + go func() { first <- store.ProbeConditionalWrites(context.Background()) }() + select { + case <-started: + case err := <-first: + t.Fatalf("the first probe ended before it reached the endpoint: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("the first probe never reached the endpoint") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + waited := make(chan error, 1) + go func() { waited <- store.ProbeConditionalWrites(ctx) }() + select { + case err := <-waited: + require.ErrorIs(t, err, context.DeadlineExceeded, "the waiter gives up at its own deadline while the first probe is in flight") + require.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + case <-time.After(5 * time.Second): + t.Fatal("the waiting probe stayed blocked after its context ended") + } + + free() + select { + case err := <-first: + require.ErrorIs(t, err, errEndpointDown, "the first probe still finishes and reports its own failure") + case <-time.After(5 * time.Second): + t.Fatal("the first probe never finished after the endpoint was released") + } +} + +func TestPutChecksContextBeforeReadingTheBody(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) // Any SDK call fails the test: nothing may be sent. + store, err := NewFromClient(api, "bucket", WithPartSize(smallPart)) + require.NoError(t, err) + reader, writer := io.Pipe() // Nothing is ever written, so reading would block forever. + defer writer.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { + _, putErr := store.Put(ctx, "k", reader, blob.PutOptions{Size: blob.UnknownSize}) + done <- putErr + }() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + case <-time.After(5 * time.Second): + t.Fatal("put blocked reading the body although its context was already canceled") + } +} + +func TestContextReaderStopsConsumingAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader := &ctxReader{ctx: ctx, reader: bytes.NewReader([]byte("abcdef"))} + buf := make([]byte, 3) + n, err := reader.Read(buf) + require.NoError(t, err) + assert.Equal(t, "abc", string(buf[:n])) + + cancel() + n, err = reader.Read(buf) + require.ErrorIs(t, err, context.Canceled) + assert.Zero(t, n) +} + +func TestMergeUploadErrorsKeepsTheWorkerFailure(t *testing.T) { + errService := errors.New("service rejected the part") + errRead := errors.New("body read failed") + live := context.Background() + canceled, cancel := context.WithCancel(context.Background()) + cancel() + + tests := []struct { + name string + parent context.Context + produceErr error + waitErr error + want error + }{ + {name: "no failure", parent: live}, + {name: "worker failure surfaces when the producer was canceled by the group", parent: live, produceErr: context.Canceled, waitErr: errService, want: errService}, + {name: "caller cancellation stays the reported error", parent: canceled, produceErr: context.Canceled, waitErr: errService, want: context.Canceled}, + {name: "unrelated producer error is kept", parent: live, produceErr: errRead, waitErr: context.Canceled, want: errRead}, + {name: "worker failure alone is reported", parent: live, waitErr: errService, want: errService}, + {name: "producer failure alone is reported", parent: live, produceErr: errRead, want: errRead}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := mergeUploadErrors(tc.parent, tc.produceErr, tc.waitErr) + if tc.want == nil { + assert.NoError(t, got) + return + } + assert.ErrorIs(t, got, tc.want) + }) + } +} + +func TestUnclassifiedFailuresCarryTheSentinel(t *testing.T) { + cause := errors.New("disk full") + err := asBlobError("put", "k", cause) + require.ErrorIs(t, err, errUtils.ErrBlobOperationFailed) + assert.ErrorIs(t, err, cause, "the original cause stays visible") +} diff --git a/pkg/artifact/blob/s3/delete.go b/pkg/artifact/blob/s3/delete.go new file mode 100644 index 00000000000..4ced8cbda2c --- /dev/null +++ b/pkg/artifact/blob/s3/delete.go @@ -0,0 +1,38 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [blob.Store]. Conditional deletes are never retried by the SDK. +func (s *Store) Delete(ctx context.Context, key string, opts blob.DeleteOptions) error { + defer perf.Track(nil, "blob.s3.Delete")() + + if err := s.validateKey(key); err != nil { + return err + } + if err := s.Capabilities().CheckDelete(storeName, key, opts); err != nil { + return err + } + conditional := opts.IfMatch != "" + if conditional { + if err := s.ensureConditional(ctx, key, blob.IfMatch(opts.IfMatch)); err != nil { + return err + } + } + api, err := s.client(ctx) + if err != nil { + return err + } + in := &s3.DeleteObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), IfMatch: optionalString(opts.IfMatch)} + if _, err := api.DeleteObject(ctx, in, retryOptions(conditional)...); err != nil { + return classify(request{op: blob.OpDelete, key: key, mutating: true, conditional: conditional}, err) + } + return nil +} diff --git a/pkg/artifact/blob/s3/doc.go b/pkg/artifact/blob/s3/doc.go new file mode 100644 index 00000000000..041de6e4550 --- /dev/null +++ b/pkg/artifact/blob/s3/doc.go @@ -0,0 +1,19 @@ +// Package s3 is the AWS SDK v2 implementation of [blob.Store], plus the shared S3 client +// construction used by the legacy CI artifact adapter. +// +// Writes stream: a body that fits one part is sent with a single PutObject, a larger or +// unknown-length body is sent as a multipart upload whose parts are uploaded by a bounded worker +// pool (default 4) from a bounded buffer pool, so memory use is independent of object size. A +// multipart upload owned by a failed or canceled operation is aborted. Conditional writes use +// If-None-Match and If-Match; conditional requests are never retried by the SDK so a lost response +// surfaces as an ambiguous error instead of a misleading precondition failure. An endpoint that +// cannot enforce conditions fails the operation with a clear error, and an optional probe detects +// endpoints that silently ignore the headers. +// +// Stat with tags pins the tag read to the version HeadObject returned when the bucket is versioned, +// which needs s3:GetObjectVersionTagging in addition to s3:GetObjectTagging. Bucket versioning is +// not required; without it, tags come from the current object and are not version-matched. +// +// Credentials are resolved lazily on first use. The generic store fails closed: a configured +// identity that cannot be resolved is an error, never a silent fall back to ambient credentials. +package s3 diff --git a/pkg/artifact/blob/s3/emulator_test.go b/pkg/artifact/blob/s3/emulator_test.go new file mode 100644 index 00000000000..51c916ff002 --- /dev/null +++ b/pkg/artifact/blob/s3/emulator_test.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "crypto/rand" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// TestEmulatorConformance runs the shared suite against a real endpoint. It is skipped unless +// ATMOS_BLOB_S3_ENDPOINT and ATMOS_BLOB_S3_BUCKET are set (see s3test.EmulatorFromEnv); the bucket +// must already exist. Every subtest uses a unique key prefix, so runs do not collide. +func TestEmulatorConformance(t *testing.T) { + emulator := s3test.EmulatorFromEnv(t) + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { + var raw [6]byte + _, err := rand.Read(raw[:]) + require.NoError(t, err) + s, err := New( + emulator.Bucket, + WithEndpoint(emulator.Endpoint), + WithRegion(emulator.Region), + WithPrefix("atmos-blob-conformance/"+hex.EncodeToString(raw[:])), + WithConditionalMode(ConditionalProbe), + WithPartSize(5<<20), + ) + require.NoError(t, err) + return s + }) +} diff --git a/pkg/artifact/blob/s3/fallback.go b/pkg/artifact/blob/s3/fallback.go new file mode 100644 index 00000000000..317f255d1de --- /dev/null +++ b/pkg/artifact/blob/s3/fallback.go @@ -0,0 +1,64 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// putFallback preserves the already-consumed prefix, spooling only after multipart is rejected. +func (s *Store) putFallback(ctx context.Context, plan *putPlan, first, carry []byte, body io.Reader) (*blob.ObjectInfo, error) { + file, err := os.CreateTemp("", "atmos-blob-fallback-*") + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("create upload spool: %w", err)) + } + defer func() { + _ = file.Close() + _ = os.Remove(file.Name()) + }() + // One byte past the limit is enough to prove the body is too large without spooling all of it. + source := io.LimitReader(&ctxReader{ctx: ctx, reader: io.MultiReader(bytes.NewReader(first), bytes.NewReader(carry), body)}, s.singlePutLimit()+1) + size, err := io.Copy(file, source) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, fmt.Errorf("spool upload: %w", err)) + } + if size > s.singlePutLimit() { + return nil, tooLargeForSinglePut(plan.key, size, s.singlePutLimit()) + } + if plan.opts.Size >= 0 && size != plan.opts.Size { + return nil, sizeMismatch(plan.key, size, plan.opts.Size) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, plan.key, err) + } + in := &s3.PutObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(plan.objectKey), Body: file, ContentLength: aws.Int64(size), ContentType: optionalString(plan.opts.ContentType), Metadata: plan.opts.Metadata, Tagging: optionalString(plan.tags)} + in.IfNoneMatch, in.IfMatch = conditionHeaders(plan.opts.Condition) + out, err := plan.api.PutObject(ctx, in, retryOptions(plan.opts.Condition.IsConditional())...) + if err != nil { + return nil, classify(plan.request(true), err) + } + return &blob.ObjectInfo{Key: plan.key, Size: size, ETag: aws.ToString(out.ETag)}, nil +} + +// ctxReader fails reads once its context is done, so a canceled operation stops consuming its body. +type ctxReader struct { + ctx context.Context + reader io.Reader +} + +func (r *ctxReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "s3.ctxReader.Read")() + + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} diff --git a/pkg/artifact/blob/s3/helpers_test.go b/pkg/artifact/blob/s3/helpers_test.go new file mode 100644 index 00000000000..6bf281bddcd --- /dev/null +++ b/pkg/artifact/blob/s3/helpers_test.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/store" +) + +// smallPart is a part size that lets tests exercise multipart with kilobytes. +const smallPart = 16 << 10 + +// newFake starts a fake S3 endpoint. +func newFake(t *testing.T, opts ...s3test.Option) *s3test.Server { + t.Helper() + return s3test.NewServer(t, opts...) +} + +// newTestStore returns a store over the fake with small parts unless overridden. +func newTestStore(t *testing.T, fake *s3test.Server, opts ...Option) *Store { + t.Helper() + all := append([]Option{WithPartSize(smallPart)}, opts...) + s, err := NewFromClient(fake.Client(), fake.Bucket(), all...) + require.NoError(t, err) + return s +} + +// putBytes stores data and fails the test on error. +func putBytes(t *testing.T, s *Store, key string, data []byte, opts blob.PutOptions) *blob.ObjectInfo { + t.Helper() + if opts.Size == 0 && len(data) > 0 { + opts.Size = int64(len(data)) + } + info, err := s.Put(context.Background(), key, bytes.NewReader(data), opts) + require.NoError(t, err) + return info +} + +// patterned returns n deterministic bytes. +func patterned(n int) []byte { + data := make([]byte, n) + for i := range data { + data[i] = byte(i*7 + i/251) + } + return data +} + +// countingReader generates size bytes and counts what the consumer has read. +type countingReader struct { + size int64 + read atomic.Int64 + onRead func(read int64) + exhaust func() + finished atomic.Bool +} + +func (r *countingReader) Read(p []byte) (int, error) { + remaining := r.size - r.read.Load() + if remaining <= 0 { + if r.finished.CompareAndSwap(false, true) && r.exhaust != nil { + r.exhaust() + } + return 0, io.EOF + } + n := int64(len(p)) + if n > remaining { + n = remaining + } + for i := int64(0); i < n; i++ { + p[i] = byte(i) + } + total := r.read.Add(n) + if r.onRead != nil { + r.onRead(total) + } + return int(n), nil +} + +// fakeResolver is an AuthResolver double. +type fakeResolver struct { + result *store.AWSAuthConfig + err error + calls atomic.Int32 +} + +func (f *fakeResolver) ResolveAWSAuthContext(_ context.Context, _ string) (*store.AWSAuthConfig, error) { + f.calls.Add(1) + return f.result, f.err +} diff --git a/pkg/artifact/blob/s3/list.go b/pkg/artifact/blob/s3/list.go new file mode 100644 index 00000000000..5b2ced61987 --- /dev/null +++ b/pkg/artifact/blob/s3/list.go @@ -0,0 +1,61 @@ +package s3 + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxListKeys is the most entries S3 returns in one page. +const maxListKeys = 1000 + +// List implements [blob.Store]. The cursor is the S3 continuation token. +func (s *Store) List(ctx context.Context, opts blob.ListOptions) (*blob.ListPage, error) { + defer perf.Track(nil, "blob.s3.List")() + + api, err := s.client(ctx) + if err != nil { + return nil, err + } + in := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.cfg.bucket), + Prefix: optionalString(s.listPrefix(opts.Prefix)), + Delimiter: optionalString(opts.Delimiter), + ContinuationToken: optionalString(opts.Cursor), + } + if opts.Limit > 0 { + in.MaxKeys = aws.Int32(int32(min(opts.Limit, maxListKeys))) + } + out, err := api.ListObjectsV2(ctx, in) + if err != nil { + return nil, classify(request{op: blob.OpList, key: opts.Prefix}, err) + } + page := &blob.ListPage{} + for _, object := range out.Contents { + page.Objects = append(page.Objects, blob.ObjectInfo{ + Key: s.storeKey(aws.ToString(object.Key)), + Size: aws.ToInt64(object.Size), + ETag: aws.ToString(object.ETag), + LastModified: aws.ToTime(object.LastModified), + }) + } + for _, prefix := range out.CommonPrefixes { + page.CommonPrefixes = append(page.CommonPrefixes, s.storeKey(aws.ToString(prefix.Prefix))) + } + if aws.ToBool(out.IsTruncated) { + page.NextCursor = aws.ToString(out.NextContinuationToken) + } + return page, nil +} + +// listPrefix maps a store prefix to the bucket prefix. +func (s *Store) listPrefix(prefix string) string { + if s.cfg.prefix == "" { + return prefix + } + return s.cfg.prefix + "/" + prefix +} diff --git a/pkg/artifact/blob/s3/mock_api_test.go b/pkg/artifact/blob/s3/mock_api_test.go new file mode 100644 index 00000000000..2141a6811a4 --- /dev/null +++ b/pkg/artifact/blob/s3/mock_api_test.go @@ -0,0 +1,242 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/blob/s3 (interfaces: API) +// +// Generated by this command: +// +// mockgen -package s3 -destination mock_api_test.go github.com/cloudposse/atmos/pkg/artifact/blob/s3 API +// + +// Package s3 is a generated GoMock package. +package s3 + +import ( + context "context" + reflect "reflect" + + s3 "github.com/aws/aws-sdk-go-v2/service/s3" + gomock "go.uber.org/mock/gomock" +) + +// MockAPI is a mock of API interface. +type MockAPI struct { + ctrl *gomock.Controller + recorder *MockAPIMockRecorder + isgomock struct{} +} + +// MockAPIMockRecorder is the mock recorder for MockAPI. +type MockAPIMockRecorder struct { + mock *MockAPI +} + +// NewMockAPI creates a new mock instance. +func NewMockAPI(ctrl *gomock.Controller) *MockAPI { + mock := &MockAPI{ctrl: ctrl} + mock.recorder = &MockAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAPI) EXPECT() *MockAPIMockRecorder { + return m.recorder +} + +// AbortMultipartUpload mocks base method. +func (m *MockAPI) AbortMultipartUpload(ctx context.Context, in *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AbortMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.AbortMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AbortMultipartUpload indicates an expected call of AbortMultipartUpload. +func (mr *MockAPIMockRecorder) AbortMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AbortMultipartUpload", reflect.TypeOf((*MockAPI)(nil).AbortMultipartUpload), varargs...) +} + +// CompleteMultipartUpload mocks base method. +func (m *MockAPI) CompleteMultipartUpload(ctx context.Context, in *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CompleteMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.CompleteMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. +func (mr *MockAPIMockRecorder) CompleteMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockAPI)(nil).CompleteMultipartUpload), varargs...) +} + +// CreateMultipartUpload mocks base method. +func (m *MockAPI) CreateMultipartUpload(ctx context.Context, in *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CreateMultipartUpload", varargs...) + ret0, _ := ret[0].(*s3.CreateMultipartUploadOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateMultipartUpload indicates an expected call of CreateMultipartUpload. +func (mr *MockAPIMockRecorder) CreateMultipartUpload(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateMultipartUpload", reflect.TypeOf((*MockAPI)(nil).CreateMultipartUpload), varargs...) +} + +// DeleteObject mocks base method. +func (m *MockAPI) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteObject", varargs...) + ret0, _ := ret[0].(*s3.DeleteObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteObject indicates an expected call of DeleteObject. +func (mr *MockAPIMockRecorder) DeleteObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObject", reflect.TypeOf((*MockAPI)(nil).DeleteObject), varargs...) +} + +// GetObject mocks base method. +func (m *MockAPI) GetObject(ctx context.Context, in *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetObject", varargs...) + ret0, _ := ret[0].(*s3.GetObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObject indicates an expected call of GetObject. +func (mr *MockAPIMockRecorder) GetObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockAPI)(nil).GetObject), varargs...) +} + +// GetObjectTagging mocks base method. +func (m *MockAPI) GetObjectTagging(ctx context.Context, in *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetObjectTagging", varargs...) + ret0, _ := ret[0].(*s3.GetObjectTaggingOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObjectTagging indicates an expected call of GetObjectTagging. +func (mr *MockAPIMockRecorder) GetObjectTagging(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObjectTagging", reflect.TypeOf((*MockAPI)(nil).GetObjectTagging), varargs...) +} + +// HeadObject mocks base method. +func (m *MockAPI) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "HeadObject", varargs...) + ret0, _ := ret[0].(*s3.HeadObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HeadObject indicates an expected call of HeadObject. +func (mr *MockAPIMockRecorder) HeadObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*MockAPI)(nil).HeadObject), varargs...) +} + +// ListObjectsV2 mocks base method. +func (m *MockAPI) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListObjectsV2", varargs...) + ret0, _ := ret[0].(*s3.ListObjectsV2Output) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObjectsV2 indicates an expected call of ListObjectsV2. +func (mr *MockAPIMockRecorder) ListObjectsV2(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjectsV2", reflect.TypeOf((*MockAPI)(nil).ListObjectsV2), varargs...) +} + +// PutObject mocks base method. +func (m *MockAPI) PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PutObject", varargs...) + ret0, _ := ret[0].(*s3.PutObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PutObject indicates an expected call of PutObject. +func (mr *MockAPIMockRecorder) PutObject(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockAPI)(nil).PutObject), varargs...) +} + +// UploadPart mocks base method. +func (m *MockAPI) UploadPart(ctx context.Context, in *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UploadPart", varargs...) + ret0, _ := ret[0].(*s3.UploadPartOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UploadPart indicates an expected call of UploadPart. +func (mr *MockAPIMockRecorder) UploadPart(ctx, in any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadPart", reflect.TypeOf((*MockAPI)(nil).UploadPart), varargs...) +} diff --git a/pkg/artifact/blob/s3/multipart.go b/pkg/artifact/blob/s3/multipart.go new file mode 100644 index 00000000000..f83fce94d19 --- /dev/null +++ b/pkg/artifact/blob/s3/multipart.go @@ -0,0 +1,278 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "sort" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "golang.org/x/sync/errgroup" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +// abortTimeout bounds the cleanup of an abandoned multipart upload, which must outlive the +// canceled operation context. +const abortTimeout = 30 * time.Second + +// multipart is one in-progress multipart upload. +type multipart struct { + store *Store + plan *putPlan + uploadID string +} + +// putMultipart streams the body as a multipart upload with bounded memory and concurrency. The +// upload is aborted on any definite failure, including cancellation. After an ambiguous +// completion it is left alone, because it may already have become the object. +func (s *Store) putMultipart(ctx context.Context, plan *putPlan, first, carry []byte, body io.Reader) (*blob.ObjectInfo, error) { + in := &s3.CreateMultipartUploadInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(plan.objectKey), + ContentType: optionalString(plan.opts.ContentType), + Metadata: plan.opts.Metadata, + Tagging: optionalString(plan.tags), + } + created, err := plan.api.CreateMultipartUpload(ctx, in, skipOptionalChecksums, noRetry) + if err != nil { + status, code := responseDetails(err) + if s.cfg.singleFallback && (status == http.StatusNotImplemented || code == "NotImplemented") { + return s.putFallback(ctx, plan, first, carry, body) + } + return nil, classify(plan.request(false), err) + } + upload := &multipart{store: s, plan: plan, uploadID: aws.ToString(created.UploadId)} + + parts, total, err := upload.uploadParts(ctx, first, carry, body) + if err == nil && plan.opts.Size >= 0 && total != plan.opts.Size { + err = sizeMismatch(plan.key, total, plan.opts.Size) + } + if err != nil { + return nil, upload.abort(ctx, asBlobError("put", plan.key, err)) + } + return upload.complete(ctx, parts, total) +} + +// partResult records an uploaded part. +type partResult struct { + mu sync.Mutex + parts []types.CompletedPart +} + +func (r *partResult) add(number int32, etag *string) { + r.mu.Lock() + r.parts = append(r.parts, types.CompletedPart{ETag: etag, PartNumber: aws.Int32(number)}) + r.mu.Unlock() +} + +// uploadParts reads the body part by part and uploads them through a bounded pool. A free-list of +// buffers caps memory at one buffer per worker. +func (m *multipart) uploadParts(parent context.Context, first, carry []byte, body io.Reader) ([]types.CompletedPart, int64, error) { + ctx, cancel := context.WithCancel(parent) + defer cancel() + group, gctx := errgroup.WithContext(ctx) + + free := make(chan []byte, m.store.cfg.concurrency) + for range m.store.cfg.concurrency { + free <- nil + } + result := &partResult{} + submit := func(number int32, data []byte) { + group.Go(func() error { + defer func() { free <- data }() + etag, err := m.uploadPart(gctx, number, data) + if err != nil { + return err + } + result.add(number, etag) + return nil + }) + } + + feed := &partFeed{free: free, submit: submit} + total, produceErr := m.produce(gctx, feed, first, carry, body) + if produceErr != nil { + cancel() + } + waitErr := group.Wait() + if err := mergeUploadErrors(parent, produceErr, waitErr); err != nil { + return nil, 0, err + } + sort.Slice(result.parts, func(i, j int) bool { + return aws.ToInt32(result.parts[i].PartNumber) < aws.ToInt32(result.parts[j].PartNumber) + }) + return result.parts, total, nil +} + +// mergeUploadErrors picks the error that explains a failed upload. When a worker fails, the group +// cancels the producer's context, so the producer reports a bare cancellation that hides the +// worker's service error. That cancellation is not the caller's own when parent is still live. +func mergeUploadErrors(parent context.Context, produceErr, waitErr error) error { + if produceErr != nil && waitErr != nil && errors.Is(produceErr, context.Canceled) && parent.Err() == nil { + return waitErr + } + if produceErr != nil { + return produceErr + } + return waitErr +} + +// partFeed hands parts to the worker pool. Its free list holds one buffer slot per worker. +type partFeed struct { + free chan []byte + submit func(number int32, data []byte) +} + +// produce feeds parts to the workers, waiting for a free buffer before reading each next part so +// the reader never runs ahead of the uploaders. +func (m *multipart) produce(ctx context.Context, feed *partFeed, first, carry []byte, body io.Reader) (int64, error) { + partSize := m.plan.partSize + free, submit := feed.free, feed.submit + if err := acquire(ctx, free); err != nil { + return 0, err + } + number := int32(1) + data := first + total := int64(0) + for { + submit(number, data) + total += int64(len(data)) + if int64(len(data)) < partSize { + return total, nil + } + buf, err := acquireBuffer(ctx, free, partSize) + if err != nil { + return 0, err + } + n, readErr := readPart(body, buf, carry) + carry = nil + if readErr != nil { + return 0, blob.NewError(blob.KindOther, blob.OpPut, m.plan.key, fmt.Errorf("read body: %w", readErr)) + } + if n == 0 { + free <- buf + return total, nil + } + number++ + if number > MaxParts { + return 0, blob.NewError(blob.KindInvalid, blob.OpPut, m.plan.key, + fmt.Errorf("%w: upload exceeds %d parts; use a larger part size", errUtils.ErrBlobInvalid, MaxParts)) + } + data = buf[:n] + } +} + +// acquire waits for a worker slot and discards its buffer. +func acquire(ctx context.Context, free chan []byte) error { + select { + case <-free: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// acquireBuffer waits for a worker slot and returns its buffer, sized to a part. +func acquireBuffer(ctx context.Context, free chan []byte, partSize int64) ([]byte, error) { + select { + case buf := <-free: + if int64(cap(buf)) < partSize { + buf = make([]byte, partSize) + } + return buf[:partSize], nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// readPart fills buf from body, starting with a carried byte when one was already consumed. It +// returns the bytes in buf; zero means the body ended exactly on the previous boundary. +func readPart(body io.Reader, buf, carry []byte) (int, error) { + filled := copy(buf, carry) + for filled < len(buf) { + n, err := body.Read(buf[filled:]) + filled += n + if errors.Is(err, io.EOF) { + return filled, nil + } + if err != nil { + // Any other error, including a source-returned io.ErrUnexpectedEOF, is a read failure + // and must abort publication. + return 0, err + } + } + return filled, nil +} + +// uploadPart sends one part and returns its ETag. +func (m *multipart) uploadPart(ctx context.Context, number int32, data []byte) (*string, error) { + out, err := m.plan.api.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + PartNumber: aws.Int32(number), + Body: bytes.NewReader(data), + ContentLength: aws.Int64(int64(len(data))), + }, skipOptionalChecksums) + if err != nil { + return nil, classify(m.plan.request(false), err) + } + return out.ETag, nil +} + +// complete assembles the parts into the object under the write condition. SDK retries are always +// disabled, conditional or not: a retried completion whose first attempt already committed would +// meet a consumed upload ID and look like a definite failure. +func (m *multipart) complete(ctx context.Context, parts []types.CompletedPart, total int64) (*blob.ObjectInfo, error) { + cond := m.plan.opts.Condition + in := &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + MultipartUpload: &types.CompletedMultipartUpload{Parts: parts}, + } + in.IfNoneMatch, in.IfMatch = conditionHeaders(cond) + out, err := m.plan.api.CompleteMultipartUpload(ctx, in, noRetry, skipOptionalChecksums) + if err != nil { + classified := classify(m.plan.request(true), err) + if blob.IsAmbiguous(classified) { + return nil, classified + } + return nil, m.abort(ctx, classified) + } + return &blob.ObjectInfo{Key: m.plan.key, Size: total, ETag: aws.ToString(out.ETag)}, nil +} + +// abort removes the upload's stored parts. It uses a context detached from cancellation so a +// canceled operation still cleans up, and reports a failed cleanup alongside the original error. +func (m *multipart) abort(ctx context.Context, cause error) error { + abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortTimeout) + defer cancel() + _, err := m.plan.api.AbortMultipartUpload(abortCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(m.store.cfg.bucket), + Key: aws.String(m.plan.objectKey), + UploadId: aws.String(m.uploadID), + }) + if err != nil { + return errors.Join(cause, blob.NewError(blob.KindOther, "abort", m.plan.key, err)) + } + return cause +} + +// asBlobError classifies err as a plain failure unless it already is a blob error. +func asBlobError(op, key string, err error) error { + var classified *blob.Error + if errors.As(err, &classified) { + return err + } + return blob.NewError(blob.KindOther, op, key, err) +} diff --git a/pkg/artifact/blob/s3/options.go b/pkg/artifact/blob/s3/options.go new file mode 100644 index 00000000000..fb3247b5c7f --- /dev/null +++ b/pkg/artifact/blob/s3/options.go @@ -0,0 +1,203 @@ +package s3 + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Defaults for streaming uploads. +const ( + // DefaultPartSize is the single-request threshold and multipart part size. S3 requires every + // part except the last to be at least 5 MiB. + DefaultPartSize int64 = 8 << 20 + + // DefaultConcurrency is the number of multipart parts uploaded at once. + DefaultConcurrency = 4 + + // MaxParts is the S3 limit on parts in one multipart upload. + MaxParts = 10000 +) + +// ConditionalMode selects how the store treats conditional writes. +type ConditionalMode int + +// Conditional modes. +const ( + // ConditionalAuto sends If-None-Match/If-Match and trusts the endpoint. An endpoint that rejects + // the headers fails the operation with ErrBlobConditionalUnsupported. + ConditionalAuto ConditionalMode = iota + + // ConditionalProbe is like Auto, but first proves once per store that the endpoint enforces + // If-None-Match and If-Match on single-object PutObject and DeleteObject requests (see + // [Store.ProbeConditionalWrites]). The probe does not exercise CompleteMultipartUpload, so + // enforcement on multipart completion is trusted, not verified. Use it for endpoints that + // might silently ignore the headers. + ConditionalProbe + + // ConditionalDisabled never sends conditions: any conditional operation fails clearly. + ConditionalDisabled +) + +// settings holds the resolved configuration of a Store. +type settings struct { + bucket string + prefix string + tags map[string]string + partSize int64 + concurrency int + conditional ConditionalMode + opaqueKeys bool + singleFallback bool + singleOnly bool + maxSinglePut int64 + client ClientConfig + api API +} + +// Option configures a [Store]. +type Option func(*settings) + +// WithPrefix prefixes every key with prefix and a slash. Leading and trailing slashes are trimmed. +func WithPrefix(prefix string) Option { + defer perf.Track(nil, "blob.s3.WithPrefix")() + + return func(s *settings) { s.prefix = trimSlashes(prefix) } +} + +// WithRegion sets the AWS region. +func WithRegion(region string) Option { + defer perf.Track(nil, "blob.s3.WithRegion")() + + return func(s *settings) { s.client.Region = region } +} + +// WithEndpoint overrides the S3 endpoint, for example an emulator. Path-style addressing is used +// unless [WithPathStyle] says otherwise. +func WithEndpoint(url string) Option { + defer perf.Track(nil, "blob.s3.WithEndpoint")() + + return func(s *settings) { s.client.EndpointURL = url } +} + +// WithPathStyle forces path-style (true) or virtual-hosted (false) addressing. +func WithPathStyle(pathStyle bool) Option { + defer perf.Track(nil, "blob.s3.WithPathStyle")() + + return func(s *settings) { s.client.PathStyle = aws.Bool(pathStyle) } +} + +// WithIdentity authenticates through an Atmos identity. Credentials resolve lazily on first use and +// an unresolvable identity fails closed. +func WithIdentity(name string, resolver AuthResolver) Option { + defer perf.Track(nil, "blob.s3.WithIdentity")() + + return func(s *settings) { + s.client.Identity = name + s.client.Resolver = resolver + } +} + +// WithHTTPClient replaces the SDK HTTP client. +func WithHTTPClient(client aws.HTTPClient) Option { + defer perf.Track(nil, "blob.s3.WithHTTPClient")() + + return func(s *settings) { s.client.HTTPClient = client } +} + +// WithTags sets default object tags copied onto every object the store writes. Tags passed to an +// individual Put override these. +func WithTags(tags map[string]string) Option { + defer perf.Track(nil, "blob.s3.WithTags")() + + return func(s *settings) { s.tags = cloneTags(tags) } +} + +// WithPartSize sets the multipart part size and the single-request threshold. Real S3 requires at +// least 5 MiB for every part but the last; smaller sizes are for emulators and tests. +func WithPartSize(size int64) Option { + defer perf.Track(nil, "blob.s3.WithPartSize")() + + return func(s *settings) { + if size > 0 { + s.partSize = size + } + } +} + +// WithConcurrency bounds the multipart parts uploaded at once. Memory use is at most +// (concurrency + 1) part buffers. +func WithConcurrency(workers int) Option { + defer perf.Track(nil, "blob.s3.WithConcurrency")() + + return func(s *settings) { + if workers > 0 { + s.concurrency = workers + } + } +} + +// WithConditionalMode selects how conditional writes are enforced; see [ConditionalMode]. +func WithConditionalMode(mode ConditionalMode) Option { + defer perf.Track(nil, "blob.s3.WithConditionalMode")() + + return func(s *settings) { s.conditional = mode } +} + +// WithOpaqueKeys accepts any non-empty key up to the S3 limit instead of requiring portable +// relative keys. The legacy CI adapter uses it to keep its historical key patterns. +func WithOpaqueKeys() Option { + defer perf.Track(nil, "blob.s3.WithOpaqueKeys")() + + return func(s *settings) { s.opaqueKeys = true } +} + +// WithAPI injects a ready S3 client (or a test double), skipping lazy client construction. +func WithAPI(api API) Option { + defer perf.Track(nil, "blob.s3.WithAPI")() + + return func(s *settings) { s.api = api } +} + +func cloneTags(tags map[string]string) map[string]string { + if len(tags) == 0 { + return nil + } + out := make(map[string]string, len(tags)) + for k, v := range tags { + out[k] = v + } + return out +} + +func trimSlashes(s string) string { + for len(s) > 0 && s[0] == '/' { + s = s[1:] + } + for len(s) > 0 && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + return s +} + +// WithSinglePutFallback retries an explicitly unsupported multipart upload as one PutObject. +// The remaining stream is spooled to disk only if the endpoint rejects multipart creation. +func WithSinglePutFallback() Option { + defer perf.Track(nil, "blob.s3.WithSinglePutFallback")() + + return func(s *settings) { s.singleFallback = true } +} + +// MaxSinglePutSize is the largest object S3 accepts in one PutObject request. +const MaxSinglePutSize int64 = 5 << 30 + +// WithSinglePutOnly never uses a multipart upload. A body that fits one part goes out from memory; +// a larger body is spooled to a temporary file and sent with one PutObject, so memory stays bounded +// at one part. Use it when the caller's credentials may be limited to PutObject: on an SSE-KMS +// bucket, multipart operations also need kms:Decrypt, which a write-only role lacks. Bodies above +// [MaxSinglePutSize] fail with a clear error instead of being sent to be rejected. +func WithSinglePutOnly() Option { + defer perf.Track(nil, "blob.s3.WithSinglePutOnly")() + + return func(s *settings) { s.singleOnly = true } +} diff --git a/pkg/artifact/blob/s3/put.go b/pkg/artifact/blob/s3/put.go new file mode 100644 index 00000000000..dd3355cf3be --- /dev/null +++ b/pkg/artifact/blob/s3/put.go @@ -0,0 +1,232 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "sort" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// putPlan is a validated write ready to send. +type putPlan struct { + api API + key string + objectKey string + opts blob.PutOptions + tags string + partSize int64 +} + +func (p *putPlan) request(mutating bool) request { + return request{op: blob.OpPut, key: p.key, mutating: mutating, conditional: p.opts.Condition.IsConditional()} +} + +// Put implements [blob.Store]. A body that fits one part is sent with a single PutObject; anything +// larger, or of unknown length, streams as a bounded multipart upload. +func (s *Store) Put(ctx context.Context, key string, body io.Reader, opts blob.PutOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.s3.Put")() + + if body == nil { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: nil body", errUtils.ErrBlobInvalid)) + } + plan, err := s.planPut(ctx, key, opts) + if err != nil { + return nil, err + } + // A canceled context must stop the put before it consumes any of the body: a source such as an + // idle pipe would otherwise block here before an SDK call could observe the cancellation. + if err := ctx.Err(); err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, key, err) + } + body = &ctxReader{ctx: ctx, reader: body} + first, carry, err := readFirstPart(body, plan.partSize, opts.Size) + if err != nil { + return nil, blob.NewError(blob.KindOther, blob.OpPut, key, fmt.Errorf("read body: %w", err)) + } + if len(carry) == 0 { + return s.putSingle(ctx, plan, first) + } + if s.cfg.singleOnly { + return s.putFallback(ctx, plan, first, carry, body) + } + return s.putMultipart(ctx, plan, first, carry, body) +} + +// planPut validates the request and prepares the client, tags, and part size. +func (s *Store) planPut(ctx context.Context, key string, opts blob.PutOptions) (*putPlan, error) { + if err := s.validateKey(key); err != nil { + return nil, err + } + if opts.Size < blob.UnknownSize { + return nil, blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: invalid size %d", errUtils.ErrBlobInvalid, opts.Size)) + } + if s.cfg.singleOnly && opts.Size > s.singlePutLimit() { + return nil, tooLargeForSinglePut(key, opts.Size, s.singlePutLimit()) + } + tags := blob.MergeTags(s.cfg.tags, opts.Tags) + if err := blob.ValidateTags(tags); err != nil { + return nil, err + } + if err := validateCondition(key, opts.Condition); err != nil { + return nil, err + } + if err := s.Capabilities().CheckPut(storeName, key, opts.Condition); err != nil { + return nil, err + } + if err := s.ensureConditional(ctx, key, opts.Condition); err != nil { + return nil, err + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + return &putPlan{ + api: api, + key: key, + objectKey: s.objectKey(key), + opts: opts, + tags: encodeTags(tags), + partSize: effectivePartSize(opts.Size, s.cfg.partSize), + }, nil +} + +// validateCondition rejects conditions the store cannot translate into request headers. Unknown +// kinds must fail before any write: sending them unconditionally would silently drop the guard. +func validateCondition(key string, cond blob.Condition) error { + switch cond.Kind { + case blob.ConditionNone, blob.ConditionIfAbsent: + return nil + case blob.ConditionIfMatch: + if cond.ETag == "" { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: if-match requires an ETag", errUtils.ErrBlobInvalid)) + } + return nil + default: + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: unknown condition kind %d", errUtils.ErrBlobInvalid, cond.Kind)) + } +} + +// effectivePartSize grows the part size so a known-size body fits within the S3 part limit. +func effectivePartSize(size, partSize int64) int64 { + if size <= 0 { + return partSize + } + needed := size / MaxParts + if size%MaxParts != 0 { + needed++ + } + return max(partSize, needed) +} + +// readFirstPart reads up to one part. When the body continues past the part, carry holds the one +// byte already consumed from the next part; an empty carry means the body ended. +func readFirstPart(body io.Reader, partSize, size int64) (first, carry []byte, err error) { + var buf bytes.Buffer + if size >= 0 && size < partSize { + buf.Grow(int(size)) + } + if _, err := io.Copy(&buf, io.LimitReader(body, partSize)); err != nil { + return nil, nil, err + } + if int64(buf.Len()) < partSize { + return buf.Bytes(), nil, nil + } + probe := make([]byte, 1) + n, err := io.ReadFull(body, probe) + switch { + case n == 1: + return buf.Bytes(), probe, nil + case errors.Is(err, io.EOF): + return buf.Bytes(), nil, nil + default: + return nil, nil, err + } +} + +// putSingle sends a body that fits one part with one PutObject. +func (s *Store) putSingle(ctx context.Context, plan *putPlan, data []byte) (*blob.ObjectInfo, error) { + size := int64(len(data)) + if plan.opts.Size >= 0 && size != plan.opts.Size { + return nil, sizeMismatch(plan.key, size, plan.opts.Size) + } + cond := plan.opts.Condition + in := &s3.PutObjectInput{ + Bucket: aws.String(s.cfg.bucket), + Key: aws.String(plan.objectKey), + Body: bytes.NewReader(data), + ContentLength: aws.Int64(size), + ContentType: optionalString(plan.opts.ContentType), + Metadata: plan.opts.Metadata, + Tagging: optionalString(plan.tags), + } + in.IfNoneMatch, in.IfMatch = conditionHeaders(cond) + out, err := plan.api.PutObject(ctx, in, retryOptions(cond.IsConditional())...) + if err != nil { + return nil, classify(plan.request(true), err) + } + return &blob.ObjectInfo{Key: plan.key, Size: size, ETag: aws.ToString(out.ETag)}, nil +} + +// conditionHeaders maps a condition to the If-None-Match and If-Match request headers. +func conditionHeaders(cond blob.Condition) (ifNoneMatch, ifMatch *string) { + switch cond.Kind { + case blob.ConditionIfAbsent: + return aws.String("*"), nil + case blob.ConditionIfMatch: + return nil, aws.String(cond.ETag) + default: + return nil, nil + } +} + +func optionalString(value string) *string { + if value == "" { + return nil + } + return aws.String(value) +} + +// encodeTags formats tags as the URL query string S3 expects, with stable ordering. +func encodeTags(tags map[string]string) string { + if len(tags) == 0 { + return "" + } + keys := make([]string, 0, len(tags)) + for key := range tags { + keys = append(keys, key) + } + sort.Strings(keys) + values := url.Values{} + for _, key := range keys { + values.Set(key, tags[key]) + } + return values.Encode() +} + +func sizeMismatch(key string, got, declared int64) error { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, fmt.Errorf("%w: body is %d bytes, declared %d", errUtils.ErrBlobInvalid, got, declared)) +} + +// tooLargeForSinglePut reports a body that one PutObject cannot carry when multipart is off. +func tooLargeForSinglePut(key string, size, limit int64) error { + return blob.NewError(blob.KindInvalid, blob.OpPut, key, + fmt.Errorf("%w: %d bytes exceeds the %d-byte single-request limit and multipart uploads are disabled", errUtils.ErrBlobInvalid, size, limit)) +} + +// singlePutLimit is the largest body one PutObject may carry. +func (s *Store) singlePutLimit() int64 { + if s.cfg.maxSinglePut > 0 { + return s.cfg.maxSinglePut + } + return MaxSinglePutSize +} diff --git a/pkg/artifact/blob/s3/put_single_only_test.go b/pkg/artifact/blob/s3/put_single_only_test.go new file mode 100644 index 00000000000..87fa1878ec5 --- /dev/null +++ b/pkg/artifact/blob/s3/put_single_only_test.go @@ -0,0 +1,186 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +// isolateTempDir points the process temp directory at an empty directory and returns it, so a test +// can prove that the upload spool is removed. +func isolateTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"TMPDIR", "TMP", "TEMP"} { + t.Setenv(name, dir) + } + return dir +} + +func requireEmptyDir(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, "the upload spool must be removed") +} + +// With multipart off, every body goes out as one PutObject, whatever its size and whether or not +// its length is declared. No multipart or read-back request is ever issued, so a write-only role +// is enough. +func TestSinglePutOnlyNeverUsesMultipartOrReadsBack(t *testing.T) { + tests := []struct { + name string + size int + // declared reports whether the caller declares the length. + declared bool + }{ + {name: "empty", size: 0, declared: true}, + {name: "below one part", size: smallPart - 1, declared: true}, + {name: "exactly one part", size: smallPart, declared: true}, + {name: "one byte past one part", size: smallPart + 1, declared: true}, + {name: "many parts", size: smallPart*5 + 17, declared: true}, + {name: "many parts of unknown length", size: smallPart*5 + 17}, + {name: "one byte past one part of unknown length", size: smallPart + 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + for _, op := range []string{ + s3test.OpGetObject, s3test.OpHeadObject, s3test.OpGetObjectTagging, s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, s3test.OpCompleteMultipartUpload, s3test.OpAbortMultipartUpload, + } { + fake.Inject(s3test.Fault{Op: op, Times: -1, Action: s3test.FailStatus, Status: http.StatusForbidden, Code: "AccessDenied"}) + } + s := newTestStore(t, fake, WithSinglePutOnly()) + data := patterned(tt.size) + opts := blob.PutOptions{Size: blob.UnknownSize} + if tt.declared { + opts.Size = int64(len(data)) + } + + info, err := s.Put(context.Background(), "obj", bytes.NewReader(data), opts) + + require.NoError(t, err) + assert.Equal(t, int64(len(data)), info.Size) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data), "stored %d bytes, want %d", len(stored.Data), len(data)) + requests := fake.Requests() + require.Len(t, requests, 1) + assert.Equal(t, s3test.OpPutObject, requests[0].Op) + requireEmptyDir(t, tmp) + }) + } +} + +// A declared size that cannot fit one request is refused before the body is read or any request +// is sent, with an error that names the cause. +func TestSinglePutOnlyRejectsDeclaredSizeAboveTheLimit(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithSinglePutOnly()) + reader := &countingReader{size: MaxSinglePutSize + 1} + + _, err := s.Put(context.Background(), "obj", reader, blob.PutOptions{Size: MaxSinglePutSize + 1}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Contains(t, err.Error(), "multipart uploads are disabled") + assert.Zero(t, reader.read.Load(), "the body is not consumed") + assert.Empty(t, fake.Requests()) +} + +// A stream of unknown length that outgrows one request stops at the limit, sends nothing, and +// removes its spool. +func TestSinglePutOnlyStopsAnUnknownLengthStreamAtTheLimit(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + s := newTestStore(t, fake, WithSinglePutOnly()) + s.cfg.maxSinglePut = smallPart * 3 + data := patterned(smallPart*3 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) + requireEmptyDir(t, tmp) + + // A body exactly at the limit is still accepted. + exact := patterned(smallPart * 3) + _, err = s.Put(context.Background(), "obj", bytes.NewReader(exact), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(exact, stored.Data)) + requireEmptyDir(t, tmp) +} + +// A refused PutObject is reported as the upload's failure; no fallback hides it, and the spool is +// removed. +func TestSinglePutOnlyReportsPutObjectDenied(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Times: -1, Action: s3test.FailStatus, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithSinglePutOnly()) + data := patterned(smallPart*2 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.Error(t, err) + assert.Empty(t, fake.Keys()) + assert.Empty(t, fake.RequestsFor(s3test.OpCreateMultipartUpload)) + requireEmptyDir(t, tmp) +} + +// The fallback option still turns an unimplemented multipart upload into one PutObject, and it +// refuses to spool without bound. +func TestSinglePutFallbackWhenMultipartUnsupported(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + data := patterned(smallPart*2 + 3) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.NoError(t, err) + stored, ok := fake.Object("obj") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data)) + assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestSinglePutFallbackDoesNotMaskOtherMultipartErrors(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusForbidden, Code: "AccessDenied"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + data := patterned(smallPart*2 + 3) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + + require.Error(t, err) + assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) +} + +func TestSinglePutFallbackStopsAtTheLimit(t *testing.T) { + tmp := isolateTempDir(t) + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Status: http.StatusNotImplemented, Code: "NotImplemented"}) + s := newTestStore(t, fake, WithSinglePutFallback()) + s.cfg.maxSinglePut = smallPart * 2 + data := patterned(smallPart*2 + 1) + + _, err := s.Put(context.Background(), "obj", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.RequestsFor(s3test.OpPutObject)) + requireEmptyDir(t, tmp) +} diff --git a/pkg/artifact/blob/s3/put_test.go b/pkg/artifact/blob/s3/put_test.go new file mode 100644 index 00000000000..3f5fe0bb6f3 --- /dev/null +++ b/pkg/artifact/blob/s3/put_test.go @@ -0,0 +1,280 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func TestSmallBodyUsesOneSignedRequest(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("tiny"), blob.PutOptions{ContentType: "text/plain", Metadata: map[string]string{"Owner": "atmos"}}) + + requests := fake.Requests() + require.Len(t, requests, 1, "no multipart machinery for a small body") + assert.Equal(t, s3test.OpPutObject, requests[0].Op) + assert.Equal(t, "text/plain", requests[0].Header.Get("Content-Type")) + assert.Equal(t, "atmos", requests[0].Header.Get("X-Amz-Meta-Owner")) + assert.Empty(t, requests[0].Header.Get("If-None-Match"), "unconditional puts carry no condition") + assert.Empty(t, fake.OpenUploads()) +} + +func TestZeroLengthObjectIsOnePutWithEmptyBody(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "empty", nil, blob.PutOptions{}) + requests := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, requests, 1) + assert.Equal(t, int64(0), requests[0].Bytes) + object, ok := fake.Object("empty") + require.True(t, ok) + assert.Zero(t, object.Size) +} + +func TestBodyExactlyOnePartIsNotMultipart(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "exact", patterned(smallPart), blob.PutOptions{}) + assert.Empty(t, fake.RequestsFor(s3test.OpCreateMultipartUpload)) + assert.Len(t, fake.RequestsFor(s3test.OpPutObject), 1) +} + +func TestMultipartRequestOrderAndContent(t *testing.T) { + fake := newFake(t, s3test.WithMinPartSize(smallPart)) + s := newTestStore(t, fake, WithConcurrency(1)) + data := patterned(smallPart*3 + 17) + info := putBytes(t, s, "big", data, blob.PutOptions{ + ContentType: "application/octet-stream", + Tags: map[string]string{"team": "platform"}, + Condition: blob.IfAbsent(), + }) + + var ops []string + for _, req := range fake.Requests() { + ops = append(ops, req.Op) + } + assert.Equal(t, []string{ + s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, s3test.OpUploadPart, s3test.OpUploadPart, s3test.OpUploadPart, + s3test.OpCompleteMultipartUpload, + }, ops, "create, then every part, then complete last") + + create := fake.RequestsFor(s3test.OpCreateMultipartUpload)[0] + assert.Equal(t, "team=platform", create.Header.Get("X-Amz-Tagging"), "tags are set when the upload is created") + assert.Empty(t, create.Header.Get("If-None-Match"), "the condition belongs to the completing request") + complete := fake.RequestsFor(s3test.OpCompleteMultipartUpload)[0] + assert.Equal(t, "*", complete.Header.Get("If-None-Match")) + for i, part := range fake.RequestsFor(s3test.OpUploadPart) { + assert.Equal(t, []string{string(rune('1' + i))}, part.Query["partNumber"], "parts are numbered in order") + } + + object, ok := fake.Object("big") + require.True(t, ok) + assert.Equal(t, data, object.Data) + assert.Equal(t, info.ETag, object.ETag) + assert.Contains(t, object.ETag, "-4", "a multipart ETag records its part count and is not a payload checksum") + assert.Equal(t, map[string]string{"team": "platform"}, object.Tags) + assert.Equal(t, "application/octet-stream", object.ContentType) + assert.Equal(t, 1, fake.CompletedUploads()) + assert.Empty(t, fake.OpenUploads()) +} + +func TestUnknownSizeStreamsAsMultipart(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + data := patterned(smallPart*2 + 5) + info, err := s.Put(context.Background(), "stream", bytes.NewReader(data), blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), info.Size) + object, _ := fake.Object("stream") + assert.Equal(t, data, object.Data) + assert.Len(t, fake.RequestsFor(s3test.OpUploadPart), 3) +} + +func TestKnownSizeGrowsPartsToStayUnderPartLimit(t *testing.T) { + assert.Equal(t, int64(10), effectivePartSize(10*MaxParts, 1)) + assert.Equal(t, int64(smallPart), effectivePartSize(100, smallPart)) + assert.Equal(t, int64(smallPart), effectivePartSize(blob.UnknownSize, smallPart)) +} + +func TestTagsAreCopiedToEveryObject(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithTags(map[string]string{"managed-by": "atmos", "env": "dev"})) + + putBytes(t, s, "single", []byte("x"), blob.PutOptions{}) + putBytes(t, s, "multi", patterned(smallPart*2+1), blob.PutOptions{}) + putBytes(t, s, "override", []byte("x"), blob.PutOptions{Tags: map[string]string{"env": "prod", "extra": "1"}}) + putBytes(t, s, "empty", nil, blob.PutOptions{}) + + want := map[string]map[string]string{ + "single": {"managed-by": "atmos", "env": "dev"}, + "multi": {"managed-by": "atmos", "env": "dev"}, + "override": {"managed-by": "atmos", "env": "prod", "extra": "1"}, + "empty": {"managed-by": "atmos", "env": "dev"}, + } + for key, tags := range want { + object, ok := fake.Object(key) + require.True(t, ok, key) + assert.Equal(t, tags, object.Tags, "tags on %s", key) + } +} + +func TestTooManyTagsRejectedBeforeAnyRequest(t *testing.T) { + fake := newFake(t) + tags := map[string]string{} + for _, key := range []string{"a", "b", "c", "d", "e", "f"} { + tags[key] = "v" + } + s := newTestStore(t, fake, WithTags(tags)) + _, err := s.Put(context.Background(), "k", bytes.NewReader(nil), blob.PutOptions{Tags: map[string]string{"g": "v", "h": "v", "i": "v", "j": "v", "k": "v"}}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) +} + +func TestSizeMismatchIsRejectedAndMultipartAborted(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "short", bytes.NewReader([]byte("ab")), blob.PutOptions{Size: 9}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests(), "a single-part size mismatch is caught before sending") + + data := patterned(smallPart*2 + 3) + _, err = s.Put(context.Background(), "long", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data) - 1)}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.OpenUploads(), "the half-built upload was aborted") + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestCanceledOperationAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Action: s3test.Delay, Delay: 200 * time.Millisecond, Times: -1}) + s := newTestStore(t, fake, WithConcurrency(2)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + source := &countingReader{size: 64 * smallPart, onRead: func(read int64) { + if read > 3*smallPart { + cancel() + } + }} + _, err := s.Put(ctx, "canceled", source, blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + assert.False(t, blob.IsAmbiguous(err), "a canceled part upload cannot have created the object") + + assert.Empty(t, fake.OpenUploads(), "no multipart upload may outlive its canceled operation") + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestFailedSourceAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + boom := errors.New("disk vanished") + body := &errAfterReader{data: patterned(smallPart*2 + 10), err: boom} + _, err := s.Put(context.Background(), "k", body, blob.PutOptions{Size: blob.UnknownSize}) + require.ErrorIs(t, err, boom) + assert.Empty(t, fake.OpenUploads()) + assert.Equal(t, 1, fake.AbortedUploads()) + assert.Empty(t, fake.Keys()) +} + +func TestFailedPartAbortsItsMultipartUpload(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Skip: 1, Times: -1, Status: 400, Code: "InvalidPart"}) + s := newTestStore(t, fake, WithConcurrency(1)) + _, err := s.Put(context.Background(), "k", bytes.NewReader(patterned(smallPart*4)), blob.PutOptions{Size: blob.UnknownSize}) + require.Error(t, err) + assert.Empty(t, fake.OpenUploads()) + assert.Equal(t, 1, fake.AbortedUploads()) +} + +func TestLostConditionAtCompletionAbortsTheUpload(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Seed("taken", []byte("existing"), nil) + + _, err := s.Put(context.Background(), "taken", bytes.NewReader(patterned(smallPart*2+1)), blob.PutOptions{Size: blob.UnknownSize, Condition: blob.IfAbsent()}) + require.Error(t, err) + assert.True(t, blob.IsPreconditionFailed(err), "%v", err) + assert.Empty(t, fake.OpenUploads(), "the losing writer cleans up its parts") + object, _ := fake.Object("taken") + assert.Equal(t, []byte("existing"), object.Data) +} + +func TestMemoryStaysBoundedWhileStreaming(t *testing.T) { + const ( + part = 1 << 20 + total = 48 << 20 + concurrency = 3 + ) + fake := newFake(t, s3test.WithDiscardBodies()) + fake.Inject(s3test.Fault{Op: s3test.OpUploadPart, Action: s3test.Delay, Delay: 2 * time.Millisecond, Times: -1}) + s := newTestStore(t, fake, WithPartSize(part), WithConcurrency(concurrency)) + + var maxInFlight atomic.Int64 + source := &countingReader{size: total} + source.onRead = func(read int64) { + inFlight := read - fake.PartBytesReceived() + for { + current := maxInFlight.Load() + if inFlight <= current || maxInFlight.CompareAndSwap(current, inFlight) { + return + } + } + } + info, err := s.Put(context.Background(), "huge", source, blob.PutOptions{Size: blob.UnknownSize}) + require.NoError(t, err) + assert.Equal(t, int64(total), info.Size) + + bound := int64(concurrency+1) * part + assert.LessOrEqual(t, maxInFlight.Load(), bound, "bytes read but not yet uploaded must stay within the buffer pool, not the object size") + assert.LessOrEqual(t, fake.MaxConcurrentParts(), concurrency, "worker concurrency is bounded") + assert.GreaterOrEqual(t, fake.MaxConcurrentParts(), 2, "parts do upload in parallel") + assert.Equal(t, int64(total), fake.PartBytesReceived()) +} + +// errAfterReader yields data and then fails. +type errAfterReader struct { + data []byte + err error +} + +func (r *errAfterReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +func TestInvalidConditionsAreRejectedBeforeAnyRequest(t *testing.T) { + cases := map[string]blob.Condition{ + "unknown kind": {Kind: blob.ConditionKind(99)}, + "negative kind": {Kind: blob.ConditionKind(-1)}, + "if-match no etag": {Kind: blob.ConditionIfMatch}, + "unknown with etag": {Kind: blob.ConditionKind(7), ETag: `"abc"`}, + } + for name, cond := range cases { + t.Run(name, func(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", bytes.NewReader([]byte("x")), blob.PutOptions{Size: 1, Condition: cond}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests(), "an invalid condition never reaches the endpoint") + }) + } +} diff --git a/pkg/artifact/blob/s3/read.go b/pkg/artifact/blob/s3/read.go new file mode 100644 index 00000000000..bc4100de55e --- /dev/null +++ b/pkg/artifact/blob/s3/read.go @@ -0,0 +1,144 @@ +package s3 + +import ( + "context" + "fmt" + "math" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Number parsing parameters for Content-Range totals. +const ( + decimalBase = 10 + bitSize64 = 64 +) + +// Get implements [blob.Store]. +func (s *Store) Get(ctx context.Context, key string, opts blob.GetOptions) (*blob.Object, error) { + defer perf.Track(nil, "blob.s3.Get")() + + if err := s.validateKey(key); err != nil { + return nil, err + } + if opts.Offset < 0 { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: negative offset", errUtils.ErrBlobInvalid)) + } + if opts.Length > 0 && opts.Offset > math.MaxInt64-(opts.Length-1) { + return nil, blob.NewError(blob.KindInvalid, blob.OpGet, key, fmt.Errorf("%w: range end overflows", errUtils.ErrBlobInvalid)) + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + in := &s3.GetObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key)), Range: rangeHeader(opts)} + out, err := api.GetObject(ctx, in) + if err != nil { + return nil, classify(request{op: blob.OpGet, key: key}, err) + } + info := blob.ObjectInfo{ + Key: key, + Size: objectSize(out), + ETag: aws.ToString(out.ETag), + ContentType: aws.ToString(out.ContentType), + LastModified: aws.ToTime(out.LastModified), + Metadata: out.Metadata, + } + return &blob.Object{ObjectInfo: info, Body: out.Body}, nil +} + +// rangeHeader renders a partial read as an HTTP Range header. +func rangeHeader(opts blob.GetOptions) *string { + if !opts.IsRange() { + return nil + } + if opts.Length > 0 { + return aws.String(fmt.Sprintf("bytes=%d-%d", opts.Offset, opts.Offset+(opts.Length-1))) + } + return aws.String(fmt.Sprintf("bytes=%d-", opts.Offset)) +} + +// objectSize returns the full object size, using Content-Range for partial responses. +func objectSize(out *s3.GetObjectOutput) int64 { + if out.ContentRange != nil { + if _, total, ok := strings.Cut(aws.ToString(out.ContentRange), "/"); ok { + if size, err := strconv.ParseInt(total, decimalBase, bitSize64); err == nil { + return size + } + } + } + return aws.ToInt64(out.ContentLength) +} + +// Stat implements [blob.Store]. +// +// With IncludeTags, the tags are read for the exact object version HeadObject returned when the +// bucket reports a real version ID, so metadata and tags describe the same object even if the key +// is overwritten between the two requests. That read requires s3:GetObjectVersionTagging; a denial +// fails the call rather than falling back to the current version's tags. Unversioned buckets and +// the replaceable "null" version read the current tags with s3:GetObjectTagging, so a concurrent +// overwrite can still pair metadata from one write with tags from another there. +func (s *Store) Stat(ctx context.Context, key string, opts blob.StatOptions) (*blob.ObjectInfo, error) { + defer perf.Track(nil, "blob.s3.Stat")() + + if err := s.validateKey(key); err != nil { + return nil, err + } + api, err := s.client(ctx) + if err != nil { + return nil, err + } + out, err := api.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))}) + if err != nil { + return nil, classify(request{op: blob.OpStat, key: key}, err) + } + info := &blob.ObjectInfo{ + Key: key, + Size: aws.ToInt64(out.ContentLength), + ETag: aws.ToString(out.ETag), + ContentType: aws.ToString(out.ContentType), + LastModified: aws.ToTime(out.LastModified), + Metadata: out.Metadata, + } + if opts.IncludeTags { + if info.Tags, err = s.tags(ctx, api, key, out.VersionId); err != nil { + return nil, err + } + } + return info, nil +} + +// nullVersionID is the version ID S3 reports for objects written while versioning was never +// enabled or was suspended. A later write replaces that version in place, so it does not identify +// one immutable object. +const nullVersionID = "null" + +// tags loads the object tags, pinned to versionID when it names an immutable object version. +func (s *Store) tags(ctx context.Context, api API, key string, versionID *string) (map[string]string, error) { + in := &s3.GetObjectTaggingInput{Bucket: aws.String(s.cfg.bucket), Key: aws.String(s.objectKey(key))} + if pinnedVersion(versionID) { + in.VersionId = versionID + } + out, err := api.GetObjectTagging(ctx, in) + if err != nil { + return nil, classify(request{op: "tags", key: key}, err) + } + tags := make(map[string]string, len(out.TagSet)) + for _, tag := range out.TagSet { + tags[aws.ToString(tag.Key)] = aws.ToString(tag.Value) + } + return tags, nil +} + +// pinnedVersion reports whether a HeadObject version ID identifies one immutable object version. +func pinnedVersion(versionID *string) bool { + id := aws.ToString(versionID) + return id != "" && id != nullVersionID +} diff --git a/pkg/artifact/blob/s3/read_version_test.go b/pkg/artifact/blob/s3/read_version_test.go new file mode 100644 index 00000000000..9027f4f47da --- /dev/null +++ b/pkg/artifact/blob/s3/read_version_test.go @@ -0,0 +1,91 @@ +package s3 + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" +) + +const versionedBucket = "versioned-bucket" + +// expectHead makes the mock report a fixed object with the given HeadObject version ID. +func expectHead(api *MockAPI, versionID *string) { + api.EXPECT(). + HeadObject(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, in *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return &s3.HeadObjectOutput{ContentLength: aws.Int64(1), ETag: aws.String(`"etag"`), VersionId: versionID}, nil + }). + Times(1) +} + +// newVersionedStore returns a store over the mocked API. +func newVersionedStore(t *testing.T, api API) *Store { + t.Helper() + s, err := NewFromClient(api, versionedBucket, WithPartSize(smallPart)) + require.NoError(t, err) + return s +} + +func TestStatPinsTagReadToTheHeadedVersion(t *testing.T) { + tests := []struct { + name string + versionID *string + want *string + }{ + {name: "versioned bucket pins the read", versionID: aws.String("3HL4kqtJlcpXroDTDmJ"), want: aws.String("3HL4kqtJlcpXroDTDmJ")}, + {name: "unversioned bucket reads current tags", versionID: nil, want: nil}, + {name: "empty version reads current tags", versionID: aws.String(""), want: nil}, + {name: "replaceable null version reads current tags", versionID: aws.String(nullVersionID), want: nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) + expectHead(api, tc.versionID) + var tagging []*s3.GetObjectTaggingInput + api.EXPECT(). + GetObjectTagging(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, in *s3.GetObjectTaggingInput, _ ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + tagging = append(tagging, in) + return &s3.GetObjectTaggingOutput{TagSet: []types.Tag{{Key: aws.String("a"), Value: aws.String("b")}}}, nil + }). + Times(1) + s := newVersionedStore(t, api) + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"a": "b"}, info.Tags) + require.Len(t, tagging, 1) + assert.Equal(t, aws.ToString(tc.want), aws.ToString(tagging[0].VersionId)) + assert.Equal(t, tc.want == nil, tagging[0].VersionId == nil) + assert.Equal(t, versionedBucket, aws.ToString(tagging[0].Bucket)) + }) + } +} + +func TestStatDoesNotFallBackWhenVersionedTagReadFails(t *testing.T) { + api := NewMockAPI(gomock.NewController(t)) + expectHead(api, aws.String("v1")) + // A denied version-specific read must not retry against the current version, so exactly one + // tagging call is allowed and it must carry the headed version. + api.EXPECT(). + GetObjectTagging(gomock.Any(), gomock.Cond(func(x any) bool { + in, ok := x.(*s3.GetObjectTaggingInput) + return ok && aws.ToString(in.VersionId) == "v1" + })). + Return(nil, errUtils.ErrBlobTaggingDenied). + Times(1) + s := newVersionedStore(t, api) + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.ErrorIs(t, err, errUtils.ErrBlobTaggingDenied) + assert.Nil(t, info) +} diff --git a/pkg/artifact/blob/s3/s3test/chunked.go b/pkg/artifact/blob/s3/s3test/chunked.go new file mode 100644 index 00000000000..7ab31e1155f --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/chunked.go @@ -0,0 +1,64 @@ +package s3test + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// hexBase is the radix of aws-chunked chunk sizes. +const hexBase = 16 + +// readBody returns the request payload, decoding aws-chunked framing when the SDK used it. +func readBody(r *http.Request) ([]byte, error) { + raw, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("%w: read request body: %w", errUtils.ErrBlobOperationFailed, err) + } + if !isChunked(r) { + return raw, nil + } + return decodeAWSChunked(raw) +} + +func isChunked(r *http.Request) bool { + return strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || + strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") +} + +// decodeAWSChunked strips the aws-chunked framing: "[;ext]\r\n\r\n" repeated, ended +// by a zero-size chunk and optional trailers. +func decodeAWSChunked(raw []byte) ([]byte, error) { + reader := bufio.NewReader(bytes.NewReader(raw)) + var out bytes.Buffer + for { + line, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("%w: read chunk header: %w", errUtils.ErrBlobInvalidChunk, err) + } + sizeText, _, _ := strings.Cut(strings.TrimSpace(line), ";") + size, err := strconv.ParseInt(sizeText, hexBase, bitSize64) + if err != nil { + return nil, fmt.Errorf("%w: parse chunk size %q: %w", errUtils.ErrBlobInvalidChunk, sizeText, err) + } + if size == 0 { + return out.Bytes(), nil + } + if _, err := io.CopyN(&out, reader, size); err != nil { + return nil, fmt.Errorf("%w: read chunk data: %w", errUtils.ErrBlobInvalidChunk, err) + } + var terminator [2]byte + if _, err := io.ReadFull(reader, terminator[:]); err != nil { + return nil, fmt.Errorf("%w: read chunk terminator: %w", errUtils.ErrBlobInvalidChunk, err) + } + if terminator != [2]byte{'\r', '\n'} { + return nil, errUtils.ErrBlobInvalidChunkTerminator + } + } +} diff --git a/pkg/artifact/blob/s3/s3test/cond.go b/pkg/artifact/blob/s3/s3test/cond.go new file mode 100644 index 00000000000..d9e472d12eb --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/cond.go @@ -0,0 +1,83 @@ +package s3test + +import ( + "net/http" + "strings" +) + +// conditionResult is the outcome of evaluating If-None-Match and If-Match. +type conditionResult int + +const ( + conditionOK conditionResult = iota + conditionFailed + conditionMissing + conditionNotImplemented +) + +// evaluate checks the conditional headers of r against the current object (nil when absent). +func (s *Server) evaluate(r *http.Request, current *object) conditionResult { + ifNoneMatch := r.Header.Get("If-None-Match") + ifMatch := r.Header.Get("If-Match") + if ifNoneMatch == "" && ifMatch == "" { + return conditionOK + } + switch s.cfg.conditional { + case ConditionalIgnore: + return conditionOK + case ConditionalNotImplemented: + return conditionNotImplemented + case ConditionalEnforce: + } + return enforce(ifNoneMatch, ifMatch, current) +} + +// enforce applies If-None-Match and If-Match to the current object (nil when absent). +func enforce(ifNoneMatch, ifMatch string, current *object) conditionResult { + if ifNoneMatch != "" && current != nil && (ifNoneMatch == "*" || etagMatches(ifNoneMatch, current.etag)) { + return conditionFailed + } + if ifMatch == "" { + return conditionOK + } + switch { + case current == nil: + return conditionMissing + case ifMatch != "*" && !strongETagMatches(ifMatch, current.etag): + return conditionFailed + default: + return conditionOK + } +} + +// strongETagMatches rejects weak validators for If-Match. +func strongETagMatches(header, etag string) bool { + if strings.HasPrefix(strings.TrimSpace(header), "W/") || strings.HasPrefix(strings.TrimSpace(etag), "W/") { + return false + } + return etagMatches(header, etag) +} + +// etagMatches compares ETags ignoring quotes and the weak prefix. +func etagMatches(header, etag string) bool { + return normalizeETag(header) == normalizeETag(etag) +} + +func normalizeETag(etag string) string { + return strings.Trim(strings.TrimPrefix(strings.TrimSpace(etag), "W/"), `"`) +} + +// reject writes the response for a failed condition and reports whether it did. +func reject(w http.ResponseWriter, result conditionResult) bool { + switch result { + case conditionFailed: + writeError(w, http.StatusPreconditionFailed, "PreconditionFailed", "At least one of the pre-conditions you specified did not hold") + case conditionMissing: + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + case conditionNotImplemented: + writeError(w, http.StatusNotImplemented, "NotImplemented", "A header you provided implies functionality that is not implemented") + case conditionOK: + return false + } + return true +} diff --git a/pkg/artifact/blob/s3/s3test/doc.go b/pkg/artifact/blob/s3/s3test/doc.go new file mode 100644 index 00000000000..e869b90bf6d --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/doc.go @@ -0,0 +1,18 @@ +// Package s3test is an in-process, S3-compatible HTTP server for tests. It speaks the S3 REST +// protocol over a real listener, so the AWS SDK exercises its real request and response handling +// against it, and it models the behavior publication code depends on rather than always succeeding: +// +// - ListObjectsV2 pagination (MaxKeys, ContinuationToken, delimiter roll-up, a configurable page cap). +// - Conditional PUT, multipart completion, and DELETE with If-None-Match and If-Match, answering +// 412 on a lost race and 404 for If-Match on a missing key, plus endpoints that ignore the +// headers or answer 501. +// - ETags (truncated SHA-256 for single puts, and a digest with a part count for multipart uploads). +// - Multipart create, upload-part, complete, and abort, with tracking of open and aborted uploads +// and of concurrent part uploads. +// - Object tags (the x-amz-tagging header and GetObjectTagging) and user metadata. +// - Byte ranges, HEAD, and zero-length objects. +// - Fault injection: fail after N requests, drop the response after the mutation committed +// (response ambiguity), drop it before the commit, and delay. +// +// Every request is recorded so tests can assert ordering and headers. Signatures are not verified. +package s3test diff --git a/pkg/artifact/blob/s3/s3test/emulator.go b/pkg/artifact/blob/s3/s3test/emulator.go new file mode 100644 index 00000000000..eccd62b9ace --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/emulator.go @@ -0,0 +1,49 @@ +package s3test + +import ( + "os" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Environment variables that opt tests in to a real S3-compatible endpoint (Floci, MinIO, AWS). +// Credentials come from the usual AWS_* variables and the bucket must already exist. +const ( + EmulatorEndpointEnv = "ATMOS_BLOB_S3_ENDPOINT" + EmulatorBucketEnv = "ATMOS_BLOB_S3_BUCKET" + EmulatorRegionEnv = "ATMOS_BLOB_S3_REGION" +) + +// Emulator describes a real S3-compatible endpoint chosen through the environment. +type Emulator struct { + Endpoint string + Bucket string + Region string +} + +// lookup returns the value of an opt-in test variable, or empty when it is unset. +func lookup(name string) string { + value, _ := os.LookupEnv(name) + return value +} + +// EmulatorFromEnv returns the opt-in emulator, or skips the test when the environment does not +// name one. Emulator-backed tests supplement the fake and must never be required by ordinary runs. +func EmulatorFromEnv(t testing.TB) Emulator { + defer perf.Track(nil, "s3test.EmulatorFromEnv")() + + t.Helper() + emulator := Emulator{ + Endpoint: lookup(EmulatorEndpointEnv), + Bucket: lookup(EmulatorBucketEnv), + Region: lookup(EmulatorRegionEnv), + } + if emulator.Endpoint == "" || emulator.Bucket == "" { + t.Skipf("set %s and %s to run against an S3-compatible endpoint", EmulatorEndpointEnv, EmulatorBucketEnv) + } + if emulator.Region == "" { + emulator.Region = "us-east-1" + } + return emulator +} diff --git a/pkg/artifact/blob/s3/s3test/fault.go b/pkg/artifact/blob/s3/s3test/fault.go new file mode 100644 index 00000000000..5a6e00bd892 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/fault.go @@ -0,0 +1,138 @@ +package s3test + +import ( + "net/http" + "strings" + "sync" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// FaultAction is what an injected fault does to a matching request. +type FaultAction int8 + +// Fault actions. +const ( + // FailStatus answers with an S3 error and does not apply the request. + FailStatus FaultAction = iota + + // DropAfterCommit applies the request, then closes the connection without answering. The client + // sees a transport error although the mutation happened: response ambiguity. + DropAfterCommit + + // DropBeforeCommit closes the connection without applying the request. + DropBeforeCommit + + // Delay sleeps before handling the request normally. + Delay +) + +// Fault describes a failure to inject. A request matches when Op and KeyPrefix match (empty +// matches anything). The first Skip matches pass through untouched ("fail after N requests"), then +// the fault applies Times times; Times zero means once and a negative Times means forever. +type Fault struct { + // Op is the S3 operation name, for example "PutObject" or "CompleteMultipartUpload". + Op string + + // KeyPrefix restricts the fault to keys beginning with it. + KeyPrefix string + + // Skip is the number of matching requests that pass before the fault applies. + Skip int32 + + // Times is how many requests the fault applies to. + Times int32 + + // Action is the fault behavior. + Action FaultAction + + // Status and Code describe the S3 error for [FailStatus]; they default to 500 InternalError. + Status int32 + Code string + + // Delay is the pause for [Delay]. + Delay time.Duration +} + +// faultRule is a registered fault with its remaining counters. +type faultRule struct { + Fault + skip int32 + times int32 +} + +// faults holds the registered rules. +type faults struct { + mu sync.Mutex + rules []*faultRule +} + +func (f *faults) add(fault *Fault) { + f.mu.Lock() + defer f.mu.Unlock() + times := fault.Times + if times == 0 { + times = 1 + } + f.rules = append(f.rules, &faultRule{Fault: *fault, skip: fault.Skip, times: times}) +} + +func (f *faults) clear() { + f.mu.Lock() + defer f.mu.Unlock() + f.rules = nil +} + +// match returns the fault to apply to req, consuming its counters. +func (f *faults) match(req *Request) *Fault { + f.mu.Lock() + defer f.mu.Unlock() + for _, rule := range f.rules { + if rule.Op != "" && rule.Op != req.Op { + continue + } + if rule.KeyPrefix != "" && !strings.HasPrefix(req.Key, rule.KeyPrefix) { + continue + } + if rule.skip > 0 { + rule.skip-- + continue + } + if rule.times == 0 { + continue + } + if rule.times > 0 { + rule.times-- + } + fault := rule.Fault + return &fault + } + return nil +} + +// status returns the HTTP status and S3 code of a FailStatus fault. +func (f *Fault) status() (int, string) { + status, code := int(f.Status), f.Code + if status == 0 { + status = http.StatusInternalServerError + } + if code == "" { + code = "InternalError" + } + return status, code +} + +// Inject registers a fault. +func (s *Server) Inject(fault Fault) { + defer perf.Track(nil, "s3test.Server.Inject")() + + s.faults.add(&fault) +} + +// ClearFaults removes every registered fault. +func (s *Server) ClearFaults() { + defer perf.Track(nil, "s3test.Server.ClearFaults")() + + s.faults.clear() +} diff --git a/pkg/artifact/blob/s3/s3test/list.go b/pkg/artifact/blob/s3/s3test/list.go new file mode 100644 index 00000000000..882a6e0fcdf --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/list.go @@ -0,0 +1,163 @@ +package s3test + +import ( + "encoding/base64" + "encoding/xml" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +type listResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Delimiter string `xml:"Delimiter,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + Contents []listContent `xml:"Contents"` + CommonPrefixes []commonPrefixXM `xml:"CommonPrefixes"` +} + +type listContent struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type commonPrefixXM struct { + Prefix string `xml:"Prefix"` +} + +// listEntry is one item of a sorted listing: an object or a rolled-up prefix. +type listEntry struct { + token string + object *object +} + +func (s *Server) listObjects(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + prefix, delimiter := query.Get("prefix"), query.Get("delimiter") + limit := min(s.cfg.pageCap, maxKeysCap) + if raw := query.Get("max-keys"); raw != "" { + requested, err := strconv.Atoi(raw) + if err != nil || requested < 0 { + writeError(w, http.StatusBadRequest, "InvalidArgument", "max-keys must be a non-negative integer") + return + } + limit = min(limit, requested) + } + cursor, err := decodeToken(query.Get("continuation-token")) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidArgument", "The continuation token provided is incorrect") + return + } + if start := query.Get("start-after"); start > cursor { + cursor = start + } + page := s.page(prefix, delimiter, cursor, limit) + + encode := func(v string) string { return v } + if query.Get("encoding-type") == "url" { + encode = func(v string) string { return strings.ReplaceAll(url.QueryEscape(v), "+", "%20") } + } + result := listResult{ + Name: s.cfg.bucket, Prefix: encode(prefix), Delimiter: delimiter, EncodingType: query.Get("encoding-type"), + MaxKeys: limit, ContinuationToken: query.Get("continuation-token"), + } + for _, entry := range page.entries { + if entry.object == nil { + result.CommonPrefixes = append(result.CommonPrefixes, commonPrefixXM{Prefix: encode(entry.token)}) + continue + } + result.Contents = append(result.Contents, listContent{ + Key: encode(entry.object.key), LastModified: entry.object.modified.UTC().Format("2006-01-02T15:04:05.000Z"), + ETag: entry.object.etag, Size: entry.object.size, StorageClass: "STANDARD", + }) + } + result.KeyCount = len(page.entries) + if page.next != "" { + result.IsTruncated = true + result.NextContinuationToken = base64.RawURLEncoding.EncodeToString([]byte(page.next)) + } + writeXML(w, result) +} + +func decodeToken(token string) (string, error) { + if token == "" { + return "", nil + } + raw, err := base64.RawURLEncoding.DecodeString(token) + return string(raw), err +} + +type listPage struct { + entries []listEntry + next string +} + +// snapshotUnder returns the sorted keys under prefix and a copy of their objects. +func (s *Server) snapshotUnder(prefix string) ([]string, map[string]object) { + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.objects)) + for key := range s.objects { + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + snapshot := make(map[string]object, len(keys)) + for _, key := range keys { + snapshot[key] = *s.objects[key] + } + return keys, snapshot +} + +// rollUp returns the ordering token of a key: the key itself, or its common prefix when the +// delimiter occurs after the listing prefix. +func rollUp(key, prefix, delimiter string) (string, bool) { + if delimiter == "" { + return key, false + } + rest := key[len(prefix):] + idx := strings.Index(rest, delimiter) + if idx < 0 { + return key, false + } + return prefix + rest[:idx+len(delimiter)], true +} + +// page builds one page: keys under prefix, delimiter roll-up, resumed after the cursor token. +func (s *Server) page(prefix, delimiter, cursor string, limit int) listPage { + keys, snapshot := s.snapshotUnder(prefix) + + var page listPage + last := "" + for _, key := range keys { + token, rolled := rollUp(key, prefix, delimiter) + if (cursor != "" && token <= cursor) || (rolled && token == last) { + continue + } + if len(page.entries) == limit { + page.next = last + return page + } + entry := listEntry{token: token} + if !rolled { + obj := snapshot[key] + entry.object = &obj + } + page.entries = append(page.entries, entry) + last = token + } + return page +} diff --git a/pkg/artifact/blob/s3/s3test/multipart.go b/pkg/artifact/blob/s3/s3test/multipart.go new file mode 100644 index 00000000000..af32fce04d0 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/multipart.go @@ -0,0 +1,196 @@ +package s3test + +import ( + "encoding/xml" + "fmt" + "net/http" + "strconv" + "strings" +) + +// upload is an open multipart upload. +type upload struct { + id string + key string + contentType string + metadata map[string]string + tags map[string]string + parts map[int]part +} + +type part struct { + etag string + digest string + size int64 + data []byte +} + +type initiateResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadID string `xml:"UploadId"` +} + +type completeRequest struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + } `xml:"Part"` +} + +type completeResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +func (s *Server) createUpload(w http.ResponseWriter, r *http.Request, key string) { + attrs, err := attributes(r) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidTag", err.Error()) + return + } + s.mu.Lock() + id := s.newID() + s.uploads[id] = &upload{id: id, key: key, contentType: attrs.contentType, metadata: attrs.metadata, tags: attrs.tags, parts: map[int]part{}} + s.mu.Unlock() + writeXML(w, initiateResult{Bucket: s.cfg.bucket, Key: key, UploadID: id}) +} + +// maxPartNumber is the highest multipart part number S3 accepts. +const maxPartNumber = 10000 + +// trackConcurrency records the peak number of UploadPart requests in flight. The returned function +// ends the request's contribution. +func (s *Server) trackConcurrency() func() { + inFlight := s.partsInFlight.Add(1) + for { + current := s.maxPartsInFlow.Load() + if inFlight <= current || s.maxPartsInFlow.CompareAndSwap(current, inFlight) { + break + } + } + return func() { s.partsInFlight.Add(-1) } +} + +func (s *Server) uploadPart(w http.ResponseWriter, r *http.Request, key string) { + defer s.trackConcurrency()() + + data, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + number, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil || number < 1 || number > maxPartNumber { + writeError(w, http.StatusBadRequest, "InvalidArgument", "Part number must be an integer between 1 and 10000") + return + } + s.partBytes.Add(int64(len(data))) + p := newPart(data, s.cfg.discard) + + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.uploads[r.URL.Query().Get("uploadId")] + if !ok || up.key != key { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + up.parts[number] = p + w.Header().Set("ETag", p.etag) + w.WriteHeader(http.StatusOK) +} + +// newPart builds a stored part, dropping the bytes when bodies are discarded. +func newPart(data []byte, discard bool) part { + p := part{etag: quotedDigest(data), digest: digestHex(data), size: int64(len(data))} + if !discard { + p.data = data + } + return p +} + +func (s *Server) completeUpload(w http.ResponseWriter, r *http.Request, key string) { + body, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + var req completeRequest + if err := xml.Unmarshal(body, &req); err != nil { + writeError(w, http.StatusBadRequest, "MalformedXML", err.Error()) + return + } + s.mu.Lock() + defer s.mu.Unlock() + up, ok := s.uploads[r.URL.Query().Get("uploadId")] + if !ok || up.key != key { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + obj, problem := s.assemble(up, req) + if problem != "" { + writeError(w, http.StatusBadRequest, "InvalidPart", problem) + return + } + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + s.objects[key] = obj + delete(s.uploads, up.id) + s.completed.Add(1) + writeXML(w, completeResult{Bucket: s.cfg.bucket, Key: key, ETag: obj.etag}) +} + +// assemble validates the requested parts and builds the final object. A non-empty string is the +// reason the parts were rejected. +func (s *Server) assemble(up *upload, req completeRequest) (*object, string) { + if len(req.Parts) == 0 { + return nil, "You must specify at least one part" + } + var ( + data []byte + sums strings.Builder + size int64 + lastNo = 0 + ) + for i, requested := range req.Parts { + stored, ok := up.parts[requested.PartNumber] + switch { + case !ok || !etagMatches(requested.ETag, stored.etag): + return nil, fmt.Sprintf("part %d is missing or its ETag does not match", requested.PartNumber) + case requested.PartNumber <= lastNo: + return nil, "parts must be in ascending order" + case i < len(req.Parts)-1 && stored.size < s.cfg.minPartSize: + return nil, "Your proposed upload is smaller than the minimum allowed size" + } + lastNo = requested.PartNumber + sums.WriteString(stored.digest) + size += stored.size + data = append(data, stored.data...) + } + obj := &object{ + key: up.key, size: size, modified: timeNow(), contentType: up.contentType, metadata: up.metadata, tags: up.tags, + etag: fmt.Sprintf(`"%s-%d"`, digestHex([]byte(sums.String())), len(req.Parts)), + } + if !s.cfg.discard { + obj.data = data + } + return obj, "" +} + +func (s *Server) abortUpload(w http.ResponseWriter, r *http.Request, key string) { + s.mu.Lock() + defer s.mu.Unlock() + id := r.URL.Query().Get("uploadId") + if up, ok := s.uploads[id]; !ok || up.key != key { + writeError(w, http.StatusNotFound, "NoSuchUpload", "The specified upload does not exist.") + return + } + delete(s.uploads, id) + s.aborted.Add(1) + w.WriteHeader(http.StatusNoContent) +} diff --git a/pkg/artifact/blob/s3/s3test/objects.go b/pkg/artifact/blob/s3/s3test/objects.go new file mode 100644 index 00000000000..a41674327b7 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/objects.go @@ -0,0 +1,218 @@ +package s3test + +import ( + "encoding/xml" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Number parsing parameters and the S3 tag limit. +const ( + maxTagCount = 10 + decimalBase = 10 + bitSize64 = 64 +) + +// objectAttrs are the attributes a write request carries. +type objectAttrs struct { + contentType string + metadata map[string]string + tags map[string]string +} + +// attributes extracts content type, user metadata, and tags from request headers. +func attributes(r *http.Request) (objectAttrs, error) { + attrs := objectAttrs{contentType: r.Header.Get("Content-Type")} + for name, values := range r.Header { + lower := strings.ToLower(name) + if strings.HasPrefix(lower, "x-amz-meta-") && len(values) > 0 { + if attrs.metadata == nil { + attrs.metadata = map[string]string{} + } + attrs.metadata[strings.TrimPrefix(lower, "x-amz-meta-")] = values[0] + } + } + raw := r.Header.Get("X-Amz-Tagging") + if raw == "" { + return attrs, nil + } + parsed, err := url.ParseQuery(raw) + if err != nil { + return objectAttrs{}, fmt.Errorf("%w: parse object tags: %w", errUtils.ErrBlobInvalid, err) + } + if len(parsed) > maxTagCount { + return objectAttrs{}, errUtils.ErrBlobTooManyTags + } + attrs.tags = make(map[string]string, len(parsed)) + for key, values := range parsed { + attrs.tags[key] = values[0] + } + return attrs, nil +} + +func (s *Server) putObject(w http.ResponseWriter, r *http.Request, key string) { + data, err := readBody(r) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + attrs, err := attributes(r) + if err != nil { + writeError(w, http.StatusBadRequest, "InvalidTag", err.Error()) + return + } + s.mu.Lock() + defer s.mu.Unlock() + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + obj := &object{ + key: key, size: int64(len(data)), etag: quotedDigest(data), modified: timeNow(), + contentType: attrs.contentType, metadata: attrs.metadata, tags: attrs.tags, + } + if !s.cfg.discard { + obj.data = data + } + s.objects[key] = obj + w.Header().Set("ETag", obj.etag) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) getObject(w http.ResponseWriter, r *http.Request, key string, head bool) { + s.mu.Lock() + obj, ok := s.objects[key] + var snapshot object + if ok { + snapshot = *obj + } + s.mu.Unlock() + if !ok { + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + return + } + window, valid := parseRange(r.Header.Get("Range"), snapshot.size) + if !valid { + writeError(w, http.StatusRequestedRangeNotSatisfiable, "InvalidRange", "The requested range is not satisfiable") + return + } + setObjectHeaders(w, &snapshot) + w.Header().Set("Content-Length", strconv.FormatInt(window.end-window.start, decimalBase)) + if window.partial { + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", window.start, window.end-1, snapshot.size)) + w.WriteHeader(http.StatusPartialContent) + } else { + w.WriteHeader(http.StatusOK) + } + if head { + return + } + if s.cfg.discard { + s.t.Errorf("s3test: cannot read object %q: bodies are discarded", key) + return + } + _, _ = w.Write(snapshot.data[window.start:window.end]) +} + +// setObjectHeaders writes the response headers shared by GET and HEAD. +func setObjectHeaders(w http.ResponseWriter, obj *object) { + header := w.Header() + header.Set("ETag", obj.etag) + header.Set("Last-Modified", obj.modified.UTC().Format(http.TimeFormat)) + if obj.contentType != "" { + header.Set("Content-Type", obj.contentType) + } + for key, value := range obj.metadata { + header.Set("X-Amz-Meta-"+key, value) + } + if len(obj.tags) > 0 { + header.Set("X-Amz-Tagging-Count", strconv.Itoa(len(obj.tags))) + } +} + +// byteRange is a half-open window into an object. +type byteRange struct { + start, end int64 + partial bool +} + +// parseRange resolves a Range header against size. It reports false for an unsatisfiable range. +func parseRange(header string, size int64) (byteRange, bool) { + if header == "" { + return byteRange{end: size}, true + } + spec, found := strings.CutPrefix(header, "bytes=") + first, last, hasDash := strings.Cut(spec, "-") + if !found || !hasDash { + return byteRange{}, false + } + start, err := strconv.ParseInt(first, decimalBase, bitSize64) + if err != nil || start < 0 || start >= size { + return byteRange{}, false + } + end, ok := rangeEnd(last, start, size) + return byteRange{start: start, end: end, partial: true}, ok +} + +// rangeEnd resolves the exclusive end of a byte range from its last-byte text. +func rangeEnd(last string, start, size int64) (int64, bool) { + if last == "" { + return size, true + } + through, err := strconv.ParseInt(last, decimalBase, bitSize64) + if err != nil || through < start { + return 0, false + } + if through >= size { + return size, true + } + return through + 1, true +} + +func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, key string) { + s.mu.Lock() + defer s.mu.Unlock() + if r.Header.Get("If-Match") != "" || r.Header.Get("If-None-Match") != "" { + if reject(w, s.evaluate(r, s.objects[key])) { + return + } + } + delete(s.objects, key) + w.WriteHeader(http.StatusNoContent) +} + +// tagging is the GetObjectTagging response. +type tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet struct { + Tags []tag `xml:"Tag"` + } `xml:"TagSet"` +} + +type tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +func (s *Server) getTagging(w http.ResponseWriter, key string) { + s.mu.Lock() + obj, ok := s.objects[key] + var tags map[string]string + if ok { + tags = cloneMap(obj.tags) + } + s.mu.Unlock() + if !ok { + writeError(w, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.") + return + } + var doc tagging + for key, value := range tags { + doc.TagSet.Tags = append(doc.TagSet.Tags, tag{Key: key, Value: value}) + } + writeXML(w, doc) +} diff --git a/pkg/artifact/blob/s3/s3test/options.go b/pkg/artifact/blob/s3/s3test/options.go new file mode 100644 index 00000000000..ef87c1513a5 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/options.go @@ -0,0 +1,79 @@ +package s3test + +import "github.com/cloudposse/atmos/pkg/perf" + +// ConditionalBehavior selects how the server treats If-None-Match and If-Match. +type ConditionalBehavior int + +// Conditional behaviors. +const ( + // ConditionalEnforce honors the conditions like S3. + ConditionalEnforce ConditionalBehavior = iota + + // ConditionalIgnore silently ignores the headers, like a non-conforming emulator. + ConditionalIgnore + + // ConditionalNotImplemented answers 501 NotImplemented to any conditional request. + ConditionalNotImplemented +) + +// defaultBucket is the bucket the server serves unless overridden. +const defaultBucket = "test-bucket" + +// maxKeysCap is the page cap S3 applies to ListObjectsV2. +const maxKeysCap = 1000 + +// settings holds server configuration. +type settings struct { + bucket string + discard bool + conditional ConditionalBehavior + pageCap int + minPartSize int64 +} + +// Option configures a [Server]. +type Option func(*settings) + +// WithBucket sets the bucket name the server serves. +func WithBucket(name string) Option { + defer perf.Track(nil, "s3test.WithBucket")() + + return func(s *settings) { s.bucket = name } +} + +// WithDiscardBodies keeps only sizes and ETags, so streaming tests can push large bodies without +// the server holding them. Reading such an object fails. +func WithDiscardBodies() Option { + defer perf.Track(nil, "s3test.WithDiscardBodies")() + + return func(s *settings) { s.discard = true } +} + +// WithConditionalBehavior selects how conditional headers are treated. +func WithConditionalBehavior(behavior ConditionalBehavior) Option { + defer perf.Track(nil, "s3test.WithConditionalBehavior")() + + return func(s *settings) { s.conditional = behavior } +} + +// WithPageCap caps the entries per ListObjectsV2 page below the client's max-keys, to force +// pagination with few objects. +func WithPageCap(n int) Option { + defer perf.Track(nil, "s3test.WithPageCap")() + + return func(s *settings) { + if n <= 0 { + panic("s3test: page cap must be positive") + } + s.pageCap = n + } +} + +// WithMinPartSize rejects completion when a non-final part is smaller than n bytes, as S3 does +// with 5 MiB. +func WithMinPartSize(n int64) Option { + defer perf.Track(nil, "s3test.WithMinPartSize")() + + return func(s *settings) { s.minPartSize = n } +} diff --git a/pkg/artifact/blob/s3/s3test/record.go b/pkg/artifact/blob/s3/s3test/record.go new file mode 100644 index 00000000000..f7836be54fb --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/record.go @@ -0,0 +1,154 @@ +package s3test + +import ( + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// S3 operation names as reported in [Request.Op]. +const ( + OpPutObject = "PutObject" + OpGetObject = "GetObject" + OpHeadObject = "HeadObject" + OpDeleteObject = "DeleteObject" + OpListObjectsV2 = "ListObjectsV2" + OpGetObjectTagging = "GetObjectTagging" + OpCreateMultipartUpload = "CreateMultipartUpload" + OpUploadPart = "UploadPart" + OpCompleteMultipartUpload = "CompleteMultipartUpload" + OpAbortMultipartUpload = "AbortMultipartUpload" + opUnknown = "Unknown" +) + +// Request is one recorded request. +type Request struct { + // Seq is the 1-based arrival order. + Seq int + + // Op is the S3 operation name. + Op string + + // Method is the HTTP method. + Method string + + // Key is the object key, empty for bucket-level requests. + Key string + + // Query holds the query parameters. + Query url.Values + + // Header holds the request headers. + Header http.Header + + // Bytes is the number of body bytes received. + Bytes int64 + + // Status is the status the server answered, or 0 when the connection was dropped. + Status int + + // Faulted reports that an injected fault affected the request. + Faulted bool + + // At is the arrival time. + At time.Time +} + +// recorder collects requests. +type recorder struct { + mu sync.Mutex + requests []Request +} + +// begin records an arriving request and returns its index. +func (r *recorder) begin(req *Request) int { + r.mu.Lock() + defer r.mu.Unlock() + req.Seq = len(r.requests) + 1 + r.requests = append(r.requests, *req) + return len(r.requests) - 1 +} + +func (r *recorder) update(index int, fn func(*Request)) { + r.mu.Lock() + defer r.mu.Unlock() + fn(&r.requests[index]) +} + +// Requests returns a copy of every recorded request in arrival order. +func (s *Server) Requests() []Request { + defer perf.Track(nil, "s3test.Server.Requests")() + + s.rec.mu.Lock() + defer s.rec.mu.Unlock() + requests := append([]Request(nil), s.rec.requests...) + for i := range requests { + query := make(url.Values, len(requests[i].Query)) + for key, values := range requests[i].Query { + query[key] = append([]string(nil), values...) + } + requests[i].Query = query + requests[i].Header = requests[i].Header.Clone() + } + return requests +} + +// RequestsFor returns the recorded requests of one operation. +func (s *Server) RequestsFor(op string) []Request { + defer perf.Track(nil, "s3test.Server.RequestsFor")() + + var matched []Request + for _, req := range s.Requests() { + if req.Op == op { + matched = append(matched, req) + } + } + return matched +} + +// ResetRequests clears the request log. +func (s *Server) ResetRequests() { + defer perf.Track(nil, "s3test.Server.ResetRequests")() + + s.rec.mu.Lock() + defer s.rec.mu.Unlock() + s.rec.requests = nil +} + +// operations maps an HTTP method and the query marker that distinguishes the operation to its +// S3 name. An empty marker is the method's default operation. +var operations = map[string]map[string]string{ + http.MethodHead: {"": OpHeadObject}, + http.MethodGet: {"tagging": OpGetObjectTagging, "": OpGetObject}, + http.MethodPut: {"uploadId": OpUploadPart, "": OpPutObject}, + http.MethodPost: {"uploads": OpCreateMultipartUpload, "uploadId": OpCompleteMultipartUpload}, + http.MethodDelete: {"uploadId": OpAbortMultipartUpload, "": OpDeleteObject}, +} + +// classifyRequest names the S3 operation for a request. +func classifyRequest(method string, query url.Values, key string) string { + if key == "" && method == http.MethodGet { + return OpListObjectsV2 + } + byMarker := operations[method] + for marker, op := range byMarker { + if marker != "" && query.Has(marker) { + return op + } + } + if op, ok := byMarker[""]; ok { + return op + } + return opUnknown +} + +// splitPath separates the bucket and key from a path-style request path. +func splitPath(path string) (bucket, key string) { + trimmed := strings.TrimPrefix(path, "/") + bucket, key, _ = strings.Cut(trimmed, "/") + return bucket, key +} diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go new file mode 100644 index 00000000000..945e0937fed --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -0,0 +1,231 @@ +package s3test + +import ( + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Server is an in-process S3-compatible endpoint serving one bucket in path-style addressing. +type Server struct { + t testing.TB + cfg settings + http *httptest.Server + rec recorder + faults faults + + mu sync.Mutex + objects map[string]*object + uploads map[string]*upload + nextID int + + aborted atomic.Int64 + completed atomic.Int64 + partBytes atomic.Int64 + partsInFlight atomic.Int64 + maxPartsInFlow atomic.Int64 +} + +// NewServer starts a server and registers its shutdown with t.Cleanup. +func NewServer(t testing.TB, opts ...Option) *Server { + defer perf.Track(nil, "s3test.NewServer")() + + t.Helper() + cfg := settings{bucket: defaultBucket, pageCap: maxKeysCap} + for _, opt := range opts { + opt(&cfg) + } + server := &Server{t: t, cfg: cfg, objects: map[string]*object{}, uploads: map[string]*upload{}} + server.http = httptest.NewServer(http.HandlerFunc(server.serve)) + t.Cleanup(server.http.Close) + return server +} + +// Endpoint returns the base URL to configure as the S3 endpoint. +func (s *Server) Endpoint() string { + defer perf.Track(nil, "s3test.Server.Endpoint")() + + return s.http.URL +} + +// Close stops the server early, for tests of connection failures. It is safe to call twice. +func (s *Server) Close() { + defer perf.Track(nil, "s3test.Server.Close")() + + s.http.Close() +} + +// Bucket returns the served bucket name. +func (s *Server) Bucket() string { + defer perf.Track(nil, "s3test.Server.Bucket")() + + return s.cfg.bucket +} + +// Client returns an SDK client wired to the server with path-style addressing, static credentials, +// and optional request checksums disabled (the settings the generic store uses). +func (s *Server) Client() *s3.Client { + defer perf.Track(nil, "s3test.Server.Client")() + + return s.client(func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) +} + +// LegacyClient returns a client that keeps the SDK's default checksum behavior, as the legacy CI +// adapter does. +func (s *Server) LegacyClient() *s3.Client { + defer perf.Track(nil, "s3test.Server.LegacyClient")() + + return s.client() +} + +func (s *Server) client(extra ...func(*s3.Options)) *s3.Client { + cfg := aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider("test", "test", ""), + } + optFns := append([]func(*s3.Options){func(o *s3.Options) { + o.BaseEndpoint = aws.String(s.http.URL) + o.UsePathStyle = true + }}, extra...) + return s3.NewFromConfig(cfg, optFns...) +} + +// serve records, applies faults to, and dispatches one request. +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + bucket, key := splitPath(r.URL.Path) + query := r.URL.Query() + req := Request{Op: classifyRequest(r.Method, query, key), Method: r.Method, Key: key, Query: query, Header: r.Header.Clone(), At: timeNow()} + index := s.rec.begin(&req) + + fault := s.faults.match(&req) + if fault != nil { + s.rec.update(index, func(rr *Request) { rr.Faulted = true }) + } + rw := &statusWriter{ResponseWriter: w} + if fault != nil && s.applyEarlyFault(rw, r, fault) { + s.finish(index, rw) + return + } + if bucket != s.cfg.bucket { + writeError(rw, http.StatusNotFound, "NoSuchBucket", "The specified bucket does not exist") + s.finish(index, rw) + return + } + if fault != nil && fault.Action == DropAfterCommit { + s.handle(&mockDiscardWriter{header: http.Header{}}, r, key, req.Op) + s.drop(w) + s.rec.update(index, func(rr *Request) { rr.Status = 0 }) + return + } + s.handle(rw, r, key, req.Op) + s.finish(index, rw) +} + +// applyEarlyFault applies faults that decide the request before it is handled. It reports whether +// the request is finished. +func (s *Server) applyEarlyFault(w *statusWriter, r *http.Request, fault *Fault) bool { + switch fault.Action { + case FailStatus: + _, _ = io.Copy(io.Discard, r.Body) + status, code := fault.status() + writeError(w, status, code, "injected fault") + return true + case DropBeforeCommit: + _, _ = io.Copy(io.Discard, r.Body) + s.drop(w.ResponseWriter) + w.dropped = true + return true + case Delay: + sleep(r.Context(), fault.Delay) + case DropAfterCommit: + } + return false +} + +// finish records the response status and body size of a handled request. +func (s *Server) finish(index int, w *statusWriter) { + s.rec.update(index, func(rr *Request) { + if !w.dropped { + rr.Status = w.status() + } + }) +} + +// drop closes the client connection without writing a response. +func (s *Server) drop(w http.ResponseWriter) { + hijacker, ok := w.(http.Hijacker) + if !ok { + s.t.Errorf("s3test: response writer cannot be hijacked") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + s.t.Errorf("s3test: hijack failed: %v", err) + return + } + _ = conn.Close() +} + +// handle dispatches a request to its operation. +func (s *Server) handle(w http.ResponseWriter, r *http.Request, key, op string) { + handlers := map[string]func(){ + OpPutObject: func() { s.putObject(w, r, key) }, + OpGetObject: func() { s.getObject(w, r, key, false) }, + OpHeadObject: func() { s.getObject(w, r, key, true) }, + OpDeleteObject: func() { s.deleteObject(w, r, key) }, + OpListObjectsV2: func() { s.listObjects(w, r) }, + OpGetObjectTagging: func() { s.getTagging(w, key) }, + OpCreateMultipartUpload: func() { s.createUpload(w, r, key) }, + OpUploadPart: func() { s.uploadPart(w, r, key) }, + OpCompleteMultipartUpload: func() { s.completeUpload(w, r, key) }, + OpAbortMultipartUpload: func() { s.abortUpload(w, r, key) }, + } + if handler, ok := handlers[op]; ok { + handler() + return + } + writeError(w, http.StatusNotImplemented, "NotImplemented", "operation is not supported by the fake") +} + +// statusWriter remembers the status code written. It deliberately has no Write method of its own: +// the embedded writer's Write is promoted, and status reports 200 when no header was written. +type statusWriter struct { + http.ResponseWriter + code int + dropped bool +} + +func (w *statusWriter) WriteHeader(code int) { + defer perf.Track(nil, "s3test.statusWriter.WriteHeader")() + + if w.code == 0 { + w.code = code + } + w.ResponseWriter.WriteHeader(code) +} + +func (w *statusWriter) status() int { + if w.code == 0 { + return http.StatusOK + } + return w.code +} + +// mockDiscardWriter swallows a response, used when the response is dropped after the commit. +type mockDiscardWriter struct{ header http.Header } + +func (d *mockDiscardWriter) Header() http.Header { return d.header } +func (d *mockDiscardWriter) Write(p []byte) (int, error) { return len(p), nil } +func (d *mockDiscardWriter) WriteHeader(int) {} diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go new file mode 100644 index 00000000000..0360c578f70 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -0,0 +1,441 @@ +package s3test + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func apiCode(t *testing.T, err error) string { + t.Helper() + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + return apiErr.ErrorCode() +} + +func putObject(t *testing.T, client *s3.Client, srv *Server, key, body string, mutate ...func(*s3.PutObjectInput)) (*s3.PutObjectOutput, error) { + t.Helper() + in := &s3.PutObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String(key), Body: strings.NewReader(body)} + for _, fn := range mutate { + fn(in) + } + return client.PutObject(context.Background(), in) +} + +func TestConditionalPutEnforced(t *testing.T) { + srv := NewServer(t) + client := srv.Client() + first, err := putObject(t, client, srv, "k", "one", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err) + + _, err = putObject(t, client, srv, "k", "two", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + + _, err = putObject(t, client, srv, "k", "three", func(in *s3.PutObjectInput) { in.IfMatch = aws.String(`"bogus"`) }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + + _, err = putObject(t, client, srv, "k", "four", func(in *s3.PutObjectInput) { in.IfMatch = first.ETag }) + require.NoError(t, err) + + _, err = putObject(t, client, srv, "ghost", "x", func(in *s3.PutObjectInput) { in.IfMatch = aws.String(`"x"`) }) + assert.Equal(t, "NoSuchKey", apiCode(t, err), "If-Match on a missing key is a 404") + object, _ := srv.Object("k") + assert.Equal(t, []byte("four"), object.Data) +} + +func TestConditionalBehaviorsOfNonConformingEndpoints(t *testing.T) { + ignore := NewServer(t, WithConditionalBehavior(ConditionalIgnore)) + for range 2 { + _, err := putObject(t, ignore.Client(), ignore, "k", "x", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err, "an ignoring endpoint overwrites silently") + } + notImplemented := NewServer(t, WithConditionalBehavior(ConditionalNotImplemented)) + _, err := putObject(t, notImplemented.Client(), notImplemented, "k", "x", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + assert.Equal(t, "NotImplemented", apiCode(t, err)) + _, err = putObject(t, notImplemented.Client(), notImplemented, "plain", "x") + require.NoError(t, err, "unconditional requests still work") +} + +func TestListPagination(t *testing.T) { + srv := NewServer(t, WithPageCap(4)) + for i := range 10 { + srv.Seed(fmt.Sprintf("p/%02d", i), nil, nil) + } + srv.Seed("q/x", nil, nil) + client := srv.Client() + + var keys []string + var token *string + pages := 0 + for { + out, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(srv.Bucket()), Prefix: aws.String("p/"), MaxKeys: aws.Int32(100), ContinuationToken: token, + }) + require.NoError(t, err) + pages++ + assert.LessOrEqual(t, len(out.Contents), 4, "the server's page cap wins over max-keys") + for _, obj := range out.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + if !aws.ToBool(out.IsTruncated) { + break + } + require.NotEmpty(t, aws.ToString(out.NextContinuationToken)) + token = out.NextContinuationToken + } + assert.Len(t, keys, 10) + assert.Equal(t, 3, pages) + + _, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(srv.Bucket()), ContinuationToken: aws.String("!!!")}) + assert.Equal(t, "InvalidArgument", apiCode(t, err)) +} + +func TestListKeysWithSpecialCharactersRoundTrip(t *testing.T) { + srv := NewServer(t) + keys := []string{"a b/c+d", "ünï/ç", "x&y=z/1", "plain"} + for _, key := range keys { + srv.Seed(key, nil, nil) + } + out, err := srv.Client().ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(srv.Bucket())}) + require.NoError(t, err) + var got []string + for _, obj := range out.Contents { + got = append(got, aws.ToString(obj.Key)) + } + assert.ElementsMatch(t, keys, got) +} + +func TestMultipartLifecycle(t *testing.T) { + srv := NewServer(t, WithMinPartSize(4)) + client := srv.Client() + ctx := context.Background() + created, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), Tagging: aws.String("a=b"), ContentType: aws.String("text/x"), + }) + require.NoError(t, err) + part := func(n int32, body string) types.CompletedPart { + out, partErr := client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, PartNumber: aws.Int32(n), Body: strings.NewReader(body), + }) + require.NoError(t, partErr) + return types.CompletedPart{ETag: out.ETag, PartNumber: aws.Int32(n)} + } + p1, p2 := part(1, "aaaa"), part(2, "bb") + assert.Len(t, srv.OpenUploads(), 1) + + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{p2, p1}}, + }) + assert.Equal(t, "InvalidPart", apiCode(t, err), "parts must be in ascending order") + + done, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{p1, p2}}, + }) + require.NoError(t, err) + assert.True(t, strings.HasSuffix(strings.Trim(aws.ToString(done.ETag), `"`), "-2")) + object, _ := srv.Object("m") + assert.Equal(t, []byte("aaaabb"), object.Data) + assert.Equal(t, map[string]string{"a": "b"}, object.Tags) + assert.Equal(t, "text/x", object.ContentType) + assert.Empty(t, srv.OpenUploads()) + + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId}) + assert.Equal(t, "NoSuchUpload", apiCode(t, err)) +} + +func TestMultipartMinimumPartSizeAndAbort(t *testing.T) { + srv := NewServer(t, WithMinPartSize(10)) + client := srv.Client() + ctx := context.Background() + created, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m")}) + require.NoError(t, err) + var parts []types.CompletedPart + for n := int32(1); n <= 2; n++ { + out, partErr := client.UploadPart(ctx, &s3.UploadPartInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, PartNumber: aws.Int32(n), Body: strings.NewReader("tiny")}) + require.NoError(t, partErr) + parts = append(parts, types.CompletedPart{ETag: out.ETag, PartNumber: aws.Int32(n)}) + } + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId, MultipartUpload: &types.CompletedMultipartUpload{Parts: parts}, + }) + assert.Equal(t, "InvalidPart", apiCode(t, err), "a non-final part below the minimum is rejected") + + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("wrong"), UploadId: created.UploadId}) + assert.Equal(t, "NoSuchUpload", apiCode(t, err)) + assert.Len(t, srv.OpenUploads(), 1) + _, err = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("m"), UploadId: created.UploadId}) + require.NoError(t, err) + assert.Equal(t, 1, srv.AbortedUploads()) + assert.Empty(t, srv.OpenUploads()) + assert.Empty(t, srv.Keys()) +} + +func TestTaggingMetadataAndHead(t *testing.T) { + srv := NewServer(t) + client := srv.Client() + _, err := putObject(t, client, srv, "k", "body", func(in *s3.PutObjectInput) { + in.Tagging = aws.String("a=1&b=2") + in.Metadata = map[string]string{"Owner": "me"} + in.ContentType = aws.String("text/plain") + }) + require.NoError(t, err) + head, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k")}) + require.NoError(t, err) + assert.Equal(t, int64(4), aws.ToInt64(head.ContentLength)) + assert.Equal(t, "text/plain", aws.ToString(head.ContentType)) + assert.Equal(t, "me", head.Metadata["owner"]) + tags, err := client.GetObjectTagging(context.Background(), &s3.GetObjectTaggingInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k")}) + require.NoError(t, err) + assert.Len(t, tags.TagSet, 2) + + _, err = putObject(t, client, srv, "toomany", "x", func(in *s3.PutObjectInput) { + in.Tagging = aws.String("a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11") + }) + assert.Equal(t, "InvalidTag", apiCode(t, err)) + _, err = client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("nope")}) + var notFound *types.NotFound + assert.ErrorAs(t, err, ¬Found) +} + +func TestRanges(t *testing.T) { + srv := NewServer(t) + srv.Seed("k", []byte("0123456789"), nil) + get := func(r string) (string, error) { + out, err := srv.Client().GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("k"), Range: aws.String(r)}) + if err != nil { + return "", err + } + defer out.Body.Close() + data, err := io.ReadAll(out.Body) + return string(data), err + } + got, err := get("bytes=2-4") + require.NoError(t, err) + assert.Equal(t, "234", got) + got, err = get("bytes=8-") + require.NoError(t, err) + assert.Equal(t, "89", got) + _, err = get("bytes=20-30") + assert.Equal(t, "InvalidRange", apiCode(t, err)) +} + +func TestUnknownBucketAndZeroLength(t *testing.T) { + srv := NewServer(t, WithBucket("mine")) + _, err := srv.Client().PutObject(context.Background(), &s3.PutObjectInput{Bucket: aws.String("other"), Key: aws.String("k"), Body: strings.NewReader("x")}) + assert.Equal(t, "NoSuchBucket", apiCode(t, err)) + + _, err = putObject(t, srv.Client(), srv, "empty", "") + require.NoError(t, err) + object, ok := srv.Object("empty") + require.True(t, ok) + assert.Zero(t, object.Size) + assert.NotEmpty(t, object.ETag) +} + +func TestFaultInjection(t *testing.T) { + t.Run("skip then fail forever", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Skip: 2, Times: -1, Status: http.StatusInternalServerError}) + client := srv.LegacyClient() + client = s3.NewFromConfig(aws.Config{Region: "us-east-1", Credentials: client.Options().Credentials, RetryMaxAttempts: 1}, func(o *s3.Options) { + o.BaseEndpoint = aws.String(srv.Endpoint()) + o.UsePathStyle = true + o.Retryer = aws.NopRetryer{} + }) + for i := range 2 { + _, err := putObject(t, client, srv, fmt.Sprint("ok", i), "x") + require.NoError(t, err) + } + _, err := putObject(t, client, srv, "boom", "x") + assert.Equal(t, "InternalError", apiCode(t, err)) + _, ok := srv.Object("boom") + assert.False(t, ok, "a failed request is not applied") + }) + t.Run("drop after commit applies the write", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: DropAfterCommit}) + client := noRetryClient(srv) + _, err := putObject(t, client, srv, "k", "x") + require.Error(t, err) + _, ok := srv.Object("k") + assert.True(t, ok) + assert.Equal(t, 0, srv.Requests()[0].Status) + assert.True(t, srv.Requests()[0].Faulted) + }) + t.Run("drop before commit does not", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: DropBeforeCommit}) + _, err := putObject(t, noRetryClient(srv), srv, "k", "x") + require.Error(t, err) + _, ok := srv.Object("k") + assert.False(t, ok) + }) + t.Run("delay", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{Op: OpPutObject, Action: Delay, Delay: 60 * time.Millisecond}) + start := time.Now() + _, err := putObject(t, srv.Client(), srv, "k", "x") + require.NoError(t, err) + assert.GreaterOrEqual(t, time.Since(start), 60*time.Millisecond) + }) + t.Run("key prefix scoping and clearing", func(t *testing.T) { + srv := NewServer(t) + srv.Inject(Fault{KeyPrefix: "bad/", Times: -1, Status: http.StatusBadRequest, Code: "Nope"}) + _, err := putObject(t, srv.Client(), srv, "good/k", "x") + require.NoError(t, err) + _, err = putObject(t, srv.Client(), srv, "bad/k", "x") + assert.Equal(t, "Nope", apiCode(t, err)) + srv.ClearFaults() + _, err = putObject(t, srv.Client(), srv, "bad/k", "x") + require.NoError(t, err) + }) +} + +func noRetryClient(srv *Server) *s3.Client { + return s3.NewFromConfig(aws.Config{Region: "us-east-1"}, func(o *s3.Options) { + o.BaseEndpoint = aws.String(srv.Endpoint()) + o.UsePathStyle = true + o.Retryer = aws.NopRetryer{} + o.Credentials = srv.Client().Options().Credentials + }) +} + +func TestRequestLogRecordsOrderAndHeaders(t *testing.T) { + srv := NewServer(t) + _, err := putObject(t, srv.Client(), srv, "a/b", "hello", func(in *s3.PutObjectInput) { in.IfNoneMatch = aws.String("*") }) + require.NoError(t, err) + _, err = srv.Client().HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("a/b")}) + require.NoError(t, err) + + requests := srv.Requests() + require.Len(t, requests, 2) + assert.Equal(t, []int{1, 2}, []int{requests[0].Seq, requests[1].Seq}) + assert.Equal(t, OpPutObject, requests[0].Op) + assert.Equal(t, "a/b", requests[0].Key) + assert.Equal(t, "*", requests[0].Header.Get("If-None-Match")) + assert.Equal(t, http.StatusOK, requests[0].Status) + requests[0].Header.Set("If-None-Match", "changed") + requests[0].Query.Set("injected", "changed") + assert.Equal(t, "*", srv.Requests()[0].Header.Get("If-None-Match")) + assert.Empty(t, srv.Requests()[0].Query.Get("injected")) + assert.Equal(t, OpHeadObject, requests[1].Op) + assert.Len(t, srv.RequestsFor(OpHeadObject), 1) + srv.ResetRequests() + assert.Empty(t, srv.Requests()) +} + +func TestDecodeAWSChunked(t *testing.T) { + raw := []byte("5;chunk-signature=abc\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:AAAA\r\n\r\n") + got, err := decodeAWSChunked(raw) + require.NoError(t, err) + assert.Equal(t, "hello world", string(got)) + + malformed := []struct { + name string + raw string + want error + cause error + }{ + {name: "missing header newline", raw: "5", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "non-hex chunk size", raw: "zz\r\n", want: errUtils.ErrBlobInvalidChunk, cause: strconv.ErrSyntax}, + {name: "chunk cut short", raw: "5\r\nhel", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "terminator missing", raw: "5\r\nhello", want: errUtils.ErrBlobInvalidChunk, cause: io.EOF}, + {name: "wrong terminator", raw: "5\r\nhelloXX0\r\n", want: errUtils.ErrBlobInvalidChunkTerminator}, + } + for _, tc := range malformed { + t.Run(tc.name, func(t *testing.T) { + _, err := decodeAWSChunked([]byte(tc.raw)) + require.ErrorIs(t, err, tc.want) + if tc.cause != nil { + require.ErrorIs(t, err, tc.cause, "the decoder keeps the underlying cause visible") + } + }) + } +} + +func TestDiscardBodiesKeepsSizesAndETags(t *testing.T) { + srv := NewServer(t, WithDiscardBodies()) + _, err := putObject(t, srv.Client(), srv, "k", "payload") + require.NoError(t, err) + object, ok := srv.Object("k") + require.True(t, ok) + assert.Equal(t, int64(7), object.Size) + assert.Empty(t, object.Data) + assert.NotEmpty(t, object.ETag) +} + +func TestEmulatorFromEnvSkipsWhenUnset(t *testing.T) { + t.Setenv(EmulatorEndpointEnv, "") + t.Setenv(EmulatorBucketEnv, "") + t.Setenv(EmulatorRegionEnv, "") + result := t.Run("inner", func(t *testing.T) { + EmulatorFromEnv(t) + t.Error("EmulatorFromEnv must skip when the environment is unset") + }) + assert.True(t, result, "a skipped subtest still counts as passed") + + t.Setenv(EmulatorEndpointEnv, "http://localhost:4566") + t.Setenv(EmulatorBucketEnv, "b") + emulator := EmulatorFromEnv(t) + assert.Equal(t, Emulator{Endpoint: "http://localhost:4566", Bucket: "b", Region: "us-east-1"}, emulator) +} + +func TestWeakIfMatchAndEmptyRange(t *testing.T) { + srv := NewServer(t) + first, err := putObject(t, srv.Client(), srv, "k", "value") + require.NoError(t, err) + _, err = putObject(t, srv.Client(), srv, "k", "new", func(in *s3.PutObjectInput) { in.IfMatch = aws.String("W/" + aws.ToString(first.ETag)) }) + assert.Equal(t, "PreconditionFailed", apiCode(t, err)) + srv.Seed("empty", nil, nil) + _, err = srv.Client().GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(srv.Bucket()), Key: aws.String("empty"), Range: aws.String("bytes=0-")}) + assert.Equal(t, "InvalidRange", apiCode(t, err)) +} + +func TestPageCapMustBePositive(t *testing.T) { + for _, cap := range []int{0, -1} { + assert.Panics(t, func() { NewServer(t, WithPageCap(cap)) }) + } +} + +func TestRangeEndMaxIntDoesNotOverflow(t *testing.T) { + got, ok := parseRange("bytes=0-9223372036854775807", 10) + require.True(t, ok) + assert.Equal(t, int64(10), got.end) +} + +func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) { + tests := []struct { + name string + tagging string + want error + }{ + {name: "malformed query", tagging: "a=%zz", want: errUtils.ErrBlobInvalid}, + {name: "too many tags", tagging: "a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8&i=9&j=10&k=11", want: errUtils.ErrBlobTooManyTags}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodPut, "http://example.invalid/k", strings.NewReader("x")) + require.NoError(t, err) + req.Header.Set("X-Amz-Tagging", tc.tagging) + + _, err = attributes(req) + require.ErrorIs(t, err, tc.want) + }) + } +} diff --git a/pkg/artifact/blob/s3/s3test/state.go b/pkg/artifact/blob/s3/s3test/state.go new file mode 100644 index 00000000000..2b8e3ca10f7 --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/state.go @@ -0,0 +1,157 @@ +package s3test + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// object is a stored object. +type object struct { + key string + data []byte + size int64 + etag string + modified time.Time + contentType string + metadata map[string]string + tags map[string]string +} + +// Object is a read-only snapshot of a stored object. +type Object struct { + Key string + Data []byte + Size int64 + ETag string + ContentType string + Metadata map[string]string + Tags map[string]string +} + +func (o *object) snapshot() Object { + return Object{ + Key: o.key, Data: append([]byte(nil), o.data...), Size: o.size, ETag: o.etag, + ContentType: o.contentType, Metadata: cloneMap(o.metadata), Tags: cloneMap(o.tags), + } +} + +func cloneMap(m map[string]string) map[string]string { + if m == nil { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// etagLength is the number of hex characters in a fake ETag, the length of an MD5 digest. +const etagLength = 32 + +// digestHex returns a short opaque digest. Real S3 ETags are MD5-based for unencrypted single +// requests, but they are concurrency tokens and callers must never rely on that, so the fake +// uses a different hash to catch code that does. +func digestHex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:])[:etagLength] +} + +// quotedDigest returns the ETag of a single-request object. +func quotedDigest(data []byte) string { + return `"` + digestHex(data) + `"` +} + +// Object returns a snapshot of the object stored under key. +func (s *Server) Object(key string) (Object, bool) { + defer perf.Track(nil, "s3test.Server.Object")() + + s.mu.Lock() + defer s.mu.Unlock() + obj, ok := s.objects[key] + if !ok { + return Object{}, false + } + return obj.snapshot(), true +} + +// Keys returns every stored key in order. +func (s *Server) Keys() []string { + defer perf.Track(nil, "s3test.Server.Keys")() + + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.objects)) + for key := range s.objects { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// Seed stores an object directly, bypassing the HTTP surface and the request log. +func (s *Server) Seed(key string, data []byte, tags map[string]string) Object { + defer perf.Track(nil, "s3test.Server.Seed")() + + s.mu.Lock() + defer s.mu.Unlock() + obj := &object{ + key: key, data: append([]byte(nil), data...), size: int64(len(data)), etag: quotedDigest(data), + modified: timeNow(), tags: cloneMap(tags), + } + s.objects[key] = obj + return obj.snapshot() +} + +// OpenUploads returns the IDs of multipart uploads that were neither completed nor aborted. +func (s *Server) OpenUploads() []string { + defer perf.Track(nil, "s3test.Server.OpenUploads")() + + s.mu.Lock() + defer s.mu.Unlock() + ids := make([]string, 0, len(s.uploads)) + for id := range s.uploads { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// AbortedUploads returns how many multipart uploads were aborted. +func (s *Server) AbortedUploads() int { + defer perf.Track(nil, "s3test.Server.AbortedUploads")() + + return int(s.aborted.Load()) +} + +// CompletedUploads returns how many multipart uploads were completed. +func (s *Server) CompletedUploads() int { + defer perf.Track(nil, "s3test.Server.CompletedUploads")() + + return int(s.completed.Load()) +} + +// PartBytesReceived returns the bytes of every UploadPart body received so far. +func (s *Server) PartBytesReceived() int64 { + defer perf.Track(nil, "s3test.Server.PartBytesReceived")() + + return s.partBytes.Load() +} + +// MaxConcurrentParts returns the most UploadPart requests that were in flight at once. +func (s *Server) MaxConcurrentParts() int { + defer perf.Track(nil, "s3test.Server.MaxConcurrentParts")() + + return int(s.maxPartsInFlow.Load()) +} + +// newID returns a unique upload ID. +func (s *Server) newID() string { + s.nextID++ + return fmt.Sprintf("upload-%d", s.nextID) +} diff --git a/pkg/artifact/blob/s3/s3test/util.go b/pkg/artifact/blob/s3/s3test/util.go new file mode 100644 index 00000000000..c3e4fb60f1e --- /dev/null +++ b/pkg/artifact/blob/s3/s3test/util.go @@ -0,0 +1,50 @@ +package s3test + +import ( + "context" + "encoding/xml" + "net/http" + "time" +) + +func timeNow() time.Time { + return time.Now() +} + +// sleep pauses for d or until ctx ends. +func sleep(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + } +} + +// errorBody is the S3 XML error document. +type errorBody struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + RequestID string `xml:"RequestId"` +} + +// writeError answers with an S3 XML error. HEAD responses carry no body. +func writeError(w http.ResponseWriter, status int, code, message string) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + body, _ := xml.Marshal(errorBody{Code: code, Message: message, RequestID: "s3test"}) + _, _ = w.Write(append([]byte(xml.Header), body...)) +} + +// writeXML answers 200 with an XML document. +func writeXML(w http.ResponseWriter, value any) { + body, err := xml.Marshal(value) + if err != nil { + writeError(w, http.StatusInternalServerError, "InternalError", err.Error()) + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(append([]byte(xml.Header), body...)) +} diff --git a/pkg/artifact/blob/s3/store.go b/pkg/artifact/blob/s3/store.go new file mode 100644 index 00000000000..994392b71b4 --- /dev/null +++ b/pkg/artifact/blob/s3/store.go @@ -0,0 +1,134 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/perf" +) + +const storeName = "s3" + +// Store is an S3 [blob.Store]. +type Store struct { + cfg settings + + mu sync.Mutex + api API + + // probeSlot admits one conditional-write probe at a time. A buffered channel instead of a mutex + // lets a waiting operation give up when its context ends. + probeSlot chan struct{} + + // probeMu guards the cached verdict below. + probeMu sync.Mutex + probed bool + probeError error +} + +// New returns a Store for bucket. No network or credential work happens until the first operation. +func New(bucket string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.s3.New")() + + if bucket == "" { + return nil, blob.NewError(blob.KindInvalid, blob.OpNew, "", fmt.Errorf("%w: bucket is required", errUtils.ErrBlobInvalid)) + } + cfg := settings{bucket: bucket, partSize: DefaultPartSize, concurrency: DefaultConcurrency} + for _, opt := range opts { + opt(&cfg) + } + return &Store{cfg: cfg, api: cfg.api, probeSlot: make(chan struct{}, 1)}, nil +} + +// NewFromClient returns a Store over an existing client. +func NewFromClient(api API, bucket string, opts ...Option) (*Store, error) { + defer perf.Track(nil, "blob.s3.NewFromClient")() + + return New(bucket, append(opts, WithAPI(api))...) +} + +// Name implements [blob.Store]. +func (s *Store) Name() string { + defer perf.Track(nil, "s3.Store.Name")() + + return storeName +} + +// Bucket returns the bucket name. +func (s *Store) Bucket() string { + defer perf.Track(nil, "s3.Store.Bucket")() + + return s.cfg.bucket +} + +// Capabilities implements [blob.Store]. +func (s *Store) Capabilities() blob.Capabilities { + defer perf.Track(nil, "s3.Store.Capabilities")() + + conditional := s.cfg.conditional != ConditionalDisabled + return blob.Capabilities{ + ConditionalPut: conditional, + ConditionalDelete: conditional, + Multipart: true, + Tags: true, + Metadata: true, + Range: true, + Listing: true, + ListDelimiter: true, + } +} + +// client returns the S3 client, building it on first use. A failed build is not cached, so a +// transient identity failure can be retried by the next operation. +func (s *Store) client(ctx context.Context) (API, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.api != nil { + return s.api, nil + } + client, err := NewClient(ctx, &s.cfg.client) + if err != nil { + return nil, blob.NewError(blob.KindIdentity, "client", "", err) + } + s.api = client + return s.api, nil +} + +// validateKey applies the configured key rules. +func (s *Store) validateKey(key string) error { + if s.cfg.opaqueKeys { + return blob.ValidateOpaqueKey(key) + } + return blob.ValidateKey(key) +} + +// objectKey maps a store key to the bucket key. +func (s *Store) objectKey(key string) string { + if s.cfg.prefix == "" { + return key + } + return s.cfg.prefix + "/" + key +} + +// storeKey maps a bucket key back to a store key. +func (s *Store) storeKey(objectKey string) string { + if s.cfg.prefix == "" { + return objectKey + } + return strings.TrimPrefix(objectKey, s.cfg.prefix+"/") +} + +// retryOptions disables SDK retries for conditional mutations. A retried conditional request whose +// first attempt already committed would otherwise answer 412 and look like a lost race. +func retryOptions(conditional bool) []func(*sdkOptions) { + if !conditional { + return nil + } + return []func(*sdkOptions){noRetry} +} + +var _ blob.Store = (*Store)(nil) diff --git a/pkg/artifact/blob/s3/store_test.go b/pkg/artifact/blob/s3/store_test.go new file mode 100644 index 00000000000..b07afcebfb2 --- /dev/null +++ b/pkg/artifact/blob/s3/store_test.go @@ -0,0 +1,185 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "math" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + "github.com/cloudposse/atmos/pkg/artifact/blob/blobtest" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" +) + +func TestConformance(t *testing.T) { + variants := map[string]struct { + fake []s3test.Option + store []Option + }{ + "single-request": {store: []Option{WithPartSize(DefaultPartSize)}}, + "multipart": {store: []Option{WithPartSize(smallPart)}}, + "prefixed": {store: []Option{WithPrefix("/team/artifacts/")}}, + "capped-pages": {fake: []s3test.Option{s3test.WithPageCap(7)}}, + } + for name, variant := range variants { + t.Run(name, func(t *testing.T) { + blobtest.RunStoreConformance(t, func(t *testing.T) blob.Store { + fake := newFake(t, variant.fake...) + return newTestStore(t, fake, variant.store...) + }) + }) + } +} + +func TestNewValidates(t *testing.T) { + _, err := New("") + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + s, err := New("bucket") + require.NoError(t, err) + assert.Equal(t, "s3", s.Name()) + assert.Equal(t, "bucket", s.Bucket()) + assert.Equal(t, DefaultPartSize, s.cfg.partSize) + assert.Equal(t, DefaultConcurrency, s.cfg.concurrency) +} + +func TestCapabilities(t *testing.T) { + enabled, err := New("b") + require.NoError(t, err) + caps := enabled.Capabilities() + assert.True(t, caps.ConditionalPut && caps.ConditionalDelete && caps.Multipart && caps.Tags && caps.Listing) + + disabled, err := New("b", WithConditionalMode(ConditionalDisabled)) + require.NoError(t, err) + assert.False(t, disabled.Capabilities().ConditionalPut) + assert.False(t, disabled.Capabilities().ConditionalDelete) +} + +func TestKeyMappingWithPrefix(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake, WithPrefix("/p/q/")) + putBytes(t, s, "a/b", []byte("x"), blob.PutOptions{}) + assert.Equal(t, []string{"p/q/a/b"}, fake.Keys()) + + objects, _, err := blob.ListAll(context.Background(), s, blob.ListOptions{}) + require.NoError(t, err) + require.Len(t, objects, 1) + assert.Equal(t, "a/b", objects[0].Key, "listed keys are relative to the configured prefix") +} + +func TestKeyValidationPortableAndOpaque(t *testing.T) { + fake := newFake(t) + strict := newTestStore(t, fake) + _, err := strict.Put(context.Background(), "a/../b", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + + opaque := newTestStore(t, fake, WithOpaqueKeys()) + putBytes(t, opaque, "a//odd/../key", []byte("x"), blob.PutOptions{}) + assert.Contains(t, fake.Keys(), "a//odd/../key", "legacy key patterns are passed through untouched") + _, err = opaque.Put(context.Background(), "", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) +} + +func TestRangedGetSendsRangeHeader(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("0123456789"), blob.PutOptions{}) + fake.ResetRequests() + + object, err := s.Get(context.Background(), "k", blob.GetOptions{Offset: 2, Length: 3}) + require.NoError(t, err) + require.NoError(t, object.Body.Close()) + assert.Equal(t, int64(10), object.Size) + require.Len(t, fake.RequestsFor(s3test.OpGetObject), 1) + assert.Equal(t, "bytes=2-4", fake.RequestsFor(s3test.OpGetObject)[0].Header.Get("Range")) + + _, err = s.Get(context.Background(), "k", blob.GetOptions{Offset: 50}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid, "416 maps to an invalid request") +} + +func TestStatLoadsTagsOnlyWhenAsked(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + putBytes(t, s, "k", []byte("x"), blob.PutOptions{Tags: map[string]string{"a": "b"}}) + fake.ResetRequests() + + info, err := s.Stat(context.Background(), "k", blob.StatOptions{}) + require.NoError(t, err) + assert.Nil(t, info.Tags) + assert.Empty(t, fake.RequestsFor(s3test.OpGetObjectTagging)) + + info, err = s.Stat(context.Background(), "k", blob.StatOptions{IncludeTags: true}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"a": "b"}, info.Tags) + assert.Len(t, fake.RequestsFor(s3test.OpGetObjectTagging), 1) +} + +func TestListPaginatesAcrossMoreThanAThousandKeys(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + const total = 2503 + for i := range total { + fake.Seed(fmt.Sprintf("many/%05d", i), nil, nil) + } + fake.ResetRequests() + + objects, _, err := blob.ListAll(context.Background(), s, blob.ListOptions{Prefix: "many/"}) + require.NoError(t, err) + require.Len(t, objects, total) + assert.Equal(t, "many/00000", objects[0].Key) + assert.Equal(t, fmt.Sprintf("many/%05d", total-1), objects[total-1].Key) + assert.Len(t, fake.RequestsFor(s3test.OpListObjectsV2), 3, "1000 + 1000 + 503 entries") + for _, object := range objects[:3] { + assert.NotEmpty(t, object.ETag, "S3 listings carry ETags") + } +} + +func TestListDefaultsToServerPageSizeWhenNoLimit(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + fake.Seed("a", nil, nil) + page, err := s.List(context.Background(), blob.ListOptions{}) + require.NoError(t, err) + assert.Empty(t, page.NextCursor) + assert.NotContains(t, fake.RequestsFor(s3test.OpListObjectsV2)[0].Query, "max-keys") +} + +func TestInvalidSizeAndOverflowingRangeFailBeforeRequest(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", strings.NewReader(""), blob.PutOptions{Size: -2}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + _, err = s.Get(context.Background(), "k", blob.GetOptions{Offset: math.MaxInt64, Length: 2}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) + assert.Equal(t, "bytes=9223372036854775807-9223372036854775807", aws.ToString(rangeHeader(blob.GetOptions{Offset: math.MaxInt64, Length: 1}))) +} + +func TestNilPutBodyIsInvalid(t *testing.T) { + fake := newFake(t) + s := newTestStore(t, fake) + _, err := s.Put(context.Background(), "k", nil, blob.PutOptions{}) + require.ErrorIs(t, err, errUtils.ErrBlobInvalid) + assert.Empty(t, fake.Requests()) +} + +func TestMultipartCreateIsNotRetriedAfterLostResponse(t *testing.T) { + fake := newFake(t) + fake.Inject(s3test.Fault{Op: s3test.OpCreateMultipartUpload, Action: s3test.DropAfterCommit}) + s := newTestStore(t, fake, WithPartSize(smallPart)) + data := bytes.Repeat([]byte("x"), int(smallPart)+1) + _, err := s.Put(context.Background(), "k", bytes.NewReader(data), blob.PutOptions{Size: int64(len(data))}) + require.Error(t, err) + assert.Len(t, fake.RequestsFor(s3test.OpCreateMultipartUpload), 1) +} + +func TestEffectivePartSizeDoesNotOverflow(t *testing.T) { + assert.Equal(t, int64(math.MaxInt64/MaxParts+1), effectivePartSize(math.MaxInt64, DefaultPartSize)) +} diff --git a/pkg/artifact/blob/store.go b/pkg/artifact/blob/store.go new file mode 100644 index 00000000000..24b457a061b --- /dev/null +++ b/pkg/artifact/blob/store.go @@ -0,0 +1,205 @@ +package blob + +import ( + "context" + "io" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate go run go.uber.org/mock/mockgen -package blob -destination mock_store.go github.com/cloudposse/atmos/pkg/artifact/blob Store + +// UnknownSize is the Size value of a [PutOptions] whose body length is not known in advance. +const UnknownSize int64 = -1 + +// Store is transport-only object I/O. Keys are slash-separated and already fully formed by the +// caller; a Store never interprets them beyond validation. Every method honors context +// cancellation. Implementations are safe for concurrent use. +type Store interface { + // Name returns the implementation name, for example "local" or "s3". + Name() string + + // Capabilities describes what the store can enforce without network access. + Capabilities() Capabilities + + // Put streams body to key. An unmet or unenforceable [PutOptions.Condition] fails the call + // without changing the object. Returned info describes the stored object, including its ETag. + Put(ctx context.Context, key string, body io.Reader, opts PutOptions) (*ObjectInfo, error) + + // Get opens key for streaming. Callers must close the returned body. A missing object fails + // with ErrBlobNotFound. + Get(ctx context.Context, key string, opts GetOptions) (*Object, error) + + // Stat returns object attributes. Some backends read the body to compute an ETag. A missing object fails with + // ErrBlobNotFound. + Stat(ctx context.Context, key string, opts StatOptions) (*ObjectInfo, error) + + // Delete removes key. Deleting a missing object without a condition succeeds. With + // [DeleteOptions.IfMatch] set, a missing object fails with ErrBlobNotFound and a different ETag + // fails with ErrBlobPreconditionFailed. + Delete(ctx context.Context, key string, opts DeleteOptions) error + + // List returns one lexicographically ordered page of objects under a prefix. Pass the returned + // NextCursor back to continue; an empty NextCursor means the listing is complete. + List(ctx context.Context, opts ListOptions) (*ListPage, error) +} + +// ConditionKind selects how a write is guarded. +type ConditionKind int + +// Condition kinds. +const ( + // ConditionNone writes unconditionally. + ConditionNone ConditionKind = iota + + // ConditionIfAbsent writes only when the key does not exist (S3 If-None-Match: *). + ConditionIfAbsent + + // ConditionIfMatch writes only when the current ETag equals Condition.ETag (S3 If-Match). + ConditionIfMatch +) + +// Condition guards a write. The zero value is unconditional. +type Condition struct { + // Kind selects the guard. + Kind ConditionKind + + // ETag is the expected current ETag for [ConditionIfMatch]. It is an opaque token. + ETag string +} + +// IfAbsent returns a condition that succeeds only when the key does not exist. +func IfAbsent() Condition { + defer perf.Track(nil, "blob.IfAbsent")() + + return Condition{Kind: ConditionIfAbsent} +} + +// IfMatch returns a condition that succeeds only when the current ETag equals etag. +func IfMatch(etag string) Condition { + defer perf.Track(nil, "blob.IfMatch")() + + return Condition{Kind: ConditionIfMatch, ETag: etag} +} + +// IsConditional reports whether the condition constrains the write. +func (c Condition) IsConditional() bool { + defer perf.Track(nil, "blob.Condition.IsConditional")() + + return c.Kind != ConditionNone +} + +// PutOptions describe one write. +type PutOptions struct { + // Size is the body length in bytes, or [UnknownSize]. A known size is verified: a body that + // is shorter or longer fails the call and leaves no object behind. + Size int64 + + // ContentType is stored with the object when non-empty. + ContentType string + + // Metadata is stored as user metadata. + Metadata map[string]string + + // Tags are explicit object tags. Stores that merge default tags apply them first; these win. + Tags map[string]string + + // Condition guards the write. The zero value is unconditional. + Condition Condition +} + +// GetOptions describe one read. +type GetOptions struct { + // Offset is the first byte to read. + Offset int64 + + // Length is the number of bytes to read from Offset. Zero or negative reads to the end. + Length int64 + + // SkipETag tells the store the caller does not need the ETag, so a store that derives it from + // the content may leave it empty instead of reading the whole object first. + SkipETag bool +} + +// IsRange reports whether the read is a partial read. +func (o GetOptions) IsRange() bool { + defer perf.Track(nil, "blob.GetOptions.IsRange")() + + return o.Offset > 0 || o.Length > 0 +} + +// StatOptions describe one head request. +type StatOptions struct { + // IncludeTags also loads the object tags, which may cost an extra request. + IncludeTags bool + + // SkipETag tells the store the caller does not need the ETag, so a store that derives it from + // the content may leave it empty instead of reading the whole object first. + SkipETag bool +} + +// DeleteOptions describe one delete. +type DeleteOptions struct { + // IfMatch deletes only when the current ETag equals this opaque token. + IfMatch string +} + +// ListOptions describe one listing page. +type ListOptions struct { + // Prefix restricts the listing to keys beginning with it. + Prefix string + + // Delimiter groups keys sharing a prefix up to the delimiter into CommonPrefixes. + Delimiter string + + // Cursor continues a previous listing. + Cursor string + + // Limit caps the entries (objects plus common prefixes) in one page; zero uses the store default. + Limit int +} + +// ObjectInfo describes a stored object. +type ObjectInfo struct { + // Key is the object key. + Key string + + // Size is the full object size in bytes, even for a ranged read. + Size int64 + + // ETag is an opaque concurrency token, never a payload checksum. + ETag string + + // ContentType is the stored content type, when known. + ContentType string + + // LastModified is the last modification time. + LastModified time.Time + + // Metadata is user metadata, when loaded. + Metadata map[string]string + + // Tags are object tags, loaded only when requested. + Tags map[string]string +} + +// Object is an open object. Callers must close Body. +type Object struct { + ObjectInfo + + // Body streams the (possibly ranged) content. + Body io.ReadCloser +} + +// ListPage is one page of a listing. +type ListPage struct { + // Objects are the keys found, lexicographically ordered. + Objects []ObjectInfo + + // CommonPrefixes are rolled-up prefixes when a delimiter was requested. + CommonPrefixes []string + + // NextCursor continues the listing; empty means complete. + NextCursor string +} diff --git a/pkg/artifact/blob/tags.go b/pkg/artifact/blob/tags.go new file mode 100644 index 00000000000..169aed4183a --- /dev/null +++ b/pkg/artifact/blob/tags.go @@ -0,0 +1,80 @@ +package blob + +import ( + "fmt" + "maps" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// S3 object tag limits, enforced for every store so tags are portable. +const ( + MaxTags = 10 + MaxTagKeyChars = 128 + MaxTagValueChars = 256 +) + +// supplementaryPlane is the first rune that needs a UTF-16 surrogate pair. +const supplementaryPlane = 0x10000 + +// MergeTags returns defaults overlaid with overrides. It returns nil when both are empty and never +// aliases either input. +func MergeTags(defaults, overrides map[string]string) map[string]string { + defer perf.Track(nil, "blob.MergeTags")() + + if len(defaults) == 0 && len(overrides) == 0 { + return nil + } + merged := make(map[string]string, len(defaults)+len(overrides)) + maps.Copy(merged, defaults) + maps.Copy(merged, overrides) + return merged +} + +// ValidateTags checks the S3 object tag limits: at most [MaxTags] tags, non-empty keys of at most +// [MaxTagKeyChars] UTF-16 code units, and values of at most [MaxTagValueChars] UTF-16 code units. +func ValidateTags(tags map[string]string) error { + defer perf.Track(nil, "blob.ValidateTags")() + + if len(tags) > MaxTags { + return tagError(fmt.Sprintf("%d tags exceed the limit of %d", len(tags), MaxTags)) + } + for key, value := range tags { + if key == "" || !utf8.ValidString(key) || utf16Len(key) > MaxTagKeyChars { + return tagError(fmt.Sprintf("tag key %q must be 1-%d characters", key, MaxTagKeyChars)) + } + if !utf8.ValidString(value) || utf16Len(value) > MaxTagValueChars { + return tagError(fmt.Sprintf("tag %q value exceeds %d characters", key, MaxTagValueChars)) + } + } + return nil +} + +// utf16Len counts UTF-16 code units, which is how S3 measures tag lengths: a supplementary +// character counts as two. +func utf16Len(s string) int { + n := 0 + for _, r := range s { + n++ + if r >= supplementaryPlane { + n++ + } + } + return n +} + +func tagError(reason string) error { + return NewError(KindInvalid, OpValidate, "", fmt.Errorf("%w: %s", errUtils.ErrBlobInvalid, reason)) +} + +// CloneMap returns a copy of m, or nil when m is empty. +func CloneMap(m map[string]string) map[string]string { + defer perf.Track(nil, "blob.CloneMap")() + + if len(m) == 0 { + return nil + } + return maps.Clone(m) +} diff --git a/pkg/artifact/compat/compat_test.go b/pkg/artifact/compat/compat_test.go index ec81d680c89..e8323e7d821 100644 --- a/pkg/artifact/compat/compat_test.go +++ b/pkg/artifact/compat/compat_test.go @@ -121,6 +121,7 @@ func TestLegacyCIFixtureStillReadsThroughCurrentStore(t *testing.T) { require.Len(t, files, 2) bodies := map[string]string{} for _, file := range files { + t.Cleanup(func() { _ = file.Data.Close() }) body, readErr := io.ReadAll(file.Data) require.NoError(t, readErr) require.NoError(t, file.Data.Close()) diff --git a/pkg/artifact/layout/admit.go b/pkg/artifact/layout/admit.go new file mode 100644 index 00000000000..b9cd8d91545 --- /dev/null +++ b/pkg/artifact/layout/admit.go @@ -0,0 +1,142 @@ +package layout + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Admission is the decision a repository takes for one publication after reading the current +// reference under the writer guard. +type Admission string + +// Admission decisions. +const ( + // AdmitCreate writes a new reference. + AdmitCreate Admission = "create" + + // AdmitReplace replaces a completed reference under force. + AdmitReplace Admission = "replace" + + // AdmitUnchanged leaves the completed reference untouched: the same intent already exists. + AdmitUnchanged Admission = "unchanged" + + // AdmitResumeCreate resumes a pending first publication of the same intent. + AdmitResumeCreate Admission = "resume-create" + + // AdmitResumeReplace resumes a pending replacement of the same intent. + AdmitResumeReplace Admission = "resume-replace" +) + +// Outcome maps an admission to the operation outcome it produces. +func (a Admission) Outcome() artifact.Outcome { + defer perf.Track(nil, "layout.Admission.Outcome")() + + switch a { + case AdmitReplace, AdmitResumeReplace: + return artifact.OutcomeReplaced + case AdmitUnchanged: + return artifact.OutcomeUnchanged + default: + return artifact.OutcomeCreated + } +} + +// Resuming reports whether the admission continues an interrupted publication. +func (a Admission) Resuming() bool { + defer perf.Track(nil, "layout.Admission.Resuming")() + + return a == AdmitResumeCreate || a == AdmitResumeReplace +} + +// AdmitInput is everything the admission rules need. +type AdmitInput struct { + // Current is the reference as stored, or nil when it does not exist. + Current *Reference + + // CurrentIntent is the intent digest of the completed Record. It is only read when Current is + // completed; empty means it could not be computed, which never counts as unchanged. + CurrentIntent digest.Digest + + // IncomingRecord and IncomingIntent identify the artifact being published. + IncomingRecord digest.Digest + IncomingIntent digest.Digest + + // ExpectedPrior is the reference the service observed; nil means the reference must not exist. + ExpectedPrior *artifact.PinnedReference + + // Force permits replacing a conflicting completed reference. + Force bool +} + +// Admit applies the compare-and-swap, intent, force, and recovery rules of the publication +// lifecycle: +// +// - A stale expected-prior reference is a conflict, never a silent overwrite. +// - The same record or intent as the completed reference is unchanged. +// - A different completed intent conflicts unless force is set. +// - A pending reference of the same intent resumes; a pending reference of a different intent +// is unavailable and names the owning operation, even under force. A pending operation is +// never stolen automatically. +func Admit(in *AdmitInput) (Admission, error) { + defer perf.Track(nil, "layout.Admit")() + + if in.Current == nil { + if in.ExpectedPrior != nil { + return "", fmt.Errorf("%w: the reference no longer exists but the writer expected %s", errUtils.ErrArtifactConflict, in.ExpectedPrior.ResolvedDigest) + } + return AdmitCreate, nil + } + if in.Current.State == StatePending { + return admitPending(in) + } + return admitCompleted(in) +} + +// admitCompleted decides publication over a completed reference. +func admitCompleted(in *AdmitInput) (Admission, error) { + if err := checkExpectedPrior(in.Current, in.ExpectedPrior); err != nil { + return "", err + } + if in.Current.RecordDigest == in.IncomingRecord || (in.CurrentIntent != "" && in.CurrentIntent == in.IncomingIntent) { + return AdmitUnchanged, nil + } + if !in.Force { + return "", fmt.Errorf("%w: %s:%s already exists with a different intent; use force to replace it", errUtils.ErrArtifactConflict, in.Current.Name, in.Current.Version) + } + return AdmitReplace, nil +} + +// admitPending decides publication over a pending reference: only the same intent, expecting the +// same prior, may resume it. +func admitPending(in *AdmitInput) (Admission, error) { + if in.Current.IntentDigest != in.IncomingIntent { + return "", fmt.Errorf("%w: %s:%s is pending for operation %s with a different intent; recover or repair it with the same intent before publishing", + errUtils.ErrArtifactPublicationUnavailable, in.Current.Name, in.Current.Version, in.Current.OperationID) + } + var expected digest.Digest + if in.ExpectedPrior != nil { + expected = in.ExpectedPrior.ResolvedDigest + } + if expected != in.Current.PriorRecordDigest { + return "", fmt.Errorf("%w: pending publication expected prior %s, caller expected %s", errUtils.ErrArtifactConflict, in.Current.PriorRecordDigest, expected) + } + if in.Current.PriorRecordDigest != "" { + return AdmitResumeReplace, nil + } + return AdmitResumeCreate, nil +} + +func checkExpectedPrior(current *Reference, expected *artifact.PinnedReference) error { + switch { + case expected == nil: + return fmt.Errorf("%w: %s:%s now exists (%s) but the writer expected it to be absent", errUtils.ErrArtifactConflict, current.Name, current.Version, current.RecordDigest) + case expected.ResolvedDigest != current.RecordDigest: + return fmt.Errorf("%w: %s:%s is %s but the writer expected %s", errUtils.ErrArtifactConflict, current.Name, current.Version, current.RecordDigest, expected.ResolvedDigest) + } + return nil +} diff --git a/pkg/artifact/layout/claims.go b/pkg/artifact/layout/claims.go new file mode 100644 index 00000000000..c42d28f65a1 --- /dev/null +++ b/pkg/artifact/layout/claims.go @@ -0,0 +1,86 @@ +package layout + +import ( + "path" + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Claim is the set of storage paths one reference owns. Two references may never own overlapping +// paths, which is how two references with a colliding key_pattern are detected: the second fails +// with a conflict and is never bound to the first reference's payload. +type Claim struct { + // ReferenceKey identifies the owning reference. + ReferenceKey string + + // Name and Version identify the owner for diagnostics. + Name string + Version string + + // OperationID is the pending operation holding the claim, empty for a completed reference. + OperationID string + + // Paths are the owned slash-separated paths relative to the repository root. + Paths []string +} + +// PayloadPaths returns the paths a payload at location owns for the repository kind: a copied +// directory owns the directory; an archive owns the file, its metadata sidecar, and its +// documents directory. +func PayloadPaths(repositoryKind, location string) []string { + defer perf.Track(nil, "layout.PayloadPaths")() + + if repositoryKind == artifact.RepositoryKindLocalArchive { + return []string{location, MetadataSidecarPath(location), DocsSidecarDir(location)} + } + return []string{location} +} + +// FoldPath returns the comparison form of a slash-separated path: cleaned and case folded, because +// common filesystems alias names that differ only by case. +func FoldPath(p string) string { + defer perf.Track(nil, "layout.FoldPath")() + + return strings.ToLower(path.Clean(p)) +} + +// PathsOverlap reports whether two paths are equal or one contains the other. +func PathsOverlap(a, b string) bool { + defer perf.Track(nil, "layout.PathsOverlap")() + + a, b = FoldPath(a), FoldPath(b) + return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/") +} + +// Overlaps reports whether any path of c overlaps any path of other. +func (c *Claim) Overlaps(other *Claim) bool { + defer perf.Track(nil, "layout.Claim.Overlaps")() + + for _, mine := range c.Paths { + for _, theirs := range other.Paths { + if PathsOverlap(mine, theirs) { + return true + } + } + } + return false +} + +// FindConflict returns the first existing claim, owned by a different reference, that overlaps the +// candidate, or nil. A reference never conflicts with itself, so replacement and same-intent +// retry keep working. +func FindConflict(candidate *Claim, existing []Claim) *Claim { + defer perf.Track(nil, "layout.FindConflict")() + + for index := range existing { + if existing[index].ReferenceKey == candidate.ReferenceKey { + continue + } + if candidate.Overlaps(&existing[index]) { + return &existing[index] + } + } + return nil +} diff --git a/pkg/artifact/layout/codec.go b/pkg/artifact/layout/codec.go new file mode 100644 index 00000000000..6db36b993db --- /dev/null +++ b/pkg/artifact/layout/codec.go @@ -0,0 +1,83 @@ +package layout + +import ( + "bytes" + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// MaxDocumentBytes bounds every layout document (reference, representation, operation journal). +const MaxDocumentBytes int64 = 1 << 20 + +// MaxEvidenceFileBytes bounds one stored evidence file: the base64 form of the largest evidence +// blob plus its envelope. +const MaxEvidenceFileBytes = 2*artifact.MaxEvidenceSize + 4096 + +// Document kinds, used in error messages. +const ( + kindReference = "reference" + kindRepresentation = "representation" + kindOperation = "operation" + kindEvidence = "evidence" +) + +// invalid wraps a malformed or inconsistent managed document. It matches both the integrity and +// the record-invalid sentinels, because a stored managed document that does not parse is, to the +// reader, corrupted state. +func invalid(kind, format string, args ...any) error { + return fmt.Errorf("%w: %w: %s: %s", errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactRecordInvalid, kind, fmt.Sprintf(format, args...)) +} + +// validator is implemented by every layout document. +type validator interface { + Validate() error +} + +// encode canonicalizes v: sorted keys, no insignificant whitespace, no trailing newline. Stored +// bytes are therefore a function of the document alone. +func encode(v validator, limit int64) ([]byte, error) { + if err := v.Validate(); err != nil { + return nil, err + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + canonical, err := artifact.CanonicalizeJSON(buf.Bytes()) + if err != nil { + return nil, err + } + if int64(len(canonical)) > limit { + return nil, fmt.Errorf("%w: layout document is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(canonical), limit) + } + return canonical, nil +} + +// decode strictly parses raw into v: it rejects oversize input, unknown fields, invalid +// documents, and any bytes that are not exactly the canonical serialization of the result. +func decode(kind string, raw []byte, v validator, limit int64) error { + if int64(len(raw)) > limit { + return invalid(kind, "%d bytes exceeds the %d byte limit", len(raw), limit) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(v); err != nil { + return invalid(kind, "%v", err) + } + if err := v.Validate(); err != nil { + return invalid(kind, "%v", err) + } + canonical, err := encode(v, limit) + if err != nil { + return invalid(kind, "%v", err) + } + if !bytes.Equal(canonical, raw) { + return invalid(kind, "bytes are not the canonical serialization") + } + return nil +} diff --git a/pkg/artifact/layout/doc.go b/pkg/artifact/layout/doc.go new file mode 100644 index 00000000000..0a49063a385 --- /dev/null +++ b/pkg/artifact/layout/doc.go @@ -0,0 +1,29 @@ +// Package layout defines the backend-neutral record layout shared by the local directory, local +// archive, and S3 artifact repositories. It is pure data and logic: it performs no filesystem, +// network, or AWS work, so every backend stores the same documents and agrees on names, paths, +// state transitions, and collision rules. +// +// Below a repository root (a directory, or a bucket prefix) the managed records live in the +// reserved ".atmos-artifacts/v1/" namespace: +// +// .atmos-artifacts/v1/refs/.json stable-root reference record +// .atmos-artifacts/v1/records/.json immutable canonical Record +// .atmos-artifacts/v1/representations/.json mutable storage description +// .atmos-artifacts/v1/operations/.json publication journal (recovery only) +// .atmos-artifacts/v1/signatures///.json detached evidence +// payload (outside the namespace) +// +// The reference key is the SHA-256 of "name:version", so names containing slashes can never +// collide with control paths. A reference record binds a name and version to the digest of its +// immutable Record and carries the pending or completed publication state. The Record never holds +// a mutable storage locator: the resolved payload location is rendered from the repository +// key_pattern exactly once, at publication, from the captured creation time, and is persisted in +// the representation record. Reads, deletes, and retries use the persisted location and never +// render the pattern again, so a date rollover cannot move an artifact. +// +// Publication order: write the pending operation journal and mark the reference pending, stage +// and install the payload, write the Record, the representation, and the adjacent documents, and +// only then flip the reference to completed. Pending references are unavailable to readers and +// absent from search results; readers recheck the reference generation after reading from a +// mutable payload location. +package layout diff --git a/pkg/artifact/layout/evidence.go b/pkg/artifact/layout/evidence.go new file mode 100644 index 00000000000..731175b5e55 --- /dev/null +++ b/pkg/artifact/layout/evidence.go @@ -0,0 +1,120 @@ +package layout + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Evidence is the stored form of one detached signature evidence blob. The file name is the +// SHA-256 of Data, so the evidence digest the verifier reports identifies exactly the signed +// bytes, and a stored file is rejected when its name and content disagree. +// +// Evidence is stored beside, never inside, the signed Record, so adding a signature cannot change +// the Record, its digest, or the reference. +type Evidence struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // Subject is the immutable subject the evidence claims to cover. + Subject artifact.SignatureSubject `json:"subject"` + + // Format is the evidence encoding. + Format artifact.EvidenceFormat `json:"format"` + + // MediaType is the evidence media type. + MediaType string `json:"media_type"` + + // Data is the evidence bytes, at most [artifact.MaxEvidenceSize]. + Data []byte `json:"data"` +} + +// NewEvidence builds the stored form of a freshly produced evidence payload. +func NewEvidence(subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (*Evidence, error) { + defer perf.Track(nil, "layout.NewEvidence")() + + evidence := &Evidence{SchemaVersion: SchemaVersion, Subject: subject, Format: payload.Format, MediaType: payload.MediaType, Data: payload.Data} + if err := evidence.Validate(); err != nil { + return nil, err + } + return evidence, nil +} + +// Digest returns the evidence digest: the SHA-256 of the evidence bytes. +func (e *Evidence) Digest() digest.Digest { + defer perf.Track(nil, "layout.Evidence.Digest")() + + return digest.FromBytes(e.Data) +} + +// Validate checks the evidence envelope. +func (e *Evidence) Validate() error { + defer perf.Track(nil, "layout.Evidence.Validate")() + + if e.SchemaVersion != SchemaVersion { + return invalid(kindEvidence, "unsupported schema_version %d", e.SchemaVersion) + } + if err := e.Subject.Validate(); err != nil { + return err + } + if e.Format == "" || e.MediaType == "" { + return invalid(kindEvidence, "format and media_type are required") + } + if len(e.Data) == 0 { + return invalid(kindEvidence, "data is empty") + } + if int64(len(e.Data)) > artifact.MaxEvidenceSize { + return fmt.Errorf("%w: evidence is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(e.Data), artifact.MaxEvidenceSize) + } + return nil +} + +// Marshal returns the canonical bytes of the envelope. +func (e *Evidence) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Evidence.Marshal")() + + return encode(e, MaxEvidenceFileBytes) +} + +// DecodeEvidence strictly parses a stored evidence file. The wantDigest argument is the evidence +// digest taken from the file name; it must equal the digest of the stored bytes. +func DecodeEvidence(raw []byte, wantDigest digest.Digest) (*Evidence, error) { + defer perf.Track(nil, "layout.DecodeEvidence")() + + var evidence Evidence + if err := decode(kindEvidence, raw, &evidence, MaxEvidenceFileBytes); err != nil { + return nil, err + } + if evidence.Digest() != wantDigest { + return nil, invalid(kindEvidence, "content digest %s does not match its file name %s", evidence.Digest(), wantDigest) + } + return &evidence, nil +} + +// Blob converts the stored evidence into an [artifact.EvidenceBlob]. The locator argument is the +// display location of the evidence file. +func (e *Evidence) Blob(locator string) artifact.EvidenceBlob { + defer perf.Track(nil, "layout.Evidence.Blob")() + + return artifact.EvidenceBlob{Evidence: e.Reference(locator), Data: append([]byte(nil), e.Data...)} +} + +// Reference returns the descriptor of the stored evidence. The locator argument is the display +// location. +func (e *Evidence) Reference(locator string) artifact.SignatureEvidence { + defer perf.Track(nil, "layout.Evidence.Reference")() + + return artifact.SignatureEvidence{ + Subject: e.Subject, + Format: e.Format, + Descriptor: ocispec.Descriptor{ + MediaType: e.MediaType, Digest: e.Digest(), Size: int64(len(e.Data)), + }, + Locator: locator, + } +} diff --git a/pkg/artifact/layout/layout_test.go b/pkg/artifact/layout/layout_test.go new file mode 100644 index 00000000000..e86a73a53df --- /dev/null +++ b/pkg/artifact/layout/layout_test.go @@ -0,0 +1,552 @@ +package layout + +import ( + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +var ( + digestA = digest.FromString("a") + digestB = digest.FromString("b") + digestC = digest.FromString("c") +) + +func TestReferenceKey(t *testing.T) { + key := ReferenceKey("components/terraform/vpc", "1.2.0") + assert.Len(t, key, 64, "lowercase hex SHA-256") + assert.Equal(t, digest.FromString("components/terraform/vpc:1.2.0").Encoded(), key, "the key is the SHA-256 of name:version") + assert.NotEqual(t, key, ReferenceKey("components/terraform", "vpc:1.2.0"), "names with slashes cannot collide with other name/version splits") + assert.NotContains(t, key, "/", "no slash can reach a control path") +} + +func TestPaths(t *testing.T) { + key := ReferenceKey("a", "1") + assert.Equal(t, ".atmos-artifacts/v1/refs/"+key+".json", ReferencePath(key)) + assert.Equal(t, ".atmos-artifacts/v1/representations/"+key+".json", RepresentationPath(key)) + assert.Equal(t, ".atmos-artifacts/v1/records/"+digestA.Encoded()+".json", RecordPath(digestA)) + assert.Equal(t, ".atmos-artifacts/v1/refs", ReferencesDir()) + assert.Equal(t, + ".atmos-artifacts/v1/signatures/record/"+digestA.Encoded()+"/"+digestB.Encoded()+".json", + SignaturePath(artifact.SubjectRecord, digestA, digestB)) + assert.Equal(t, "x/y.tgz.metadata.json", MetadataSidecarPath("x/y.tgz")) + assert.Equal(t, "x/y.tgz.docs/README.md", DocumentSidecarPath("x/y.tgz", artifact.DocumentReadme)) + assert.Equal(t, "x/y.tgz.docs/CHANGELOG.md", DocumentSidecarPath("x/y.tgz", artifact.DocumentChangelog)) + assert.Equal(t, "x/y/.atmos/artifact/manifest.json", DistributionPath("x/y", ManifestName)) + assert.True(t, strings.HasPrefix(LockPath("any resource/with:odd chars"), ManagedRoot+"/locks/")) + assert.NotEqual(t, LockPath("a"), LockPath("b")) +} + +func TestOperationFileNameCannotCollideWithHashedNames(t *testing.T) { + hashed := OperationFileName("op/with/slash") + require.True(t, strings.HasPrefix(hashed, "sha256-")) + assert.NotEqual(t, hashed, OperationFileName(hashed), "a literal id equal to another id's hashed name is hashed itself") +} + +func TestOperationFileName(t *testing.T) { + tests := []struct { + name string + id string + safe bool + }{ + {"uuid", "9f1c6d52-48b6-4c3c-9f0e-3a5b7f1d1c11", true}, + {"dotted", "op.1_x-2", true}, + {"slash", "op-team/app-1.0.0", false}, + {"space", "op one", false}, + {"traversal", "../x", false}, + {"overlong", strings.Repeat("a", 200), false}, + {"empty", "", false}, + {"reserved hash prefix", "sha256-deadbeef", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := OperationFileName(tt.id) + if tt.safe { + assert.Equal(t, tt.id, got) + } else { + assert.True(t, strings.HasPrefix(got, "sha256-"), got) + } + assert.NotContains(t, got, "/") + assert.Equal(t, got, OperationFileName(tt.id), "stable") + assert.Equal(t, ManagedRoot+"/operations/"+got+".json", OperationPath(tt.id)) + assert.Equal(t, ManagedRoot+"/staging/"+got, StagingPath(tt.id)) + }) + } +} + +func TestInManagedNamespace(t *testing.T) { + assert.True(t, InManagedNamespace(".atmos-artifacts/v1/refs/x.json")) + assert.True(t, InManagedNamespace("./.ATMOS-Artifacts")) + assert.False(t, InManagedNamespace("team/.atmos-artifacts"), "only the first segment is the root namespace") + assert.False(t, InManagedNamespace("atmos-artifacts/x")) +} + +func TestRenderLocation(t *testing.T) { + captured := time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + in := &PatternInput{Name: "components/terraform/vpc", Version: "1.2.0", Kind: "component/terraform", CreatedAt: captured} + tests := []struct { + name string + pattern string + packaging artifact.Packaging + want string + wantErr error + }{ + {"default copied files", "", artifact.PackagingNone, "components/terraform/vpc/1.2.0", nil}, + {"default tgz", "", artifact.PackagingTgz, "components/terraform/vpc/1.2.0.tgz", nil}, + {"default tar", "", artifact.PackagingTar, "components/terraform/vpc/1.2.0.tar", nil}, + {"default zip", "", artifact.PackagingZip, "components/terraform/vpc/1.2.0.zip", nil}, + {"dated root from the PRD", `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, artifact.PackagingTgz, "2026/10/30/components/terraform/vpc/1.2.0.tgz", nil}, + {"kind field", "{{ .Kind }}/{{ .Version }}", artifact.PackagingNone, "component/terraform/1.2.0", nil}, + {"trailing slash is trimmed", "x/{{ .Version }}/", artifact.PackagingNone, "x/1.2.0", nil}, + {"sprig function", "{{ .Name | upper }}/{{ .Version }}", artifact.PackagingNone, "COMPONENTS/TERRAFORM/VPC/1.2.0", nil}, + {"mismatched archive extension", "{{ .Name }}.zip", artifact.PackagingTgz, "", errUtils.ErrArtifactPackagingInvalid}, + {"tar.gz agrees with tgz", "{{ .Name }}.tar.gz", artifact.PackagingTgz, "components/terraform/vpc.tar.gz", nil}, + {"no extension is not renamed", "{{ .Name }}", artifact.PackagingTgz, "components/terraform/vpc", nil}, + {"root escape", "../{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"rendered root escape", "{{ \"..\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"absolute", "/abs/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"empty segment", "a//b", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"reserved namespace", ".atmos-artifacts/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"reserved namespace any case", "x/.ATMOS-ARTIFACTS/y", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"windows reserved character", "a:b/{{ .Name }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + {"unknown field", "{{ .Stack }}", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"syntax error", "{{ .Name", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"clock is not available", "{{ now | date \"2006\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"environment is not available", "{{ env \"HOME\" }}/x", artifact.PackagingNone, "", errUtils.ErrArtifactConfigInvalid}, + {"empty result", "{{ \"\" }}", artifact.PackagingNone, "", errUtils.ErrArtifactUnsafePath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := RenderLocation(tt.pattern, in, tt.packaging) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRenderLocation_RejectsRandomAndClockDependentFunctions(t *testing.T) { + in := &PatternInput{Name: "app", Version: "1", Kind: "x", CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)} + for _, name := range []string{ + "now", "ago", "env", "randAlphaNum", "randAlpha", "randNumeric", "randAscii", "randInt", "uuidv4", + "shuffle", "dateInZone", "date_in_zone", "htmlDate", "htmlDateInZone", + } { + t.Run(name, func(t *testing.T) { + _, err := RenderLocation("{{ "+name+" }}", in, artifact.PackagingNone) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "%s must not be defined in key_pattern", name) + assert.Contains(t, err.Error(), "not defined") + }) + } + got, err := RenderLocation("{{ .Name }}/{{ date \"2006\" .CreatedAt }}", in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, "app/2026", got, "the deterministic UTC date function remains available") +} + +func TestRenderLocation_DateIsUTCAndStable(t *testing.T) { + // 23:30 at UTC-5 is already the next day in UTC; the captured instant decides, not the zone. + zone := time.FixedZone("UTC-5", -5*60*60) + local := time.Date(2026, 10, 30, 23, 30, 0, 0, zone) + in := &PatternInput{Name: "a", Version: "1", Kind: "files", CreatedAt: local} + pattern := `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}` + first, err := RenderLocation(pattern, in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, "2026/10/31/a", first) + + // Rendering again with the same captured time is stable even if the wall clock moved on. + second, err := RenderLocation(pattern, in, artifact.PackagingNone) + require.NoError(t, err) + assert.Equal(t, first, second) + + rolled := *in + rolled.CreatedAt = local.Add(48 * time.Hour) + third, err := RenderLocation(pattern, &rolled, artifact.PackagingNone) + require.NoError(t, err) + assert.NotEqual(t, first, third, "a different captured time renders a different dated root; callers persist the first") +} + +func TestCleanLocation(t *testing.T) { + for _, valid := range []string{"a", "a/b", "a/b/", "a/b.tgz", "team/app/1.0.0", "ünï/çode"} { + _, err := CleanLocation(valid) + require.NoError(t, err, valid) + } + for _, bad := range []string{"", "/", "/a", "a/../b", "./a", "a/./b", `a\b`, "a\x00b", "a\nb", ".atmos-artifacts", "a/.Atmos-Artifacts/b", "a|b", "a\xff", strings.Repeat("a", 2000)} { + _, err := CleanLocation(bad) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, "%q", bad) + } +} + +func TestClaims(t *testing.T) { + archive := func(key, location string) Claim { + return Claim{ReferenceKey: key, Paths: PayloadPaths(artifact.RepositoryKindLocalArchive, location)} + } + dir := func(key, location string) Claim { + return Claim{ReferenceKey: key, Paths: PayloadPaths(artifact.RepositoryKindLocalDir, location)} + } + assert.Equal(t, []string{"x/1.tgz", "x/1.tgz.metadata.json", "x/1.tgz.docs"}, archive("k", "x/1.tgz").Paths) + assert.Equal(t, []string{"x/1"}, dir("k", "x/1").Paths) + + tests := []struct { + name string + candidate Claim + existing []Claim + conflict bool + }{ + {"identical location", dir("b", "x/1"), []Claim{dir("a", "x/1")}, true}, + {"case-folded location", dir("b", "X/1"), []Claim{dir("a", "x/1")}, true}, + {"candidate inside existing directory", dir("b", "x/1/y"), []Claim{dir("a", "x/1")}, true}, + {"existing inside candidate directory", dir("b", "x"), []Claim{dir("a", "x/1")}, true}, + {"archive sidecar collides with another archive", archive("b", "x/1.tgz.metadata.json"), []Claim{archive("a", "x/1.tgz")}, true}, + {"archive docs directory collides", archive("b", "x/1.tgz.docs/README.md"), []Claim{archive("a", "x/1.tgz")}, true}, + {"siblings do not collide", dir("b", "x/2"), []Claim{dir("a", "x/1")}, false}, + {"prefix without a separator is not an overlap", dir("b", "x/10"), []Claim{dir("a", "x/1")}, false}, + {"the same reference never conflicts with itself", dir("a", "x/1"), []Claim{dir("a", "x/1")}, false}, + {"no claims", dir("a", "x/1"), nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + found := FindConflict(&tt.candidate, tt.existing) + assert.Equal(t, tt.conflict, found != nil) + if tt.conflict { + assert.Equal(t, "a", found.ReferenceKey) + } + }) + } +} + +func TestReference_StateModel(t *testing.T) { + first := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + require.NoError(t, first.Validate()) + assert.False(t, first.Available(), "a pending reference is unavailable to readers") + assert.EqualValues(t, 1, first.Generation) + assert.Empty(t, first.PriorRecordDigest) + assert.Equal(t, ReferenceKey("team/app", "1.0.0"), first.Key()) + + done := first.Completed(digestB) + require.NoError(t, done.Validate()) + assert.True(t, done.Available()) + assert.EqualValues(t, 2, done.Generation, "every state change bumps the generation") + assert.Equal(t, digestB, done.RecordDigest) + assert.Equal(t, StatePending, first.State, "Completed does not mutate its receiver") + + replacing := NewPendingReference("team/app", "1.0.0", "op-2", digestC, &done) + require.NoError(t, replacing.Validate()) + assert.False(t, replacing.Available()) + assert.EqualValues(t, 3, replacing.Generation) + assert.Equal(t, digestB, replacing.PriorRecordDigest, "a replacement remembers what it replaces") + assert.Empty(t, replacing.RecordDigest, "the pending reference must not advertise either record") + + again := NewPendingReference("team/app", "1.0.0", "op-3", digestC, &replacing) + assert.Equal(t, digestB, again.PriorRecordDigest, "a re-pending interrupted replacement keeps the original prior") + assert.EqualValues(t, 4, again.Generation) + + finished := replacing.Completed(digestC) + assert.Empty(t, finished.PriorRecordDigest) + assert.EqualValues(t, 4, finished.Generation) +} + +func TestReference_RoundTripAndStrictness(t *testing.T) { + ref := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + ref = ref.Completed(digestB) + raw, err := ref.Marshal() + require.NoError(t, err) + assert.NotContains(t, string(raw), "\n") + + decoded, err := DecodeReference(raw) + require.NoError(t, err) + assert.Equal(t, ref, *decoded) + + again, err := decoded.Marshal() + require.NoError(t, err) + assert.Equal(t, raw, again, "serialization is deterministic") + + tests := []struct { + name string + mutate func(string) string + }{ + {"unknown field", func(s string) string { return strings.Replace(s, `{"`, `{"extra":1,"`, 1) }}, + {"not canonical (whitespace)", func(s string) string { return strings.Replace(s, `,"`, `, "`, 1) }}, + {"truncated", func(s string) string { return s[:len(s)-2] }}, + {"wrong schema", func(s string) string { return strings.Replace(s, `"schema_version":1`, `"schema_version":2`, 1) }}, + {"garbage", func(string) string { return "not json" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DecodeReference([]byte(tt.mutate(string(raw)))) + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } +} + +func TestReference_ValidateRejectsInconsistentStates(t *testing.T) { + base := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + completed := base.Completed(digestB) + tests := []struct { + name string + ref func() Reference + wantOK bool + }{ + {"pending without record", func() Reference { return base }, true}, + {"completed", func() Reference { return completed }, true}, + {"pending naming a record", func() Reference { r := base; r.RecordDigest = digestB; return r }, false}, + {"completed without a record", func() Reference { r := completed; r.RecordDigest = ""; return r }, false}, + {"completed with a prior", func() Reference { r := completed; r.PriorRecordDigest = digestA; return r }, false}, + {"zero generation", func() Reference { r := base; r.Generation = 0; return r }, false}, + {"missing operation", func() Reference { r := base; r.OperationID = ""; return r }, false}, + {"unknown state", func() Reference { r := base; r.State = "weird"; return r }, false}, + {"bad intent digest", func() Reference { r := base; r.IntentDigest = "sha256:zz"; return r }, false}, + {"invalid name", func() Reference { r := base; r.Name = "../x"; return r }, false}, + {"invalid version", func() Reference { r := base; r.Version = ""; return r }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref := tt.ref() + err := ref.Validate() + if tt.wantOK { + require.NoError(t, err) + return + } + require.Error(t, err) + }) + } +} + +func sampleRepresentation() Representation { + return Representation{ + SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalArchive, Name: "team/app", Version: "1.0.0", + RecordDigest: digestA, OperationID: "op-1", Packaging: artifact.PackagingTgz, Location: "2026/10/30/team/app/1.0.0.tgz", + Payload: []ocispec.Descriptor{{MediaType: artifact.MediaTypeTarGzip, Digest: digestB, Size: 10}}, + Documents: []ocispec.Descriptor{}, + RepresentationDigest: digestB, + } +} + +func TestRepresentation_PersistsResolvedLocation(t *testing.T) { + rep := sampleRepresentation() + raw, err := rep.Marshal() + require.NoError(t, err) + decoded, err := DecodeRepresentation(raw) + require.NoError(t, err) + assert.Equal(t, rep.Location, decoded.Location, "the dated location survives serialization verbatim") + assert.Equal(t, rep.Payload, decoded.Payload) + assert.Equal(t, []string{rep.Location, rep.Location + ".metadata.json", rep.Location + ".docs"}, decoded.Claim().Paths) + + converted := decoded.Artifact("/abs/root/" + rep.Location) + assert.Equal(t, artifact.PackagingTgz, converted.Packaging) + assert.Equal(t, "/abs/root/"+rep.Location, converted.Locator) + assert.Equal(t, digestA, converted.RecordDigest) + converted.Payload[0].Size = 99 + assert.EqualValues(t, 10, decoded.Payload[0].Size, "the conversion copies the descriptors") +} + +func TestRepresentation_RejectsUnsafeStoredLocation(t *testing.T) { + for _, location := range []string{"", "../escape", "/abs", "a/../b", ".atmos-artifacts/x", "a//b", "a/"} { + rep := sampleRepresentation() + rep.Location = location + _, err := rep.Marshal() + require.Error(t, err, "%q", location) + } + missing := sampleRepresentation() + missing.Payload = nil + _, err := missing.Marshal() + require.Error(t, err) +} + +func TestRepresentation_RejectsMalformedDescriptors(t *testing.T) { + tests := []struct { + name string + mutate func(rep *Representation) + }{ + {name: "payload digest that is not a digest", mutate: func(rep *Representation) { rep.Payload[0].Digest = "x" }}, + {name: "payload digest with an unsupported algorithm", mutate: func(rep *Representation) { + rep.Payload[0].Digest = digest.SHA512.FromString("b") + }}, + {name: "payload digest empty", mutate: func(rep *Representation) { rep.Payload[0].Digest = "" }}, + {name: "payload size negative", mutate: func(rep *Representation) { rep.Payload[0].Size = -1 }}, + {name: "document digest malformed", mutate: func(rep *Representation) { + rep.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: "sha256:zz", Size: 2}} + }}, + {name: "document size negative", mutate: func(rep *Representation) { + rep.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: digestC, Size: -5}} + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rep := sampleRepresentation() + tc.mutate(&rep) + require.NotPanics(t, func() { _ = rep.Validate() }) + require.ErrorIs(t, rep.Validate(), errUtils.ErrArtifactRecordInvalid) + + _, err := rep.Marshal() + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a malformed descriptor never reaches storage") + }) + } + + valid := sampleRepresentation() + valid.Documents = []ocispec.Descriptor{{MediaType: "application/json", Digest: digestC, Size: 0}} + require.NoError(t, valid.Validate(), "well-formed descriptors, including an empty document, stay valid") + + // A stored record with a hand-edited digest fails decoding instead of reaching stream verification. + good := sampleRepresentation() + raw, err := good.Marshal() + require.NoError(t, err) + tampered := strings.Replace(string(raw), digestB.String(), "x", 1) + require.NotEqual(t, string(raw), tampered) + _, err = DecodeRepresentation([]byte(tampered)) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) +} + +func sampleOperation() Operation { + return Operation{ + SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op/1", Name: "team/app", Version: "1.0.0", + IntentDigest: digestA, RecordDigest: digestB, CapturedAt: artifact.NewTimestamp(time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC)), + Packaging: artifact.PackagingNone, Location: "2026/10/30/team/app/1.0.0", ExpectedPrior: digestC, + Prior: &Prior{RecordDigest: digestC, Location: "2026/10/29/team/app/1.0.0"}, + Payload: []ocispec.Descriptor{}, Documents: []ocispec.Descriptor{}, Owner: "host:1", + } +} + +func TestOperation_JournalRoundTrip(t *testing.T) { + op := sampleOperation() + raw, err := op.Marshal() + require.NoError(t, err) + decoded, err := DecodeOperation(raw) + require.NoError(t, err) + assert.Equal(t, op.CapturedAt.Time(), decoded.CapturedAt.Time(), "the captured time is journaled") + assert.Equal(t, op.Location, decoded.Location) + assert.Equal(t, op.Prior, decoded.Prior) + assert.Equal(t, op.ExpectedPrior, decoded.ExpectedPrior) + + claim := decoded.Claim() + assert.Equal(t, ReferenceKey("team/app", "1.0.0"), claim.ReferenceKey) + assert.Equal(t, "op/1", claim.OperationID) + assert.Equal(t, []string{"2026/10/30/team/app/1.0.0"}, claim.Paths) + + broken := sampleOperation() + broken.CapturedAt = artifact.Timestamp{} + _, err = broken.Marshal() + require.Error(t, err, "the captured time is required") + broken = sampleOperation() + broken.Prior = &Prior{} + _, err = broken.Marshal() + require.Error(t, err) + for _, malformed := range []digest.Digest{"x", "sha256:short", digest.Digest("md5:" + digestA.Encoded()), digest.Digest(strings.Repeat("g", 71))} { + broken = sampleOperation() + broken.Prior = &Prior{RecordDigest: malformed, Location: "2026/10/29/team/app/1.0.0"} + _, err = broken.Marshal() + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "malformed prior digest %q", malformed) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + } +} + +func TestEvidence_EnvelopeBindsFileNameToContent(t *testing.T) { + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digestA, MediaType: artifact.MediaTypeConfig, Size: 12} + payload := &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte(`{"bundle":true}`)} + evidence, err := NewEvidence(subject, payload) + require.NoError(t, err) + assert.Equal(t, digest.FromBytes(payload.Data), evidence.Digest()) + + raw, err := evidence.Marshal() + require.NoError(t, err) + decoded, err := DecodeEvidence(raw, evidence.Digest()) + require.NoError(t, err) + assert.Equal(t, payload.Data, decoded.Data) + + blob := decoded.Blob("loc") + assert.Equal(t, subject, blob.Evidence.Subject) + assert.Equal(t, evidence.Digest(), blob.Evidence.Descriptor.Digest) + assert.EqualValues(t, len(payload.Data), blob.Evidence.Descriptor.Size) + assert.Equal(t, "loc", blob.Evidence.Locator) + blob.Data[0] = 'X' + assert.Equal(t, byte('{'), decoded.Data[0], "Blob copies the bytes") + + _, err = DecodeEvidence(raw, digestB) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a file whose name disagrees with its content is rejected") + + for _, bad := range []*artifact.EvidencePayload{ + {Format: "", MediaType: "x", Data: []byte("x")}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "", Data: []byte("x")}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: nil}, + {Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: make([]byte, artifact.MaxEvidenceSize+1)}, + } { + _, err := NewEvidence(subject, bad) + require.Error(t, err) + } + _, err = NewEvidence(artifact.SignatureSubject{Kind: "bogus", Digest: digestA}, payload) + require.Error(t, err) +} + +func TestAdmit(t *testing.T) { + completed := NewPendingReference("team/app", "1.0.0", "op-0", digestA, nil) + completed = completed.Completed(digestB) + pendingFirst := NewPendingReference("team/app", "1.0.0", "op-1", digestA, nil) + pendingReplace := NewPendingReference("team/app", "1.0.0", "op-2", digestA, &completed) + expectCurrent := &artifact.PinnedReference{ResolvedDigest: digestB} + + tests := []struct { + name string + in AdmitInput + want Admission + outcome artifact.Outcome + wantErr error + }{ + {"create when absent", AdmitInput{IncomingRecord: digestC, IncomingIntent: digestA}, AdmitCreate, artifact.OutcomeCreated, nil}, + {"absent but a prior was expected", AdmitInput{IncomingRecord: digestC, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"same record is unchanged", AdmitInput{Current: &completed, IncomingRecord: digestB, ExpectedPrior: expectCurrent}, AdmitUnchanged, artifact.OutcomeUnchanged, nil}, + {"same intent is unchanged", AdmitInput{Current: &completed, CurrentIntent: digestA, IncomingRecord: digestC, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, AdmitUnchanged, artifact.OutcomeUnchanged, nil}, + {"unknown current intent is never unchanged", AdmitInput{Current: &completed, IncomingRecord: digestC, IncomingIntent: "", ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"different intent conflicts without force", AdmitInput{Current: &completed, CurrentIntent: digestC, IncomingRecord: digestA, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"different intent replaces under force", AdmitInput{Current: &completed, CurrentIntent: digestC, IncomingRecord: digestA, IncomingIntent: digestA, ExpectedPrior: expectCurrent, Force: true}, AdmitReplace, artifact.OutcomeReplaced, nil}, + {"stale expected prior", AdmitInput{Current: &completed, IncomingRecord: digestC, ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digestC}, Force: true}, "", "", errUtils.ErrArtifactConflict}, + {"exists but writer expected none", AdmitInput{Current: &completed, IncomingRecord: digestC, Force: true}, "", "", errUtils.ErrArtifactConflict}, + {"same-intent pending first publication resumes", AdmitInput{Current: &pendingFirst, IncomingIntent: digestA}, AdmitResumeCreate, artifact.OutcomeCreated, nil}, + {"same-intent pending replacement resumes", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, AdmitResumeReplace, artifact.OutcomeReplaced, nil}, + {"stale pending replacement", AdmitInput{Current: &pendingReplace, IncomingIntent: digestA}, "", "", errUtils.ErrArtifactConflict}, + {"unexpected prior for pending creation", AdmitInput{Current: &pendingFirst, IncomingIntent: digestA, ExpectedPrior: expectCurrent}, "", "", errUtils.ErrArtifactConflict}, + {"different-intent pending names the operation", AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}, "", "", errUtils.ErrArtifactPublicationUnavailable}, + {"force never steals a different pending intent", AdmitInput{Current: &pendingReplace, IncomingIntent: digestC, Force: true}, "", "", errUtils.ErrArtifactPublicationUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Admit(&tt.in) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.outcome, got.Outcome()) + assert.Equal(t, tt.want == AdmitResumeCreate || tt.want == AdmitResumeReplace, got.Resuming()) + }) + } + + _, err := Admit(&AdmitInput{Current: &pendingFirst, IncomingIntent: digestC}) + require.ErrorContains(t, err, "op-1", "the recovery diagnostic includes the owning operation ID") +} + +func TestPortableWindowsNames(t *testing.T) { + for _, name := range []string{"CON", "aux.txt", "com1", "lPt9.tar", "trailing.", "trailing "} { + _, err := CleanLocation("team/" + name) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, name) + } + for _, id := range []string{"CON", "aux.txt", "COM1", "lpt9.tar"} { + assert.NotEqual(t, id, OperationFileName(id)) + } + assert.Equal(t, "COM10", OperationFileName("COM10")) +} + +func TestOperationRejectsUnsafePriorLocation(t *testing.T) { + op := &Operation{SchemaVersion: SchemaVersion, RepositoryKind: artifact.RepositoryKindLocalDir, OperationID: "op", Name: "team/app", Version: "1", CapturedAt: artifact.NewTimestamp(time.Now()), Location: "safe", Payload: []ocispec.Descriptor{}, Documents: []ocispec.Descriptor{}, Prior: &Prior{RecordDigest: digestA, Location: artifact.ReservedNamespace + "/unsafe"}} + require.Error(t, op.Validate()) +} diff --git a/pkg/artifact/layout/location.go b/pkg/artifact/layout/location.go new file mode 100644 index 00000000000..a516d1769e6 --- /dev/null +++ b/pkg/artifact/layout/location.go @@ -0,0 +1,236 @@ +package layout + +import ( + "bytes" + "fmt" + "strings" + "text/template" + "time" + "unicode/utf8" + + "github.com/Masterminds/sprig/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/templatefuncs" +) + +// maxLocationLength bounds one resolved location, in bytes. +const maxLocationLength = 1024 + +// windowsReserved lists characters that are invalid in Windows file names. Locations avoid them on +// every platform so a repository root can be moved between hosts. +const windowsReserved = `<>:"|?*` + +// deleteRune is the ASCII DEL control character. +const deleteRune = 0x7f + +// nonHermeticFunctions are Sprig functions removed from key_pattern rendering. A location must +// depend only on the captured publication inputs: not on the clock, the process environment, the +// network, or randomness. +var nonHermeticFunctions = []string{ + "now", "ago", "env", "expandenv", "getHostByName", "uuidv4", "randAlphaNum", "randAlpha", "randNumeric", + "randAscii", "randBytes", "randInt", "shuffle", "genPrivateKey", "genCA", "genCAWithKey", "genSelfSignedCert", + "genSelfSignedCertWithKey", "genSignedCert", "genSignedCertWithKey", "buildCustomCert", "derivePassword", + "htpasswd", "bcrypt", "encryptAES", "decryptAES", + // Sprig's zone-aware date helpers fall back to the current time for unsupported input and + // format in the host's local zone. The "date" function is replaced by a UTC variant instead. + "dateInZone", "date_in_zone", "htmlDate", "htmlDateInZone", +} + +// PatternInput holds the values a key_pattern may reference: .Name, .Version, .Kind, and +// .CreatedAt. CreatedAt is the creation time captured once for a publication. +type PatternInput struct { + // Name is the artifact name. + Name string + + // Version is the artifact version. + Version string + + // Kind is the artifact kind. + Kind string + + // CreatedAt is the captured creation time; it is rendered in UTC. + CreatedAt time.Time +} + +// patternData is the template data. CreatedAt is a time.Time so date formatting applies. +type patternData struct { + Name string + Version string + Kind string + CreatedAt time.Time +} + +// DefaultLocation returns the default payload location below the repository root for the +// packaging: "/" for copied files, "/." for an archive. +func DefaultLocation(name, version string, packaging artifact.Packaging) string { + defer perf.Track(nil, "layout.DefaultLocation")() + + return name + "/" + version + ArchiveExtension(packaging) +} + +// ArchiveExtension returns the file extension (with the dot) of an archive packaging, or the empty +// string for copied files and unknown packagings. +func ArchiveExtension(packaging artifact.Packaging) string { + defer perf.Track(nil, "layout.ArchiveExtension")() + + switch packaging { + case artifact.PackagingTar: + return ".tar" + case artifact.PackagingTgz: + return ".tgz" + case artifact.PackagingZip: + return ".zip" + default: + return "" + } +} + +// RenderLocation resolves the payload location for one publication. An empty pattern selects +// [DefaultLocation]. A non-empty pattern is rendered as a Go template with the Sprig functions +// (minus the non-hermetic ones) and a UTC "date". The result is validated: it must stay below the +// repository root, avoid the reserved namespace, and for archives agree with the packaging. +// +// Callers render once, at publication, persist the result in the representation record, and use +// the persisted value for every later read, delete, and retry; they never render again. +func RenderLocation(pattern string, in *PatternInput, packaging artifact.Packaging) (string, error) { + defer perf.Track(nil, "layout.RenderLocation")() + + raw := DefaultLocation(in.Name, in.Version, packaging) + if pattern != "" { + rendered, err := renderPattern(pattern, in) + if err != nil { + return "", err + } + raw = rendered + } + location, err := CleanLocation(raw) + if err != nil { + return "", fmt.Errorf("key_pattern %q rendered %q: %w", pattern, raw, err) + } + if err := checkExtension(location, packaging); err != nil { + return "", fmt.Errorf("key_pattern %q rendered %q: %w", pattern, raw, err) + } + return location, nil +} + +func renderPattern(pattern string, in *PatternInput) (string, error) { + funcs := sprig.TxtFuncMap() + for _, name := range nonHermeticFunctions { + delete(funcs, name) + } + funcs["date"] = utcDate + for name, fn := range templatefuncs.FuncMap() { + funcs[name] = fn + } + tmpl, err := template.New("key_pattern").Funcs(funcs).Option("missingkey=error").Parse(pattern) + if err != nil { + return "", fmt.Errorf("%w: key_pattern %q: %w", errUtils.ErrArtifactConfigInvalid, pattern, err) + } + data := patternData{Name: in.Name, Version: in.Version, Kind: in.Kind, CreatedAt: in.CreatedAt.UTC()} + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("%w: key_pattern %q: %w", errUtils.ErrArtifactConfigInvalid, pattern, err) + } + return out.String(), nil +} + +// utcDate formats a time in UTC. Sprig's date uses the local zone, which would make the persisted +// location depend on the host that published it. +func utcDate(format string, value any) string { + switch typed := value.(type) { + case time.Time: + return typed.UTC().Format(format) + case *time.Time: + if typed != nil { + return typed.UTC().Format(format) + } + } + return "" +} + +// CleanLocation validates a resolved or persisted payload location and returns it in canonical +// form (a single trailing slash is removed). A location is relative, slash separated, free of +// empty, "." and ".." segments, control and Windows-reserved characters, and never enters the +// reserved namespace, however the case is written. Locations are never rewritten otherwise. +func CleanLocation(location string) (string, error) { + defer perf.Track(nil, "layout.CleanLocation")() + + location = strings.TrimSuffix(location, "/") + if err := checkLocationShape(location); err != nil { + return "", err + } + for _, segment := range strings.Split(location, "/") { + switch { + case segment == "" || segment == "." || segment == "..": + return "", fmt.Errorf("%w: location %q has an empty, '.', or '..' segment", errUtils.ErrArtifactUnsafePath, location) + case strings.HasSuffix(segment, ".") || strings.HasSuffix(segment, " ") || windowsDeviceName(segment): + return "", fmt.Errorf("%w: location %q contains a Windows-reserved name", errUtils.ErrArtifactUnsafePath, location) + case strings.EqualFold(segment, artifact.ReservedNamespace): + return "", fmt.Errorf("%w: location %q enters the reserved %s namespace", errUtils.ErrArtifactUnsafePath, location, artifact.ReservedNamespace) + } + } + return location, nil +} + +// checkLocationShape checks length, encoding, rootedness, separators, and character classes. +func checkLocationShape(location string) error { + switch { + case location == "" || len(location) > maxLocationLength: + return fmt.Errorf("%w: location must be 1-%d bytes", errUtils.ErrArtifactUnsafePath, maxLocationLength) + case !utf8.ValidString(location): + return fmt.Errorf("%w: location %q is not valid UTF-8", errUtils.ErrArtifactUnsafePath, location) + case strings.HasPrefix(location, "/") || strings.Contains(location, `\`): + return fmt.Errorf("%w: location %q must be relative and slash-separated", errUtils.ErrArtifactUnsafePath, location) + case strings.ContainsAny(location, windowsReserved) || strings.IndexFunc(location, isControl) >= 0: + return fmt.Errorf("%w: location %q contains control or reserved characters", errUtils.ErrArtifactUnsafePath, location) + } + return nil +} + +// windowsDeviceName identifies reserved device basenames, including names with extensions. +func windowsDeviceName(segment string) bool { + base, _, _ := strings.Cut(segment, ".") + base = strings.ToUpper(base) + switch base { + case "CON", "PRN", "AUX", "NUL": + return true + } + return len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9' +} + +func isControl(r rune) bool { + return r < ' ' || r == deleteRune +} + +// archiveSuffixes maps archive file extensions to their packaging, longest first. +var archiveSuffixes = []struct { + suffix string + packaging artifact.Packaging +}{ + {".tar.gz", artifact.PackagingTgz}, + {".tgz", artifact.PackagingTgz}, + {".tar", artifact.PackagingTar}, + {".zip", artifact.PackagingZip}, +} + +// checkExtension rejects an archive extension that disagrees with the packaging. A location +// without an archive extension is accepted; it is never silently renamed. +func checkExtension(location string, packaging artifact.Packaging) error { + if ArchiveExtension(packaging) == "" { + return nil + } + lower := strings.ToLower(location) + for _, candidate := range archiveSuffixes { + if !strings.HasSuffix(lower, candidate.suffix) { + continue + } + if candidate.packaging != packaging { + return fmt.Errorf("%w: %w: location ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, candidate.suffix, packaging) + } + return nil + } + return nil +} diff --git a/pkg/artifact/layout/paths.go b/pkg/artifact/layout/paths.go new file mode 100644 index 00000000000..4095c4a15d4 --- /dev/null +++ b/pkg/artifact/layout/paths.go @@ -0,0 +1,207 @@ +package layout + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Names of the managed layout. +const ( + // SchemaVersion is the schema version of every layout document. + SchemaVersion = 1 + + // Version is the layout version directory below the reserved namespace. + Version = "v1" + + // ManagedRoot is the slash-separated managed directory below a repository root. + ManagedRoot = artifact.ReservedNamespace + "/" + Version + + // MetadataSuffix is appended to an archive payload path for its adjacent metadata sidecar. + MetadataSuffix = ".metadata.json" + + // DocsSuffix is appended to an archive payload path for its adjacent documents directory. + DocsSuffix = ".docs" + + // DistributionDir is the distribution directory below a copied-files payload, and inside an + // archive, that holds README, CHANGELOG, and manifest.json. Managed readers omit it when + // materializing user content. + DistributionDir = ".atmos/artifact" + + // ManifestName is the generated manifest file name inside the distribution directory. + ManifestName = "manifest.json" +) + +// Directory names inside the managed root. +const ( + refsDir = "refs" + recordsDir = "records" + representationsDir = "representations" + operationsDir = "operations" + signaturesDir = "signatures" + locksDir = "locks" + stagingDirName = "staging" + jsonSuffix = ".json" + lockSuffix = ".lock" + + // The longest operation identifier used verbatim as a file name. + maxSafeOperationID = 128 + + // The prefix of a file name derived from an unsafe operation identifier. + hashedOperationPrefix = "sha256-" +) + +// safeOperationID matches identifiers that are safe path segments on every platform. +var safeOperationID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// ReferenceKey returns the private reference key for a name and version: the lowercase hex +// SHA-256 of "name:version". The colon is unambiguous because names never contain one. +func ReferenceKey(name, version string) string { + defer perf.Track(nil, "layout.ReferenceKey")() + + sum := sha256.Sum256([]byte(name + ":" + version)) + return hex.EncodeToString(sum[:]) +} + +// ReferencePath returns the slash-separated path of the reference record for key. +func ReferencePath(key string) string { + defer perf.Track(nil, "layout.ReferencePath")() + + return ManagedRoot + "/" + refsDir + "/" + key + jsonSuffix +} + +// ReferencesDir returns the directory that holds every reference record. +func ReferencesDir() string { + defer perf.Track(nil, "layout.ReferencesDir")() + + return ManagedRoot + "/" + refsDir +} + +// RecordPath returns the slash-separated path of the immutable Record with the given digest. +func RecordPath(recordDigest digest.Digest) string { + defer perf.Track(nil, "layout.RecordPath")() + + return ManagedRoot + "/" + recordsDir + "/" + recordDigest.Encoded() + jsonSuffix +} + +// RepresentationPath returns the slash-separated path of the representation record for key. +func RepresentationPath(key string) string { + defer perf.Track(nil, "layout.RepresentationPath")() + + return ManagedRoot + "/" + representationsDir + "/" + key + jsonSuffix +} + +// OperationFileName returns the file stem for an operation identifier. Safe identifiers are used +// verbatim; anything else (slashes, spaces, overlong values) maps to a hash so that no identifier +// can address a path outside the operations directory. The hashed prefix is reserved: an +// identifier that starts with it is hashed too, so a literal identifier can never collide with +// the hashed file name of another identifier. +func OperationFileName(operationID string) string { + defer perf.Track(nil, "layout.OperationFileName")() + + if len(operationID) <= maxSafeOperationID && !strings.HasPrefix(operationID, hashedOperationPrefix) && safeOperationID.MatchString(operationID) && !windowsDeviceName(operationID) { + return operationID + } + sum := sha256.Sum256([]byte(operationID)) + return hashedOperationPrefix + hex.EncodeToString(sum[:]) +} + +// OperationPath returns the slash-separated path of the journal for an operation. +func OperationPath(operationID string) string { + defer perf.Track(nil, "layout.OperationPath")() + + return ManagedRoot + "/" + operationsDir + "/" + OperationFileName(operationID) + jsonSuffix +} + +// StagingPath returns the slash-separated staging directory owned by an operation. Staging lives +// below the managed root so it shares a filesystem with the payload for atomic renames. +func StagingPath(operationID string) string { + defer perf.Track(nil, "layout.StagingPath")() + + return ManagedRoot + "/" + stagingDirName + "/" + OperationFileName(operationID) +} + +// TempDir returns the directory for short-lived temporary files used by atomic replacement. +func TempDir() string { + defer perf.Track(nil, "layout.TempDir")() + + return ManagedRoot + "/tmp" +} + +// LockPath returns the slash-separated path of the lock file for a resource. The file name is the +// hash of the resource so any resource string is safe. +func LockPath(resource string) string { + defer perf.Track(nil, "layout.LockPath")() + + sum := sha256.Sum256([]byte(resource)) + return ManagedRoot + "/" + locksDir + "/" + hex.EncodeToString(sum[:]) + lockSuffix +} + +// SignatureDir returns the directory that holds the detached evidence for a subject. +func SignatureDir(kind artifact.SubjectKind, subject digest.Digest) string { + defer perf.Track(nil, "layout.SignatureDir")() + + return ManagedRoot + "/" + signaturesDir + "/" + string(kind) + "/" + subject.Encoded() +} + +// SignaturePath returns the slash-separated path of one detached evidence file: +// .atmos-artifacts/v1/signatures///.json, with +// the digests written as lowercase hex. +func SignaturePath(kind artifact.SubjectKind, subject, evidence digest.Digest) string { + defer perf.Track(nil, "layout.SignaturePath")() + + return SignatureDir(kind, subject) + "/" + evidence.Encoded() + jsonSuffix +} + +// MetadataSidecarPath returns the adjacent metadata sidecar of an archive payload location. +func MetadataSidecarPath(location string) string { + defer perf.Track(nil, "layout.MetadataSidecarPath")() + + return location + MetadataSuffix +} + +// DocsSidecarDir returns the adjacent documents directory of an archive payload location. +func DocsSidecarDir(location string) string { + defer perf.Track(nil, "layout.DocsSidecarDir")() + + return location + DocsSuffix +} + +// DocumentFileName returns the file name of a distribution document. +func DocumentFileName(role artifact.DocumentRole) string { + defer perf.Track(nil, "layout.DocumentFileName")() + + if role == artifact.DocumentChangelog { + return "CHANGELOG.md" + } + return "README.md" +} + +// DocumentSidecarPath returns the adjacent document file of an archive payload location. +func DocumentSidecarPath(location string, role artifact.DocumentRole) string { + defer perf.Track(nil, "layout.DocumentSidecarPath")() + + return DocsSidecarDir(location) + "/" + DocumentFileName(role) +} + +// DistributionPath returns the distribution file of a copied-files payload location. +func DistributionPath(location, name string) string { + defer perf.Track(nil, "layout.DistributionPath")() + + return location + "/" + DistributionDir + "/" + name +} + +// InManagedNamespace reports whether a slash-separated path enters the reserved namespace. The +// comparison folds case because common filesystems alias names that differ only by case. +func InManagedNamespace(p string) bool { + defer perf.Track(nil, "layout.InManagedNamespace")() + + first, _, _ := strings.Cut(strings.TrimPrefix(p, "./"), "/") + return strings.EqualFold(first, artifact.ReservedNamespace) +} diff --git a/pkg/artifact/layout/records.go b/pkg/artifact/layout/records.go new file mode 100644 index 00000000000..eb0d38fb82c --- /dev/null +++ b/pkg/artifact/layout/records.go @@ -0,0 +1,469 @@ +package layout + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// State is the publication state of a reference. +type State string + +// Reference states. A pending reference is unavailable to managed readers and absent from +// completed search results. The completed state is written last in the publication sequence. +const ( + // StatePending marks a reference whose publication started and has not completed. It stays + // pending after an interruption until the same intent retries and repairs it. + StatePending State = "pending" + + // StateCompleted marks a reference whose Record, payload, and documents are all in place. + StateCompleted State = "completed" +) + +// Reference is the stable-root reference record: it binds a name and version to the digest of the +// immutable Record. It is the only mutable document readers consult, and it is replaced +// atomically. Generation increases on every state change so a reader can detect that the +// reference moved while it was reading from a mutable payload location. +type Reference struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // State is pending or completed. + State State `json:"state"` + + // Generation increases by one on every state change; the first record has generation 1. + Generation int64 `json:"generation"` + + // OperationID is the operation that wrote this state: the pending operation, or the operation + // that completed the publication. + OperationID string `json:"operation_id"` + + // IntentDigest is the intent digest of the publication that wrote this state. + IntentDigest digest.Digest `json:"intent_digest"` + + // RecordDigest is the digest of the canonical Record. It is required when completed and empty + // while a first publication is pending. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // PriorRecordDigest is the Record this pending publication replaces, kept for recovery + // diagnostics. It is empty for a first publication and once completed. + PriorRecordDigest digest.Digest `json:"prior_record_digest,omitempty"` +} + +// Available reports whether managed readers may use the reference. +func (r *Reference) Available() bool { + defer perf.Track(nil, "layout.Reference.Available")() + + return r.State == StateCompleted +} + +// Key returns the reference key. +func (r *Reference) Key() string { + defer perf.Track(nil, "layout.Reference.Key")() + + return ReferenceKey(r.Name, r.Version) +} + +// Validate checks the reference's shape and state invariants. +func (r *Reference) Validate() error { + defer perf.Track(nil, "layout.Reference.Validate")() + + if err := r.validateIdentity(); err != nil { + return err + } + if err := r.validateState(); err != nil { + return err + } + return validateOptionalDigests(r.RecordDigest, r.PriorRecordDigest) +} + +func (r *Reference) validateIdentity() error { + if r.SchemaVersion != SchemaVersion { + return invalid(kindReference, "unsupported schema_version %d", r.SchemaVersion) + } + if err := artifact.ValidateName(r.Name); err != nil { + return err + } + if err := artifact.ValidateVersion(r.Version); err != nil { + return err + } + if r.Generation < 1 || r.OperationID == "" { + return invalid(kindReference, "generation and operation_id are required") + } + return artifact.ValidateDigest(r.IntentDigest) +} + +func (r *Reference) validateState() error { + switch r.State { + case StateCompleted: + if r.RecordDigest == "" || r.PriorRecordDigest != "" { + return invalid(kindReference, "a completed reference needs record_digest and no prior_record_digest") + } + case StatePending: + if r.RecordDigest != "" { + return invalid(kindReference, "a pending reference must not name a record_digest") + } + default: + return invalid(kindReference, "unsupported state %q", r.State) + } + return nil +} + +func validateOptionalDigests(digests ...digest.Digest) error { + for _, d := range digests { + if d == "" { + continue + } + if err := artifact.ValidateDigest(d); err != nil { + return err + } + } + return nil +} + +// NewPendingReference returns the pending state that replaces prior (nil for a first publication). +func NewPendingReference(name, version, operationID string, intent digest.Digest, prior *Reference) Reference { + defer perf.Track(nil, "layout.NewPendingReference")() + + pending := Reference{ + SchemaVersion: SchemaVersion, Name: name, Version: version, State: StatePending, + Generation: 1, OperationID: operationID, IntentDigest: intent, + } + if prior != nil { + pending.Generation = prior.Generation + 1 + if prior.State == StateCompleted { + pending.PriorRecordDigest = prior.RecordDigest + } else { + pending.PriorRecordDigest = prior.PriorRecordDigest + } + } + return pending +} + +// Completed returns the completed state that follows a pending reference once the Record, +// payload, and documents are in place. +func (r *Reference) Completed(recordDigest digest.Digest) Reference { + defer perf.Track(nil, "layout.Reference.Completed")() + + done := *r + done.State = StateCompleted + done.Generation = r.Generation + 1 + done.RecordDigest = recordDigest + done.PriorRecordDigest = "" + return done +} + +// Marshal returns the canonical bytes of the reference. +func (r *Reference) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Reference.Marshal")() + + return encode(r, MaxDocumentBytes) +} + +// DecodeReference strictly parses a stored reference record. +func DecodeReference(raw []byte) (*Reference, error) { + defer perf.Track(nil, "layout.DecodeReference")() + + var ref Reference + if err := decode(kindReference, raw, &ref, MaxDocumentBytes); err != nil { + return nil, err + } + return &ref, nil +} + +// Representation is the mutable storage description of one reference's current Record. It holds +// the resolved payload location rendered once at publication; later reads and deletes use it and +// never render the key_pattern again. +type Representation struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // RepositoryKind is the repository kind that wrote it, for example "local/archive". + RepositoryKind string `json:"repository_kind"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // RecordDigest is the digest of the Record this representation belongs to. + RecordDigest digest.Digest `json:"record_digest"` + + // OperationID is the operation that wrote the representation. + OperationID string `json:"operation_id"` + + // Packaging is the payload packaging. + Packaging artifact.Packaging `json:"packaging"` + + // Location is the resolved payload location, relative to the repository root. + Location string `json:"location"` + + // Payload lists the payload blob descriptors: one archive, or one per file. + Payload []ocispec.Descriptor `json:"payload"` + + // Documents lists the document blob descriptors stored alongside the payload. + Documents []ocispec.Descriptor `json:"documents"` + + // RepresentationDigest is the checksum of the archive, or of the per-file payload digests. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` +} + +// Validate checks the representation's shape. +func (r *Representation) Validate() error { + defer perf.Track(nil, "layout.Representation.Validate")() + + if r.SchemaVersion != SchemaVersion { + return invalid(kindRepresentation, "unsupported schema_version %d", r.SchemaVersion) + } + if err := validateNameVersion(r.Name, r.Version); err != nil { + return err + } + if err := artifact.ValidateDigest(r.RecordDigest); err != nil { + return err + } + if r.RepositoryKind == "" || r.OperationID == "" { + return invalid(kindRepresentation, "repository_kind and operation_id are required") + } + if _, err := CleanLocation(r.Location); err != nil || r.Location != cleanedOrSelf(r.Location) { + return invalid(kindRepresentation, "location %q is not a canonical relative location", r.Location) + } + if err := r.validateLists(); err != nil { + return err + } + return validateOptionalDigests(r.RepresentationDigest) +} + +// validateLists checks that both blob descriptor lists are present and well formed. +func (r *Representation) validateLists() error { + if r.Payload == nil || r.Documents == nil { + return invalid(kindRepresentation, "payload and documents lists are required (use empty lists)") + } + if err := validateDescriptors("payload", r.Payload); err != nil { + return err + } + return validateDescriptors("documents", r.Documents) +} + +// validateDescriptors checks that every blob descriptor carries a sha256 digest and a size that +// stream verification can enforce. A malformed digest must be rejected here: later reads hand it +// straight to the verifying reader, which has no error path for an unparsable algorithm. +func validateDescriptors(list string, descriptors []ocispec.Descriptor) error { + for index := range descriptors { + if err := artifact.ValidateDigest(descriptors[index].Digest); err != nil { + return invalid(kindRepresentation, "%s[%d] has an invalid digest: %v", list, index, err) + } + if descriptors[index].Size < 0 { + return invalid(kindRepresentation, "%s[%d] has a negative size %d", list, index, descriptors[index].Size) + } + } + return nil +} + +// validateNameVersion checks the name and version of a document. +func validateNameVersion(name, version string) error { + if err := artifact.ValidateName(name); err != nil { + return err + } + return artifact.ValidateVersion(version) +} + +// cleanedOrSelf returns the canonical form of location, or location itself when it is invalid. +func cleanedOrSelf(location string) string { + if cleaned, err := CleanLocation(location); err == nil { + return cleaned + } + return location +} + +// Marshal returns the canonical bytes of the representation. +func (r *Representation) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Representation.Marshal")() + + return encode(r, MaxDocumentBytes) +} + +// DecodeRepresentation strictly parses a stored representation record. +func DecodeRepresentation(raw []byte) (*Representation, error) { + defer perf.Track(nil, "layout.DecodeRepresentation")() + + var rep Representation + if err := decode(kindRepresentation, raw, &rep, MaxDocumentBytes); err != nil { + return nil, err + } + return &rep, nil +} + +// Artifact converts the stored description into the repository-neutral [artifact.Representation]. +// The locator argument is the display locator of the payload, for example an absolute path or an +// s3:// URI. +func (r *Representation) Artifact(locator string) artifact.Representation { + defer perf.Track(nil, "layout.Representation.Artifact")() + + return artifact.Representation{ + Packaging: r.Packaging, + Payload: append([]ocispec.Descriptor(nil), r.Payload...), + Documents: append([]ocispec.Descriptor(nil), r.Documents...), + Locator: locator, + RecordDigest: r.RecordDigest, + RepresentationDigest: r.RepresentationDigest, + } +} + +// Prior describes the state an operation replaces. +type Prior struct { + // RecordDigest is the replaced Record. + RecordDigest digest.Digest `json:"record_digest"` + + // Location is where the replaced payload lives, so a recovery can retire it. + Location string `json:"location,omitempty"` + + // Packaging is the packaging of the replaced payload, for backends such as S3 whose object + // keys depend on it. Local backends leave it empty. + Packaging artifact.Packaging `json:"packaging,omitempty"` +} + +// Operation is the publication journal: an implementation recovery record, not a public catalog. +// It is written before any payload path changes and records everything a same-intent retry needs +// to resume or repair the publication: the captured time, the intent, the prior reference, the +// resolved location, and the prepared representation checksums. +type Operation struct { + // SchemaVersion is [SchemaVersion]. + SchemaVersion int `json:"schema_version"` + + // RepositoryKind is the repository kind performing the publication. + RepositoryKind string `json:"repository_kind"` + + // OperationID is the logical operation identifier, stable across retries. + OperationID string `json:"operation_id"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the artifact version. + Version string `json:"version"` + + // IntentDigest is the intent digest of the artifact being published. + IntentDigest digest.Digest `json:"intent_digest"` + + // RecordDigest is the digest of the Record being published. + RecordDigest digest.Digest `json:"record_digest"` + + // CapturedAt is the creation time captured once for the publication. + CapturedAt artifact.Timestamp `json:"captured_at"` + + // Packaging is the payload packaging being written. + Packaging artifact.Packaging `json:"packaging"` + + // Location is the resolved payload location reserved for this operation. + Location string `json:"location"` + + // ExpectedPrior is the record digest the writer observed under the guard; empty means none. + ExpectedPrior digest.Digest `json:"expected_prior,omitempty"` + + // Prior describes the state being replaced, if any. + Prior *Prior `json:"prior,omitempty"` + + // Payload and Documents are the prepared representation checksums. + Payload []ocispec.Descriptor `json:"payload"` + Documents []ocispec.Descriptor `json:"documents"` + + // Owner identifies the writer for recovery diagnostics, for example host and process. + Owner string `json:"owner,omitempty"` +} + +// Validate checks the journal's shape. +func (o *Operation) Validate() error { + defer perf.Track(nil, "layout.Operation.Validate")() + + if o.SchemaVersion != SchemaVersion { + return invalid(kindOperation, "unsupported schema_version %d", o.SchemaVersion) + } + if o.OperationID == "" || o.RepositoryKind == "" { + return invalid(kindOperation, "operation_id and repository_kind are required") + } + if err := validateNameVersion(o.Name, o.Version); err != nil { + return err + } + if err := o.validateState(); err != nil { + return err + } + return validateOptionalDigests(o.IntentDigest, o.RecordDigest, o.ExpectedPrior) +} + +func (o *Operation) validateState() error { + if o.CapturedAt.IsZero() { + return invalid(kindOperation, "captured_at is required") + } + if _, err := CleanLocation(o.Location); err != nil { + return invalid(kindOperation, "location %q is not a valid relative location", o.Location) + } + if o.Payload == nil || o.Documents == nil { + return invalid(kindOperation, "payload and documents lists are required (use empty lists)") + } + return validatePrior(o.Prior) +} + +func validatePrior(prior *Prior) error { + if prior == nil { + return nil + } + if prior.RecordDigest == "" { + return invalid(kindOperation, "prior needs a record_digest") + } + if err := artifact.ValidateDigest(prior.RecordDigest); err != nil { + return invalid(kindOperation, "prior record_digest is malformed: %v", err) + } + if prior.Location != "" { + if _, err := CleanLocation(prior.Location); err != nil { + return invalid(kindOperation, "prior location %q is not a valid relative location", prior.Location) + } + } + return nil +} + +// Claim returns the paths the operation reserves. +func (o *Operation) Claim() Claim { + defer perf.Track(nil, "layout.Operation.Claim")() + + return Claim{ + ReferenceKey: ReferenceKey(o.Name, o.Version), Name: o.Name, Version: o.Version, + OperationID: o.OperationID, Paths: PayloadPaths(o.RepositoryKind, o.Location), + } +} + +// Marshal returns the canonical bytes of the journal. +func (o *Operation) Marshal() ([]byte, error) { + defer perf.Track(nil, "layout.Operation.Marshal")() + + return encode(o, MaxDocumentBytes) +} + +// DecodeOperation strictly parses a stored operation journal. +func DecodeOperation(raw []byte) (*Operation, error) { + defer perf.Track(nil, "layout.DecodeOperation")() + + var op Operation + if err := decode(kindOperation, raw, &op, MaxDocumentBytes); err != nil { + return nil, err + } + return &op, nil +} + +// Claim returns the paths a completed representation owns. +func (r *Representation) Claim() Claim { + defer perf.Track(nil, "layout.Representation.Claim")() + + return Claim{ + ReferenceKey: ReferenceKey(r.Name, r.Version), Name: r.Name, Version: r.Version, + Paths: PayloadPaths(r.RepositoryKind, r.Location), + } +} diff --git a/pkg/artifact/local/conformance_test.go b/pkg/artifact/local/conformance_test.go new file mode 100644 index 00000000000..8db2036a99b --- /dev/null +++ b/pkg/artifact/local/conformance_test.go @@ -0,0 +1,54 @@ +package local + +import ( + "errors" + "testing" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" +) + +// errCrash is the error the fault hooks simulate a crash with. +var errCrash = errors.New("simulated crash") + +func TestDirRepositoryConformance(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRepo(t, &repoSpec{kind: artifact.RepositoryKindLocalDir, opts: []Option{WithVerifier(&artifacttest.FakeVerifier{})}}) + }, + Packaging: artifact.PackagingNone, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) +} + +func TestArchiveRepositoryConformance(t *testing.T) { + for _, packaging := range []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip} { + t.Run(string(packaging), func(t *testing.T) { + signer := newSigner() + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { + return newRepo(t, &repoSpec{kind: artifact.RepositoryKindLocalArchive, packaging: packaging, opts: []Option{WithVerifier(&artifacttest.FakeVerifier{})}}) + }, + Packaging: packaging, + Signer: signer, + TrustedPolicy: signer.TrustedPolicy(), + UntrustedPolicy: signer.UntrustedPolicy(), + }) + }) + } +} + +// Without a verifier the repositories advertise no evidence formats and Sign returns a capability +// error instead of a successful no-op. +func TestRepositoryConformance_WithoutVerifier(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + artifacttest.RunRepositoryConformance(t, &artifacttest.Harness{ + New: func(t *testing.T) artifact.Repository { return newRepo(t, spec) }, + }) + }) + } +} diff --git a/pkg/artifact/local/delete.go b/pkg/artifact/local/delete.go new file mode 100644 index 00000000000..5c3b7255cf1 --- /dev/null +++ b/pkg/artifact/local/delete.go @@ -0,0 +1,82 @@ +package local + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Delete implements [artifact.Repository]. Deletion is scoped to the selected artifact: its +// reference, its Record, its representation, its detached evidence, and the payload and sidecars +// it exclusively owns. Nothing outside the repository root is touched, and files the repository +// does not manage (a user's extra file inside a payload directory) stay in place. +// +// The reference record is removed first, so the artifact disappears atomically; an interruption +// afterwards leaves garbage that a later publication of the same reference clears, never a +// reference that points at missing bytes. A pending publication is never deleted: its owner +// must repair it with the same intent. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Delete(ctx context.Context, pinned artifact.PinnedReference, options artifact.DeleteOptions) (*artifact.DeleteResult, error) { + defer perf.Track(nil, "local.Repository.Delete")() + + if options.Scope != "" && options.Scope != artifact.DeleteObject { + return nil, artifact.NewUnsupportedCapability(r.alias, "delete", "delete/"+string(options.Scope), + "local repositories delete the reference and the payload it owns together (object scope)") + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + record, _, err := t.loadRecord(pinned.RecordDigest) + if err != nil { + return nil, err + } + var held *fileLock + if options.Lock == nil { + if held, err = r.acquire(ctx, record.Name+":"+record.Version, r.settings.owner, ""); err != nil { + return nil, err + } + } + result, err := r.deleteLocked(ctx, t, &pinned, &options) + if held != nil { + if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { + err = releaseErr + } + } + return result, err +} + +func (r *Repository) deleteLocked(ctx context.Context, t *tree, pinned *artifact.PinnedReference, options *artifact.DeleteOptions) (*artifact.DeleteResult, error) { + v, err := r.openView(ctx, t, pinned) + if err != nil { + return nil, err + } + if options.ExpectedCurrent != nil && options.ExpectedCurrent.ResolvedDigest != v.ref.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is %s but the delete expected %s", errUtils.ErrArtifactConflict, v.ref.Name, v.ref.Version, v.ref.RecordDigest, options.ExpectedCurrent.ResolvedDigest) + } + if err := t.Remove(native(layout.ReferencePath(v.ref.Key()))); err != nil { + return nil, fmt.Errorf("%w: remove reference: %w", errUtils.ErrArtifactDeleteFailed, err) + } + removed, err := r.store.remove(t, v) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) + } + for _, owned := range []string{ + layout.SignatureDir(artifact.SubjectRecord, v.ref.RecordDigest), + layout.RecordPath(v.ref.RecordDigest), + layout.RepresentationPath(v.ref.Key()), + } { + if err := t.removeAll(owned); err != nil { + return nil, fmt.Errorf("%w: remove %s: %w", errUtils.ErrArtifactDeleteFailed, owned, err) + } + } + r.tidyOperation(t, v.ref.OperationID) + t.removeEmptyParents(v.rep.Location) + return &artifact.DeleteResult{Deleted: removed, Digest: v.ref.RecordDigest}, nil +} diff --git a/pkg/artifact/local/delete_test.go b/pkg/artifact/local/delete_test.go new file mode 100644 index 00000000000..b67d76eb0b5 --- /dev/null +++ b/pkg/artifact/local/delete_test.go @@ -0,0 +1,228 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +func TestDelete_IsScopedToTheSelectedArtifact(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{name: "team/keep", version: "1.0.0"}, artifact.PublishOptions{}) + publish(t, repo, &prepareSpec{name: "team/keep", version: "2.0.0", files: map[string]string{"data.txt": "two"}}, artifact.PublishOptions{}) + target := publish(t, repo, &prepareSpec{name: "team/drop", version: "1.0.0", files: map[string]string{"data.txt": "drop"}}, artifact.PublishOptions{}) + + userFile := filepath.Join(repo.root, "notes", "user.txt") + require.NoError(t, os.MkdirAll(filepath.Dir(userFile), 0o755)) + require.NoError(t, os.WriteFile(userFile, []byte("mine"), 0o644)) + outside := filepath.Join(filepath.Dir(repo.root), "outside.txt") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + + pinned := resolve(t, repo, ref("team/drop", "1.0.0")) + result, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &pinned, Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.Equal(t, pinned.RecordDigest, result.Digest) + require.NotEmpty(t, result.Deleted) + assert.Equal(t, target.Location, result.Deleted[0]) + + _, err = repo.Resolve(context.Background(), ref("team/drop", "1.0.0")) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assertIntact(t, repo, ref("team/keep", "1.0.0")) + assertIntact(t, repo, ref("team/keep", "2.0.0")) + assert.FileExists(t, userFile, "files the repository does not manage are never touched") + assert.FileExists(t, outside, "nothing outside the root is touched") + assert.DirExists(t, repo.root) + assert.NoDirExists(t, filepath.Join(repo.root, "team", "drop"), "empty parents of the deleted artifact are pruned") + assert.DirExists(t, filepath.Join(repo.root, "team", "keep")) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.RecordPath(pinned.RecordDigest)))) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(layout.ReferenceKey("team/drop", "1.0.0"))))) + assert.Equal(t, []string{"team/keep:1.0.0", "team/keep:2.0.0"}, listNames(t, repo)) + + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "deleting twice reports not found") + }) + } +} + +func TestDelete_LeavesUnmanagedFilesInsideADirectoryPayload(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + extra := filepath.Join(result.Location, "mine.txt") + require.NoError(t, os.WriteFile(extra, []byte("user data"), 0o644)) + + pinned := resolve(t, repo, defaultRef()) + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.FileExists(t, extra, "delete removes what the artifact owns, not a user's extra file") + assert.NoFileExists(t, filepath.Join(result.Location, "README.md")) + assert.NoDirExists(t, filepath.Join(result.Location, ".atmos")) + assert.Equal(t, []string{"team/app/1.0.0/mine.txt"}, payloadFiles(t, repo.root)) +} + +func TestDelete_RejectsStaleTargetsAndScopes(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + + stale := pinned + stale.ResolvedDigest = digest.FromString("other") + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{ExpectedCurrent: &stale, Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assertIntact(t, repo, defaultRef()) + + for _, scope := range []artifact.DeleteSemantics{artifact.DeleteNone, artifact.DeleteManifest, artifact.DeleteReference} { + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: scope}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, scope) + } + assertIntact(t, repo, defaultRef()) +} + +func TestDelete_DoesNotDeletePendingPublications(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Delete(context.Background(), old, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.Error(t, err, "a pending operation is never deleted out from under its owner") + require.ErrorContains(t, err, "op-2") +} + +// A payload location replaced by a symlink to a directory outside the root must not let a delete +// reach the outside files: every operation is confined to the repository root. +func TestDelete_NeverEscapesTheRootThroughASymlink(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + outside := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(outside, "README.md"), []byte("outside readme"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "empty.txt"), nil, 0o644)) + require.NoError(t, os.RemoveAll(result.Location)) + if err := os.Symlink(outside, result.Location); err != nil { + t.Skipf("symlinks are unavailable: %v", err) + } + + _, err := repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.Error(t, err, "the swapped location cannot be verified or removed") + assert.FileExists(t, filepath.Join(outside, "README.md")) + assert.FileExists(t, filepath.Join(outside, "empty.txt")) +} + +// A stored representation that names a location outside the root is rejected on load. +func TestStoredLocationOutsideTheRootIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + key := layout.ReferenceKey(testName, testVersion) + + tr, err := repo.openTree(false) + require.NoError(t, err) + t.Cleanup(func() { _ = tr.Close() }) + rep, err := tr.loadRepresentation(key) + require.NoError(t, err) + require.NoError(t, tr.Close()) + + rep.Location = "../escape" + _, marshalErr := rep.Marshal() + require.Error(t, marshalErr, "the layout refuses to serialize an escaping location") + + // Write the document by hand to simulate tampering. + raw := `{"documents":[],"location":"../escape","name":"team/app","operation_id":"x","packaging":"none","payload":[],"record_digest":"` + pinned.RecordDigest.String() + `","repository_kind":"local/dir","schema_version":1,"version":"1.0.0"}` + require.NoError(t, os.WriteFile(filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(key))), []byte(raw), 0o644)) + _, err = repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestList_EnumeratesManagedReferencesOnly(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + var fault armedFault + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "2"}, {"a/app", "1"}} { + publish(t, repo, &prepareSpec{name: item.name, version: item.version, labels: map[string]string{"team": item.name}}, artifact.PublishOptions{}) + } + // Unmanaged files, a stray directory, and a pending publication are all ignored. + require.NoError(t, os.WriteFile(filepath.Join(repo.root, "user.txt"), []byte("x"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(repo.root, "a", "stray"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(repo.root, "a", "stray", "file.json"), []byte("{}"), 0o644)) + pending := prepare(t, repo, &prepareSpec{name: "c/pending", opID: "op-pending"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), pending, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + assert.Equal(t, []string{"a/app:1", "a/app:2", "b/app:1"}, listNames(t, repo)) + + named, err := repo.List(context.Background(), artifact.ListQuery{Name: "a/app"}) + require.NoError(t, err) + require.Len(t, named.Entries, 2) + assert.Equal(t, "1", named.Entries[0].Version) + assert.Equal(t, "2", named.Entries[1].Version) + + labeled, err := repo.List(context.Background(), artifact.ListQuery{Labels: []artifact.Label{{Key: "team", Value: "b/app"}}}) + require.NoError(t, err) + require.Len(t, labeled.Entries, 1) + assert.Equal(t, "b/app", labeled.Entries[0].Name) + assert.Equal(t, artifact.KindFiles, labeled.Entries[0].Kind) + assert.Equal(t, repo.root, labeled.Entries[0].Pinned.Repository) + }) + } +} + +func TestList_PaginatesInLexicalOrder(t *testing.T) { + repo := newRepo(t, dirSpec()) + for _, item := range []struct{ name, version string }{{"b/app", "1"}, {"a/app", "10"}, {"a/app", "2"}, {"a/app-x", "1"}, {"c/app", "1"}} { + publish(t, repo, &prepareSpec{name: item.name, version: item.version}, artifact.PublishOptions{}) + } + var seen []string + cursor := "" + for pages := 0; pages < 10; pages++ { + page, err := repo.List(context.Background(), artifact.ListQuery{PageSize: 2, Cursor: cursor}) + require.NoError(t, err) + for index := range page.Entries { + seen = append(seen, page.Entries[index].Name+":"+page.Entries[index].Version) + } + if page.NextCursor == "" { + break + } + require.NotEqual(t, cursor, page.NextCursor) + cursor = page.NextCursor + } + assert.Equal(t, []string{"a/app:10", "a/app:2", "a/app-x:1", "b/app:1", "c/app:1"}, seen, "name then version, every entry exactly once") +} + +func TestList_EmptyOrMissingRoot(t *testing.T) { + repo := newRepo(t, dirSpec()) + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + assert.Empty(t, page.Entries) + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "listing never creates the root") + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, statErr = os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "reads never create the root") +} diff --git a/pkg/artifact/local/doc.go b/pkg/artifact/local/doc.go new file mode 100644 index 00000000000..87ac02a6fff --- /dev/null +++ b/pkg/artifact/local/doc.go @@ -0,0 +1,48 @@ +// Package local implements the local directory (local/dir) and local archive (local/archive) +// artifact repositories behind [artifact.Repository]. They share one record layout (package +// layout), one publication lifecycle, one lock implementation, and one detached-evidence store; +// they differ only in how a payload is stored. +// +// # Directory repositories +// +// A local/dir payload is a tree of ordinary copied files below its resolved location, with the +// distribution documents and manifest.json adjacent at /.atmos/artifact/. Managed +// readers omit that distribution directory when they materialize user content. +// +// # Archive repositories +// +// A local/archive payload is one tar, tgz, or zip file. The repository stores adjacent +// .metadata.json and .docs/README.md|CHANGELOG.md sidecars, so metadata and +// documents are readable without opening the archive, while the archive itself embeds portable +// copies for transport. The configured quoted-octal permissions (default 0600) apply to the +// archive file and its sidecars only; they are distinct from member modes inside the archive. An +// explicit request the host cannot enforce fails with an unsupported-capability error instead of +// silently weakening the mode. +// +// # Publication and recovery +// +// A publication follows the contracts sequence: reserve the resolved location and record a pending +// operation journal, mark the reference pending, stage the payload on the same filesystem, write +// the immutable Record, install the payload and sidecars, write the representation, and only then +// flip the reference to completed. Pending references are unavailable to readers and invisible to +// listing. Readers recheck the reference generation after reading from the mutable payload +// location, so replaced bytes are never served under an old digest. +// +// An interrupted or failed force replacement leaves the reference unavailable until a retry with +// the same intent repairs it; there is no rollback. A pending operation of a different intent is +// never stolen: publication fails with a recovery diagnostic naming the operation ID. +// +// # Locks +// +// Lock returns an OS-backed file lock (host scope only) with an ownership token. It is held for +// the reference and, during publication, for the resolved location. The OS releases the lock when +// its owner exits. Acquisition is cancel-safe: a canceled wait leaks no ownership. +// +// # Signatures +// +// Signature evidence is stored as detached files below the reserved namespace and is bound to the +// canonical Record digest. Adding evidence never changes the Record, the reference, or the +// archive: evidence added after publication does not claim that the original archive bytes were +// signed, only the Record that describes them. Cryptographic verification is delegated to an +// injected [artifact.Verifier]; without one, the repository advertises no evidence formats. +package local diff --git a/pkg/artifact/local/edge_test.go b/pkg/artifact/local/edge_test.go new file mode 100644 index 00000000000..36be45ce9ad --- /dev/null +++ b/pkg/artifact/local/edge_test.go @@ -0,0 +1,350 @@ +package local + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// cloneWithBlobs returns a prepared artifact sharing the Record but with different blobs, without +// copying the original's once-guard. +func cloneWithBlobs(p *artifact.PreparedArtifact, blobs []artifact.StagedBlob) *artifact.PreparedArtifact { + return &artifact.PreparedArtifact{ + Record: p.Record, IntentDigest: p.IntentDigest, OperationID: p.OperationID, PreparedAt: p.PreparedAt, + Packaging: p.Packaging, Blobs: blobs, + } +} + +func fileBlob(path string, data []byte) artifact.StagedBlob { + return artifact.StagedBlob{ + Descriptor: artifact.StagedBlob{}.Descriptor, Role: artifact.BlobPayload, Path: path, + Open: func(context.Context) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(data)), nil }, + } +} + +func TestPublish_RejectsInconsistentPreparedBlobs(t *testing.T) { + files := map[string]string{"a.txt": "alpha", "b/c.txt": "gamma"} + docFiles := map[string]string{"a.txt": "alpha", ".atmos/artifact/README.md": "docs\n"} + tests := []struct { + name string + spec *repoSpec + files map[string]string + edit func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact + want error + }{ + {"unopenable blob", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[0].Open = nil + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactConfigInvalid}, + {"inventory file without a blob", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, p.Blobs[:1]) + }, errUtils.ErrArtifactIntegrityFailed}, + {"blob that is not in the inventory", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + extra := p.Blobs[0] + extra.Path = "unlisted.txt" + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), extra)) + }, errUtils.ErrArtifactIntegrityFailed}, + {"blob without a path", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[0].Path = "" + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate path", dirSpec(), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[0])) + }, errUtils.ErrArtifactIntegrityFailed}, + {"document blob missing", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + var blobs []artifact.StagedBlob + for _, blob := range p.Blobs { + if blob.Role == artifact.BlobPayload { + blobs = append(blobs, blob) + } + } + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"duplicate document blob", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[len(p.Blobs)-1])) + }, errUtils.ErrArtifactRecordInvalid}, + {"document blob with the wrong digest", dirSpec(), docFiles, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + blobs := append([]artifact.StagedBlob(nil), p.Blobs...) + blobs[len(blobs)-1].Descriptor.Digest = digest.FromString("other") + return cloneWithBlobs(p, blobs) + }, errUtils.ErrArtifactIntegrityFailed}, + {"archive without a payload blob", archiveSpec(artifact.PackagingTgz), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, nil) + }, errUtils.ErrArtifactPackagingInvalid}, + {"archive with two payload blobs", archiveSpec(artifact.PackagingTgz), files, func(p *artifact.PreparedArtifact) *artifact.PreparedArtifact { + return cloneWithBlobs(p, append(append([]artifact.StagedBlob(nil), p.Blobs...), p.Blobs[0])) + }, errUtils.ErrArtifactPackagingInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newRepo(t, tt.spec) + prepared := tt.edit(prepare(t, repo, &prepareSpec{files: tt.files})) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, tt.want) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a rejected preparation leaves no pending reference behind") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "and creates nothing") + }) + } +} + +// A blob whose bytes disagree with its descriptor fails the copy; the reference stays unavailable +// until a correct retry repairs it, and the bad bytes are never visible. +func TestPublish_CorruptStagedBlobIsNeverCommitted(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + good := prepare(t, repo, &prepareSpec{}) + for _, lie := range []string{"digest", "short", "long"} { + blobs := append([]artifact.StagedBlob(nil), good.Blobs...) + original := blobs[0].Open + switch lie { + case "digest": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(reader) + _ = reader.Close() + if len(data) > 0 { + data[0] ^= 0xff + } + return io.NopCloser(bytes.NewReader(data)), readErr + } + case "short": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, _ := io.ReadAll(reader) + _ = reader.Close() + return io.NopCloser(bytes.NewReader(data[:len(data)-1])), nil + } + case "long": + blobs[0].Open = func(ctx context.Context) (io.ReadCloser, error) { + reader, err := original(ctx) + if err != nil { + return nil, err + } + data, _ := io.ReadAll(reader) + _ = reader.Close() + return io.NopCloser(bytes.NewReader(append(data, 'x'))), nil + } + } + _, err := repo.Publish(context.Background(), cloneWithBlobs(good, blobs), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, lie) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "%s: no partial success is advertised", lie) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(good.OperationID))), "%s: a failed attempt cleans its staging", lie) + } + _, err := repo.Publish(context.Background(), good, artifact.PublishOptions{}) + require.NoError(t, err, "the correct retry repairs the reference") + assertIntact(t, repo, defaultRef()) + }) + } +} + +// Cancellation mid-publication surfaces as a context error and leaves the same repairable state. +func TestPublish_CancellationLeavesARepairableState(t *testing.T) { + repo := newRepo(t, dirSpec()) + good := prepare(t, repo, &prepareSpec{files: map[string]string{"a.txt": "a", "b.txt": "b"}}) + ctx, cancel := context.WithCancel(context.Background()) + blobs := append([]artifact.StagedBlob(nil), good.Blobs...) + original := blobs[0].Open + blobs[0].Open = func(openCtx context.Context) (io.ReadCloser, error) { + cancel() + return original(openCtx) + } + _, err := repo.Publish(ctx, cloneWithBlobs(good, blobs), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + + _, err = repo.Publish(context.Background(), good, artifact.PublishOptions{}) + require.NoError(t, err) + assertIntact(t, repo, defaultRef()) +} + +// If the journal was lost, a same-intent retry still repairs the pending reference. +func TestPendingReference_RepairsWithoutItsJournal(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + require.NoError(t, os.RemoveAll(filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations"))) + retry, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeReplaced, retry.Outcome) + assertIntact(t, repo, defaultRef()) + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestAbsentRootReportsNotFoundEverywhere(t *testing.T) { + repo := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + ctx := context.Background() + pinned := artifact.PinnedReference{Ref: defaultRef(), RecordDigest: digest.FromString("x"), ResolvedDigest: digest.FromString("x")} + subject := artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("x")} + + _, err := repo.Resolve(ctx, defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Pin(ctx, defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Inspect(ctx, pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Read(ctx, pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.OpenDocument(ctx, pinned, artifact.DocumentReadme) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Delete(ctx, pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Sign(ctx, subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.Verify(ctx, subject, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.ReadEvidence(ctx, &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "no read or failed mutation created the root") +} + +func TestCanceledContextStopsReads(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := repo.Resolve(ctx, defaultRef()) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.Read(ctx, pinned) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.ReadEvidence(ctx, &pinned) + require.ErrorIs(t, err, context.Canceled) + _, err = repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, context.Canceled) +} + +func TestRead_OpeningFailsCleanlyWhenAPayloadFileIsMissing(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location, "nested", "deep", "file.txt"))) + content, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.Nil(t, content) + + require.NoError(t, os.RemoveAll(result.Location)) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a missing payload directory is damage, not absence") +} + +func TestRead_ReaderReportsTheReferenceMovingOrDisappearing(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + // Remove only the reference record: payload handles stay valid on every platform. + require.NoError(t, os.Remove(filepath.Join(repo.root, filepath.FromSlash(layout.ReferencePath(layout.ReferenceKey(testName, testVersion)))))) + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactNotFound, "a deleted reference is reported, not served") + }) + } +} + +func TestRead_PayloadThatGrowsIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + result := publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "short"}}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, filepath.Join(result.Location, "data.txt"))), appendJunk) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed, "bytes beyond the declared size are never served") +} + +func TestUnchangedRepublishDetectsAMissingRepresentation(t *testing.T) { + repo := newRepo(t, dirSpec()) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + current := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(repo.root, filepath.FromSlash(layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)))))) + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{ExpectedPrior: ¤t}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestRootThatIsAFileFailsClearly(t *testing.T) { + file := filepath.Join(t.TempDir(), "root") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + repo := newRepo(t, &repoSpec{root: file}) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repo.Lock(context.Background(), lockRequest("r")) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = repo.List(context.Background(), artifact.ListQuery{}) + require.Error(t, err) +} + +func TestInventoryInsideTheGeneratedDistributionDirIsRejected(t *testing.T) { + repo := newRepo(t, dirSpec()) + prepared := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "x"}}) + record := prepared.Record + record.Inventory = append(append([]artifact.Entry(nil), record.Inventory...), artifact.Entry{ + Path: ".atmos/artifact/extra.txt", Type: artifact.EntryFile, Size: 1, Mode: 0o644, Digest: digest.FromString("x"), + }) + require.NoError(t, record.Seal()) + conflicting := &artifact.PreparedArtifact{ + Record: record, IntentDigest: prepared.IntentDigest, OperationID: "op-conflict", PreparedAt: prepared.PreparedAt, + Packaging: artifact.PackagingNone, + Blobs: append(append([]artifact.StagedBlob(nil), prepared.Blobs...), fileBlob(".atmos/artifact/extra.txt", []byte("x"))), + } + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor = prepared.Blobs[0].Descriptor + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor.Digest = digest.FromString("x") + conflicting.Blobs[len(conflicting.Blobs)-1].Descriptor.Size = 1 + _, err := repo.Publish(context.Background(), conflicting, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestFormattedErrorsNameTheOperationAndReference(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + fault.hook = crashAt(faultBeforePayload) + prepared := prepare(t, repo, &prepareSpec{opID: "op-visible"}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorContains(t, err, fmt.Sprintf("%s:%s", testName, testVersion)) + require.ErrorContains(t, err, "op-visible") + require.ErrorContains(t, err, "same intent") +} diff --git a/pkg/artifact/local/evidence.go b/pkg/artifact/local/evidence.go new file mode 100644 index 00000000000..0ce11ee6117 --- /dev/null +++ b/pkg/artifact/local/evidence.go @@ -0,0 +1,396 @@ +package local + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxMalformedEvidence bounds diagnostics and reads for invalid evidence entries. +const maxMalformedEvidence = 64 + +// loadedEvidence is one stored evidence file that parsed and matched its file name. +type loadedEvidence struct { + evidence *layout.Evidence + locator string +} + +// loadEvidence reads the evidence stored for a Record subject. Malformed or oversize files are +// reported as diagnostics and skipped; they can never override valid matching evidence. +func (t *tree) loadEvidence(subject digest.Digest) ([]loadedEvidence, []artifact.Diagnostic, error) { + dir := layout.SignatureDir(artifact.SubjectRecord, subject) + file, err := t.Open(native(dir)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, nil + } + if err != nil { + return nil, nil, fmt.Errorf("open evidence directory: %w", err) + } + defer file.Close() + var loaded []loadedEvidence + var diags []artifact.Diagnostic + ignored := 0 + for { + entries, readErr := file.ReadDir(maxMalformedEvidence) + for _, entry := range entries { + item, diag, ok := t.evidenceEntry(dir, entry) + if ok { + loaded = append(loaded, item) + continue + } + ignored++ + if diag != nil { + diags = append(diags, *diag) + } + if ignored > maxMalformedEvidence { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-truncated", Message: "too many invalid evidence entries; remaining entries were not read"}) + return nil, diags, fmt.Errorf("%w: more than %d invalid entries for %s", errUtils.ErrArtifactEvidenceMalformed, maxMalformedEvidence, subject) + } + } + if errors.Is(readErr, io.EOF) { + return loaded, diags, nil + } + if readErr != nil { + return nil, diags, fmt.Errorf("read evidence directory: %w", readErr) + } + } +} + +// evidenceEntry reads one directory entry of an evidence directory. It reports ok for valid +// evidence; otherwise the entry is ignored, with a diagnostic when it looked like evidence. +func (t *tree) evidenceEntry(dir string, entry fs.DirEntry) (loadedEvidence, *artifact.Diagnostic, bool) { + stem, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + return loadedEvidence{}, nil, false + } + locator := dir + "/" + entry.Name() + item, problem := t.readEvidenceFile(locator, stem) + if problem != nil { + return loadedEvidence{}, &artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "malformed-evidence", Message: fmt.Sprintf("%s was ignored: %v", locator, problem)}, false + } + return loadedEvidence{evidence: item, locator: locator}, nil, true +} + +func (t *tree) readEvidenceFile(locator, hexDigest string) (*layout.Evidence, error) { + want := digest.NewDigestFromEncoded(digest.SHA256, hexDigest) + if err := want.Validate(); err != nil { + return nil, err + } + raw, err := t.readBounded(locator, layout.MaxEvidenceFileBytes) + if err != nil { + return nil, err + } + return layout.DecodeEvidence(raw, want) +} + +// listEvidence returns the stored evidence descriptors for a subject, skipping malformed files. +func (t *tree) listEvidence(subject digest.Digest) []artifact.SignatureEvidence { + loaded, _, _ := t.loadEvidence(subject) + out := make([]artifact.SignatureEvidence, 0, len(loaded)) + for index := range loaded { + out = append(out, loaded[index].evidence.Reference(loaded[index].locator)) + } + return out +} + +// requireCurrentRecord checks that digest names the current completed Record of its reference and +// returns the canonical bytes. Evidence only ever binds to a live subject. +func (r *Repository) requireCurrentRecord(t *tree, recordDigest digest.Digest) ([]byte, error) { + record, raw, err := t.loadRecord(recordDigest) + if err != nil { + return nil, err + } + if _, err := r.resolveRef(t, artifact.Ref{Name: record.Name, Digest: recordDigest}); err != nil { + return nil, err + } + return raw, nil +} + +// checkSubject validates a signature subject for this repository. +func (r *Repository) checkSubject(operation string, subject artifact.SignatureSubject) error { + if err := subject.Validate(); err != nil { + return err + } + if caps := r.Capabilities(); !caps.HasSubjectKind(subject.Kind) { + return artifact.NewUnsupportedCapability(r.alias, operation, "subject/"+string(subject.Kind), + "local repositories bind detached evidence to the canonical Record only") + } + return nil +} + +// Sign implements [artifact.Repository]: it asks the signer for evidence over the Record subject +// and stores it as a detached file named by the evidence digest. Evidence lives beside the Record, +// so signing never changes the Record, the reference, or the payload. A signature added after +// publication covers the Record that describes the archive, not the original archive bytes. +func (r *Repository) Sign(ctx context.Context, subject artifact.SignatureSubject, signer artifact.Signer, opts artifact.SignOptions) ([]artifact.SignatureEvidence, error) { + defer perf.Track(nil, "local.Repository.Sign")() + + if err := r.checkSignRequest(subject, signer); err != nil { + return nil, err + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, artifact.Ref{Name: "(subject)", Digest: subject.Digest}) + } + defer t.Close() + raw, err := r.requireCurrentRecord(t, subject.Digest) + if err != nil { + return nil, err + } + if subject.Size != 0 && subject.Size != int64(len(raw)) { + return nil, fmt.Errorf("%w: the subject declares %d bytes but the Record is %d", errUtils.ErrArtifactRefInvalid, subject.Size, len(raw)) + } + payload, err := signer.Sign(ctx, subject) + if err != nil { + return nil, err + } + if payload == nil { + return nil, fmt.Errorf("%w: the signer returned no evidence", errUtils.ErrArtifactVerificationFailed) + } + // The signer may have waited arbitrarily long, so a concurrent Delete can have removed the Record + // since the first check. Store the evidence under the reference lock, against a re-checked Record. + var stored artifact.SignatureEvidence + err = r.withEvidenceLock(ctx, t, subject.Digest, opts.Lock, func() error { + var storeErr error + stored, storeErr = r.storeEvidence(t, subject, payload) + return storeErr + }) + if err != nil { + return nil, err + } + return []artifact.SignatureEvidence{stored}, nil +} + +// withEvidenceLock runs fn while holding the reference lock of the Record, after confirming the +// Record is still current. A matching caller lock is reused; otherwise the lock is acquired and +// released here, so evidence is never written for a Record a concurrent Delete removed. +func (r *Repository) withEvidenceLock(ctx context.Context, t *tree, recordDigest digest.Digest, lock artifact.LockHandle, fn func() error) (err error) { + held, err := r.lockEvidenceReference(ctx, t, recordDigest, lock) + if err != nil { + return err + } + if held != nil { + defer func() { + if releaseErr := held.Release(context.WithoutCancel(ctx)); err == nil { + err = releaseErr + } + }() + } + if _, err := r.requireCurrentRecord(t, recordDigest); err != nil { + return err + } + return fn() +} + +// checkSignRequest refuses an unsupported subject, a missing signer, and a signer whose evidence +// formats the repository cannot store, before anything is opened or signed. +func (r *Repository) checkSignRequest(subject artifact.SignatureSubject, signer artifact.Signer) error { + if err := r.checkSubject("sign", subject); err != nil { + return err + } + if signer == nil { + return fmt.Errorf("%w: no signer supplied", errUtils.ErrArtifactConfigInvalid) + } + caps := r.Capabilities() + return caps.CheckSignerFormats(r.alias, signer) +} + +// storeEvidence validates and writes one evidence payload. +func (r *Repository) storeEvidence(t *tree, subject artifact.SignatureSubject, payload *artifact.EvidencePayload) (artifact.SignatureEvidence, error) { + if caps := r.Capabilities(); !caps.HasEvidenceFormat(payload.Format) { + return artifact.SignatureEvidence{}, artifact.NewUnsupportedCapability(r.alias, "sign", "evidence/"+string(payload.Format), + fmt.Sprintf("the repository stores and verifies %v evidence; configure a verifier for this format", caps.EvidenceFormats)) + } + evidence, err := layout.NewEvidence(subject, payload) + if err != nil { + return artifact.SignatureEvidence{}, err + } + raw, err := evidence.Marshal() + if err != nil { + return artifact.SignatureEvidence{}, err + } + locator := layout.SignaturePath(subject.Kind, subject.Digest, evidence.Digest()) + if err := t.writeAtomic(locator, raw); err != nil { + return artifact.SignatureEvidence{}, fmt.Errorf("%w: store evidence: %w", errUtils.ErrArtifactUploadFailed, err) + } + return evidence.Reference(locator), nil +} + +// Verify implements [artifact.Repository]. It authenticates the stored evidence with the injected +// verifier; it never trusts a cached verdict. No evidence, an unverifiable format, or an +// unsatisfied policy is a failure. +func (r *Repository) Verify(ctx context.Context, subject artifact.SignatureSubject, policy artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + defer perf.Track(nil, "local.Repository.Verify")() + + if err := r.checkSubject("verify", subject); err != nil { + return nil, err + } + verifier := r.settings.verifier + if verifier == nil { + return nil, artifact.NewUnsupportedCapability(r.alias, "verify", "evidence/verify", "no signature verifier is configured for this repository") + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, artifact.Ref{Name: "(subject)", Digest: subject.Digest}) + } + defer t.Close() + if _, err := r.requireCurrentRecord(t, subject.Digest); err != nil { + return nil, err + } + loaded, diags, loadErr := t.loadEvidence(subject.Digest) + if loadErr != nil { + diags = append(diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "evidence-read-failed", Message: loadErr.Error()}) + } + blobs := evidenceBlobs(subject, loaded, &diags) + result := &artifact.VerificationResult{Subject: subject, Status: artifact.VerificationFailed, Diagnostics: diags} + if loadErr != nil { + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, loadErr) + } + if fingerprint, fingerprintErr := policy.Fingerprint(); fingerprintErr == nil { + result.PolicyFingerprint = fingerprint + } + if len(blobs) == 0 { + result.Diagnostics = append(result.Diagnostics, artifact.Diagnostic{Severity: artifact.SeverityError, Code: "no-evidence", Message: "no signature evidence is stored for the subject"}) + return result, fmt.Errorf("%w: no signature evidence is stored for %s", errUtils.ErrArtifactVerificationFailed, subject.Digest) + } + verified, verifyErr := verifier.Verify(ctx, subject, blobs, policy) + return r.finishVerification(result, verified, verifyErr) +} + +// evidenceBlobs converts the evidence bound to the subject, reporting any bound to another subject. +func evidenceBlobs(subject artifact.SignatureSubject, loaded []loadedEvidence, diags *[]artifact.Diagnostic) []artifact.EvidenceBlob { + blobs := make([]artifact.EvidenceBlob, 0, len(loaded)) + for index := range loaded { + stored := loaded[index].evidence + if stored.Subject.Kind != subject.Kind || stored.Subject.Digest != subject.Digest { + *diags = append(*diags, artifact.Diagnostic{Severity: artifact.SeverityWarning, Code: "subject-mismatch", Message: loaded[index].locator + " is bound to a different subject and was ignored"}) + continue + } + blobs = append(blobs, stored.Blob(loaded[index].locator)) + } + return blobs +} + +// finishVerification merges the verifier's result with the storage diagnostics and turns anything +// but a verified status into an error. +func (r *Repository) finishVerification(base, verified *artifact.VerificationResult, verifyErr error) (*artifact.VerificationResult, error) { + result := base + if verified != nil { + merged := *verified + merged.Subject = base.Subject + merged.Diagnostics = append(append([]artifact.Diagnostic(nil), base.Diagnostics...), verified.Diagnostics...) + if merged.PolicyFingerprint == "" { + merged.PolicyFingerprint = base.PolicyFingerprint + } + result = &merged + } + switch { + case verifyErr != nil && errors.Is(verifyErr, errUtils.ErrArtifactVerificationFailed): + return result, verifyErr + case verifyErr != nil: + return result, fmt.Errorf("%w: %w", errUtils.ErrArtifactVerificationFailed, verifyErr) + case result.Status != artifact.VerificationVerified: + return result, fmt.Errorf("%w: no valid permitted signature satisfies the policy for %s", errUtils.ErrArtifactVerificationFailed, result.Subject.Digest) + } + return result, nil +} + +// ReadEvidence implements [artifact.EvidenceReader]: it exports the stored evidence blobs of a +// pinned artifact so a copy through the shared staging path can carry them along. The evidence is +// untrusted until verified. +func (r *Repository) ReadEvidence(ctx context.Context, pinned *artifact.PinnedReference) ([]artifact.EvidenceBlob, error) { + defer perf.Track(nil, "local.Repository.ReadEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + if _, err := r.requireCurrentRecord(t, pinned.RecordDigest); err != nil { + return nil, err + } + loaded, _, err := t.loadEvidence(pinned.RecordDigest) + if err != nil { + return nil, err + } + // Evidence bound to another subject can share the directory. Verification ignores it, so the + // export must too: the destination rejects a foreign envelope and would stop before importing + // the valid signatures that follow it. + var ignored []artifact.Diagnostic + return evidenceBlobs(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: pinned.RecordDigest}, loaded, &ignored), nil +} + +// WriteEvidence implements [artifact.EvidenceWriter]: it imports evidence for an artifact this +// repository already holds. The evidence must bind to the pinned Record and carry matching bytes; +// importing never changes the Record or the reference. +func (r *Repository) WriteEvidence(ctx context.Context, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob, lock artifact.LockHandle) (stored []artifact.SignatureEvidence, err error) { + defer perf.Track(nil, "local.Repository.WriteEvidence")() + + if err := ctx.Err(); err != nil { + return nil, err + } + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + err = r.withEvidenceLock(ctx, t, pinned.RecordDigest, lock, func() error { + var writeErr error + stored, writeErr = r.writeEvidenceLocked(t, pinned, evidence) + return writeErr + }) + return stored, err +} + +// writeEvidenceLocked validates and stores each evidence blob under the reference lock. +func (r *Repository) writeEvidenceLocked(t *tree, pinned *artifact.PinnedReference, evidence []artifact.EvidenceBlob) ([]artifact.SignatureEvidence, error) { + stored := make([]artifact.SignatureEvidence, 0, len(evidence)) + for index := range evidence { + blob := &evidence[index] + subject := blob.Evidence.Subject + if err := r.checkSubject("write evidence", subject); err != nil { + return stored, err + } + if subject.Digest != pinned.RecordDigest { + return stored, fmt.Errorf("%w: evidence is bound to %s, not to the pinned Record %s", errUtils.ErrArtifactRefInvalid, subject.Digest, pinned.RecordDigest) + } + if blob.Evidence.Descriptor.Digest != digest.FromBytes(blob.Data) { + return stored, fmt.Errorf("%w: evidence bytes do not match their descriptor %s", errUtils.ErrArtifactIntegrityFailed, blob.Evidence.Descriptor.Digest) + } + item, err := r.storeEvidence(t, subject, &artifact.EvidencePayload{Format: blob.Evidence.Format, MediaType: blob.Evidence.Descriptor.MediaType, Data: blob.Data}) + if err != nil { + return stored, err + } + stored = append(stored, item) + } + return stored, nil +} + +// lockEvidenceReference reuses a matching caller lock or acquires the reference lock. +func (r *Repository) lockEvidenceReference(ctx context.Context, t *tree, recordDigest digest.Digest, lock artifact.LockHandle) (*fileLock, error) { + record, _, err := t.loadRecord(recordDigest) + if err != nil { + return nil, err + } + resource := record.Name + ":" + record.Version + if lock != nil { + if lock.Resource() != resource { + return nil, fmt.Errorf("%w: supplied lock does not cover reference %s", errUtils.ErrArtifactConflict, resource) + } + return nil, nil + } + return r.acquire(ctx, resource, r.settings.owner, "") +} diff --git a/pkg/artifact/local/faults.go b/pkg/artifact/local/faults.go new file mode 100644 index 00000000000..cd07592fccc --- /dev/null +++ b/pkg/artifact/local/faults.go @@ -0,0 +1,62 @@ +package local + +import ( + "errors" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// faultPoint names a point in the publication lifecycle at which tests simulate a crash. +type faultPoint string + +// Fault points, in lifecycle order. +const ( + // The before-payload point fires after the pending state was recorded and before any payload + // byte is staged. + faultBeforePayload faultPoint = "before-payload" + + // The during-payload point fires while the first payload blob is being staged. + faultDuringPayload faultPoint = "during-payload" + + // The after-metadata point fires after the Record, payload, and representation are in place + // and before the reference is completed. + faultAfterMetadata faultPoint = "after-metadata" + + // The after-commit point fires after the reference was completed, before cleanup and before + // the caller sees the response. + faultAfterCommit faultPoint = "after-commit" +) + +// faultHook returns a non-nil error to simulate a crash at the point. +type faultHook func(point faultPoint) error + +// crashError marks an injected crash. A crashed process cannot clean up after itself, so the +// publication leaves its staging and journal exactly as a real interruption would. +type crashError struct{ err error } + +func (c *crashError) Error() string { + defer perf.Track(nil, "local.crashError.Error")() + + return c.err.Error() +} + +func (c *crashError) Unwrap() error { + defer perf.Track(nil, "local.crashError.Unwrap")() + + return c.err +} + +func (s *settings) fault(point faultPoint) error { + if s.faults == nil { + return nil + } + if err := s.faults(point); err != nil { + return &crashError{err: err} + } + return nil +} + +func isCrash(err error) bool { + var crash *crashError + return errors.As(err, &crash) +} diff --git a/pkg/artifact/local/fsys.go b/pkg/artifact/local/fsys.go new file mode 100644 index 00000000000..ffd8b9dad96 --- /dev/null +++ b/pkg/artifact/local/fsys.go @@ -0,0 +1,255 @@ +package local + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Modes of directories and files this package creates itself. +const ( + dirMode fs.FileMode = 0o755 + privateDir fs.FileMode = 0o700 + privateFile fs.FileMode = 0o600 + publicFile fs.FileMode = 0o644 + randomBytes = 8 + copyBufBytes = 32 * 1024 +) + +// errRootMissing reports that the repository root directory does not exist yet. Read paths treat +// it as an empty repository; only writes create the root. +var errRootMissing = errors.New("repository root does not exist") + +// tree is the repository root opened as an os.Root, so no operation can leave the root even +// through a symlink planted inside it. Slash-separated relative paths are converted for the host. +type tree struct { + *os.Root + + abs string +} + +// openTree opens the repository root. With create set, a missing root is created first. +func (r *Repository) openTree(create bool) (*tree, error) { + if create { + if err := os.MkdirAll(r.root, dirMode); err != nil { + return nil, fmt.Errorf("%w: create repository root %q: %w", errUtils.ErrArtifactConfigInvalid, r.root, err) + } + } + root, err := os.OpenRoot(r.root) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, errRootMissing + case err != nil: + return nil, fmt.Errorf("%w: open repository root %q: %w", errUtils.ErrArtifactConfigInvalid, r.root, err) + } + return &tree{Root: root, abs: r.root}, nil +} + +// native converts a slash-separated relative path to the host form. +func native(rel string) string { + return filepath.FromSlash(rel) +} + +// absolute returns the absolute host path of a slash-separated relative path. +func (t *tree) absolute(rel string) string { + return filepath.Join(t.abs, native(rel)) +} + +// exists reports whether a path exists without following a final symlink. +func (t *tree) exists(rel string) (bool, error) { + _, err := t.Lstat(native(rel)) + switch { + case err == nil: + return true, nil + case errors.Is(err, fs.ErrNotExist): + return false, nil + default: + return false, err + } +} + +// readBounded reads a whole regular file of at most limit bytes. A missing file returns an error +// that wraps fs.ErrNotExist. +func (t *tree) readBounded(rel string, limit int64) ([]byte, error) { + file, err := t.Open(native(rel)) + if err != nil { + return nil, err + } + defer file.Close() + return artifact.ReadBounded(file, limit) +} + +// ensureParent creates the parent directories of rel. +func (t *tree) ensureParent(rel string) error { + parent := path.Dir(rel) + if parent == "." || parent == "" { + return nil + } + return t.MkdirAll(native(parent), dirMode) +} + +// writeAtomic writes data at rel through a temporary file in the managed temp directory, syncs +// it, applies the public file mode, and renames it over the destination. Readers see the old or the new file, +// never a partial one. +func (t *tree) writeAtomic(rel string, data []byte) error { + if err := t.MkdirAll(native(layout.TempDir()), privateDir); err != nil { + return err + } + if err := t.ensureParent(rel); err != nil { + return err + } + suffix, err := randomSuffix() + if err != nil { + return err + } + tmp := layout.TempDir() + "/" + path.Base(rel) + "." + suffix + ".tmp" + file, err := t.OpenFile(native(tmp), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + _, writeErr := file.Write(data) + if writeErr == nil { + writeErr = file.Sync() + } + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr == nil { + writeErr = t.Chmod(native(tmp), publicFile) + } + if writeErr == nil { + writeErr = t.Rename(native(tmp), native(rel)) + } + if writeErr != nil { + _ = t.Remove(native(tmp)) + return writeErr + } + return nil +} + +func randomSuffix() (string, error) { + buf := make([]byte, randomBytes) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} + +// removeAll deletes a path and everything below it inside the root. Directories that lack owner +// permissions (a payload may legitimately contain mode 0555 directories) are made writable first +// so removal cannot be blocked by the artifact's own modes. +func (t *tree) removeAll(rel string) error { + if ok, err := t.exists(rel); err != nil || !ok { + return err + } + t.makeWritable(rel) + return t.RemoveAll(native(rel)) +} + +// makeWritable gives every directory below rel owner access. Failures are ignored: removal reports +// the real error. +func (t *tree) makeWritable(rel string) { + info, err := t.Lstat(native(rel)) + if err != nil || !info.IsDir() { + return + } + _ = t.Chmod(native(rel), info.Mode().Perm()|privateDir) + children, err := fs.ReadDir(t.FS(), filepath.ToSlash(native(rel))) + if err != nil { + return + } + for _, child := range children { + if child.IsDir() { + t.makeWritable(rel + "/" + child.Name()) + } + } +} + +// removeEmptyParents removes the now-empty parent directories of rel up to, never including, the +// root. A non-empty directory stops the walk without error. +func (t *tree) removeEmptyParents(rel string) { + for parent := path.Dir(rel); parent != "." && parent != "" && !layout.InManagedNamespace(parent); parent = path.Dir(parent) { + if err := t.Remove(native(parent)); err != nil { + return + } + } +} + +// copyVerified copies src to dst, enforcing the declared size and SHA-256 digest. It never writes +// more than size bytes plus the one extra byte needed to detect an oversize source. +func copyVerified(dst io.Writer, src io.Reader, want digest.Digest, size int64) error { + hasher := want.Algorithm().Hash() + written, err := io.CopyBuffer(io.MultiWriter(dst, hasher), io.LimitReader(src, size+1), make([]byte, copyBufBytes)) + switch { + case err != nil: + return err + case written != size: + return fmt.Errorf("%w: blob is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, written, size) + } + if got := digest.NewDigest(want.Algorithm(), hasher); got != want { + return fmt.Errorf("%w: blob digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, want) + } + return nil +} + +// stageFile creates rel exclusively in the tree and copies the verified blob into it. +func (t *tree) stageFile(rel string, src io.Reader, want digest.Digest, size int64) error { + if err := t.ensureParent(rel); err != nil { + return err + } + file, err := t.OpenFile(native(rel), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + copyErr := copyVerified(file, src, want, size) + if copyErr == nil { + copyErr = file.Sync() + } + if closeErr := file.Close(); copyErr == nil { + copyErr = closeErr + } + return copyErr +} + +// writeNew creates rel exclusively with data and mode. +func (t *tree) writeNew(rel string, data []byte, mode fs.FileMode) error { + if err := t.ensureParent(rel); err != nil { + return err + } + file, err := t.OpenFile(native(rel), os.O_WRONLY|os.O_CREATE|os.O_EXCL, privateFile) + if err != nil { + return err + } + _, writeErr := file.Write(data) + if closeErr := file.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr == nil { + writeErr = t.Chmod(native(rel), mode) + } + return writeErr +} + +// moveAside renames rel to dest when rel exists, creating the parent of dest. +func (t *tree) moveAside(rel, dest string) error { + ok, err := t.exists(rel) + if err != nil || !ok { + return err + } + if err := t.ensureParent(dest); err != nil { + return err + } + return t.Rename(native(rel), native(dest)) +} diff --git a/pkg/artifact/local/helpers_test.go b/pkg/artifact/local/helpers_test.go new file mode 100644 index 00000000000..40987ca3d49 --- /dev/null +++ b/pkg/artifact/local/helpers_test.go @@ -0,0 +1,262 @@ +package local + +import ( + "context" + "io" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guards for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Packaging: "none", Permissions: "0600"} + +const ( + testName = "team/app" + testVersion = "1.0.0" +) + +// capturedAt is the fixed preparation time of the test artifacts. +var capturedAt = time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + +// repoSpec configures a repository under test. +type repoSpec struct { + kind string + packaging artifact.Packaging + permissions string + keyPattern string + root string + opts []Option +} + +func (s *repoSpec) config(t *testing.T) *schema.ArtifactRepositoryConfig { + t.Helper() + options := map[string]any{"path": s.root} + if s.keyPattern != "" { + options["key_pattern"] = s.keyPattern + } + cfg := &schema.ArtifactRepositoryConfig{Kind: s.kind, Options: options, Permissions: s.permissions} + if s.packaging != "" { + cfg.Packaging = string(s.packaging) + } + return cfg +} + +// newRepo builds a repository under a fresh temporary root unless the spec names one. +func newRepo(t *testing.T, spec *repoSpec) *Repository { + t.Helper() + spec = spec.clone() + if spec.root == "" { + spec.root = filepath.Join(t.TempDir(), "repo") + } + if spec.kind == "" { + spec.kind = artifact.RepositoryKindLocalDir + } + repo, err := newRepository("test", spec.kind, spec.config(t), spec.opts) + require.NoError(t, err) + return repo +} + +func dirSpec() *repoSpec { return &repoSpec{kind: artifact.RepositoryKindLocalDir} } + +func archiveSpec(packaging artifact.Packaging) *repoSpec { + return &repoSpec{kind: artifact.RepositoryKindLocalArchive, packaging: packaging} +} + +// clone returns an independent copy, so a case can adjust its spec without leaking into others. +func (s *repoSpec) clone() *repoSpec { + c := *s + c.opts = append([]Option(nil), s.opts...) + return &c +} + +func (s *repoSpec) withOpts(opts ...Option) *repoSpec { + c := s.clone() + c.opts = append(c.opts, opts...) + return c +} + +func (s *repoSpec) withRoot(root string) *repoSpec { + c := s.clone() + c.root = root + return c +} + +func (s *repoSpec) withPattern(pattern string) *repoSpec { + c := s.clone() + c.keyPattern = pattern + return c +} + +func (s *repoSpec) withPermissions(permissions string) *repoSpec { + c := s.clone() + c.permissions = permissions + return c +} + +// prepareSpec describes the artifact a test prepares with the production files producer. +type prepareSpec struct { + name string + version string + packaging artifact.Packaging + files map[string]string + at time.Time + opID string + labels map[string]string +} + +func (p *prepareSpec) clone() *prepareSpec { + c := *p + return &c +} + +func (p *prepareSpec) withDefaults(repo *Repository) *prepareSpec { + p = p.clone() + if p.name == "" { + p.name = testName + } + if p.version == "" { + p.version = testVersion + } + if p.packaging == "" { + p.packaging = repo.packaging + } + if p.at.IsZero() { + p.at = capturedAt + } + if p.opID == "" { + p.opID = "op-" + p.name + "-" + p.version + } + if p.files == nil { + p.files = map[string]string{"data.txt": "hello\n"} + } + return p +} + +func prepare(t *testing.T, repo *Repository, spec *prepareSpec) *artifact.PreparedArtifact { + t.Helper() + spec = spec.withDefaults(repo) + src := t.TempDir() + for name, content := range spec.files { + fixtures.WriteFile(t, src, name, content, 0o644) + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: spec.name, Version: spec.version, Packaging: spec.packaging, + Metadata: artifact.Metadata{Labels: spec.labels}, + OperationID: spec.opID, PreparedAt: spec.at, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +// publish prepares and publishes, failing the test on error. +func publish(t *testing.T, repo *Repository, spec *prepareSpec, options artifact.PublishOptions) *artifact.PublishResult { + t.Helper() + prepared := prepare(t, repo, spec) + result, err := repo.Publish(context.Background(), prepared, options) + require.NoError(t, err) + return result +} + +func ref(name, version string) artifact.Ref { return artifact.Ref{Name: name, Version: version} } + +func defaultRef() artifact.Ref { return ref(testName, testVersion) } + +func resolve(t *testing.T, repo *Repository, r artifact.Ref) artifact.PinnedReference { + t.Helper() + pinned, err := repo.Resolve(context.Background(), r) + require.NoError(t, err) + return pinned +} + +// readAll drains every stream of the content, returning the first read error. +func readAll(t *testing.T, content *artifact.Content) error { + t.Helper() + for index := range content.Streams { + if _, err := io.Copy(io.Discard, content.Streams[index].Reader); err != nil { + return err + } + } + return nil +} + +// listFiles returns the sorted slash-separated relative paths of every file below dir. +func listFiles(t *testing.T, dir string) []string { + t.Helper() + var out []string + err := filepath.WalkDir(dir, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() { + rel, relErr := filepath.Rel(dir, p) + require.NoError(t, relErr) + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + require.NoError(t, err) + sort.Strings(out) + return out +} + +// payloadFiles lists the files outside the managed namespace. +func payloadFiles(t *testing.T, root string) []string { + t.Helper() + var out []string + for _, p := range listFiles(t, root) { + if !layout.InManagedNamespace(p) { + out = append(out, p) + } + } + return out +} + +func newSigner() *artifacttest.FakeSigner { + return &artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"} +} + +func withFaults(hook faultHook) Option { + return func(s *settings) { s.faults = hook } +} + +// crashAt returns a fault hook that fails once at the point. +func crashAt(point faultPoint) faultHook { + fired := false + return func(p faultPoint) error { + if p == point && !fired { + fired = true + return errCrash + } + return nil + } +} + +// armedFault is a fault hook that does nothing until a test arms it. +type armedFault struct{ hook faultHook } + +func (a *armedFault) option() Option { + return withFaults(func(point faultPoint) error { + if a.hook == nil { + return nil + } + return a.hook(point) + }) +} + +// helperConfig is the configuration the lock helper process builds its repository from. +func helperConfig(root string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": root}} +} diff --git a/pkg/artifact/local/hold_test.go b/pkg/artifact/local/hold_test.go new file mode 100644 index 00000000000..1e10cba80d1 --- /dev/null +++ b/pkg/artifact/local/hold_test.go @@ -0,0 +1,36 @@ +package local + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// holdLock takes a repository lock for a test. The lock is also released when the test ends, so an +// assertion that aborts the test never leaves the resource held. Releasing it explicitly first is +// fine: the cleanup's second release only reports that the handle is gone, and that is ignored. +func holdLock(t *testing.T, repo *Repository, request artifact.LockRequest) artifact.LockHandle { + t.Helper() + handle, err := repo.Lock(context.Background(), request) + require.NoError(t, err) + t.Cleanup(func() { _ = handle.Release(context.Background()) }) + return handle +} + +func TestHoldLockReleasesWhenTheTestEnds(t *testing.T) { + repo := newRepo(t, dirSpec()) + + t.Run("a subtest that never releases", func(t *testing.T) { + _ = holdLock(t, repo, lockRequest("r")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + next, err := repo.Lock(ctx, lockRequest("r")) + require.NoError(t, err, "the lock a finished subtest still held is free for the next one") + require.NoError(t, next.Release(context.Background())) +} diff --git a/pkg/artifact/local/integrity_test.go b/pkg/artifact/local/integrity_test.go new file mode 100644 index 00000000000..b7f9da4aaca --- /dev/null +++ b/pkg/artifact/local/integrity_test.go @@ -0,0 +1,258 @@ +package local + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// tamper rewrites one file below the repository root after applying edit to its bytes. +func tamper(t *testing.T, root, rel string, edit func([]byte) []byte) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + original, err := os.ReadFile(path) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.Chmod(path, 0o644)) + require.NoError(t, os.WriteFile(path, edit(original), info.Mode().Perm())) +} + +func flipFirstByte(data []byte) []byte { + out := append([]byte(nil), data...) + out[0] ^= 0xff + return out +} + +func appendJunk(data []byte) []byte { + return append(append([]byte(nil), data...), []byte("junk")...) +} + +func truncate(data []byte) []byte { return data[:len(data)/2] } + +// integrityCase names one corruption and where it surfaces. +type integrityCase struct { + name string + // target returns the file to corrupt, relative to the root, given the stored location. + target func(location string, record *artifact.Record) string + edit func([]byte) []byte + // surfaces lists the operations that must fail with an integrity error. + surfaces []string +} + +func TestTamperedState_FailsIntegrity(t *testing.T) { + ref := defaultRef() + managed := func(path func(key string, recordDigest string) string) func(string, *artifact.Record) string { + return func(_ string, record *artifact.Record) string { + digestValue, err := record.RecordDigest() + require.NoError(t, err) + return path(layout.ReferenceKey(record.Name, record.Version), digestValue.Encoded()) + } + } + cases := []integrityCase{ + { + name: "record file bytes", + target: managed(func(_, recordHex string) string { + return ".atmos-artifacts/v1/records/" + recordHex + ".json" + }), + edit: appendJunk, surfaces: []string{"resolve", "list"}, + }, + { + name: "reference file garbage", + target: managed(func(key, _ string) string { return layout.ReferencePath(key) }), + edit: truncate, + surfaces: []string{"resolve", "list"}, + }, + { + name: "representation file", + target: managed(func(key, _ string) string { return layout.RepresentationPath(key) }), + edit: flipFirstByte, + surfaces: []string{"inspect", "read", "document"}, + }, + } + for _, spec := range allSpecs { + for _, tc := range cases { + t.Run(specName(spec)+"/"+tc.name, func(t *testing.T) { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, ref) + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + tamper(t, repo.root, tc.target(result.Location, inspection.Record), tc.edit) + assertIntegrityFailure(t, repo, ref, &pinned, tc.surfaces) + }) + } + } +} + +func assertIntegrityFailure(t *testing.T, repo *Repository, r artifact.Ref, pinned *artifact.PinnedReference, surfaces []string) { + t.Helper() + ctx := context.Background() + for _, surface := range surfaces { + var err error + switch surface { + case "resolve": + _, err = repo.Resolve(ctx, r) + case "list": + _, err = repo.List(ctx, artifact.ListQuery{}) + case "inspect": + _, err = repo.Inspect(ctx, *pinned) + case "read": + _, err = repo.Read(ctx, *pinned) + case "document": + _, err = repo.OpenDocument(ctx, *pinned, artifact.DocumentReadme) + } + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, surface) + } +} + +func TestTamperedMetadataSidecar_FailsIntegrity(t *testing.T) { + for _, tc := range []struct { + spec *repoSpec + sidecar func(location string) string + describe string + }{ + {archiveSpec(artifact.PackagingTgz), func(l string) string { return l + ".metadata.json" }, "archive metadata sidecar"}, + {dirSpec(), func(l string) string { return l + "/.atmos/artifact/manifest.json" }, "directory manifest"}, + } { + t.Run(tc.describe, func(t *testing.T) { + repo := newRepo(t, tc.spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + rel := filepath.ToSlash(mustRel(t, repo.root, result.Location)) + tamper(t, repo.root, tc.sidecar(rel), appendJunk) + + _, err := repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + } +} + +func mustRel(t *testing.T, base, target string) string { + t.Helper() + rel, err := filepath.Rel(base, target) + require.NoError(t, err) + return rel +} + +func TestMissingSidecars_FailExplicitly(t *testing.T) { + t.Run("archive metadata sidecar", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(result.Location+".metadata.json")) + _, err := repo.Inspect(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a completed reference promised the sidecar exists") + _, err = repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("archive document sidecar", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location+".docs", "README.md"))) + _, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + assert.NotErrorIs(t, err, errUtils.ErrArtifactDocumentMissing, "a lost sidecar is damage, not an absent optional document") + _, err = repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.NoError(t, err, "the other sidecar is unaffected") + }) + t.Run("directory distribution", func(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.Remove(filepath.Join(result.Location, ".atmos", "artifact", "CHANGELOG.md"))) + _, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +func TestTamperedPayloadAndDocuments_FailAtTheDeclaredDigest(t *testing.T) { + t.Run("archive bytes", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, result.Location)), flipFirstByteInPlace) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err, "opening is cheap; verification happens as the bytes are consumed") + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("archive truncated", func(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingZip)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, result.Location)), truncate) + _, err := repo.Read(context.Background(), pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "a size mismatch is caught before any byte is read") + }) + t.Run("directory file", func(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, filepath.Join(result.Location, "nested", "deep", "file.txt"))), sameLengthEdit) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.ErrorIs(t, readAll(t, content), errUtils.ErrArtifactIntegrityFailed) + }) + t.Run("document", func(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + docPath := filepath.Join(result.Location, ".atmos", "artifact", "README.md") + if spec.kind == artifact.RepositoryKindLocalArchive { + docPath = filepath.Join(result.Location+".docs", "README.md") + } + tamper(t, repo.root, filepath.ToSlash(mustRel(t, repo.root, docPath)), sameLengthEdit) + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentReadme) + require.NoError(t, err) + _, err = io.Copy(io.Discard, stream.Reader) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, spec.kind) + } + }) +} + +func flipFirstByteInPlace(data []byte) []byte { + out := append([]byte(nil), data...) + out[len(out)/2] ^= 0xff + return out +} + +// sameLengthEdit changes content without changing its size, so only the digest can catch it. +func sameLengthEdit(data []byte) []byte { + out := append([]byte(nil), data...) + out[0] = 'X' + if len(data) > 0 && data[0] == 'X' { + out[0] = 'Y' + } + return out +} + +func TestExtraFilesInsideAPayloadAreNotAdopted(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + require.NoError(t, os.WriteFile(filepath.Join(result.Location, "injected.txt"), []byte("not in the inventory"), 0o644)) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + for index := range content.Streams { + assert.NotEqual(t, "injected.txt", content.Streams[index].Path, "only inventory files are ever served") + } +} diff --git a/pkg/artifact/local/job.go b/pkg/artifact/local/job.go new file mode 100644 index 00000000000..13687cd5ab2 --- /dev/null +++ b/pkg/artifact/local/job.go @@ -0,0 +1,247 @@ +package local + +import ( + "fmt" + "io/fs" + "runtime" + "sort" + + "github.com/google/uuid" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// publishJob is everything one publication attempt derives from the prepared artifact, computed +// once before any state changes. +type publishJob struct { + prepared *artifact.PreparedArtifact + record *artifact.Record + recordBytes []byte + recordDigest digest.Digest + inventory []artifact.Entry + key string + opID string + location string + staging string + payload []ocispec.Descriptor + documents []ocispec.Descriptor + staged int + installing bool +} + +// stagedPayload is the staging path that becomes the payload directory. +func (j *publishJob) stagedPayload() string { + return j.staging + "/payload" +} + +// retiredDir is where replaced content is moved aside until the new reference is completed. +func (j *publishJob) retiredDir() string { + return j.staging + "/retired" +} + +// blobStaged records one staged payload blob and fires the during-payload fault after the first. +func (j *publishJob) blobStaged(r *Repository) error { + j.staged++ + if j.staged == 1 { + return r.settings.fault(faultDuringPayload) + } + return nil +} + +// newJob validates a prepared artifact against the repository and derives the publication inputs. +func (r *Repository) newJob(prepared *artifact.PreparedArtifact) (*publishJob, error) { + if prepared == nil { + return nil, fmt.Errorf("%w: no prepared artifact to publish", errUtils.ErrArtifactConfigInvalid) + } + if !r.store.supports(prepared.Packaging) { + return nil, artifact.NewUnsupportedCapability(r.alias, "publish", "packaging/"+string(prepared.Packaging), + fmt.Sprintf("%s repositories store %v payloads", r.kind, r.Capabilities().Packagings)) + } + if err := r.checkArchivePermissions(); err != nil { + return nil, err + } + canonical, err := prepared.Record.Canonical() + if err != nil { + return nil, err + } + record := prepared.Record + opID := prepared.OperationID + if opID == "" { + opID = uuid.NewString() + } + job := &publishJob{ + prepared: prepared, record: &record, recordBytes: canonical, recordDigest: digest.FromBytes(canonical), + inventory: sortedEntries(record.Inventory), key: layout.ReferenceKey(record.Name, record.Version), + opID: opID, staging: layout.StagingPath(opID), + } + if err := job.describeBlobs(); err != nil { + return nil, err + } + if err := r.store.validate(job); err != nil { + return nil, err + } + return job, nil +} + +func sortedEntries(entries []artifact.Entry) []artifact.Entry { + out := append([]artifact.Entry(nil), entries...) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// describeBlobs derives the representation descriptors from the staged blobs and checks that the +// blobs agree with the Record before anything is written. +func (j *publishJob) describeBlobs() error { + j.payload, j.documents = []ocispec.Descriptor{}, []ocispec.Descriptor{} + docs := map[artifact.DocumentRole]bool{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Open == nil { + return fmt.Errorf("%w: staged blob %q cannot be opened", errUtils.ErrArtifactConfigInvalid, blob.Path) + } + descriptor := blob.Descriptor + if blob.Role == artifact.BlobDocument { + if docs[blob.DocumentRole] { + return fmt.Errorf("%w: duplicate %s document blob", errUtils.ErrArtifactRecordInvalid, blob.DocumentRole) + } + docs[blob.DocumentRole] = true + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: layout.DocumentFileName(blob.DocumentRole)} + j.documents = append(j.documents, descriptor) + continue + } + if blob.Path != "" { + descriptor.Annotations = map[string]string{ocispec.AnnotationTitle: blob.Path} + } + j.payload = append(j.payload, descriptor) + } + for index := range j.record.Documents { + if !docs[j.record.Documents[index].Role] { + return fmt.Errorf("%w: the Record lists a %s document with no staged blob", errUtils.ErrArtifactIntegrityFailed, j.record.Documents[index].Role) + } + } + if len(docs) != len(j.record.Documents) { + return fmt.Errorf("%w: %d document blobs but %d documents in the Record", errUtils.ErrArtifactIntegrityFailed, len(docs), len(j.record.Documents)) + } + return nil +} + +// indexBlobs maps file payload blobs by inventory path and document blobs by role. +func (j *publishJob) indexBlobs() (map[string]*artifact.StagedBlob, map[artifact.DocumentRole]*artifact.StagedBlob, error) { + files := map[string]*artifact.StagedBlob{} + docs := map[artifact.DocumentRole]*artifact.StagedBlob{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Role == artifact.BlobDocument { + docs[blob.DocumentRole] = blob + continue + } + if _, duplicate := files[blob.Path]; duplicate || blob.Path == "" { + return nil, nil, fmt.Errorf("%w: payload blob path %q is missing or duplicated", errUtils.ErrArtifactIntegrityFailed, blob.Path) + } + files[blob.Path] = blob + } + if err := j.matchDocuments(docs); err != nil { + return nil, nil, err + } + return files, docs, nil +} + +// archiveBlobs returns the single archive payload blob and the document blobs. +func (j *publishJob) archiveBlobs() (*artifact.StagedBlob, map[artifact.DocumentRole]*artifact.StagedBlob, error) { + var payload *artifact.StagedBlob + docs := map[artifact.DocumentRole]*artifact.StagedBlob{} + for index := range j.prepared.Blobs { + blob := &j.prepared.Blobs[index] + if blob.Role == artifact.BlobDocument { + docs[blob.DocumentRole] = blob + continue + } + if payload != nil { + return nil, nil, fmt.Errorf("%w: an archive publication needs exactly one payload blob", errUtils.ErrArtifactPackagingInvalid) + } + payload = blob + } + if payload == nil { + return nil, nil, fmt.Errorf("%w: an archive publication needs exactly one payload blob", errUtils.ErrArtifactPackagingInvalid) + } + if err := j.matchDocuments(docs); err != nil { + return nil, nil, err + } + return payload, docs, nil +} + +// matchDocuments checks every document blob against the Record's descriptor for its role. +func (j *publishJob) matchDocuments(docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + for role, blob := range docs { + doc, ok := documentDescriptor(j.record, role) + if !ok || doc.Digest != blob.Descriptor.Digest || doc.Size != blob.Descriptor.Size { + return fmt.Errorf("%w: the staged %s document does not match the Record", errUtils.ErrArtifactIntegrityFailed, role) + } + } + return nil +} + +// representationDigest is the checksum of the transferred payload representation. +func (j *publishJob) representationDigest(packaging artifact.Packaging) digest.Digest { + if packaging == artifact.PackagingNone { + return dirRepresentationDigest(j.record) + } + if len(j.payload) == 1 { + return j.payload[0].Digest + } + return "" +} + +// permissionBits holds the ordinary permission bits of a mode. +const permissionBits fs.FileMode = 0o777 + +// fsMode converts a Record mode to a file mode. +func fsMode(mode uint32) fs.FileMode { + return fs.FileMode(mode) & permissionBits +} + +// checkArchivePermissions fails before any state changes when an explicitly requested archive +// mode cannot be enforced on this host. +func (r *Repository) checkArchivePermissions() error { + if r.kind != artifact.RepositoryKindLocalArchive || !r.modeExplicit || r.hostModes { + return nil + } + return artifact.NewUnsupportedCapability(r.alias, "publish", "permissions/archive-file-mode", + fmt.Sprintf("the host (%s) cannot enforce permissions %04o on the archive file; remove the permissions setting to accept the host default, or publish from a POSIX host", runtime.GOOS, r.archiveMode)) +} + +// chmodIfSupported applies a mode when the host can enforce one. +func (r *Repository) chmodIfSupported(t *tree, rel string, mode fs.FileMode) error { + if !r.hostModes { + return nil + } + return t.Chmod(native(rel), mode) +} + +// setArchiveMode applies the archive permissions to a staged archive file, sidecar, or sidecar +// directory. For an explicit request it also reads the mode back, because a filesystem that +// silently ignores chmod must not be reported as enforcing the configured permissions. +func (r *Repository) setArchiveMode(t *tree, rel string, mode fs.FileMode) error { + if !r.hostModes { + return nil + } + if err := t.Chmod(native(rel), mode); err != nil { + return err + } + if !r.modeExplicit { + return nil + } + info, err := t.Lstat(native(rel)) + if err != nil { + return err + } + if info.Mode().Perm() != mode { + return artifact.NewUnsupportedCapability(r.alias, "publish", "permissions/archive-file-mode", + fmt.Sprintf("the filesystem under %q did not keep mode %04o (got %04o)", r.root, mode, info.Mode().Perm())) + } + return nil +} diff --git a/pkg/artifact/local/kind_archive.go b/pkg/artifact/local/kind_archive.go new file mode 100644 index 00000000000..d841aeec9c1 --- /dev/null +++ b/pkg/artifact/local/kind_archive.go @@ -0,0 +1,210 @@ +package local + +import ( + "context" + "fmt" + "io" + "io/fs" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Staging names inside an operation's staging directory. +const ( + stagedArchive = "archive" + stagedMetadata = "metadata.json" + stagedDocs = "docs" +) + +// archiveStore stores a payload as one tar, tgz, or zip file with adjacent metadata and document +// sidecars. The configured archive permissions apply to the file and its sidecars, never to the +// members inside the archive. +type archiveStore struct { + repo *Repository +} + +var _ payloadStore = archiveStore{} + +func (archiveStore) supports(packaging artifact.Packaging) bool { + switch packaging { + case artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip: + return true + default: + return false + } +} + +// validate checks that the publication carries exactly one archive blob matching the Record's +// documents, before any state changes. +func (archiveStore) validate(job *publishJob) error { + _, _, err := job.archiveBlobs() + return err +} + +// stage copies the archive and the document sidecars into the staging directory and applies the +// archive permissions before anything becomes visible at the final location. +func (s archiveStore) stage(ctx context.Context, t *tree, job *publishJob) error { + payload, docs, err := job.archiveBlobs() + if err != nil { + return err + } + archivePath := job.staging + "/" + stagedArchive + if err := s.copyBlob(ctx, t, payload, archivePath); err != nil { + return err + } + if err := job.blobStaged(s.repo); err != nil { + return err + } + if err := s.repo.setArchiveMode(t, archivePath, s.repo.archiveMode); err != nil { + return err + } + if err := s.stageDocuments(ctx, t, job, docs); err != nil { + return err + } + metadata := job.staging + "/" + stagedMetadata + if err := t.writeNew(metadata, job.recordBytes, privateFile); err != nil { + return err + } + return s.repo.setArchiveMode(t, metadata, s.repo.archiveMode) +} + +// stageDocuments copies the document sidecars and gives their directory the matching mode. +func (s archiveStore) stageDocuments(ctx context.Context, t *tree, job *publishJob, docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + for role, blob := range docs { + docPath := job.staging + "/" + stagedDocs + "/" + layout.DocumentFileName(role) + if err := s.copyBlob(ctx, t, blob, docPath); err != nil { + return err + } + if err := s.repo.setArchiveMode(t, docPath, s.repo.archiveMode); err != nil { + return err + } + } + if len(docs) == 0 { + return nil + } + return s.repo.setArchiveMode(t, job.staging+"/"+stagedDocs, directoryMode(s.repo.archiveMode)) +} + +func (archiveStore) copyBlob(ctx context.Context, t *tree, blob *artifact.StagedBlob, dest string) error { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(dest, reader, blob.Descriptor.Digest, blob.Descriptor.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %s: %w", dest, copyErr) + } + return nil +} + +// install retires the previous archive and sidecars, then renames the staged ones into place. +func (archiveStore) install(t *tree, job *publishJob) error { + retired := job.retiredDir() + for _, move := range []struct{ from, to string }{ + {job.location, retired + "/archive"}, + {layout.DocsSidecarDir(job.location), retired + "/docs"}, + {layout.MetadataSidecarPath(job.location), retired + "/metadata.json"}, + } { + if err := t.moveAside(move.from, move.to); err != nil { + return err + } + } + if err := t.ensureParent(job.location); err != nil { + return err + } + installs := []struct{ from, to string }{{job.staging + "/" + stagedArchive, job.location}} + if ok, err := t.exists(job.staging + "/" + stagedDocs); err != nil { + return err + } else if ok { + installs = append(installs, struct{ from, to string }{job.staging + "/" + stagedDocs, layout.DocsSidecarDir(job.location)}) + } + installs = append(installs, struct{ from, to string }{job.staging + "/" + stagedMetadata, layout.MetadataSidecarPath(job.location)}) + for _, install := range installs { + if err := t.Rename(native(install.from), native(install.to)); err != nil { + return err + } + } + return nil +} + +// streams returns the archive as the single payload stream plus the document sidecars. +func (s archiveStore) streams(_ context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) { + if len(v.rep.Payload) != 1 { + return nil, fmt.Errorf("%w: an archive representation must have exactly one payload, found %d", errUtils.ErrArtifactIntegrityFailed, len(v.rep.Payload)) + } + descriptor := v.rep.Payload[0] + file, info, err := t.openFile(v.rep.Location, "archive") + if err != nil { + return nil, err + } + if info.Size() != descriptor.Size { + _ = file.Close() + return nil, fmt.Errorf("%w: archive %q is %d bytes; the representation declares %d", errUtils.ErrArtifactIntegrityFailed, v.rep.Location, info.Size(), descriptor.Size) + } + streams := []artifact.ContentStream{{ + Descriptor: descriptor, Role: artifact.BlobPayload, + Reader: newVerifyingReader(file, descriptor.Digest, descriptor.Size, guard), + }} + for index := range v.record.Documents { + doc := &v.record.Documents[index] + reader, err := s.document(t, v, doc, guard) + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{Descriptor: blobDescriptor(doc), Role: artifact.BlobDocument, Reader: reader}) + } + return streams, nil +} + +// document opens a document from its adjacent sidecar. A missing sidecar is an integrity failure: +// reading it must not fall back to opening the archive, and the completed reference promised the +// sidecar exists. +func (archiveStore) document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) { + file, _, err := t.openFile(layout.DocumentSidecarPath(v.rep.Location, doc.Role), string(doc.Role)+" sidecar") + if err != nil { + return nil, err + } + return newVerifyingReader(file, doc.Digest, doc.Size, guard), nil +} + +func (archiveStore) verifySidecars(t *tree, v *view) error { + return t.verifyBytes(layout.MetadataSidecarPath(v.rep.Location), "metadata sidecar", v.recordBytes) +} + +// remove deletes the archive, its metadata sidecar, and its document sidecars. +func (archiveStore) remove(t *tree, v *view) ([]string, error) { + location := v.rep.Location + removed := []string{} + targets := []string{location, layout.MetadataSidecarPath(location)} + for index := range v.record.Documents { + targets = append(targets, layout.DocumentSidecarPath(location, v.record.Documents[index].Role)) + } + for _, target := range targets { + ok, err := t.exists(target) + if err != nil { + return removed, err + } + if !ok { + continue + } + if err := t.Remove(native(target)); err != nil { + return removed, err + } + removed = append(removed, t.absolute(target)) + } + _ = t.Remove(native(layout.DocsSidecarDir(location))) + return removed, nil +} + +// readBits holds the read permission bits; execute follows read for a directory. +const readBits fs.FileMode = 0o444 + +// directoryMode derives the directory mode that goes with a file mode: execute follows read. +func directoryMode(file fs.FileMode) fs.FileMode { + return file | (file&readBits)>>2 +} diff --git a/pkg/artifact/local/kind_dir.go b/pkg/artifact/local/kind_dir.go new file mode 100644 index 00000000000..ad4b444aadc --- /dev/null +++ b/pkg/artifact/local/kind_dir.go @@ -0,0 +1,300 @@ +package local + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// filePayloadMediaType is the media type of one copied file. +const filePayloadMediaType = "application/octet-stream" + +// dirStore stores a payload as ordinary copied files below its resolved location, with the +// distribution documents and manifest adjacent at /.atmos/artifact/. +type dirStore struct { + repo *Repository +} + +var _ payloadStore = dirStore{} + +func (dirStore) supports(packaging artifact.Packaging) bool { + return packaging == artifact.PackagingNone +} + +// validate checks the staged blobs against the inventory before the publication starts, so a +// malformed preparation never leaves a pending reference behind. +func (dirStore) validate(job *publishJob) error { + blobs, _, err := job.indexBlobs() + if err != nil { + return err + } + files := 0 + for index := range job.inventory { + entry := &job.inventory[index] + if entry.Path == layout.DistributionDir || strings.HasPrefix(entry.Path, layout.DistributionDir+"/") { + return fmt.Errorf("%w: inventory path %q is inside the generated %s directory", errUtils.ErrArtifactUnsafePath, entry.Path, layout.DistributionDir) + } + if entry.Type != artifact.EntryFile { + continue + } + blob, ok := blobs[entry.Path] + if !ok || blob.Descriptor.Digest != entry.Digest || blob.Descriptor.Size != entry.Size { + return fmt.Errorf("%w: no staged blob matches inventory file %q", errUtils.ErrArtifactIntegrityFailed, entry.Path) + } + files++ + } + if files != len(blobs) { + return fmt.Errorf("%w: %d staged payload blobs but %d inventory files", errUtils.ErrArtifactIntegrityFailed, len(blobs), files) + } + return nil +} + +// stage copies every inventory entry and the distribution directory into the staging tree. +func (s dirStore) stage(ctx context.Context, t *tree, job *publishJob) error { + root := job.stagedPayload() + if err := t.MkdirAll(native(root), privateDir); err != nil { + return err + } + blobs, docs, err := job.indexBlobs() + if err != nil { + return err + } + if err := s.writeInventory(ctx, t, job, blobs); err != nil { + return err + } + if err := s.writeDistribution(ctx, t, job, docs); err != nil { + return err + } + return s.applyModes(t, job) +} + +func (s dirStore) writeInventory(ctx context.Context, t *tree, job *publishJob, blobs map[string]*artifact.StagedBlob) error { + root := job.stagedPayload() + for index := range job.inventory { + entry := &job.inventory[index] + target := root + "/" + entry.Path + switch entry.Type { + case artifact.EntryDirectory: + if err := t.MkdirAll(native(target), privateDir); err != nil { + return err + } + case artifact.EntrySymlink: + if err := t.ensureParent(target); err != nil { + return err + } + if err := t.Symlink(native(entry.Target), native(target)); err != nil { + return fmt.Errorf("%w: create link %q: %w", errUtils.ErrArtifactPackagingInvalid, entry.Path, err) + } + default: + if err := s.stageEntry(ctx, t, blobs[entry.Path], target, entry); err != nil { + return err + } + if err := job.blobStaged(s.repo); err != nil { + return err + } + } + } + return nil +} + +func (s dirStore) stageEntry(ctx context.Context, t *tree, blob *artifact.StagedBlob, target string, entry *artifact.Entry) error { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(target, reader, entry.Digest, entry.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %q: %w", entry.Path, copyErr) + } + return s.repo.chmodIfSupported(t, target, fsMode(entry.Mode)) +} + +// writeDistribution writes the documents and the generated manifest.json that the payload +// carries so it stays self-describing. The manifest is the canonical Record. +func (s dirStore) writeDistribution(ctx context.Context, t *tree, job *publishJob, docs map[artifact.DocumentRole]*artifact.StagedBlob) error { + dist := job.stagedPayload() + "/" + layout.DistributionDir + if err := t.MkdirAll(native(dist), dirMode); err != nil { + return err + } + for role, blob := range docs { + reader, err := blob.Open(ctx) + if err != nil { + return err + } + copyErr := t.stageFile(dist+"/"+layout.DocumentFileName(role), reader, blob.Descriptor.Digest, blob.Descriptor.Size) + if closeErr := reader.Close(); copyErr == nil { + copyErr = closeErr + } + if copyErr != nil { + return fmt.Errorf("stage %s: %w", role, copyErr) + } + if err := s.repo.chmodIfSupported(t, dist+"/"+layout.DocumentFileName(role), publicFile); err != nil { + return err + } + } + return t.writeNew(dist+"/"+layout.ManifestName, job.recordBytes, publicFile) +} + +// applyModes sets directory modes last, deepest first, so a read-only directory never blocks the +// writes below it. +func (s dirStore) applyModes(t *tree, job *publishJob) error { + if !s.repo.hostModes { + return nil + } + root := job.stagedPayload() + for _, generated := range []string{root, root + "/.atmos", root + "/" + layout.DistributionDir} { + if err := t.Chmod(native(generated), dirMode); err != nil { + return err + } + } + dirs := make([]artifact.Entry, 0) + for index := range job.inventory { + if job.inventory[index].Type == artifact.EntryDirectory { + dirs = append(dirs, job.inventory[index]) + } + } + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i].Path) > len(dirs[j].Path) }) + for index := range dirs { + if err := t.Chmod(native(root+"/"+dirs[index].Path), fsMode(dirs[index].Mode)); err != nil { + return err + } + } + return nil +} + +// install retires the previous payload directory and renames the staged tree into place. +func (dirStore) install(t *tree, job *publishJob) error { + if err := t.moveAside(job.location, job.retiredDir()+"/payload"); err != nil { + return err + } + if err := t.ensureParent(job.location); err != nil { + return err + } + return t.Rename(native(job.stagedPayload()), native(job.location)) +} + +// streams returns one verified stream per inventory file plus the documents. +func (s dirStore) streams(_ context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) { + if ok, err := t.exists(v.rep.Location); err != nil || !ok { + return nil, fmt.Errorf("%w: payload directory %q is missing", errUtils.ErrArtifactIntegrityFailed, v.rep.Location) + } + streams := make([]artifact.ContentStream, 0, len(v.record.Inventory)+len(v.record.Documents)) + for index := range v.record.Inventory { + entry := &v.record.Inventory[index] + if entry.Type != artifact.EntryFile { + continue + } + file, _, err := t.openFile(v.rep.Location+"/"+entry.Path, "payload file") + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{ + Descriptor: ocispec.Descriptor{MediaType: filePayloadMediaType, Digest: entry.Digest, Size: entry.Size}, + Role: artifact.BlobPayload, + Path: entry.Path, + Reader: newVerifyingReader(file, entry.Digest, entry.Size, guard), + }) + } + for index := range v.record.Documents { + doc := &v.record.Documents[index] + reader, err := s.document(t, v, doc, guard) + if err != nil { + return streams, err + } + streams = append(streams, artifact.ContentStream{Descriptor: blobDescriptor(doc), Role: artifact.BlobDocument, Reader: reader}) + } + return streams, nil +} + +func (dirStore) document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) { + file, _, err := t.openFile(layout.DistributionPath(v.rep.Location, layout.DocumentFileName(doc.Role)), string(doc.Role)+" document") + if err != nil { + return nil, err + } + return newVerifyingReader(file, doc.Digest, doc.Size, guard), nil +} + +func (dirStore) verifySidecars(t *tree, v *view) error { + return t.verifyBytes(layout.DistributionPath(v.rep.Location, layout.ManifestName), "manifest", v.recordBytes) +} + +// remove deletes the inventory files, the distribution directory, and then the directories that +// became empty. Files the artifact does not own stay in place and keep their directory alive. +func (dirStore) remove(t *tree, v *view) ([]string, error) { + location := v.rep.Location + if ok, err := t.exists(location); err != nil || !ok { + return nil, err + } + t.makeWritable(location) + if err := removeFiles(t, location, v.record); err != nil { + return nil, err + } + removeDirectories(t, location, v.record) + return []string{t.absolute(location)}, nil +} + +// removeFiles deletes every non-directory inventory entry and the generated distribution files. +func removeFiles(t *tree, location string, record *artifact.Record) error { + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type != artifact.EntryDirectory { + if err := removeIfPresent(t, location+"/"+entry.Path); err != nil { + return err + } + } + } + for _, name := range []string{layout.ManifestName, "README.md", "CHANGELOG.md"} { + if err := removeIfPresent(t, layout.DistributionPath(location, name)); err != nil { + return err + } + } + return nil +} + +// removeDirectories removes the artifact's directories deepest first. A directory that still holds +// unmanaged files is left in place. +func removeDirectories(t *tree, location string, record *artifact.Record) { + var dirs []string + for index := range record.Inventory { + if record.Inventory[index].Type == artifact.EntryDirectory { + dirs = append(dirs, location+"/"+record.Inventory[index].Path) + } + } + dirs = append(dirs, location+"/"+layout.DistributionDir, location+"/.atmos", location) + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + for _, dir := range dirs { + _ = t.Remove(native(dir)) + } +} + +// removeIfPresent removes a path, ignoring a path that is already gone. +func removeIfPresent(t *tree, rel string) error { + if ok, err := t.exists(rel); err != nil || !ok { + return err + } + return t.Remove(native(rel)) +} + +// dirRepresentationDigest is the checksum of the per-file payload: the SHA-256 of the sorted +// "path NUL digest LF" lines. +func dirRepresentationDigest(record *artifact.Record) digest.Digest { + hasher := sha256.New() + for index := range record.Inventory { + if entry := &record.Inventory[index]; entry.Type == artifact.EntryFile { + _, _ = io.WriteString(hasher, entry.Path+"\x00"+entry.Digest.String()+"\n") + } + } + return digest.NewDigest(digest.SHA256, hasher) +} diff --git a/pkg/artifact/local/location_test.go b/pkg/artifact/local/location_test.go new file mode 100644 index 00000000000..f61897e9e01 --- /dev/null +++ b/pkg/artifact/local/location_test.go @@ -0,0 +1,241 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +const datedPattern = `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}` + +func datedSpec(kind string, packaging artifact.Packaging) *repoSpec { + suffix := "" + if kind == artifact.RepositoryKindLocalArchive { + suffix = layout.ArchiveExtension(packaging) + } + return &repoSpec{kind: kind, packaging: packaging, keyPattern: datedPattern + suffix} +} + +var datedSpecs = []*repoSpec{ + datedSpec(artifact.RepositoryKindLocalDir, ""), + datedSpec(artifact.RepositoryKindLocalArchive, artifact.PackagingTgz), +} + +func storedLocation(t *testing.T, repo *Repository, name, version string) string { + t.Helper() + tr, err := repo.openTree(false) + require.NoError(t, err) + defer tr.Close() + rep, err := tr.loadRepresentation(layout.ReferenceKey(name, version)) + require.NoError(t, err) + require.NotNil(t, rep) + return rep.Location +} + +func TestDatedRoot_LocationIsRenderedOnceAndPersisted(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + result := publish(t, repo, &prepareSpec{at: time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC)}, artifact.PublishOptions{}) + want := layout.DefaultLocation(testName, testVersion, repo.packaging) + want = "2026/10/30/" + want + assert.Equal(t, filepath.Join(repo.root, filepath.FromSlash(want)), result.Location) + assert.Equal(t, want, storedLocation(t, repo, testName, testVersion)) + + // A new repository object at the same root with a different pattern (and the wall + // clock long past the capture date) still reads and deletes the persisted location. + reader := newRepo(t, spec.withRoot(repo.root).withPattern("somewhere/else/{{ .Version }}")) + assertIntact(t, reader, defaultRef()) + pinned := resolve(t, reader, defaultRef()) + deleted, err := reader.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + require.NotEmpty(t, deleted.Deleted) + assert.Equal(t, result.Location, deleted.Deleted[0], "delete uses the persisted dated location, never a re-rendered one") + assert.NoDirExists(t, filepath.Join(repo.root, "2026"), "empty dated parents are pruned") + assert.NoDirExists(t, filepath.Join(repo.root, "somewhere")) + }) + } +} + +func TestDatedRoot_RolloverReusesTheCapturedLocation(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + dayOne := time.Date(2026, 10, 30, 23, 59, 59, 0, time.UTC) + first := prepare(t, repo, &prepareSpec{at: dayOne, opID: "op-day-one"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + // The same prepared artifact, retried after midnight, lands in the day-one location. + result, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Contains(t, filepath.ToSlash(result.Location), "2026/10/30/") + + // A later invocation (new preparation time) of the same intent after an interruption + // also resumes at the journaled location instead of creating a second dated tree. + repo2 := newRepo(t, spec.withRoot(repo.root).withOpts(withFaults(crashAt(faultAfterMetadata)))) + second := prepare(t, repo2, &prepareSpec{name: "team/other", at: dayOne, opID: "op-other-1"}) + _, err = repo2.Publish(context.Background(), second, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + dayTwo := prepare(t, repo2, &prepareSpec{name: "team/other", at: dayOne.Add(26 * time.Hour), opID: "op-other-2"}) + resumed, err := repo2.Publish(context.Background(), dayTwo, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Contains(t, filepath.ToSlash(resumed.Location), "2026/10/30/", "the journaled location is reused, not re-rendered for the new day") + assert.NoDirExists(t, filepath.Join(repo.root, "2026", "10", "31")) + assertIntact(t, repo2, ref("team/other", testVersion)) + }) + } +} + +func TestDatedRoot_ForceReplacementAcrossDaysRetiresTheOldLocation(t *testing.T) { + for _, spec := range datedSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + dayOne := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + first := publish(t, repo, &prepareSpec{at: dayOne, files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + second := publish(t, repo, &prepareSpec{at: dayOne.Add(48 * time.Hour), opID: "op-2", files: map[string]string{"data.txt": "v2\n"}}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + assert.Equal(t, artifact.OutcomeReplaced, second.Outcome) + assert.Contains(t, filepath.ToSlash(second.Location), "2026/11/01/") + assert.NoFileExists(t, first.Location, "the replaced dated payload is retired") + assert.NoDirExists(t, filepath.Join(repo.root, "2026", "10"), "its empty dated parents are pruned") + assertIntact(t, repo, defaultRef()) + }) + } +} + +func collidingSpec(kind string) *repoSpec { + spec := &repoSpec{kind: kind, keyPattern: "shared/{{ .Version }}"} + if kind == artifact.RepositoryKindLocalArchive { + spec.packaging = artifact.PackagingTgz + spec.keyPattern += ".tgz" + } + return spec +} + +// Two references whose key_pattern resolves to one path: the second fails with a conflict and is +// never bound to the first reference's payload. +func TestCollidingKeyPattern_SecondReferenceConflicts(t *testing.T) { + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + t.Run(kind, func(t *testing.T) { + repo := newRepo(t, collidingSpec(kind)) + first := publish(t, repo, &prepareSpec{name: "a/one", files: map[string]string{"data.txt": "one\n"}}, artifact.PublishOptions{}) + before := resolve(t, repo, ref("a/one", testVersion)) + + second := prepare(t, repo, &prepareSpec{name: "b/two", files: map[string]string{"data.txt": "two\n"}}) + _, err := repo.Publish(context.Background(), second, artifact.PublishOptions{Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "a/one:1.0.0", "the conflict names the owning reference") + + _, err = repo.Resolve(context.Background(), ref("b/two", testVersion)) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "the loser has no reference at all") + assert.Equal(t, before.RecordDigest, resolve(t, repo, ref("a/one", testVersion)).RecordDigest) + content, err := repo.Read(context.Background(), before) + require.NoError(t, err) + defer content.Close() + require.NoError(t, readAll(t, content)) + assert.Equal(t, []string{"a/one:1.0.0"}, listNames(t, repo)) + assert.True(t, pathExists(first.Location), "the first payload is untouched") + assert.Empty(t, listFiles(t, filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations")), "a rejected reservation leaves no journal behind") + }) + } +} + +func TestCollidingKeyPattern_AncestorAndCaseAliases(t *testing.T) { + spec := &repoSpec{kind: artifact.RepositoryKindLocalDir, keyPattern: "{{ .Kind }}/{{ .Name }}"} + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{name: "tools"}, artifact.PublishOptions{}) + nested := prepare(t, repo, &prepareSpec{name: "tools/cli"}) + _, err := repo.Publish(context.Background(), nested, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a payload directory nested in another artifact's directory is a collision") + + caseSpec := &repoSpec{kind: artifact.RepositoryKindLocalDir, keyPattern: "same/{{ .Version }}"} + caseRepo := newRepo(t, caseSpec) + publish(t, caseRepo, &prepareSpec{name: "a/x", version: "RC1"}, artifact.PublishOptions{}) + aliased := prepare(t, caseRepo, &prepareSpec{name: "b/y", version: "rc1"}) + _, err = caseRepo.Publish(context.Background(), aliased, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "locations that differ only by case alias on common filesystems") +} + +// Content that exists at the location but is not managed by this repository is never adopted or +// overwritten. +func TestUnmanagedContentAtTheLocationIsNeverOverwritten(t *testing.T) { + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + t.Run(kind, func(t *testing.T) { + spec := &repoSpec{kind: kind} + if kind == artifact.RepositoryKindLocalArchive { + spec.packaging = artifact.PackagingTgz + } + repo := newRepo(t, spec) + location := layout.DefaultLocation(testName, testVersion, repo.packaging) + userFile := filepath.Join(repo.root, filepath.FromSlash(location)) + if kind == artifact.RepositoryKindLocalDir { + userFile = filepath.Join(userFile, "mine.txt") + } + require.NoError(t, os.MkdirAll(filepath.Dir(userFile), 0o755)) + require.NoError(t, os.WriteFile(userFile, []byte("precious"), 0o644)) + + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.ErrorContains(t, err, "not managed") + got, readErr := os.ReadFile(userFile) + require.NoError(t, readErr) + assert.Equal(t, "precious", string(got)) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// Concurrent publications of colliding references: exactly one wins. +func TestCollidingKeyPattern_ConcurrentWritersExactlyOneWins(t *testing.T) { + repo := newRepo(t, collidingSpec(artifact.RepositoryKindLocalDir)) + names := []string{"a/one", "b/two", "c/three", "d/four"} + prepared := make([]*artifact.PreparedArtifact, len(names)) + for index, name := range names { + prepared[index] = prepare(t, repo, &prepareSpec{name: name, files: map[string]string{"data.txt": name}}) + } + errs := make([]error, len(names)) + var wg sync.WaitGroup + for index := range names { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[index] = repo.Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wg.Wait() + winners := 0 + for _, err := range errs { + if err == nil { + winners++ + } else { + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + } + assert.Equal(t, 1, winners, "ownership of the shared location is exclusive") + assert.Len(t, listNames(t, repo), 1) + for _, name := range listNames(t, repo) { + parsed, err := artifact.ParseRef(name) + require.NoError(t, err) + assertIntact(t, repo, parsed) + } +} + +func pathExists(p string) bool { + _, err := os.Stat(p) + return err == nil +} diff --git a/pkg/artifact/local/lock.go b/pkg/artifact/local/lock.go new file mode 100644 index 00000000000..953743d752d --- /dev/null +++ b/pkg/artifact/local/lock.go @@ -0,0 +1,177 @@ +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/filelock" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Internal lock resources. They start with a NUL byte, which no artifact reference can contain, +// so they can never collide with a caller's reference lock. +const ( + claimsResource = "\x00claims" + locationResource = "\x00location\x00" +) + +// heldLocks tracks the locks this process holds, keyed by lock file path. The OS lock excludes +// other processes; this table supplies the ownership-token check on release and diagnostics. +var ( + heldMu sync.Mutex + heldLocks = map[string]*fileLock{} +) + +// fileLock is an OS file lock held until Release. It implements [artifact.LockHandle]. +type fileLock struct { + path string + resource string + owner string + operationID string + token string + release chan struct{} + done chan error + released bool +} + +var _ artifact.LockHandle = (*fileLock)(nil) + +// Lock implements [artifact.LockProvider] with an OS-backed file lock of host scope. It waits for +// a competing owner until ctx ends; a canceled wait leaks no ownership. Leases are rejected +// because a paused writer could not be fenced, and repository scope is rejected because a local +// file lock cannot exclude writers on other hosts. +func (r *Repository) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + defer perf.Track(nil, "local.Repository.Lock")() + + if request.Lease != 0 { + return nil, artifact.NewUnsupportedCapability(r.alias, "lock", "lease", "local file locks do not expire, so a lease cannot be fenced") + } + if request.Required != "" && !artifact.LockScopeHost.Satisfies(request.Required) { + return nil, artifact.NewUnsupportedCapability(r.alias, "lock", "lock/"+string(request.Required), + "local repositories provide host-scope locks only; use an S3 repository or a lock delegate for repository scope") + } + if request.Resource == "" { + return nil, fmt.Errorf("%w: a lock needs a resource", errUtils.ErrArtifactRefInvalid) + } + owner := request.Owner + if owner == "" { + owner = r.settings.owner + } + return r.acquire(ctx, request.Resource, owner, request.OperationID) +} + +// acquire takes the OS lock for resource. +func (r *Repository) acquire(ctx context.Context, resource, owner, operationID string) (*fileLock, error) { + lockPath := filepath.Join(r.root, native(layout.LockPath(resource))) + if err := os.MkdirAll(filepath.Dir(lockPath), dirMode); err != nil { + return nil, fmt.Errorf("%w: create lock directory for %q: %w", errUtils.ErrArtifactConfigInvalid, resource, err) + } + held := make(chan struct{}) + lock := &fileLock{ + path: lockPath, resource: resource, owner: owner, operationID: operationID, + token: uuid.NewString(), release: make(chan struct{}), done: make(chan error, 1), + } + go func() { + lock.done <- filelock.New(lockPath).WithExclusive(ctx, func() error { + close(held) + <-lock.release + return nil + }) + }() + select { + case <-held: + heldMu.Lock() + heldLocks[lockPath] = lock + heldMu.Unlock() + return lock, nil + case err := <-lock.done: + return nil, fmt.Errorf("waiting for lock %q in repository %q: %w", resource, r.alias, err) + } +} + +// Resource implements [artifact.LockHandle]. +func (l *fileLock) Resource() string { + defer perf.Track(nil, "local.fileLock.Resource")() + + return l.resource +} + +// Owner implements [artifact.LockHandle]. +func (l *fileLock) Owner() string { + defer perf.Track(nil, "local.fileLock.Owner")() + + return l.owner +} + +// Scope implements [artifact.LockHandle]; local locks are host scope. +func (l *fileLock) Scope() artifact.LockScope { + defer perf.Track(nil, "local.fileLock.Scope")() + + return artifact.LockScopeHost +} + +// Token implements [artifact.LockHandle]. +func (l *fileLock) Token() string { + defer perf.Track(nil, "local.fileLock.Token")() + + return l.token +} + +// ExpiresAt implements [artifact.LockHandle]; local locks never expire. +func (l *fileLock) ExpiresAt() (time.Time, bool) { + defer perf.Track(nil, "local.fileLock.ExpiresAt")() + + return time.Time{}, false +} + +// Renew implements [artifact.LockHandle]; local locks have no leases. +func (l *fileLock) Renew(context.Context, time.Duration) error { + defer perf.Track(nil, "local.fileLock.Renew")() + + return artifact.NewUnsupportedCapability("", "lock", "lease", "local file locks do not expire") +} + +// Release implements [artifact.LockHandle]. It checks the ownership token against the lock this +// process holds, so a stale or foreign handle can never release another operation's lock. The OS +// also releases the lock if its owner exits. +func (l *fileLock) Release(context.Context) error { + defer perf.Track(nil, "local.fileLock.Release")() + + heldMu.Lock() + current, owned := heldLocks[l.path] + if !owned || current != l || l.released || current.token != l.token { + heldMu.Unlock() + return fmt.Errorf("%w: lock %q is not held by token %s", errUtils.ErrArtifactConflict, l.resource, l.token) + } + delete(heldLocks, l.path) + l.released = true + heldMu.Unlock() + close(l.release) + if err := <-l.done; err != nil { + return fmt.Errorf("release lock %q: %w", l.resource, err) + } + return nil +} + +// releaseQuietly releases a lock acquired internally and returns the first error alongside err. +func releaseAll(ctx context.Context, locks ...*fileLock) error { + var first error + for index := len(locks) - 1; index >= 0; index-- { + if locks[index] == nil { + continue + } + if err := locks[index].Release(context.WithoutCancel(ctx)); err != nil && first == nil { + first = err + } + } + return first +} diff --git a/pkg/artifact/local/lock_test.go b/pkg/artifact/local/lock_test.go new file mode 100644 index 00000000000..2e1193e33a9 --- /dev/null +++ b/pkg/artifact/local/lock_test.go @@ -0,0 +1,396 @@ +package local + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// Environment variables that turn the test binary into a lock helper process. The test binary +// doubles as a cross-platform subprocess: no Unix-only helper programs are involved. +const ( + helperModeEnv = "_ATMOS_LOCAL_LOCK_HELPER" + helperRootEnv = "_ATMOS_LOCAL_LOCK_ROOT" + helperResourceEnv = "_ATMOS_LOCAL_LOCK_RESOURCE" + + helperExitAcquired = 0 + helperExitBusy = 3 + helperExitError = 4 +) + +func TestMain(m *testing.M) { + if mode := os.Getenv(helperModeEnv); mode != "" { + os.Exit(runLockHelper(mode)) + } + os.Exit(m.Run()) +} + +// runLockHelper is the body of the helper process. +// +// try: attempt the lock for a short time; exit 0 when acquired, 3 when busy. +// hold: acquire the lock, announce it on stdout, and hold it until stdin closes or the process is killed. +func runLockHelper(mode string) int { + repo, err := newRepository("helper", artifact.RepositoryKindLocalDir, helperConfig(os.Getenv(helperRootEnv)), nil) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + timeout := 400 * time.Millisecond + if mode == "hold" { + timeout = 20 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + handle, err := repo.Lock(ctx, artifact.LockRequest{Resource: os.Getenv(helperResourceEnv), Owner: "helper-process"}) + switch { + case errors.Is(err, context.DeadlineExceeded): + return helperExitBusy + case err != nil: + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + if mode == "hold" { + fmt.Println("locked") + _, _ = bufio.NewReader(os.Stdin).ReadString('\n') + } + if err := handle.Release(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + return helperExitError + } + return helperExitAcquired +} + +func helperCommand(t *testing.T, mode, root, resource string) *exec.Cmd { + t.Helper() + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command(exe, "-test.run=^$") + cmd.Env = append(os.Environ(), helperModeEnv+"="+mode, helperRootEnv+"="+root, helperResourceEnv+"="+resource) + return cmd +} + +func exitCode(t *testing.T, cmd *exec.Cmd) int { + t.Helper() + err := cmd.Run() + if err == nil { + return 0 + } + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + return exitErr.ExitCode() +} + +func lockRequest(resource string) artifact.LockRequest { + return artifact.LockRequest{Resource: resource, Owner: "test-owner", OperationID: "op-lock", Required: artifact.LockScopeHost} +} + +func TestLock_HandleContract(t *testing.T) { + repo := newRepo(t, dirSpec()) + handle := holdLock(t, repo, lockRequest("team/app:1.0.0")) + assert.Equal(t, "team/app:1.0.0", handle.Resource()) + assert.Equal(t, "test-owner", handle.Owner()) + assert.Equal(t, artifact.LockScopeHost, handle.Scope()) + assert.NotEmpty(t, handle.Token()) + _, expires := handle.ExpiresAt() + assert.False(t, expires, "local locks never expire") + require.ErrorIs(t, handle.Renew(context.Background(), time.Minute), errUtils.ErrArtifactUnsupportedCapability) + assert.FileExists(t, lockFile(repo, "team/app:1.0.0"), "the lock file lives in the managed namespace and is never removed") + + require.NoError(t, handle.Release(context.Background())) + require.ErrorIs(t, handle.Release(context.Background()), errUtils.ErrArtifactConflict, "a handle cannot be released twice") + assert.FileExists(t, lockFile(repo, "team/app:1.0.0")) + + other := holdLock(t, repo, artifact.LockRequest{Resource: "x"}) + assert.NotEmpty(t, other.Owner(), "an empty owner defaults to the repository owner") + require.NoError(t, other.Release(context.Background())) +} + +func TestLock_RejectsUnsupportedGuarantees(t *testing.T) { + repo := newRepo(t, dirSpec()) + request := lockRequest("r") + request.Lease = time.Minute + _, err := repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a lease cannot be fenced") + + request = lockRequest("r") + request.Required = artifact.LockScopeRepository + _, err = repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a host file lock is never reported as repository scope") + capErr, ok := artifact.AsCapabilityError(err) + require.True(t, ok) + assert.Equal(t, "lock/repository", capErr.Capability) + + _, err = repo.Lock(context.Background(), artifact.LockRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + caps := repo.Capabilities() + assert.True(t, caps.HasLockScope(artifact.LockScopeHost)) + assert.False(t, caps.HasLockScope(artifact.LockScopeRepository)) +} + +func TestLock_ReleaseChecksTheOwnershipToken(t *testing.T) { + repo := newRepo(t, dirSpec()) + handle := holdLock(t, repo, lockRequest("r")) + real, ok := handle.(*fileLock) + require.True(t, ok) + + forged := &fileLock{path: real.path, resource: real.resource, token: "not-the-token", release: make(chan struct{}), done: make(chan error, 1)} + require.ErrorIs(t, forged.Release(context.Background()), errUtils.ErrArtifactConflict, "a wrong token never releases another operation's lock") + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, err := repo.Lock(ctx, lockRequest("r")) + require.ErrorIs(t, err, context.DeadlineExceeded, "the real owner still holds the lock") + require.NoError(t, handle.Release(context.Background())) +} + +func TestLock_CompetingHandlesAreMutuallyExclusive(t *testing.T) { + // Separate repository objects over the same root model separate writers in one process. + root := dirSpec().withRoot(filepath.Join(t.TempDir(), "shared")) + writers := make([]*Repository, 4) + for index := range writers { + writers[index] = newRepo(t, root) + } + var ( + inside, maxInside, total atomic.Int32 + wg sync.WaitGroup + ) + for _, writer := range writers { + wg.Add(1) + go func() { + defer wg.Done() + for range 5 { + handle, err := writer.Lock(context.Background(), lockRequest("team/app:1.0.0")) + if !assert.NoError(t, err) { + return + } + now := inside.Add(1) + for { + seen := maxInside.Load() + if now <= seen || maxInside.CompareAndSwap(seen, now) { + break + } + } + time.Sleep(2 * time.Millisecond) + total.Add(1) + inside.Add(-1) + assert.NoError(t, handle.Release(context.Background())) + } + }() + } + wg.Wait() + assert.EqualValues(t, 1, maxInside.Load(), "never two owners inside the critical section") + assert.EqualValues(t, 20, total.Load()) +} + +func TestLock_CanceledAcquisitionLeaksNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder := holdLock(t, repo, lockRequest("r")) + + var wg sync.WaitGroup + for range 5 { + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + handle, err := repo.Lock(ctx, lockRequest("r")) + assert.Nil(t, handle) + assert.ErrorIs(t, err, context.DeadlineExceeded) + }() + } + wg.Wait() + require.NoError(t, holder.Release(context.Background())) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + next, err := repo.Lock(ctx, lockRequest("r")) + require.NoError(t, err, "the canceled waiters took no ownership") + require.NoError(t, next.Release(context.Background())) + + canceled, cancelNow := context.WithCancel(context.Background()) + cancelNow() + _, err = repo.Lock(canceled, lockRequest("r2")) + if err != nil { + require.ErrorIs(t, err, context.Canceled) + } + again, err := repo.Lock(context.Background(), lockRequest("r2")) + require.NoError(t, err, "an already-canceled attempt leaves the resource free") + require.NoError(t, again.Release(context.Background())) +} + +// Two writers to one reference: the OS lock excludes them across the comparison and commit, so +// exactly one creates the reference and every other sees the conflict. +func TestTwoWritersToOneReference(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + const writers = 6 + prepared := make([]*artifact.PreparedArtifact, writers) + for index := range writers { + prepared[index] = prepare(t, repo, &prepareSpec{opID: fmt.Sprintf("op-%d", index)}) + } + outcomes := make([]error, writers) + var wg sync.WaitGroup + for index := range writers { + wg.Add(1) + go func() { + defer wg.Done() + _, outcomes[index] = repo.Publish(context.Background(), prepared[index], artifact.PublishOptions{}) + }() + } + wg.Wait() + created := 0 + for _, err := range outcomes { + if err == nil { + created++ + } else { + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + } + assert.Equal(t, 1, created) + assertIntact(t, repo, defaultRef()) + assert.Empty(t, listFiles(t, filepath.Join(repo.root, filepath.FromSlash(layout.ManagedRoot), "operations")), "no journal outlives its publication") + }) + } +} + +// A publication without a caller-held lock takes the reference lock itself, so it waits for an +// existing holder; with a caller-held lock it proceeds without deadlocking on itself. +func TestPublishHonorsTheReferenceLock(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder := holdLock(t, repo, lockRequest(testName+":"+testVersion)) + + prepared := prepare(t, repo, &prepareSpec{}) + done := make(chan error, 1) + go func() { + _, publishErr := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + done <- publishErr + }() + select { + case err := <-done: + t.Fatalf("the publication did not wait for the held lock (returned %v)", err) + case <-time.After(200 * time.Millisecond): + } + require.NoError(t, holder.Release(context.Background())) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("the publication never resumed after the lock was released") + } + + second := newRepo(t, dirSpec()) + held := holdLock(t, second, lockRequest("team/other:1.0.0")) + _, err := second.Publish(context.Background(), prepare(t, second, &prepareSpec{name: "team/other"}), artifact.PublishOptions{Lock: held}) + require.NoError(t, err, "a service-held lock is honored, not re-acquired") +} + +func TestPublishCancellationWhileWaitingLeaksNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + holder := holdLock(t, repo, lockRequest(testName+":"+testVersion)) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err := repo.Publish(ctx, prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{}) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.NoError(t, holder.Release(context.Background())) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "nothing was written") + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) +} + +func TestLock_ExcludesOtherProcesses(t *testing.T) { + repo := newRepo(t, dirSpec()) + resource := "team/app:1.0.0" + handle := holdLock(t, repo, lockRequest(resource)) + + assert.Equal(t, helperExitBusy, exitCode(t, helperCommand(t, "try", repo.root, resource)), "another process cannot take a held lock") + other := resource + "-other" + assert.Equal(t, helperExitAcquired, exitCode(t, helperCommand(t, "try", repo.root, other)), "locks on other resources are independent") + + require.NoError(t, handle.Release(context.Background())) + assert.Equal(t, helperExitAcquired, exitCode(t, helperCommand(t, "try", repo.root, resource)), "the lock is free after release") +} + +func TestLock_ReleasesWhenItsOwnerProcessExits(t *testing.T) { + repo := newRepo(t, dirSpec()) + resource := "team/app:1.0.0" + cmd := helperCommand(t, "hold", repo.root, resource) + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + stdin, err := cmd.StdinPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + exited := false + defer func() { + if !exited { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + + line, err := bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + require.Equal(t, "locked", strings.TrimSpace(line)) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err = repo.Lock(ctx, lockRequest(resource)) + require.ErrorIs(t, err, context.DeadlineExceeded, "the helper process holds the lock") + + // Kill the owner while its stdin is still open, so it cannot leave its read loop and release + // the lock on the normal path: the OS alone must free it. Stdin closes only after the wait. + require.NoError(t, cmd.Process.Kill()) + waitErr := cmd.Wait() + exited = true + _ = stdin.Close() + require.Error(t, waitErr, "the owner was terminated rather than exiting on its own") + require.NotNil(t, cmd.ProcessState) + assert.False(t, cmd.ProcessState.Success(), "the owner did not exit cleanly through its release path") + + acquire, cancelAcquire := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelAcquire() + handle, err := repo.Lock(acquire, lockRequest(resource)) + require.NoError(t, err, "the lock is released when its owner exits") + require.NoError(t, handle.Release(context.Background())) +} + +func lockFile(repo *Repository, resource string) string { + return filepath.Join(repo.root, filepath.FromSlash(layout.LockPath(resource))) +} + +func TestPublishRejectsLockForAnotherReference(t *testing.T) { + repo := newRepo(t, dirSpec()) + held := holdLock(t, repo, lockRequest("other:1.0.0")) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{}), artifact.PublishOptions{Lock: held}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) +} + +func TestWriteEvidenceHonorsReferenceLock(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + held := holdLock(t, repo, lockRequest(testName+":"+testVersion)) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err := repo.WriteEvidence(ctx, &pinned, nil, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + _, err = repo.WriteEvidence(context.Background(), &pinned, nil, held) + require.NoError(t, err) + require.NoError(t, held.Release(context.Background())) +} diff --git a/pkg/artifact/local/options.go b/pkg/artifact/local/options.go new file mode 100644 index 00000000000..be3b0f6cd4f --- /dev/null +++ b/pkg/artifact/local/options.go @@ -0,0 +1,57 @@ +package local + +import ( + "fmt" + "os" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// settings holds the construction options of a local repository. +type settings struct { + verifier artifact.Verifier + owner string + faults faultHook +} + +// Option configures a local repository or its registration. +type Option func(*settings) + +// WithVerifier injects the cryptographic verifier used by Verify. The formats it understands are +// the evidence formats the repository advertises. Without a verifier the repository stores no +// evidence and Sign and Verify return unsupported-capability errors. +func WithVerifier(verifier artifact.Verifier) Option { + defer perf.Track(nil, "local.WithVerifier")() + + return func(s *settings) { s.verifier = verifier } +} + +// WithOwner sets the owner recorded in lock requests and operation journals when the caller does +// not supply one. The default is ":". +func WithOwner(owner string) Option { + defer perf.Track(nil, "local.WithOwner")() + + return func(s *settings) { s.owner = owner } +} + +func applySettings(opts []Option) settings { + var applied settings + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + if applied.owner == "" { + applied.owner = defaultOwner() + } + return applied +} + +func defaultOwner() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s:%d", host, os.Getpid()) +} diff --git a/pkg/artifact/local/permissions_test.go b/pkg/artifact/local/permissions_test.go new file mode 100644 index 00000000000..42be34f0a53 --- /dev/null +++ b/pkg/artifact/local/permissions_test.go @@ -0,0 +1,166 @@ +package local + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" +) + +func modeOf(t *testing.T, path string) fs.FileMode { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err) + return info.Mode().Perm() +} + +// tgzMemberMode returns the mode recorded in the archive for one member. +func tgzMemberMode(t *testing.T, archivePath, member string) int64 { + t.Helper() + file, err := os.Open(archivePath) + require.NoError(t, err) + defer file.Close() + gz, err := gzip.NewReader(file) + require.NoError(t, err) + reader := tar.NewReader(gz) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + t.Fatalf("member %q not found in %s", member, archivePath) + } + require.NoError(t, err) + if header.Name == member { + return header.Mode & 0o777 + } + } +} + +func TestArchivePermissions_DefaultIs0600OnTheArchiveAndSidecars(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + + assert.Equal(t, fs.FileMode(0o600), modeOf(t, result.Location), "default archive permissions") + assert.Equal(t, fs.FileMode(0o600), modeOf(t, result.Location+".metadata.json")) + assert.Equal(t, fs.FileMode(0o600), modeOf(t, filepath.Join(result.Location+".docs", "README.md"))) + assert.Equal(t, fs.FileMode(0o700), modeOf(t, result.Location+".docs"), "the sidecar directory keeps owner traversal") + assert.False(t, repo.modeExplicit, "a default is not an explicit request") +} + +func TestArchivePermissions_ExplicitModeIsDistinctFromMemberModes(t *testing.T) { + fixtures.RequirePOSIXModes(t) + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0640") + repo := newRepo(t, spec) + require.True(t, repo.modeExplicit) + result, _ := publishFilePackage(t, repo) + + assert.Equal(t, fs.FileMode(0o640), modeOf(t, result.Location), "the configured mode applies to the archive file") + assert.Equal(t, fs.FileMode(0o640), modeOf(t, result.Location+".metadata.json")) + assert.Equal(t, fs.FileMode(0o750), modeOf(t, result.Location+".docs")) + assert.EqualValues(t, 0o755, tgzMemberMode(t, result.Location, "bin/helper.sh"), "member modes are independent of the archive file mode") + assert.EqualValues(t, 0o644, tgzMemberMode(t, result.Location, "README.md")) + assert.EqualValues(t, 0o640, tgzMemberMode(t, result.Location, fixtures.UnicodeName)) +} + +func TestArchivePermissions_AppliedBeforeAnythingIsVisible(t *testing.T) { + fixtures.RequirePOSIXModes(t) + spec := archiveSpec(artifact.PackagingTar) + spec = spec.withPermissions("0400") + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + archivePath := filepath.Join(repo.root, "team", "app", "1.0.0.tar") + assert.Equal(t, fs.FileMode(0o400), modeOf(t, archivePath), "even an interrupted publication never exposes a wrongly-permissioned archive") +} + +// An explicit permission request the host cannot enforce fails before anything is written. +func TestArchivePermissions_UnsupportedHostFailsExplicitly(t *testing.T) { + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0600") + repo := newRepo(t, spec) + repo.hostModes = false + + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + capErr, ok := artifact.AsCapabilityError(err) + require.True(t, ok) + assert.Equal(t, "permissions/archive-file-mode", capErr.Capability) + assert.Contains(t, capErr.Reason, "0600") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "nothing was created before the capability check") +} + +// Without an explicit request, a host that cannot enforce modes publishes with the host default. +func TestArchivePermissions_DefaultWorksWhereModesCannotBeEnforced(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + repo.hostModes = false + result, _ := publishFilePackage(t, repo) + assert.FileExists(t, result.Location) + assertIntact(t, repo, defaultRef()) +} + +// On the real host the behavior matches its capability: Windows rejects an explicit request, +// POSIX hosts honor it. +func TestArchivePermissions_RealHostBehavior(t *testing.T) { + spec := archiveSpec(artifact.PackagingTgz) + spec = spec.withPermissions("0600") + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "Windows cannot enforce POSIX archive permissions") + return + } + require.NoError(t, err) + assert.Equal(t, fs.FileMode(0o600), modeOf(t, filepath.Join(repo.root, "team", "app", "1.0.0.tgz"))) +} + +func TestPermissions_InvalidConfigurationIsRejected(t *testing.T) { + for _, bad := range []string{"preserve", "0o600", "9999", "4755", "6", "rw-------", "600.0"} { + spec := archiveSpec(artifact.PackagingTgz).withPermissions(bad).withRoot(filepath.Join(t.TempDir(), "r")) + _, err := newRepository("x", spec.kind, spec.config(t), nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, bad) + } + dir := dirSpec().withPermissions("0600").withRoot(filepath.Join(t.TempDir(), "r")) + _, err := newRepository("x", dir.kind, dir.config(t), nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "permissions apply only to local archive files") +} + +func TestCapabilities_AreHonest(t *testing.T) { + dir := newRepo(t, dirSpec()) + archive := newRepo(t, archiveSpec(artifact.PackagingTgz)) + + dirCaps, archiveCaps := dir.Capabilities(), archive.Capabilities() + assert.Equal(t, []artifact.Packaging{artifact.PackagingNone}, dirCaps.Packagings) + assert.ElementsMatch(t, []artifact.Packaging{artifact.PackagingTar, artifact.PackagingTgz, artifact.PackagingZip}, archiveCaps.Packagings) + for _, caps := range []artifact.Capabilities{dirCaps, archiveCaps} { + assert.True(t, caps.Read && caps.Write && caps.EnumerateNamed && caps.EnumerateCollection && caps.ConditionalUpdate) + assert.Equal(t, artifact.DeleteObject, caps.Delete) + assert.Equal(t, []artifact.LockScope{artifact.LockScopeHost}, caps.LockScopes, "host scope only") + assert.Equal(t, []artifact.SubjectKind{artifact.SubjectRecord}, caps.SubjectKinds) + assert.Empty(t, caps.EvidenceFormats, "no verifier is configured, so no evidence format is advertised") + } + assert.Equal(t, hostEnforcesModes(), dirCaps.PreservesModes) + assert.True(t, archiveCaps.PreservesModes, "member modes live inside the archive on every host") + + dir.hostModes = false + assert.False(t, dir.Capabilities().PreservesModes) + assert.Equal(t, dir.Capabilities(), dir.Capabilities(), "capabilities are stable") +} diff --git a/pkg/artifact/local/publish.go b/pkg/artifact/local/publish.go new file mode 100644 index 00000000000..1fa036beb4a --- /dev/null +++ b/pkg/artifact/local/publish.go @@ -0,0 +1,415 @@ +package local + +import ( + "context" + "errors" + "fmt" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// publishPlan is what the admission decision and the existing state say about one publication. +type publishPlan struct { + admission layout.Admission + current *layout.Reference + prior *layout.Prior + owned []string + abandoned string + oldOp *layout.Operation + expected *artifact.PinnedReference +} + +// Publish implements [artifact.Repository] with the publication sequence of the contracts: +// +// 1. Hold the reference lock (the service's, or one taken here) and the resolved-location lock. +// 2. Read the current reference and admit the publication: conflict, force, unchanged, or resume. +// 3. Reserve the location and record the pending operation journal and pending reference before +// any payload path changes. +// 4. Stage the payload and sidecars on the same filesystem, write the immutable Record, install +// the payload, and write the representation. +// 5. Complete the reference last, with an atomic replacement. +// 6. Clean up only the temporary resources this operation owns. +// +// A failure after step 3 leaves the reference pending: unavailable to readers, repaired by a retry +// with the same intent. The caller reconciles an ambiguous response by reading the reference. +func (r *Repository) Publish(ctx context.Context, prepared *artifact.PreparedArtifact, options artifact.PublishOptions) (result *artifact.PublishResult, err error) { + defer perf.Track(nil, "local.Repository.Publish")() + + job, err := r.newJob(prepared) + if err != nil { + return nil, err + } + t, err := r.openTree(true) + if err != nil { + return nil, err + } + defer t.Close() + var held []*fileLock + defer func() { + if releaseErr := releaseAll(ctx, held...); err == nil { + err = releaseErr + } + }() + if options.Lock != nil && options.Lock.Resource() != prepared.Record.Name+":"+prepared.Record.Version { + return nil, fmt.Errorf("%w: supplied lock does not cover reference %s:%s", errUtils.ErrArtifactConflict, prepared.Record.Name, prepared.Record.Version) + } + if options.Lock == nil { + lock, lockErr := r.acquire(ctx, prepared.Record.Name+":"+prepared.Record.Version, r.settings.owner, job.opID) + if lockErr != nil { + return nil, lockErr + } + held = append(held, lock) + } + return r.publishLocked(ctx, t, job, &options, &held) +} + +// publishLocked runs the publication while the reference lock is held. +func (r *Repository) publishLocked(ctx context.Context, t *tree, job *publishJob, options *artifact.PublishOptions, held *[]*fileLock) (*artifact.PublishResult, error) { + plan, err := r.admit(t, job, options) + if err != nil { + return nil, err + } + if plan.admission == layout.AdmitUnchanged { + return r.unchanged(t, job, plan.current) + } + if err := r.resolveLocation(t, job, plan); err != nil { + return nil, err + } + locationLock, err := r.acquire(ctx, locationResource+layout.FoldPath(job.location), r.settings.owner, job.opID) + if err != nil { + return nil, err + } + *held = append(*held, locationLock) + pending, err := r.begin(ctx, t, job, plan) + if err != nil { + return nil, err + } + result, err := r.build(ctx, t, job, plan, pending) + if err != nil && !isCrash(err) && !job.installing { + _ = t.removeAll(job.staging) + } + return result, err +} + +// admit reads the current reference and applies the admission rules. +func (r *Repository) admit(t *tree, job *publishJob, options *artifact.PublishOptions) (*publishPlan, error) { + current, err := t.loadReference(job.key) + if err != nil { + return nil, err + } + input := layout.AdmitInput{ + Current: current, IncomingRecord: job.recordDigest, IncomingIntent: job.prepared.IntentDigest, + ExpectedPrior: options.ExpectedPrior, Force: options.Force, + } + if current != nil && current.Available() { + record, _, loadErr := t.loadRecord(current.RecordDigest) + if loadErr != nil { + return nil, loadErr + } + if intent, intentErr := record.IntentDigest(); intentErr == nil { + input.CurrentIntent = intent + } + } + admission, err := layout.Admit(&input) + if err != nil { + return nil, err + } + return &publishPlan{admission: admission, current: current, expected: options.ExpectedPrior}, nil +} + +// unchanged returns the existing artifact for an identical republication and tidies leftovers of +// the operation that completed it. +func (r *Repository) unchanged(t *tree, job *publishJob, current *layout.Reference) (*artifact.PublishResult, error) { + record, _, err := t.loadRecord(current.RecordDigest) + if err != nil { + return nil, err + } + rep, err := t.loadRepresentation(job.key) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != current.RecordDigest { + return nil, fmt.Errorf("%w: %s:%s is completed but has no matching representation", errUtils.ErrArtifactIntegrityFailed, current.Name, current.Version) + } + r.tidyOperation(t, current.OperationID) + return r.result(t, record, current.RecordDigest, rep, layout.AdmitUnchanged.Outcome()), nil +} + +// tidyOperation removes the journal and staging directory of a finished operation. It is +// best-effort: a leftover is harmless and is retried by the next publication. +func (r *Repository) tidyOperation(t *tree, operationID string) { + if operationID == "" { + return + } + _ = t.removeAll(layout.StagingPath(operationID)) + _ = t.removeAll(layout.OperationPath(operationID)) +} + +// resolveLocation decides where the payload lives. A resumed publication reuses the location in +// its journal; otherwise the key_pattern is rendered exactly once, here, from the Record's +// captured creation time. The result is persisted; it is never rendered again. +func (r *Repository) resolveLocation(t *tree, job *publishJob, plan *publishPlan) error { + rep, err := t.loadRepresentation(job.key) + if err != nil { + return err + } + if err := r.loadPendingState(t, plan, rep); err != nil { + return err + } + if plan.admission.Resuming() && plan.oldOp != nil && plan.oldOp.IntentDigest == job.prepared.IntentDigest && plan.oldOp.Packaging == job.prepared.Packaging { + job.location = plan.oldOp.Location + return nil + } + if plan.oldOp != nil { + plan.abandoned = plan.oldOp.Location + } + job.location, err = layout.RenderLocation(r.keyPattern, &layout.PatternInput{ + Name: job.record.Name, Version: job.record.Version, Kind: job.record.Kind, CreatedAt: job.record.CreatedAt.Time(), + }, job.prepared.Packaging) + return err +} + +// loadPendingState records the locations this reference already owns and the content a +// publication replaces. The interrupted operation's journal is read when one is pending. +func (r *Repository) loadPendingState(t *tree, plan *publishPlan, rep *layout.Representation) error { + if rep != nil { + plan.owned = append(plan.owned, rep.Location) + } + if plan.current != nil && !plan.current.Available() { + oldOp, err := t.loadOperation(plan.current.OperationID) + if err != nil { + return err + } + plan.oldOp = oldOp + } + plan.prior = priorState(plan, rep) + if plan.oldOp != nil { + plan.owned = append(plan.owned, plan.oldOp.Location) + if plan.oldOp.Prior != nil { + plan.owned = append(plan.owned, plan.oldOp.Prior.Location) + } + } + return nil +} + +// priorState describes the content a publication replaces, so a later cleanup can retire it. +func priorState(plan *publishPlan, rep *layout.Representation) *layout.Prior { + if plan.admission == layout.AdmitReplace { + prior := &layout.Prior{RecordDigest: plan.current.RecordDigest} + if rep != nil && rep.RecordDigest == prior.RecordDigest { + prior.Location = rep.Location + } + return prior + } + if !plan.admission.Resuming() { + return nil + } + if plan.oldOp != nil && plan.oldOp.Prior != nil { + prior := *plan.oldOp.Prior + return &prior + } + if plan.current.PriorRecordDigest == "" { + return nil + } + prior := &layout.Prior{RecordDigest: plan.current.PriorRecordDigest} + if rep != nil && rep.RecordDigest == prior.RecordDigest { + prior.Location = rep.Location + } + return prior +} + +// begin reserves the location, writes the operation journal, and marks the reference pending. +// It holds the repository-wide claims lock so two references can never both reserve overlapping +// locations, and it never changes a payload path. +func (r *Repository) begin(ctx context.Context, t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + claims, err := r.acquire(ctx, claimsResource, r.settings.owner, job.opID) + if err != nil { + return nil, err + } + pending, beginErr := r.reserve(t, job, plan) + if releaseErr := claims.Release(context.WithoutCancel(ctx)); beginErr == nil { + beginErr = releaseErr + } + if beginErr != nil { + return nil, beginErr + } + if plan.oldOp != nil && plan.oldOp.OperationID != job.opID { + r.tidyOperation(t, plan.oldOp.OperationID) + } + return pending, r.settings.fault(faultBeforePayload) +} + +// reserve checks for location collisions and unmanaged content, then records the journal and the +// pending reference. +func (r *Repository) reserve(t *tree, job *publishJob, plan *publishPlan) (*layout.Reference, error) { + if err := r.rejectUnmanagedPrior(t, job, plan); err != nil { + return nil, err + } + existing, err := t.loadOperation(job.opID) + if err != nil { + return nil, err + } + if existing != nil && layout.ReferenceKey(existing.Name, existing.Version) != job.key { + return nil, fmt.Errorf("%w: operation %s belongs to %s:%s", errUtils.ErrArtifactConflict, job.opID, existing.Name, existing.Version) + } + if err := r.checkCollision(t, job, plan); err != nil { + return nil, err + } + journal := &layout.Operation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: r.kind, OperationID: job.opID, + Name: job.record.Name, Version: job.record.Version, IntentDigest: job.prepared.IntentDigest, + RecordDigest: job.recordDigest, CapturedAt: job.record.CreatedAt, Packaging: job.prepared.Packaging, + Location: job.location, Prior: plan.prior, Payload: job.payload, Documents: job.documents, Owner: r.settings.owner, + } + if plan.expected != nil { + journal.ExpectedPrior = plan.expected.ResolvedDigest + } + if err := t.writeOperation(journal); err != nil { + return nil, fmt.Errorf("%w: record operation %s: %w", errUtils.ErrArtifactUploadFailed, job.opID, err) + } + pending := layout.NewPendingReference(job.record.Name, job.record.Version, job.opID, job.prepared.IntentDigest, plan.current) + if err := t.writeReference(&pending); err != nil { + return nil, fmt.Errorf("%w: mark %s:%s pending: %w", errUtils.ErrArtifactUploadFailed, job.record.Name, job.record.Version, err) + } + return &pending, nil +} + +// build stages, writes, installs, and completes. It is the part of the sequence after the pending +// state is durable. +func (r *Repository) build(ctx context.Context, t *tree, job *publishJob, plan *publishPlan, pending *layout.Reference) (*artifact.PublishResult, error) { + if err := t.removeAll(job.staging); err != nil { + return nil, fmt.Errorf("%w: clear staging: %w", errUtils.ErrArtifactUploadFailed, err) + } + if err := r.store.stage(ctx, t, job); err != nil { + return nil, wrapWrite("stage", err) + } + if err := t.writeRecord(job.recordDigest, job.recordBytes); err != nil { + return nil, wrapWrite("write record", err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + job.installing = true + if err := r.store.install(t, job); err != nil { + return nil, wrapWrite("install payload", err) + } + rep := r.representation(job) + if err := t.writeRepresentation(rep); err != nil { + return nil, wrapWrite("write representation", err) + } + if err := r.settings.fault(faultAfterMetadata); err != nil { + return nil, err + } + completed := pending.Completed(job.recordDigest) + if err := t.writeReference(&completed); err != nil { + return nil, wrapWrite("complete reference", err) + } + if err := r.settings.fault(faultAfterCommit); err != nil { + return nil, err + } + r.retire(t, job, plan) + return r.result(t, job.record, job.recordDigest, rep, plan.admission.Outcome()), nil +} + +// wrapWrite classifies a storage failure. Context errors and already-typed artifact errors pass +// through so callers keep matching on them. +func wrapWrite(step string, err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isArtifactError(err) { + return err + } + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactUploadFailed, step, err) +} + +func isArtifactError(err error) bool { + for _, sentinel := range []error{ + errUtils.ErrArtifactConflict, errUtils.ErrArtifactUnsupportedCapability, errUtils.ErrArtifactIntegrityFailed, + errUtils.ErrArtifactPackagingInvalid, errUtils.ErrArtifactUnsafePath, errUtils.ErrArtifactTooLarge, + errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactRecordInvalid, + } { + if errors.Is(err, sentinel) { + return true + } + } + return isCrash(err) +} + +// representation builds the representation record for the job. +func (r *Repository) representation(job *publishJob) *layout.Representation { + return &layout.Representation{ + SchemaVersion: layout.SchemaVersion, RepositoryKind: r.kind, Name: job.record.Name, Version: job.record.Version, + RecordDigest: job.recordDigest, OperationID: job.opID, Packaging: job.prepared.Packaging, Location: job.location, + Payload: job.payload, Documents: job.documents, RepresentationDigest: job.representationDigest(job.prepared.Packaging), + } +} + +// retire removes what the completed publication replaced and the temporary resources it owned. +// Everything here is best-effort: the reference is already complete, and a leftover is garbage, +// not an inconsistency. +func (r *Repository) retire(t *tree, job *publishJob, plan *publishPlan) { + removeStaging := true + if r.kind == artifact.RepositoryKindLocalDir && plan.prior != nil { + if record, _, err := t.loadRecord(plan.prior.RecordDigest); err != nil || checkOwnedDirectory(t, job.retiredDir()+"/payload", record) != nil { + removeStaging = false + } + } + if removeStaging { + _ = t.removeAll(job.staging) + } + if plan.abandoned != "" && layout.FoldPath(plan.abandoned) != layout.FoldPath(job.location) && r.kind != artifact.RepositoryKindLocalDir { + r.removePayload(t, plan.abandoned) + } + if plan.prior != nil { + r.retirePrior(t, job, plan.prior) + } + _ = t.removeAll(layout.OperationPath(job.opID)) +} + +// retirePrior removes what the replaced reference left behind: its payload at a location the new +// publication no longer uses, and its record and evidence when the record changed. +func (r *Repository) retirePrior(t *tree, job *publishJob, prior *layout.Prior) { + if prior.Location != "" && layout.FoldPath(prior.Location) != layout.FoldPath(job.location) { + r.retirePriorPayload(t, prior) + } + if prior.RecordDigest != "" && prior.RecordDigest != job.recordDigest { + _ = t.removeAll(layout.RecordPath(prior.RecordDigest)) + _ = t.removeAll(layout.SignatureDir(artifact.SubjectRecord, prior.RecordDigest)) + } +} + +// retirePriorPayload removes the replaced payload. A local directory repository removes only the +// files its record owns, so unrelated files at that location survive. +func (r *Repository) retirePriorPayload(t *tree, prior *layout.Prior) { + if r.kind != artifact.RepositoryKindLocalDir { + r.removePayload(t, prior.Location) + return + } + if record, _, err := t.loadRecord(prior.RecordDigest); err == nil { + _, _ = r.store.remove(t, &view{record: record, rep: &layout.Representation{Location: prior.Location}}) + t.removeEmptyParents(prior.Location) + } +} + +// removePayload deletes the payload paths this repository owns at a location it no longer uses. +func (r *Repository) removePayload(t *tree, location string) { + for _, owned := range layout.PayloadPaths(r.kind, location) { + _ = t.removeAll(owned) + } + t.removeEmptyParents(location) +} + +// result builds the publish result for a stored artifact. +func (r *Repository) result(t *tree, record *artifact.Record, recordDigest digest.Digest, rep *layout.Representation, outcome artifact.Outcome) *artifact.PublishResult { + locator := t.absolute(rep.Location) + return &artifact.PublishResult{ + Outcome: outcome, + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, recordDigest), + Representation: rep.Artifact(locator), + Location: locator, + LockScope: artifact.LockScopeHost, + } +} diff --git a/pkg/artifact/local/read.go b/pkg/artifact/local/read.go new file mode 100644 index 00000000000..e4342244a0d --- /dev/null +++ b/pkg/artifact/local/read.go @@ -0,0 +1,231 @@ +package local + +import ( + "context" + "errors" + "fmt" + "hash" + "io" + "io/fs" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" +) + +// payloadStore abstracts how one repository kind lays a payload out on disk. +type payloadStore interface { + // supports reports whether the kind can store and read the packaging. + supports(packaging artifact.Packaging) bool + + // validate checks, before any state changes, that the staged blobs agree with the Record. + validate(job *publishJob) error + + // stage writes the payload, documents, and sidecars of a publication into its staging directory. + stage(ctx context.Context, t *tree, job *publishJob) error + + // install moves the staged payload into place at the resolved location, moving any previous + // content aside first. + install(t *tree, job *publishJob) error + + // streams opens the verified payload and document streams of a completed artifact. + streams(ctx context.Context, t *tree, v *view, guard func() error) ([]artifact.ContentStream, error) + + // document opens one distribution document without touching the payload. + document(t *tree, v *view, doc *artifact.Document, guard func() error) (io.ReadCloser, error) + + // verifySidecars checks the adjacent metadata copy against the stored Record bytes. + verifySidecars(t *tree, v *view) error + + // remove deletes the files exclusively owned by the artifact and returns what it removed. + remove(t *tree, v *view) ([]string, error) +} + +// Read implements [artifact.Repository]. It opens the verified streams of exactly the pinned +// Record. The reference must still name that Record; after the caller reads the streams, the +// reference generation is rechecked, so bytes that changed underneath the reader are reported +// instead of served under the old digest. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Read(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Content, error) { + defer perf.Track(nil, "local.Repository.Read")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + if err := r.store.verifySidecars(t, v); err != nil { + return nil, err + } + guard := r.guard(v.ref) + streams, err := r.store.streams(ctx, t, v, guard) + if err != nil { + closeStreams(streams) + return nil, err + } + return &artifact.Content{ + Record: v.record, + Representation: v.rep.Artifact(t.absolute(v.rep.Location)), + Streams: streams, + }, nil +} + +// missingRoot maps an absent repository root to not found and passes other errors through. +func (r *Repository) missingRoot(err error, ref artifact.Ref) error { + if errors.Is(err, errRootMissing) { + return fmt.Errorf("%w: %s (the repository directory %q does not exist)", errUtils.ErrArtifactNotFound, ref, r.root) + } + return err +} + +func closeStreams(streams []artifact.ContentStream) { + for index := range streams { + if streams[index].Reader != nil { + _ = streams[index].Reader.Close() + } + } +} + +// guard returns the generation recheck for a reader that observed ref. It reads the reference +// again and fails when the generation, state, or Record changed, because the payload location is +// mutable and a changed reference means the bytes may belong to a different version. +func (r *Repository) guard(observed *layout.Reference) func() error { + return func() error { + t, err := r.openTree(false) + if err != nil { + return fmt.Errorf("%w: the repository changed while reading %s:%s: %w", errUtils.ErrArtifactPublicationUnavailable, observed.Name, observed.Version, err) + } + defer t.Close() + current, err := t.loadReference(observed.Key()) + switch { + case err != nil: + return err + case current == nil: + return fmt.Errorf("%w: %s:%s was deleted while it was being read", errUtils.ErrArtifactNotFound, observed.Name, observed.Version) + case current.Generation != observed.Generation || !current.Available() || current.RecordDigest != observed.RecordDigest: + return fmt.Errorf("%w: %s:%s changed while it was being read (generation %d to %d); the bytes read are discarded rather than served under the old digest", + errUtils.ErrArtifactPublicationUnavailable, observed.Name, observed.Version, observed.Generation, current.Generation) + } + return nil + } +} + +// verifyingReader verifies size and digest as the content is read. At EOF it compares the digest +// and then runs the generation recheck, so a clean EOF means the bytes were intact and still +// belonged to the pinned reference. +type verifyingReader struct { + source io.ReadCloser + hasher hash.Hash + want digest.Digest + size int64 + read int64 + guard func() error + finished bool + failure error +} + +func newVerifyingReader(source io.ReadCloser, want digest.Digest, size int64, guard func() error) *verifyingReader { + return &verifyingReader{source: source, hasher: want.Algorithm().Hash(), want: want, size: size, guard: guard} +} + +// Read implements io.Reader. +func (v *verifyingReader) Read(p []byte) (int, error) { + defer perf.Track(nil, "local.verifyingReader.Read")() + + if v.failure != nil { + return 0, v.failure + } + n, err := v.source.Read(p) + v.read += int64(n) + _, _ = v.hasher.Write(p[:n]) + switch { + case v.read > v.size: + v.failure = fmt.Errorf("%w: content is larger than its declared %d bytes", errUtils.ErrArtifactIntegrityFailed, v.size) + return n, v.failure + case errors.Is(err, io.EOF) && !v.finished: + v.finished = true + if verr := v.finish(); verr != nil { + v.failure = verr + return n, verr + } + case err != nil && !errors.Is(err, io.EOF): + v.failure = fmt.Errorf("%w: %w", errUtils.ErrArtifactIntegrityFailed, err) + return n, v.failure + } + return n, err +} + +func (v *verifyingReader) finish() error { + if v.read != v.size { + return fmt.Errorf("%w: content is %d bytes; its descriptor declares %d", errUtils.ErrArtifactIntegrityFailed, v.read, v.size) + } + if got := digest.NewDigest(v.want.Algorithm(), v.hasher); got != v.want { + return fmt.Errorf("%w: content digest %s does not match its descriptor %s", errUtils.ErrArtifactIntegrityFailed, got, v.want) + } + if v.guard != nil { + return v.guard() + } + return nil +} + +// Close implements io.Closer. +func (v *verifyingReader) Close() error { + defer perf.Track(nil, "local.verifyingReader.Close")() + + return v.source.Close() +} + +// openFile opens a payload or sidecar file, mapping a missing file to an integrity failure: the +// completed reference promised it exists. +func (t *tree) openFile(rel, what string) (io.ReadCloser, fs.FileInfo, error) { + file, err := t.Open(native(rel)) + if err != nil { + return nil, nil, fmt.Errorf("%w: %s %q is missing or unreadable: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, fmt.Errorf("%w: %s %q: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, nil, fmt.Errorf("%w: %s %q is not a regular file", errUtils.ErrArtifactIntegrityFailed, what, rel) + } + return file, info, nil +} + +// verifyBytes compares a stored sidecar with the expected bytes. +func (t *tree) verifyBytes(rel, what string, want []byte) error { + got, err := t.readBounded(rel, artifact.MaxRecordSize) + if err != nil { + return fmt.Errorf("%w: %s %q is missing or unreadable: %w", errUtils.ErrArtifactIntegrityFailed, what, rel, err) + } + if string(got) != string(want) { + return fmt.Errorf("%w: %s %q does not match the stored Record", errUtils.ErrArtifactIntegrityFailed, what, rel) + } + return nil +} + +// documentDescriptor returns the Record's descriptor for a role. +func documentDescriptor(record *artifact.Record, role artifact.DocumentRole) (*artifact.Document, bool) { + for index := range record.Documents { + if record.Documents[index].Role == role { + return &record.Documents[index], true + } + } + return nil, false +} + +// blobDescriptor converts a Record document to the OCI-style descriptor used in streams. +func blobDescriptor(doc *artifact.Document) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: doc.MediaType, Digest: doc.Digest, Size: doc.Size} +} diff --git a/pkg/artifact/local/recovery_test.go b/pkg/artifact/local/recovery_test.go new file mode 100644 index 00000000000..daf7c459fea --- /dev/null +++ b/pkg/artifact/local/recovery_test.go @@ -0,0 +1,347 @@ +package local + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +var allSpecs = []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz), archiveSpec(artifact.PackagingZip)} + +func specName(spec *repoSpec) string { return spec.kind + "/" + string(spec.packaging) } + +func listNames(t *testing.T, repo *Repository) []string { + t.Helper() + page, err := repo.List(context.Background(), artifact.ListQuery{}) + require.NoError(t, err) + names := make([]string, 0, len(page.Entries)) + for index := range page.Entries { + names = append(names, page.Entries[index].Name+":"+page.Entries[index].Version) + } + return names +} + +// An interruption before the commit never advertises partial success: the reference is +// unavailable (not found, not complete-looking), invisible to listing, and a same-intent retry +// repairs it completely. +func TestInterruptedPublication_RetryRepairs(t *testing.T) { + points := []faultPoint{faultBeforePayload, faultDuringPayload, faultAfterMetadata} + for _, spec := range allSpecs { + for _, point := range points { + t.Run(specName(spec)+"/"+string(point), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(point))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "a pending reference is unavailable to managed readers") + require.ErrorContains(t, err, prepared.OperationID, "the diagnostic names the owning operation") + assert.Empty(t, listNames(t, repo), "a pending reference is absent from search results") + _, err = repo.Pin(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + recordDigest, err := prepared.Record.RecordDigest() + require.NoError(t, err) + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: recordDigest}) + require.Error(t, err, "even the digest does not expose a pending record") + assert.True(t, errors.Is(err, errUtils.ErrArtifactPublicationUnavailable) || errors.Is(err, errUtils.ErrArtifactNotFound), "%v", err) + + retry, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, retry.Outcome) + assertIntact(t, repo, defaultRef()) + assert.Equal(t, []string{testName + ":" + testVersion}, listNames(t, repo)) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(prepared.OperationID))), "the retry cleans the operation's staging") + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath(prepared.OperationID)))) + }) + } + } +} + +// assertIntact reads the artifact end to end and checks it verifies. +func assertIntact(t *testing.T, repo *Repository, r artifact.Ref) { + t.Helper() + pinned := resolve(t, repo, r) + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + require.NoError(t, readAll(t, content)) +} + +// A crash after the commit but before the response leaves a completed reference: a retry is a +// no-op, and the caller can prove success by reading it. +func TestInterruptedPublication_AfterCommitIsComplete(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterCommit))) + repo := newRepo(t, spec) + prepared := prepare(t, repo, &prepareSpec{}) + + _, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + assertIntact(t, repo, defaultRef()) + digestWant, err := prepared.Record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, digestWant, resolve(t, repo, defaultRef()).RecordDigest) + + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a writer that expects the reference to be absent is stale") + current := resolve(t, repo, defaultRef()) + retry, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{ExpectedPrior: ¤t}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, retry.Outcome) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath(prepared.OperationID))), "an unchanged republish tidies the finished operation") + }) + } +} + +// A different intent never steals a pending operation, with or without force. The same intent +// from a later invocation (new operation ID and preparation time) repairs it. +func TestPendingOperation_IsNeverStolen(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterMetadata))) + repo := newRepo(t, spec) + first := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "first\n"}, opID: "op-first"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + + other := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "different\n"}, opID: "op-other"}) + for _, force := range []bool{false, true} { + _, err := repo.Publish(context.Background(), other, artifact.PublishOptions{Force: force}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "force=%v", force) + require.ErrorContains(t, err, "op-first", "the recovery diagnostic includes the operation ID") + } + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the failed attempt did not disturb the pending state") + + later := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "first\n"}, opID: "op-later", at: capturedAt.Add(72 * time.Hour)}) + result, err := repo.Publish(context.Background(), later, artifact.PublishOptions{}) + require.NoError(t, err, "the same intent from a later invocation repairs the pending operation") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath("op-first"))), "the abandoned journal is retired") + }) + } +} + +// A failed or interrupted force replacement leaves the reference unavailable until the same-intent +// retry repairs it. The old digest never serves the replacement's bytes and is not found after. +func TestInterruptedForceReplacement(t *testing.T) { + for _, spec := range allSpecs { + for _, point := range []faultPoint{faultDuringPayload, faultAfterMetadata} { + t.Run(specName(spec)+"/"+string(point), func(t *testing.T) { + var armed faultHook + spec.opts = []Option{withFaults(func(p faultPoint) error { + if armed != nil { + return armed(p) + } + return nil + })} + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}, opID: "op-1"}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}) + armed = crashAt(point) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable, "the interrupted replacement leaves the reference unavailable") + _, err = repo.Read(context.Background(), old) + require.Error(t, err, "the old digest is never served while the replacement is in flight") + assert.Empty(t, listNames(t, repo)) + + repo = newRepo(t, spec.withRoot(repo.root)) + result, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.NoError(t, err, "the same-intent retry repairs the reference") + assert.Equal(t, artifact.OutcomeReplaced, result.Outcome) + current := resolve(t, repo, defaultRef()) + assert.NotEqual(t, old.RecordDigest, current.RecordDigest) + assertIntact(t, repo, defaultRef()) + + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "retired bytes report not found, never the current bytes") + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: testName, Digest: old.RecordDigest}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } + } +} + +// Old-digest reads never return the replacement's bytes, even when the physical location is the +// same, and the guard catches a reader that is mid-stream when the replacement lands. +func TestReplacementNeverServesNewBytesUnderOldDigest(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + content, err := repo.Read(context.Background(), old) + require.NoError(t, err) + defer content.Close() + + // Windows cannot rename a directory that contains open files, so a directory replacement + // cannot land under a mid-stream reader there. Close the reader first and keep the + // old-digest assertions; the mid-stream guard is exercised on POSIX hosts. + midStream := runtime.GOOS != "windows" || spec.kind != artifact.RepositoryKindLocalDir + if !midStream { + require.NoError(t, content.Close()) + } + + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + + if midStream { + err = readAll(t, content) + require.Error(t, err, "a reader that was open when the reference moved must not report success") + assert.True(t, errorsIsAny(err, errUtils.ErrArtifactPublicationUnavailable, errUtils.ErrArtifactIntegrityFailed, errUtils.ErrArtifactNotFound), "%v", err) + } + + _, err = repo.Read(context.Background(), old) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// The recovery path triggers only for the matching condition: without an interruption a changed +// intent is a plain conflict and nothing is repaired or replaced. +func TestNoRecoveryWithoutInterruption(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1\n"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + + changed := prepare(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2\n"}, opID: "op-2"}) + _, err := repo.Publish(context.Background(), changed, artifact.PublishOptions{ExpectedPrior: &old}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + assert.Equal(t, old.RecordDigest, resolve(t, repo, defaultRef()).RecordDigest, "the stored artifact is untouched") + assertIntact(t, repo, defaultRef()) + + stale := artifact.PinnedReference{ResolvedDigest: digest.FromString("stale")} + _, err = repo.Publish(context.Background(), changed, artifact.PublishOptions{Force: true, ExpectedPrior: &stale}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a stale expected prior is a conflict even under force") + + _, err = repo.Publish(context.Background(), changed, artifact.PublishOptions{Force: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "the reference exists but the writer expected none") +} + +// Identical republication is unchanged even when time and operation differ. +func TestRepublishIsUnchanged(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, spec) + first := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + before := resolve(t, repo, defaultRef()) + pinned := before + second := publish(t, repo, &prepareSpec{at: capturedAt.Add(48 * time.Hour), opID: "op-again"}, artifact.PublishOptions{ExpectedPrior: &pinned}) + assert.Equal(t, artifact.OutcomeUnchanged, second.Outcome) + assert.Equal(t, first.Pinned.RecordDigest, second.Pinned.RecordDigest, "the existing Record is returned unchanged") + assert.Equal(t, first.Location, second.Location) + assert.Equal(t, before.RecordDigest, resolve(t, repo, defaultRef()).RecordDigest) + }) + } +} + +func errorsIsAny(err error, targets ...error) bool { + for _, target := range targets { + if errors.Is(err, target) { + return true + } + } + return false +} + +// A failed publication before any state change (validation, unsupported packaging) writes nothing. +func TestRejectedPublicationWritesNothing(t *testing.T) { + repo := newRepo(t, dirSpec()) + wrong := prepare(t, newRepo(t, archiveSpec(artifact.PackagingTgz)), &prepareSpec{}) + _, err := repo.Publish(context.Background(), wrong, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a dir repository does not accept an archive payload") + _, statErr := os.Stat(repo.root) + assert.True(t, os.IsNotExist(statErr), "the root was not even created") + + _, err = repo.Publish(context.Background(), nil, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +// A same-intent retry under a different packaging cannot reuse the journaled location (its +// extension would disagree), so it publishes at a freshly rendered location and retires the +// abandoned attempt's partial payload instead of orphaning it. +func TestResumedPublicationRetiresAnAbandonedLocation(t *testing.T) { + var fault armedFault + root := filepath.Join(t.TempDir(), "repo") + tgz := archiveSpec(artifact.PackagingTgz).withRoot(root).withOpts(fault.option()) + repo := newRepo(t, tgz) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), prepare(t, repo, &prepareSpec{opID: "op-tgz"}), artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + abandoned := filepath.Join(root, "team", "app", "1.0.0.tgz") + assert.FileExists(t, abandoned, "the crashed attempt left its payload behind") + + zipRepo := newRepo(t, archiveSpec(artifact.PackagingZip).withRoot(root)) + result, err := zipRepo.Publish(context.Background(), prepare(t, zipRepo, &prepareSpec{opID: "op-zip"}), artifact.PublishOptions{}) + require.NoError(t, err, "the same intent repairs the pending reference whatever the packaging") + assert.Equal(t, filepath.Join(root, "team", "app", "1.0.0.zip"), result.Location) + assert.NoFileExists(t, abandoned, "the abandoned payload was retired, not orphaned") + assert.NoFileExists(t, abandoned+".metadata.json") + assertIntact(t, zipRepo, defaultRef()) + assert.Equal(t, []string{"team/app/1.0.0.zip", "team/app/1.0.0.zip.metadata.json"}, payloadFiles(t, root)) +} + +func TestPendingOperationIDCannotBeReusedForAnotherReference(t *testing.T) { + var fault armedFault + repo := newRepo(t, dirSpec().withOpts(fault.option())) + fault.hook = crashAt(faultBeforePayload) + first := prepare(t, repo, &prepareSpec{name: "team/first", opID: "shared-op"}) + _, err := repo.Publish(context.Background(), first, artifact.PublishOptions{}) + require.ErrorIs(t, err, errCrash) + second := prepare(t, repo, &prepareSpec{name: "team/second", opID: "shared-op"}) + _, err = repo.Publish(context.Background(), second, artifact.PublishOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + _, err = repo.Resolve(context.Background(), artifact.Ref{Name: "team/first", Version: testVersion}) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) +} + +func TestReplacementRejectsUnmanagedFiles(t *testing.T) { + repo := newRepo(t, dirSpec()) + first := publish(t, repo, &prepareSpec{files: map[string]string{"owned": "old"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + unmanaged := filepath.Join(first.Location, "user-file") + require.NoError(t, os.WriteFile(unmanaged, []byte("keep"), 0o644)) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"owned": "new"}, opID: "other"}) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + data, err := os.ReadFile(unmanaged) + require.NoError(t, err) + assert.Equal(t, "keep", string(data)) + assertIntact(t, repo, defaultRef()) +} + +func TestReplacementRecoveryWithChangedInventory(t *testing.T) { + var fault armedFault + repo := newRepo(t, dirSpec().withOpts(fault.option())) + publish(t, repo, &prepareSpec{files: map[string]string{"old-name": "old"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + replacement := prepare(t, repo, &prepareSpec{files: map[string]string{"new-name": "new"}, opID: "replacement"}) + fault.hook = crashAt(faultAfterMetadata) + _, err := repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.ErrorIs(t, err, errCrash) + _, err = repo.Publish(context.Background(), replacement, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + require.NoError(t, err) + assertIntact(t, repo, defaultRef()) +} diff --git a/pkg/artifact/local/register.go b/pkg/artifact/local/register.go new file mode 100644 index 00000000000..75f5a3f2354 --- /dev/null +++ b/pkg/artifact/local/register.go @@ -0,0 +1,72 @@ +package local + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Register registers the local/dir and local/archive repository kinds in the process-wide +// registry. It is called explicitly by a composition root, never from init. Options such as +// [WithVerifier] apply to every repository the registered factories construct. It does not touch +// the legacy CI artifact registry, whose own "local/dir" store is a different thing. +func Register(opts ...Option) error { + defer perf.Track(nil, "local.Register")() + + return RegisterWith(artifact.DefaultRepositoryRegistry(), opts...) +} + +// RegisterWith registers the local kinds in the given registry. A kind that is already +// registered is an error, so double registration is caught instead of silently replaced. +func RegisterWith(registry *artifact.RepositoryRegistry, opts ...Option) error { + defer perf.Track(nil, "local.RegisterWith")() + + for _, kind := range []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} { + desc := Descriptor(kind, opts...) + if err := registry.Register(&desc, factoryFor(kind, opts)); err != nil { + return err + } + } + return nil +} + +// Descriptor returns the static descriptor of a local kind: the built-in descriptor refined with +// the capabilities the registered backend actually provides (deletion, enumeration, conditional +// updates, detached evidence for the verifier's formats). It needs no I/O. +func Descriptor(kind string, opts ...Option) artifact.KindDescriptor { + defer perf.Track(nil, "local.Descriptor")() + + desc, _ := artifact.BuiltinKind(kind) + applied := applySettings(opts) + desc.Capabilities = staticCapabilities(kind, hostEnforcesModes(), applied.verifier) + return *desc +} + +func factoryFor(kind string, opts []Option) artifact.RepositoryFactory { + return func(_ context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, _ ...artifact.RepositoryOption) (artifact.Repository, error) { + repo, err := newRepository(alias, kind, cfg, opts) + if err != nil { + return nil, err + } + return repo, nil + } +} + +// Kinds lists the repository kinds this package registers. +func Kinds() []string { + defer perf.Track(nil, "local.Kinds")() + + return []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive} +} + +// ensureKind rejects a configuration that names another kind. +func ensureKind(want string, cfg *schema.ArtifactRepositoryConfig) error { + if cfg.Kind != "" && cfg.Kind != want { + return fmt.Errorf("%w: configuration kind %q cannot construct a %q repository", errUtils.ErrArtifactConfigInvalid, cfg.Kind, want) + } + return nil +} diff --git a/pkg/artifact/local/register_test.go b/pkg/artifact/local/register_test.go new file mode 100644 index 00000000000..0be61b12ac9 --- /dev/null +++ b/pkg/artifact/local/register_test.go @@ -0,0 +1,126 @@ +package local + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestRegisterWith_RegistersBothKindsAsSeparateBackends(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + assert.Equal(t, []string{artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive}, Kinds()) + for _, kind := range Kinds() { + assert.True(t, registry.IsRegistered(kind), kind) + } + + dirDesc, ok := registry.Lookup(artifact.RepositoryKindLocalDir) + require.True(t, ok) + archiveDesc, ok := registry.Lookup(artifact.RepositoryKindLocalArchive) + require.True(t, ok) + assert.Equal(t, []artifact.Packaging{artifact.PackagingNone}, dirDesc.Capabilities.Packagings) + assert.Contains(t, archiveDesc.Capabilities.Packagings, artifact.PackagingZip) + assert.Equal(t, artifact.DeleteObject, dirDesc.Capabilities.Delete, "the registered descriptor carries the backend's real capabilities") + assert.True(t, archiveDesc.AllowsPermissions) + assert.False(t, dirDesc.AllowsPermissions) + assert.Equal(t, artifact.PackagingTgz, archiveDesc.DefaultPackaging) + assert.Equal(t, []string{artifact.OptionPath}, archiveDesc.PathOptions) + + require.ErrorIs(t, RegisterWith(registry), errUtils.ErrArtifactConflict, "double registration is an error, not a silent replacement") +} + +func TestRegisteredFactories_ConstructRepositories(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + root := t.TempDir() + + dir, err := registry.New(context.Background(), "dir", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "dir")}, + }) + require.NoError(t, err) + archive, err := registry.New(context.Background(), "archive", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Permissions: "0640", Packaging: "zip", + Options: map[string]any{"path": filepath.Join(root, "archive"), "key_pattern": `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.zip`}, + }) + require.NoError(t, err) + assert.IsType(t, dirStore{}, dir.(*Repository).store) + assert.IsType(t, archiveStore{}, archive.(*Repository).store) + assert.Equal(t, artifact.PackagingZip, archive.(*Repository).packaging) + assert.EqualValues(t, 0o640, archive.(*Repository).archiveMode) + assert.True(t, archive.(*Repository).modeExplicit) + + _, statErr := filepath.Glob(filepath.Join(root, "*")) + require.NoError(t, statErr) + entries, _ := filepath.Glob(filepath.Join(root, "*")) + assert.Empty(t, entries, "construction creates no directories") + + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Packaging: "tgz", Options: map[string]any{"path": root}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a directory repository does not accept archive packaging") + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Permissions: "0600", Options: map[string]any{"path": root}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = registry.New(context.Background(), "bad", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": root, "key_pattern": "../{{ .Name }}.tgz"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a key_pattern that escapes the root is rejected before anything is built") +} + +func TestRegisterWithVerifier_AdvertisesEvidenceFormats(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry, WithVerifier(&artifacttest.FakeVerifier{}), WithOwner("custom-owner"))) + desc, _ := registry.Lookup(artifact.RepositoryKindLocalArchive) + assert.Equal(t, []artifact.EvidenceFormat{artifacttest.FakeEvidenceFormat}, desc.Capabilities.EvidenceFormats) + + repo, err := registry.New(context.Background(), "a", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": t.TempDir()}, + }) + require.NoError(t, err) + caps := repo.Capabilities() + assert.Equal(t, desc.Capabilities.EvidenceFormats, caps.EvidenceFormats, "static and constructed capabilities agree") + assert.Equal(t, "custom-owner", repo.(*Repository).settings.owner) +} + +func TestDescriptor_ReflectsHostAndVerifier(t *testing.T) { + plain := Descriptor(artifact.RepositoryKindLocalDir) + assert.Empty(t, plain.Capabilities.EvidenceFormats) + assert.NotNil(t, plain.NewOptions) + assert.Equal(t, hostEnforcesModes(), plain.Capabilities.PreservesModes) +} + +func TestDefaultRegistryRegistration(t *testing.T) { + artifact.ResetRepositoryKindsForTesting() + t.Cleanup(artifact.ResetRepositoryKindsForTesting) + require.NoError(t, Register()) + repo, err := artifact.NewRepository(context.Background(), "x", &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": t.TempDir()}, + }) + require.NoError(t, err) + assert.IsType(t, &Repository{}, repo) + require.ErrorIs(t, Register(), errUtils.ErrArtifactConflict) +} + +func TestNewRepository_RejectsBadInputs(t *testing.T) { + _, err := NewDir("x", nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = NewDir("x", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalArchive, Options: map[string]any{"path": t.TempDir()}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "a dir constructor refuses another kind's configuration") + _, err = NewArchive("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": ""}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = NewArchive("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": t.TempDir(), "unknown": 1}}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + + repo, err := NewDir("x", &schema.ArtifactRepositoryConfig{Options: map[string]any{"path": "relative/root"}}) + require.NoError(t, err) + assert.True(t, filepath.IsAbs(repo.Root()), "a relative root is made absolute at construction") +} diff --git a/pkg/artifact/local/repository.go b/pkg/artifact/local/repository.go new file mode 100644 index 00000000000..38b4d329c0c --- /dev/null +++ b/pkg/artifact/local/repository.go @@ -0,0 +1,288 @@ +package local + +import ( + "context" + "fmt" + "io/fs" + "path/filepath" + "runtime" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Repository is a local directory or local archive artifact repository. Construct it with +// [NewDir] or [NewArchive]. It is safe for concurrent use: all coordination happens through +// OS file locks and atomic file replacement on disk. +type Repository struct { + alias string + kind string + root string + keyPattern string + packaging artifact.Packaging + store payloadStore + settings settings + + // archiveMode and modeExplicit hold the quoted-octal archive permissions and whether the + // configuration requested them explicitly. They are meaningful for archives only. + archiveMode fs.FileMode + modeExplicit bool + + // hostModes reports whether the host can enforce permission bits. It is a field so tests can + // exercise the unsupported-platform path on any host. + hostModes bool +} + +var ( + _ artifact.Repository = (*Repository)(nil) + _ artifact.EvidenceReader = (*Repository)(nil) + _ artifact.EvidenceWriter = (*Repository)(nil) +) + +// NewDir returns a local/dir repository for the configuration. It performs no I/O: the root is +// created by the first write. +func NewDir(alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "local.NewDir")() + + return newRepository(alias, artifact.RepositoryKindLocalDir, cfg, opts) +} + +// NewArchive returns a local/archive repository for the configuration. It performs no I/O. +func NewArchive(alias string, cfg *schema.ArtifactRepositoryConfig, opts ...Option) (*Repository, error) { + defer perf.Track(nil, "local.NewArchive")() + + return newRepository(alias, artifact.RepositoryKindLocalArchive, cfg, opts) +} + +func newRepository(alias, kind string, cfg *schema.ArtifactRepositoryConfig, opts []Option) (*Repository, error) { + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, ok := artifact.BuiltinKind(kind) + if !ok { + return nil, fmt.Errorf("%w: unknown repository kind %q", errUtils.ErrArtifactConfigInvalid, kind) + } + if err := ensureKind(kind, cfg); err != nil { + return nil, err + } + checked := *cfg + checked.Kind = kind + if err := artifact.ValidateRepositoryConfig(desc, &checked); err != nil { + return nil, err + } + options := desc.NewOptions() + if err := artifact.DecodeOptions(cfg.Options, options); err != nil { + return nil, err + } + root, keyPattern, err := rootAndPattern(options) + if err != nil { + return nil, err + } + repo := &Repository{ + alias: alias, kind: kind, root: root, keyPattern: keyPattern, + packaging: desc.EffectivePackaging(cfg.Packaging), settings: applySettings(opts), + archiveMode: artifact.DefaultArchivePermissions, hostModes: hostEnforcesModes(), + } + if err := repo.configureStore(cfg.Permissions); err != nil { + return nil, err + } + return repo, nil +} + +// configureStore selects the payload store of the kind and, for archives, parses the permissions. +// It records whether the permissions were explicit, because an explicit request that the host +// cannot enforce is an error while a default is not. +func (r *Repository) configureStore(permissions string) error { + if r.kind != artifact.RepositoryKindLocalArchive { + r.store = dirStore{repo: r} + return nil + } + r.store = archiveStore{repo: r} + if permissions == "" { + return nil + } + mode, err := artifact.ParsePermissions(permissions) + if err != nil { + return err + } + r.archiveMode, r.modeExplicit = mode, true + return nil +} + +// rootAndPattern extracts the absolute root and key pattern from decoded options. +func rootAndPattern(options artifact.Options) (string, string, error) { + var path, pattern string + switch typed := options.(type) { + case *artifact.LocalDirOptions: + path, pattern = typed.Path, typed.KeyPattern + case *artifact.LocalArchiveOptions: + path, pattern = typed.Path, typed.KeyPattern + } + root, err := filepath.Abs(filepath.Clean(path)) + if err != nil { + return "", "", fmt.Errorf("%w: repository path %q: %w", errUtils.ErrArtifactConfigInvalid, path, err) + } + return root, pattern, nil +} + +// hostEnforcesModes reports whether the host honors Unix permission bits. Windows maps modes onto +// a read-only attribute and ACLs, so it cannot enforce an arbitrary mode. +func hostEnforcesModes() bool { + return runtime.GOOS != "windows" && runtime.GOOS != "plan9" +} + +// Root returns the absolute repository root. +func (r *Repository) Root() string { + defer perf.Track(nil, "local.Repository.Root")() + + return r.root +} + +// Capabilities implements [artifact.Repository]. It is computed from configuration alone. +func (r *Repository) Capabilities() artifact.Capabilities { + defer perf.Track(nil, "local.Repository.Capabilities")() + + return staticCapabilities(r.kind, r.hostModes, r.settings.verifier) +} + +// staticCapabilities describes a local kind without constructing a repository. +func staticCapabilities(kind string, hostModes bool, verifier artifact.Verifier) artifact.Capabilities { + desc, _ := artifact.BuiltinKind(kind) + caps := desc.Capabilities + caps.Read, caps.Write = true, true + caps.EnumerateNamed, caps.EnumerateCollection = true, true + caps.Delete = artifact.DeleteObject + caps.ConditionalUpdate = true + caps.PreservesModes = hostModes || kind == artifact.RepositoryKindLocalArchive + caps.SubjectKinds = []artifact.SubjectKind{artifact.SubjectRecord} + caps.LockScopes = []artifact.LockScope{artifact.LockScopeHost} + caps.EvidenceFormats = nil + if verifier != nil { + caps.EvidenceFormats = append(caps.EvidenceFormats, verifier.Formats()...) + } + return caps +} + +// pinned builds the immutable reference for a stored Record. +func (r *Repository) pinned(ref artifact.Ref, record *artifact.Record, recordDigest digest.Digest) artifact.PinnedReference { + return artifact.PinnedReference{ + Ref: ref, + Repository: r.root, + ResolvedDigest: recordDigest, + RecordDigest: recordDigest, + ContentDigest: record.ContentDigest, + } +} + +// view is a consistent, verified read of one completed artifact: its reference, Record, and +// representation. +type view struct { + ref *layout.Reference + record *artifact.Record + recordBytes []byte + rep *layout.Representation +} + +// unavailable builds the error for a reference that exists but cannot be used right now. +func (r *Repository) unavailable(ref *layout.Reference) error { + return fmt.Errorf("%w: %s:%s has a pending publication by operation %s; retry the publication with the same intent to repair it", + errUtils.ErrArtifactPublicationUnavailable, ref.Name, ref.Version, ref.OperationID) +} + +// resolved is a completed reference together with the verified Record it names. +type resolved struct { + ref *layout.Reference + record *artifact.Record + recordBytes []byte +} + +// resolveRef finds the reference and Record a request names. A version reference must be +// completed. A digest reference checks the Record's embedded name and succeeds only while that +// Record is still the current completed one: retired digests are not found, never redirected to +// the current bytes. +func (r *Repository) resolveRef(t *tree, ref artifact.Ref) (*resolved, error) { + if err := ref.Validate(); err != nil { + return nil, err + } + if ref.IsDigest() { + return r.resolveDigest(t, ref) + } + stored, err := t.loadReference(layout.ReferenceKey(ref.Name, ref.Version)) + if err != nil { + return nil, err + } + if stored == nil { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + if !stored.Available() { + return nil, r.unavailable(stored) + } + record, raw, err := t.loadRecord(stored.RecordDigest) + if err != nil { + return nil, err + } + if err := matchesReference(stored, record); err != nil { + return nil, err + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +func (r *Repository) resolveDigest(t *tree, ref artifact.Ref) (*resolved, error) { + record, raw, err := t.loadRecord(ref.Digest) + if err != nil { + return nil, err + } + if record.Name != ref.Name { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, ref) + } + stored, err := t.loadReference(layout.ReferenceKey(record.Name, record.Version)) + if err != nil { + return nil, err + } + switch { + case stored == nil: + return nil, fmt.Errorf("%w: %s (the reference was deleted)", errUtils.ErrArtifactNotFound, ref) + case !stored.Available(): + return nil, r.unavailable(stored) + case stored.RecordDigest != ref.Digest: + return nil, fmt.Errorf("%w: %s is retired; %s:%s now resolves to another record", errUtils.ErrArtifactNotFound, ref, record.Name, record.Version) + } + return &resolved{ref: stored, record: record, recordBytes: raw}, nil +} + +// matchesReference checks that a Record agrees with the reference that names it. +func matchesReference(ref *layout.Reference, record *artifact.Record) error { + if record.Name != ref.Name || record.Version != ref.Version { + return fmt.Errorf("%w: record names %s:%s but its reference is %s:%s", errUtils.ErrArtifactIntegrityFailed, record.Name, record.Version, ref.Name, ref.Version) + } + return nil +} + +// openView loads the verified view of a pinned artifact. The pinned Record must still be the +// current completed Record of its reference. +func (r *Repository) openView(ctx context.Context, t *tree, pinned *artifact.PinnedReference) (*view, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + current, err := r.resolveRef(t, artifact.Ref{Name: pinned.Ref.Name, Digest: pinned.RecordDigest}) + if err != nil { + return nil, err + } + stored := current.ref + rep, err := t.loadRepresentation(stored.Key()) + if err != nil { + return nil, err + } + if rep == nil || rep.RecordDigest != pinned.RecordDigest || rep.RepositoryKind != r.kind { + return nil, fmt.Errorf("%w: %s:%s has no valid %s representation for record %s", errUtils.ErrArtifactIntegrityFailed, stored.Name, stored.Version, r.kind, pinned.RecordDigest) + } + if !r.store.supports(rep.Packaging) { + return nil, fmt.Errorf("%w: %s:%s is stored as %q, which this repository cannot read", errUtils.ErrArtifactPackagingInvalid, stored.Name, stored.Version, rep.Packaging) + } + return &view{ref: stored, record: current.record, recordBytes: current.recordBytes, rep: rep}, nil +} diff --git a/pkg/artifact/local/reserve.go b/pkg/artifact/local/reserve.go new file mode 100644 index 00000000000..23f3e8316fa --- /dev/null +++ b/pkg/artifact/local/reserve.go @@ -0,0 +1,91 @@ +package local + +import ( + "fmt" + "slices" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// checkCollision rejects a location that another reference owns or that holds content this +// repository does not manage. A second reference with a colliding key_pattern therefore fails +// with a conflict and is never bound to the first reference's payload, and a user's own files at +// the location are never overwritten. The caller holds the claims lock, so no other publication +// can reserve between the check and the journal write. +func (r *Repository) checkCollision(t *tree, job *publishJob, plan *publishPlan) error { + claims, err := r.collectClaims(t, job.key) + if err != nil { + return err + } + candidate := layout.Claim{ + ReferenceKey: job.key, Name: job.record.Name, Version: job.record.Version, + OperationID: job.opID, Paths: layout.PayloadPaths(r.kind, job.location), + } + if other := layout.FindConflict(&candidate, claims); other != nil { + return fmt.Errorf("%w: location %q for %s:%s is already claimed by %s:%s%s; give the repository a key_pattern that keeps references apart", + errUtils.ErrArtifactConflict, job.location, job.record.Name, job.record.Version, other.Name, other.Version, pendingSuffix(other)) + } + return r.checkUnmanaged(t, &candidate, plan) +} + +func pendingSuffix(claim *layout.Claim) string { + if claim.OperationID == "" { + return "" + } + return " (pending operation " + claim.OperationID + ")" +} + +// collectClaims lists the paths owned by every other reference: the representation of each +// completed reference and the journal of each pending one. +func (r *Repository) collectClaims(t *tree, ownKey string) ([]layout.Claim, error) { + refs, err := t.listReferences() + if err != nil { + return nil, err + } + var claims []layout.Claim + for _, ref := range refs { + if ref.Key() == ownKey { + continue + } + if ref.Available() { + rep, err := t.loadRepresentation(ref.Key()) + if err != nil { + return nil, err + } + if rep != nil { + claims = append(claims, rep.Claim()) + } + continue + } + op, err := t.loadOperation(ref.OperationID) + if err != nil { + return nil, err + } + if op != nil { + claims = append(claims, op.Claim()) + } + } + return claims, nil +} + +// checkUnmanaged fails when a path the publication would write already exists and this reference +// does not own it. +func (r *Repository) checkUnmanaged(t *tree, candidate *layout.Claim, plan *publishPlan) error { + var owned []string + for _, location := range plan.owned { + for _, path := range layout.PayloadPaths(r.kind, location) { + owned = append(owned, layout.FoldPath(path)) + } + } + for _, path := range candidate.Paths { + exists, err := t.exists(path) + if err != nil { + return fmt.Errorf("%w: inspect %q: %w", errUtils.ErrArtifactUploadFailed, path, err) + } + if exists && !slices.Contains(owned, layout.FoldPath(path)) { + return fmt.Errorf("%w: %q already exists and is not managed by this repository; remove it or choose another key_pattern", errUtils.ErrArtifactConflict, path) + } + } + return nil +} diff --git a/pkg/artifact/local/resolve.go b/pkg/artifact/local/resolve.go new file mode 100644 index 00000000000..a4b024edc98 --- /dev/null +++ b/pkg/artifact/local/resolve.go @@ -0,0 +1,232 @@ +package local + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// defaultPageSize is the list page size when the query does not set one. +const defaultPageSize = 100 + +// Resolve implements [artifact.Repository]. A version reference resolves once to the digest of +// its immutable Record, which every later operation uses. A pending reference is unavailable; a +// digest reference checks the Record's embedded name and names only the current Record. +func (r *Repository) Resolve(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "local.Repository.Resolve")() + + if err := ctx.Err(); err != nil { + return artifact.PinnedReference{}, err + } + t, err := r.openTree(false) + if err != nil { + return artifact.PinnedReference{}, r.missingRoot(err, ref) + } + defer t.Close() + current, err := r.resolveRef(t, ref) + if err != nil { + return artifact.PinnedReference{}, err + } + return r.pinned(ref, current.record, current.ref.RecordDigest), nil +} + +// Pin implements [artifact.Repository]. It resolves the reference and lists the evidence +// descriptors known now. It takes no lock and grants no trust. +func (r *Repository) Pin(ctx context.Context, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "local.Repository.Pin")() + + pinned, err := r.Resolve(ctx, ref) + if err != nil { + return pinned, err + } + t, err := r.openTree(false) + if err != nil { + return artifact.PinnedReference{}, err + } + defer t.Close() + pinned.Evidence = descriptorsOf(t.listEvidence(pinned.RecordDigest)) + return pinned, nil +} + +// Inspect implements [artifact.Repository]. It reads only managed records and the small adjacent +// metadata copy; it never opens the payload. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) Inspect(ctx context.Context, pinned artifact.PinnedReference) (*artifact.Inspection, error) { + defer perf.Track(nil, "local.Repository.Inspect")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + if err := r.store.verifySidecars(t, v); err != nil { + return nil, err + } + record := *v.record + inspection := &artifact.Inspection{ + Pinned: r.pinned(pinned.Ref, &record, v.ref.RecordDigest), + Record: &record, + Representation: v.rep.Artifact(t.absolute(v.rep.Location)), + } + inspection.Evidence = t.listEvidence(v.ref.RecordDigest) + return inspection, nil +} + +// OpenDocument implements [artifact.Repository] without reading the payload. The reader verifies +// the document digest and size and rechecks the reference generation at EOF. +// +//nolint:gocritic // The pinned reference is passed by value because artifact.Repository fixes the signature. +func (r *Repository) OpenDocument(ctx context.Context, pinned artifact.PinnedReference, role artifact.DocumentRole) (*artifact.DocumentStream, error) { + defer perf.Track(nil, "local.Repository.OpenDocument")() + + t, err := r.openTree(false) + if err != nil { + return nil, r.missingRoot(err, pinned.Ref) + } + defer t.Close() + v, err := r.openView(ctx, t, &pinned) + if err != nil { + return nil, err + } + doc, ok := documentDescriptor(v.record, role) + if !ok { + return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactDocumentMissing, role) + } + reader, err := r.store.document(t, v, doc, r.guard(v.ref)) + if err != nil { + return nil, err + } + return &artifact.DocumentStream{Document: *doc, Reader: reader}, nil +} + +// List implements [artifact.Repository]. It enumerates managed completed references only: never +// arbitrary user files, pending publications, staging, or journals. Entries are sorted by name +// then version. +func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + defer perf.Track(nil, "local.Repository.List")() + + t, err := r.openTree(false) + if err != nil { + if errors.Is(err, errRootMissing) { + return &artifact.ListPage{}, nil + } + return nil, err + } + defer t.Close() + refs, err := t.listReferences() + if err != nil { + return nil, err + } + candidates := make([]listCandidate, 0, len(refs)) + for _, ref := range refs { + if ref.Available() && (query.Name == "" || ref.Name == query.Name) { + candidates = append(candidates, listCandidate{name: ref.Name, version: ref.Version, digest: ref.RecordDigest}) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].less(&candidates[j]) }) + return r.page(ctx, t, candidates, &query) +} + +// listCandidate is one completed reference awaiting its Record. +type listCandidate struct { + name string + version string + digest digest.Digest +} + +func (c *listCandidate) less(other *listCandidate) bool { + if c.name != other.name { + return c.name < other.name + } + return c.version < other.version +} + +// page loads the Records for one page. Label selectors need every Record, so they are applied +// before paging; without them only the page's Records are read. +func (r *Repository) page(ctx context.Context, t *tree, candidates []listCandidate, query *artifact.ListQuery) (*artifact.ListPage, error) { + size := query.PageSize + if size <= 0 { + size = defaultPageSize + } + if query.Cursor != "" { + start := sort.Search(len(candidates), func(i int) bool { return cursorAfter(&candidates[i], query.Cursor) }) + candidates = candidates[start:] + } + page := &artifact.ListPage{} + for index := range candidates { + if err := ctx.Err(); err != nil { + return nil, err + } + entry, matched, err := r.listEntry(t, &candidates[index], query.Labels) + if err != nil { + return nil, err + } + if !matched { + continue + } + if len(page.Entries) == size { + page.NextCursor = page.Entries[len(page.Entries)-1].Name + ":" + page.Entries[len(page.Entries)-1].Version + return page, nil + } + page.Entries = append(page.Entries, entry) + } + return page, nil +} + +// cursorAfter reports whether the candidate sorts strictly after the cursor's name and version. +func cursorAfter(candidate *listCandidate, cursor string) bool { + index := strings.LastIndex(cursor, ":") + if index < 0 { + return true + } + after := listCandidate{name: cursor[:index], version: cursor[index+1:]} + return after.less(candidate) +} + +func (r *Repository) listEntry(t *tree, candidate *listCandidate, labels []artifact.Label) (artifact.ListEntry, bool, error) { + record, _, err := t.loadRecord(candidate.digest) + if err != nil { + return artifact.ListEntry{}, false, err + } + if record.Name != candidate.name || record.Version != candidate.version { + return artifact.ListEntry{}, false, fmt.Errorf("%w: record %s describes %s:%s but its reference is %s:%s", + errUtils.ErrArtifactIntegrityFailed, candidate.digest, record.Name, record.Version, candidate.name, candidate.version) + } + for _, label := range labels { + if value, ok := record.Metadata.Labels[label.Key]; !ok || value != label.Value { + return artifact.ListEntry{}, false, nil + } + } + return artifact.ListEntry{ + Pinned: r.pinned(artifact.Ref{Name: record.Name, Version: record.Version}, record, candidate.digest), + Name: record.Name, + Version: record.Version, + Kind: record.Kind, + Description: record.Metadata.Description, + Labels: record.Metadata.Labels, + Documents: record.Documents, + }, true, nil +} + +// descriptorsOf lists evidence descriptors. +func descriptorsOf(evidence []artifact.SignatureEvidence) []ocispec.Descriptor { + out := make([]ocispec.Descriptor, 0, len(evidence)) + for index := range evidence { + out = append(out, evidence[index].Descriptor) + } + return out +} diff --git a/pkg/artifact/local/roundtrip_test.go b/pkg/artifact/local/roundtrip_test.go new file mode 100644 index 00000000000..f8b566418ab --- /dev/null +++ b/pkg/artifact/local/roundtrip_test.go @@ -0,0 +1,303 @@ +package local + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/layout" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +// publishFilePackage publishes the shared "File package" fixture: nested files, an executable +// helper, an empty file, an empty directory, a confined symlink, a Unicode filename, and distinct +// package README and CHANGELOG documents. +func publishFilePackage(t *testing.T, repo *Repository) (*artifact.PublishResult, fixtures.FilePackage) { + t.Helper() + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: testName, Version: testVersion, Packaging: repo.packaging, + OperationID: "op-file-package", PreparedAt: capturedAt, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + result, err := repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + return result, pkg +} + +func TestRoundTrip_EveryAllowedPackaging(t *testing.T) { + specs := []*repoSpec{ + dirSpec(), + archiveSpec(artifact.PackagingTar), + archiveSpec(artifact.PackagingTgz), + archiveSpec(artifact.PackagingZip), + } + for _, spec := range specs { + name := spec.kind + "/" + string(spec.packaging) + t.Run(name, func(t *testing.T) { + repo := newRepo(t, spec) + result, pkg := publishFilePackage(t, repo) + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assert.Equal(t, artifact.LockScopeHost, result.LockScope) + assert.Equal(t, repo.packaging, result.Representation.Packaging) + + pinned := resolve(t, repo, defaultRef()) + assert.Equal(t, pinned.RecordDigest, pinned.ResolvedDigest) + assert.Equal(t, result.Pinned.RecordDigest, pinned.RecordDigest) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state, "the pulled tree matches the inventory, modes included") + + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), listInventoryPaths(content.Record)) + assert.NoDirExists(t, filepath.Join(out, ".atmos"), "the distribution directory never leaks into pulled user content") + assert.FileExists(t, filepath.Join(out, "README.md"), "a root README is ordinary payload") + t.Run("POSIX modes", func(t *testing.T) { + fixtures.RequirePOSIXModes(t) + info, err := os.Stat(filepath.Join(out, "bin", "helper.sh")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&0o111, "the executable bit survives the round trip") + }) + }) + } +} + +func listInventoryPaths(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for index := range record.Inventory { + paths = append(paths, record.Inventory[index].Path) + } + sort.Strings(paths) + return paths +} + +func TestDirLayout_PayloadIsOrdinaryFilesWithAdjacentDistribution(t *testing.T) { + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + location := filepath.Join(repo.root, "team", "app", "1.0.0") + assert.Equal(t, location, result.Location) + + assert.FileExists(t, filepath.Join(location, "README.md")) + data, err := os.ReadFile(filepath.Join(location, "nested", "deep", "file.txt")) + require.NoError(t, err) + assert.Equal(t, "deep content\n", string(data), "payload files are ordinary copied files") + assert.DirExists(t, filepath.Join(location, "emptydir")) + + dist := filepath.Join(location, ".atmos", "artifact") + for name, want := range map[string]string{"README.md": fixtures.PackageReadme, "CHANGELOG.md": fixtures.PackageChangelog} { + got, err := os.ReadFile(filepath.Join(dist, name)) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + } + manifest, err := os.ReadFile(filepath.Join(dist, "manifest.json")) + require.NoError(t, err) + record, err := artifact.DecodeRecord(manifest) + require.NoError(t, err) + assert.Equal(t, testName, record.Name) + + for _, managed := range []string{ + layout.ReferencePath(layout.ReferenceKey(testName, testVersion)), + layout.RepresentationPath(layout.ReferenceKey(testName, testVersion)), + layout.RecordPath(result.Pinned.RecordDigest), + } { + assert.FileExists(t, filepath.Join(repo.root, filepath.FromSlash(managed))) + } + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.StagingPath("op-file-package"))), "staging is cleaned") + assert.NoFileExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.OperationPath("op-file-package"))), "the finished journal is removed") +} + +func TestArchiveLayout_SidecarsAdjacentAndPortableCopiesEmbedded(t *testing.T) { + repo := newRepo(t, archiveSpec(artifact.PackagingTgz)) + result, _ := publishFilePackage(t, repo) + archivePath := filepath.Join(repo.root, "team", "app", "1.0.0.tgz") + assert.Equal(t, archivePath, result.Location) + + assert.FileExists(t, archivePath) + metadata, err := os.ReadFile(archivePath + ".metadata.json") + require.NoError(t, err) + record, err := artifact.DecodeRecord(metadata) + require.NoError(t, err) + assert.Equal(t, result.Pinned.ContentDigest, record.ContentDigest) + for name, want := range map[string]string{"README.md": fixtures.PackageReadme, "CHANGELOG.md": fixtures.PackageChangelog} { + got, err := os.ReadFile(filepath.Join(archivePath+".docs", name)) + require.NoError(t, err) + assert.Equal(t, want, string(got)) + } + + members := tgzMembers(t, archivePath) + assert.Contains(t, members, ".atmos/artifact/manifest.json", "the archive embeds a portable manifest") + assert.Contains(t, members, ".atmos/artifact/README.md") + assert.Equal(t, string(metadata), members[".atmos/artifact/manifest.json"], "the embedded manifest equals the adjacent metadata") +} + +// tgzMembers returns file member contents by name. +func tgzMembers(t *testing.T, archivePath string) map[string]string { + t.Helper() + file, err := os.Open(archivePath) + require.NoError(t, err) + defer file.Close() + gz, err := gzip.NewReader(file) + require.NoError(t, err) + members := map[string]string{} + reader := tar.NewReader(gz) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + if header.Typeflag == tar.TypeReg { + data, err := io.ReadAll(reader) + require.NoError(t, err) + members[header.Name] = string(data) + } + } + return members +} + +func TestDocumentsAreReadWithoutThePayload(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + // Make the main payload unreadable: documents and metadata must not need it. + if spec.kind == artifact.RepositoryKindLocalArchive { + require.NoError(t, os.Remove(result.Location)) + } else { + require.NoError(t, os.Remove(filepath.Join(result.Location, "README.md"))) + require.NoError(t, os.Remove(filepath.Join(result.Location, "nested", "deep", "file.txt"))) + } + + stream, err := repo.OpenDocument(context.Background(), pinned, artifact.DocumentChangelog) + require.NoError(t, err) + t.Cleanup(func() { _ = stream.Reader.Close() }) + data, err := io.ReadAll(stream.Reader) + require.NoError(t, err) + require.NoError(t, stream.Reader.Close()) + assert.Equal(t, fixtures.PackageChangelog, string(data)) + + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + assert.Equal(t, testName, inspection.Record.Name) + + _, err = repo.OpenDocument(context.Background(), pinned, "unknown-role") + require.ErrorIs(t, err, errUtils.ErrArtifactDocumentMissing) + + content, err := repo.Read(context.Background(), pinned) + if err == nil { + defer content.Close() + err = readAll(t, content) + } + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "the payload itself is verified and reports the loss") + }) + } +} + +func TestPullReplacementRemovesStaleFilesInsideTheOutputOnly(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + + parent := t.TempDir() + out := filepath.Join(parent, "out") + require.NoError(t, os.MkdirAll(filepath.Join(out, "stale"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(out, "stale", "old.txt"), []byte("old"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(out, "README.md"), []byte("old readme"), 0o644)) + sibling := filepath.Join(parent, "sibling.txt") + require.NoError(t, os.WriteFile(sibling, []byte("keep"), 0o644)) + + content, err := repo.Read(context.Background(), pinned) + require.NoError(t, err) + defer content.Close() + state, err := materialize.InspectDestination(out, content.Record) + require.NoError(t, err) + assert.NotEqual(t, materialize.DestinationMatches, state) + err = materialize.Materialize(context.Background(), content, out) + if runtime.GOOS == "windows" { + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + old, readErr := os.ReadFile(filepath.Join(out, "stale", "old.txt")) + require.NoError(t, readErr) + assert.Equal(t, "old", string(old), "unsupported replacement preserves the original output") + readme, readErr := os.ReadFile(filepath.Join(out, "README.md")) + require.NoError(t, readErr) + assert.Equal(t, "old readme", string(readme)) + assert.FileExists(t, sibling) + return + } + require.NoError(t, err) + + assert.NoFileExists(t, filepath.Join(out, "stale", "old.txt"), "stale files inside the output are removed, not merged") + assert.FileExists(t, sibling, "nothing outside the requested output root is touched") + got, err := os.ReadFile(filepath.Join(out, "README.md")) + require.NoError(t, err) + assert.Equal(t, fixtures.RootReadme, string(got)) + }) + } +} + +func TestSafeRelativeSymlinkRoundTrips(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTar)} { + t.Run(spec.kind, func(t *testing.T) { + repo := newRepo(t, spec) + src := t.TempDir() + fixtures.WriteFile(t, src, "real/target.txt", "target\n", 0o644) + if !fixtures.Symlink(t, src, "alias.txt", "real/target.txt") { + t.Skip("symlinks are unavailable on this host") + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: testName, Version: testVersion, Packaging: repo.packaging, OperationID: "op-link", PreparedAt: capturedAt, + }) + require.NoError(t, err) + defer prepared.Close() + _, err = repo.Publish(context.Background(), prepared, artifact.PublishOptions{}) + require.NoError(t, err) + + content, err := repo.Read(context.Background(), resolve(t, repo, defaultRef())) + require.NoError(t, err) + defer content.Close() + out := filepath.Join(t.TempDir(), "out") + require.NoError(t, materialize.Materialize(context.Background(), content, out)) + target, err := os.Readlink(filepath.Join(out, "alias.txt")) + require.NoError(t, err) + assert.Equal(t, filepath.FromSlash("real/target.txt"), target, "a contained relative link stays a link") + }) + } +} + +func TestExecutableBitsSurviveAsOrdinaryFiles(t *testing.T) { + fixtures.RequirePOSIXModes(t) + repo := newRepo(t, dirSpec()) + result, _ := publishFilePackage(t, repo) + info, err := os.Stat(filepath.Join(result.Location, "bin", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), "the stored payload keeps the ordinary executable mode") + info, err = os.Stat(filepath.Join(result.Location, fixtures.UnicodeName)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o640), info.Mode().Perm()) + assert.True(t, repo.Capabilities().PreservesModes) +} diff --git a/pkg/artifact/local/service_test.go b/pkg/artifact/local/service_test.go new file mode 100644 index 00000000000..2ec6603cc27 --- /dev/null +++ b/pkg/artifact/local/service_test.go @@ -0,0 +1,151 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/materialize" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// newConfiguredService wires the registered local kinds through the configuration-driven +// repository table, the way a composition root does. +func newConfiguredService(t *testing.T) (*service.Service, string) { + t.Helper() + root := t.TempDir() + registry := artifact.NewRepositoryRegistry() + require.NoError(t, RegisterWith(registry)) + packed := schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalArchive, Packaging: "tgz", + Options: map[string]any{"path": filepath.Join(root, "packed")}, + } + // Windows cannot enforce archive file modes, and the repository rejects the setting there. + if fixtures.POSIXModes() { + packed.Permissions = "0600" + } + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "files": {Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": filepath.Join(root, "files")}}, + "packed": packed, + }} + svc, err := service.New(service.FromConfig(cfg, registry), nil, + service.WithStagingRoot(t.TempDir()), service.WithRetry(2, time.Millisecond), service.WithSleep(func(context.Context, time.Duration) error { return nil })) + require.NoError(t, err) + return svc, root +} + +func TestService_DrivesBothLocalBackends(t *testing.T) { + svc, root := newConfiguredService(t) + ctx := context.Background() + src := t.TempDir() + pkg := fixtures.WriteFilePackage(t, src) + + published, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, published.Outcome) + assert.Equal(t, artifact.LockScopeHost, published.LockScope) + assert.Equal(t, filepath.Join(root, "files", "team", "app", "1.0.0"), published.Location) + + again, err := svc.Publish(ctx, artifact.PublishRequest{Repository: "files", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeUnchanged, again.Outcome, "an identical republication is a no-op") + + // Local directory to local archive: Record identity survives repackaging. + copied, err := svc.Copy(ctx, artifact.CopyRequest{Ref: defaultRef(), From: "files", To: "packed"}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeCreated, copied.Outcome) + assert.Equal(t, filepath.Join(root, "packed", "team", "app", "1.0.0.tgz"), copied.Location) + + listed, err := svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + require.Len(t, listed.Artifacts, 2) + assert.Equal(t, listed.Artifacts[0].Pinned.RecordDigest, listed.Artifacts[1].Pinned.RecordDigest, "the same Record in both packagings") + + document, err := svc.ShowDocument(ctx, artifact.ShowDocumentRequest{Repository: "packed", Ref: defaultRef(), Role: artifact.DocumentReadme}) + require.NoError(t, err) + assert.Equal(t, fixtures.PackageReadme, string(document.Content)) + + out := filepath.Join(t.TempDir(), "pulled") + pulled, err := svc.Pull(ctx, artifact.PullRequest{Repository: "packed", Ref: defaultRef(), Destination: out}) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeRead, pulled.Outcome) + inspection, err := svc.Inspect(ctx, artifact.InspectRequest{Repository: "files", Ref: defaultRef()}) + require.NoError(t, err) + state, err := materialize.InspectDestination(out, inspection.Inspection.Record) + require.NoError(t, err) + assert.Equal(t, materialize.DestinationMatches, state) + assert.Equal(t, fixtures.FilePackagePaths(pkg.Symlinks), listInventoryPaths(inspection.Inspection.Record)) + + pin, err := svc.Pin(ctx, artifact.PinRequest{Repository: "packed", Ref: defaultRef()}) + require.NoError(t, err) + assert.Equal(t, inspection.Inspection.Pinned.RecordDigest, pin.Pinned.RecordDigest) + + _, err = svc.Delete(ctx, artifact.DeleteRequest{Repository: "files", Ref: defaultRef()}) + require.NoError(t, err) + listed, err = svc.List(ctx, artifact.ListRequest{}) + require.NoError(t, err) + require.Len(t, listed.Artifacts, 1) + assert.Equal(t, "packed", listed.Artifacts[0].Repository) + assert.NoDirExists(t, filepath.Join(root, "files", "team")) +} + +// The service reconciles a commit whose response was lost by reading the completed reference. +func TestService_ReconcilesALostResponseAfterCommit(t *testing.T) { + for _, spec := range []*repoSpec{dirSpec(), archiveSpec(artifact.PackagingTgz)} { + t.Run(spec.kind, func(t *testing.T) { + spec = spec.withOpts(withFaults(crashAt(faultAfterCommit))) + repo := newRepo(t, spec) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"r": repo}}, nil, service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload\n", 0o644) + + result, err := svc.Publish(context.Background(), artifact.PublishRequest{Repository: "r", Source: src, Name: testName, Version: testVersion}) + require.NoError(t, err, "the committed reference proves success") + var reconciled bool + for _, diagnostic := range result.Diagnostics { + reconciled = reconciled || diagnostic.Code == "reconciled" + } + assert.True(t, reconciled, "success was confirmed by reading the reference: %v", result.Diagnostics) + assertIntact(t, repo, defaultRef()) + }) + } +} + +// An interruption before the commit is surfaced as unavailable, and the service's own retry of the +// same intent repairs it. +func TestService_RetryRepairsAnInterruptedPublication(t *testing.T) { + var fault armedFault + spec := dirSpec() + spec = spec.withOpts(fault.option()) + repo := newRepo(t, spec) + fault.hook = crashAt(faultAfterMetadata) + svc, err := service.New(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"r": repo}}, nil, + service.WithStagingRoot(t.TempDir()), service.WithRetry(1, time.Millisecond)) + require.NoError(t, err) + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", "payload\n", 0o644) + request := artifact.PublishRequest{Repository: "r", Source: src, Name: testName, Version: testVersion} + + _, err = svc.Publish(context.Background(), request) + require.Error(t, err) + _, err = repo.Resolve(context.Background(), defaultRef()) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + + result, err := svc.Publish(context.Background(), request) + require.NoError(t, err, "a later invocation with the same intent repairs the pending operation") + assert.Equal(t, artifact.OutcomeCreated, result.Outcome) + assertIntact(t, repo, defaultRef()) + _, statErr := os.Stat(filepath.Join(repo.root, ".atmos-artifacts", "v1", "operations")) + require.NoError(t, statErr) + assert.Empty(t, listFiles(t, filepath.Join(repo.root, ".atmos-artifacts", "v1", "operations"))) +} diff --git a/pkg/artifact/local/signature_test.go b/pkg/artifact/local/signature_test.go new file mode 100644 index 00000000000..c56d7976c61 --- /dev/null +++ b/pkg/artifact/local/signature_test.go @@ -0,0 +1,401 @@ +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +func signedSpec(spec *repoSpec) *repoSpec { + return spec.withOpts(WithVerifier(&artifacttest.FakeVerifier{})) +} + +// recordSubject builds the Record signature subject the service uses for the default reference. +func recordSubject(t *testing.T, repo *Repository) artifact.SignatureSubject { + t.Helper() + inspection, err := repo.Inspect(context.Background(), resolve(t, repo, defaultRef())) + require.NoError(t, err) + canonical, err := inspection.Record.Canonical() + require.NoError(t, err) + return artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromBytes(canonical), MediaType: artifact.MediaTypeConfig, Size: int64(len(canonical))} +} + +// snapshot reads every file below dir into a map for before/after comparison. +func snapshot(t *testing.T, dir string, skip func(rel string) bool) map[string]string { + t.Helper() + out := map[string]string{} + for _, rel := range listFiles(t, dir) { + if skip != nil && skip(rel) { + continue + } + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + out[rel] = string(data) + } + return out +} + +func notEvidence(rel string) bool { + return len(rel) >= len(layout.ManagedRoot+"/signatures") && rel[:len(layout.ManagedRoot+"/signatures")] == layout.ManagedRoot+"/signatures" +} + +func TestSignature_StorageNeverMutatesTheSignedRecord(t *testing.T) { + for _, spec := range allSpecs { + t.Run(specName(spec), func(t *testing.T) { + repo := newRepo(t, signedSpec(spec)) + result, _ := publishFilePackage(t, repo) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo) + assert.Equal(t, pinned.RecordDigest, subject.Digest) + before := snapshot(t, repo.root, nil) + + signer := newSigner() + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{OperationID: "sign-op"}) + require.NoError(t, err) + require.Len(t, evidence, 1) + assert.Equal(t, layout.SignaturePath(artifact.SubjectRecord, subject.Digest, evidence[0].Descriptor.Digest), evidence[0].Locator) + assert.FileExists(t, filepath.Join(repo.root, filepath.FromSlash(evidence[0].Locator))) + + after := snapshot(t, repo.root, nil) + added := 0 + for rel, content := range after { + if notEvidence(rel) { + added++ + continue + } + assert.Equal(t, before[rel], content, "%s must be byte-identical after signing", rel) + } + assert.Equal(t, 1, added, "exactly one detached evidence file was added") + assert.Len(t, after, len(before)+1) + + again := resolve(t, repo, defaultRef()) + assert.Equal(t, pinned.RecordDigest, again.RecordDigest, "the signed Record, and so its digest, did not change") + assert.Equal(t, result.Pinned.RecordDigest, again.RecordDigest) + assertIntact(t, repo, defaultRef()) + + inspection, err := repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.Len(t, inspection.Evidence, 1) + assert.Equal(t, evidence[0].Descriptor.Digest, inspection.Evidence[0].Descriptor.Digest) + pin, err := repo.Pin(context.Background(), defaultRef()) + require.NoError(t, err) + require.Len(t, pin.Evidence, 1, "a pin carries the evidence descriptors known now") + + verified, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + assert.NotEmpty(t, verified.PolicyFingerprint) + _, err = repo.Verify(context.Background(), subject, signer.UntrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) + } +} + +// Signing an archive after publication leaves the archive bytes untouched: the evidence covers the +// Record that describes it, and no claim is made about the original archive bytes. +func TestSignature_ArchiveBytesAreNeverRewritten(t *testing.T) { + repo := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + result, _ := publishFilePackage(t, repo) + archiveBefore, err := os.ReadFile(result.Location) + require.NoError(t, err) + + _, err = repo.Sign(context.Background(), recordSubject(t, repo), newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + + archiveAfter, err := os.ReadFile(result.Location) + require.NoError(t, err) + assert.Equal(t, archiveBefore, archiveAfter) + assert.Equal(t, digest.FromBytes(archiveBefore), result.Representation.RepresentationDigest, "the representation checksum still describes the unsigned archive") +} + +func TestSignature_VerifyFailsClosed(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + signer := newSigner() + + result, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "no evidence is never verified") + require.NotNil(t, result) + assert.Equal(t, artifact.VerificationFailed, result.Status) + + _, err = repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + // A malformed file and a file bound to another subject are reported and ignored; they can + // neither break nor override the valid evidence. + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + junk := digest.FromString("junk") + require.NoError(t, os.WriteFile(filepath.Join(dir, junk.Encoded()+".json"), []byte("not json"), 0o644)) + foreign := digest.FromString("foreign subject") + evidence, err := layout.NewEvidence(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: foreign}, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte("{}")}) + require.NoError(t, err) + raw, err := evidence.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, evidence.Digest().Encoded()+".json"), raw, 0o644)) + + verified, err := repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + codes := map[string]bool{} + for _, diagnostic := range verified.Diagnostics { + codes[diagnostic.Code] = true + } + assert.True(t, codes["malformed-evidence"], "malformed evidence is bounded and reported: %v", verified.Diagnostics) + assert.True(t, codes["subject-mismatch"], "foreign evidence is reported: %v", verified.Diagnostics) + + // Evidence whose bytes were edited no longer matches its file name and is ignored. + files := listFiles(t, dir) + for _, name := range files { + if name != junk.Encoded()+".json" && name != evidence.Digest().Encoded()+".json" { + tamper(t, dir, name, appendJunk) + } + } + _, err = repo.Verify(context.Background(), subject, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed, "tampered evidence cannot verify") +} + +func TestSignature_SubjectBinding(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + signer := newSigner() + + t.Run("unsupported subject kind", func(t *testing.T) { + other := subject + other.Kind = artifact.SubjectOCIRoot + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + _, err = repo.Verify(context.Background(), other, signer.TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("unknown record", func(t *testing.T) { + other := subject + other.Digest = digest.FromString("missing") + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + t.Run("wrong declared size", func(t *testing.T) { + other := subject + other.Size++ + _, err := repo.Sign(context.Background(), other, signer, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) + t.Run("nil signer", func(t *testing.T) { + _, err := repo.Sign(context.Background(), subject, nil, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("format the repository cannot verify", func(t *testing.T) { + // Sign has no expectation: an unusable signer must be refused before it runs. + foreign := artifact.NewMockSigner(gomock.NewController(t)) + foreign.EXPECT().Formats().Return([]artifact.EvidenceFormat{"other"}).AnyTimes() + _, err := repo.Sign(context.Background(), subject, foreign, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + t.Run("signer produces nothing", func(t *testing.T) { + empty := artifact.NewMockSigner(gomock.NewController(t)) + empty.EXPECT().Formats().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}).AnyTimes() + empty.EXPECT().Sign(gomock.Any(), gomock.Any()).Return(nil, nil) + _, err := repo.Sign(context.Background(), subject, empty, artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + }) +} + +func TestSignature_WithoutVerifierIsACapabilityError(t *testing.T) { + repo := newRepo(t, dirSpec()) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "no advertised evidence format means signing is unavailable, not a silent no-op") + _, err = repo.Verify(context.Background(), subject, newSigner().TrustedPolicy()) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestSignature_ExportAndImportEvidenceAcrossRepositories(t *testing.T) { + source := newRepo(t, signedSpec(archiveSpec(artifact.PackagingTgz))) + publish(t, source, &prepareSpec{}, artifact.PublishOptions{}) + sourcePinned := resolve(t, source, defaultRef()) + subject := recordSubject(t, source) + signer := newSigner() + _, err := source.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + blobs, err := source.ReadEvidence(context.Background(), &sourcePinned) + require.NoError(t, err) + require.Len(t, blobs, 1) + + // The destination stores the same Record in a different packaging. The Record, and so the + // signed subject, is identical, so imported evidence verifies there without re-signing. + destination := newRepo(t, signedSpec(dirSpec())) + publish(t, destination, &prepareSpec{}, artifact.PublishOptions{}) + destPinned := resolve(t, destination, defaultRef()) + assert.Equal(t, subject.Digest, destPinned.RecordDigest, "packaging does not change Record identity") + _, err = destination.WriteEvidence(context.Background(), &destPinned, blobs, nil) + require.NoError(t, err) + verified, err := destination.Verify(context.Background(), recordSubject(t, destination), signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) + + different := newRepo(t, signedSpec(dirSpec())) + publish(t, different, &prepareSpec{files: map[string]string{"data.txt": "other content"}}, artifact.PublishOptions{}) + differentPinned := resolve(t, different, defaultRef()) + _, err = different.WriteEvidence(context.Background(), &differentPinned, blobs, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid, "evidence for another Record is never imported") + + imported, err := source.WriteEvidence(context.Background(), &sourcePinned, blobs, nil) + require.NoError(t, err) + require.Len(t, imported, 1) + assert.Equal(t, blobs[0].Evidence.Descriptor.Digest, imported[0].Descriptor.Digest, "re-import is idempotent") + again, err := source.ReadEvidence(context.Background(), &sourcePinned) + require.NoError(t, err) + assert.Len(t, again, 1) + + corrupted := blobs[0] + corrupted.Data = append([]byte(nil), corrupted.Data...) + corrupted.Data[0] ^= 0xff + _, err = source.WriteEvidence(context.Background(), &sourcePinned, []artifact.EvidenceBlob{corrupted}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) +} + +func TestSignature_EvidenceFollowsTheRecordLifecycle(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v1"}}, artifact.PublishOptions{}) + old := resolve(t, repo, defaultRef()) + oldSubject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + evidenceDir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, oldSubject.Digest))) + assert.DirExists(t, evidenceDir) + + publish(t, repo, &prepareSpec{files: map[string]string{"data.txt": "v2"}, opID: "op-2"}, artifact.PublishOptions{Force: true, ExpectedPrior: &old}) + assert.NoDirExists(t, evidenceDir, "evidence of a retired Record is retired with it") + _, err = repo.Sign(context.Background(), oldSubject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a retired Record cannot be signed") + + current := resolve(t, repo, defaultRef()) + currentSubject := recordSubject(t, repo) + _, err = repo.Sign(context.Background(), currentSubject, newSigner(), artifact.SignOptions{}) + require.NoError(t, err) + currentDir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, currentSubject.Digest))) + assert.DirExists(t, currentDir) + _, err = repo.Delete(context.Background(), current, artifact.DeleteOptions{Scope: artifact.DeleteObject}) + require.NoError(t, err) + assert.NoDirExists(t, currentDir, "delete removes the evidence the artifact exclusively owns") +} + +func TestSignature_ReadEvidenceExportsOnlyThePinnedSubject(t *testing.T) { + source := newRepo(t, signedSpec(dirSpec())) + publish(t, source, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, source, defaultRef()) + subject := recordSubject(t, source) + signer := newSigner() + signed, err := source.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.NoError(t, err) + + // A valid envelope for another subject sits in the same directory, as the verification test + // above explicitly permits. Exporting it would make the destination reject the whole batch. + dir := filepath.Join(source.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + foreign, err := layout.NewEvidence(artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: digest.FromString("foreign subject")}, + &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte("{}")}) + require.NoError(t, err) + raw, err := foreign.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, foreign.Digest().Encoded()+".json"), raw, 0o644)) + require.Len(t, listFiles(t, dir), 2, "both envelopes are on disk") + + blobs, err := source.ReadEvidence(context.Background(), &pinned) + require.NoError(t, err) + require.Len(t, blobs, 1, "only evidence bound to the pinned Record is exported") + assert.Equal(t, signed[0].Descriptor.Digest, blobs[0].Evidence.Descriptor.Digest) + assert.Equal(t, subject.Digest, blobs[0].Evidence.Subject.Digest) + + destination := newRepo(t, signedSpec(dirSpec())) + publish(t, destination, &prepareSpec{}, artifact.PublishOptions{}) + destPinned := resolve(t, destination, defaultRef()) + stored, err := destination.WriteEvidence(context.Background(), &destPinned, blobs, nil) + require.NoError(t, err, "the destination accepts the exported batch") + require.Len(t, stored, 1) + verified, err := destination.Verify(context.Background(), recordSubject(t, destination), signer.TrustedPolicy()) + require.NoError(t, err) + assert.Equal(t, artifact.VerificationVerified, verified.Status) +} + +func TestSignature_ReadEvidenceReportsExhaustedMalformedEntries(t *testing.T) { + repo, subject := signedWith(t, &artifacttest.FakeVerifier{}) + pinned := resolve(t, repo, defaultRef()) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + for i := range maxMalformedEvidence + 1 { + require.NoError(t, os.WriteFile(filepath.Join(dir, fmt.Sprintf("bad-%03d.json", i)), []byte("bad"), 0o644)) + } + + blobs, err := repo.ReadEvidence(context.Background(), &pinned) + require.ErrorIs(t, err, errUtils.ErrArtifactEvidenceMalformed) + assert.Empty(t, blobs) +} + +// A signer may wait long enough for the Record to be deleted. Signing must then fail instead of +// recreating the evidence directory for a Record that no longer exists. +func TestSignature_SignDoesNotResurrectEvidenceForADeletedRecord(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + pinned := resolve(t, repo, defaultRef()) + subject := recordSubject(t, repo) + + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}).AnyTimes() + signer.EXPECT().Sign(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + _, deleteErr := repo.Delete(ctx, pinned, artifact.DeleteOptions{}) + require.NoError(t, deleteErr, "the Record is deleted while the signer is working") + return &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: []byte(`{"late":true}`)}, nil + }) + + evidence, err := repo.Sign(context.Background(), subject, signer, artifact.SignOptions{}) + require.Error(t, err, "signing a deleted Record must not report success") + assert.Empty(t, evidence) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))), + "no evidence directory is recreated for the deleted Record") +} + +func TestSignature_SignStoresEvidenceUnderTheReferenceLock(t *testing.T) { + repo := newRepo(t, signedSpec(dirSpec())) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + resource := testName + ":" + testVersion + + handle := holdLock(t, repo, lockRequest(resource)) + + t.Run("another writer holds the lock, so storing waits and honors the deadline", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, signErr := repo.Sign(ctx, subject, newSigner(), artifact.SignOptions{}) + require.ErrorIs(t, signErr, context.DeadlineExceeded) + assert.NoDirExists(t, filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest)))) + }) + t.Run("a matching caller lock is reused", func(t *testing.T) { + evidence, signErr := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{Lock: handle}) + require.NoError(t, signErr) + require.Len(t, evidence, 1) + }) + require.NoError(t, handle.Release(context.Background())) + + t.Run("the lock is released after signing", func(t *testing.T) { + _, signErr := repo.Sign(context.Background(), subject, newSigner(), artifact.SignOptions{}) + require.NoError(t, signErr) + reacquired, lockErr := repo.Lock(context.Background(), lockRequest(resource)) + require.NoError(t, lockErr) + require.NoError(t, reacquired.Release(context.Background())) + }) +} diff --git a/pkg/artifact/local/state.go b/pkg/artifact/local/state.go new file mode 100644 index 00000000000..3054d3b9fab --- /dev/null +++ b/pkg/artifact/local/state.go @@ -0,0 +1,167 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// readManaged reads a managed document, mapping a missing file to (nil, nil). +func (t *tree) readManaged(rel string, limit int64) ([]byte, error) { + raw, err := t.readBounded(rel, limit) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, nil + case err != nil: + return nil, fmt.Errorf("%w: read %s: %w", errUtils.ErrArtifactIntegrityFailed, rel, err) + } + return raw, nil +} + +// loadReference returns the stored reference, or nil when it does not exist. +func (t *tree) loadReference(key string) (*layout.Reference, error) { + raw, err := t.readManaged(layout.ReferencePath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + ref, err := layout.DecodeReference(raw) + if err != nil { + return nil, fmt.Errorf("reference %s: %w", key, err) + } + if ref.Key() != key { + return nil, fmt.Errorf("%w: reference file %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, ref.Name, ref.Version) + } + return ref, nil +} + +// loadRepresentation returns the stored representation, or nil when it does not exist. +func (t *tree) loadRepresentation(key string) (*layout.Representation, error) { + raw, err := t.readManaged(layout.RepresentationPath(key), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + rep, err := layout.DecodeRepresentation(raw) + if err != nil { + return nil, fmt.Errorf("representation %s: %w", key, err) + } + if layout.ReferenceKey(rep.Name, rep.Version) != key { + return nil, fmt.Errorf("%w: representation file %s describes %s:%s", errUtils.ErrArtifactIntegrityFailed, key, rep.Name, rep.Version) + } + return rep, nil +} + +// loadOperation returns the stored journal, or nil when it does not exist. +func (t *tree) loadOperation(operationID string) (*layout.Operation, error) { + raw, err := t.readManaged(layout.OperationPath(operationID), layout.MaxDocumentBytes) + if err != nil || raw == nil { + return nil, err + } + op, err := layout.DecodeOperation(raw) + if err != nil { + return nil, fmt.Errorf("operation %s: %w", operationID, err) + } + if op.OperationID != operationID { + return nil, fmt.Errorf("%w: operation journal for %q describes %q", errUtils.ErrArtifactIntegrityFailed, operationID, op.OperationID) + } + return op, nil +} + +// loadRecord reads and verifies the immutable Record with the given digest. A missing Record is +// not found; a Record whose bytes do not hash to its file name, or that is not canonical, fails +// integrity. +func (t *tree) loadRecord(recordDigest digest.Digest) (*artifact.Record, []byte, error) { + if err := artifact.ValidateDigest(recordDigest); err != nil { + return nil, nil, err + } + raw, err := t.readManaged(layout.RecordPath(recordDigest), artifact.MaxRecordSize) + if err != nil { + return nil, nil, err + } + if raw == nil { + return nil, nil, fmt.Errorf("%w: record %s", errUtils.ErrArtifactNotFound, recordDigest) + } + if got := digest.FromBytes(raw); got != recordDigest { + return nil, nil, fmt.Errorf("%w: record file %s holds bytes that hash to %s", errUtils.ErrArtifactIntegrityFailed, recordDigest, got) + } + record, err := artifact.DecodeRecord(raw) + if err != nil { + return nil, nil, fmt.Errorf("%w: record %s: %w", errUtils.ErrArtifactIntegrityFailed, recordDigest, err) + } + return record, raw, nil +} + +// writeReference replaces the reference record atomically. +func (t *tree) writeReference(ref *layout.Reference) error { + raw, err := ref.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.ReferencePath(ref.Key()), raw) +} + +// writeRepresentation replaces the representation record atomically. +func (t *tree) writeRepresentation(rep *layout.Representation) error { + raw, err := rep.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.RepresentationPath(layout.ReferenceKey(rep.Name, rep.Version)), raw) +} + +// writeOperation replaces the operation journal atomically. +func (t *tree) writeOperation(op *layout.Operation) error { + raw, err := op.Marshal() + if err != nil { + return err + } + return t.writeAtomic(layout.OperationPath(op.OperationID), raw) +} + +// writeRecord stores the immutable canonical Record. An existing file with identical bytes is +// left alone; different bytes under the same digest name are corruption. +func (t *tree) writeRecord(recordDigest digest.Digest, canonical []byte) error { + existing, err := t.readManaged(layout.RecordPath(recordDigest), artifact.MaxRecordSize) + if err != nil { + return err + } + if existing != nil && string(existing) != string(canonical) { + return fmt.Errorf("%w: record file %s already exists with different bytes", errUtils.ErrArtifactIntegrityFailed, recordDigest) + } + if existing != nil { + return nil + } + return t.writeAtomic(layout.RecordPath(recordDigest), canonical) +} + +// listReferences returns every stored reference, sorted by file name, failing on a corrupt one. +func (t *tree) listReferences() ([]*layout.Reference, error) { + entries, err := fs.ReadDir(t.FS(), layout.ReferencesDir()) + switch { + case errors.Is(err, fs.ErrNotExist): + return nil, nil + case err != nil: + return nil, fmt.Errorf("%w: list references: %w", errUtils.ErrArtifactIntegrityFailed, err) + } + refs := make([]*layout.Reference, 0, len(entries)) + for _, entry := range entries { + key, ok := strings.CutSuffix(entry.Name(), ".json") + if entry.IsDir() || !ok { + continue + } + ref, err := t.loadReference(key) + if err != nil { + return nil, err + } + if ref != nil { + refs = append(refs, ref) + } + } + return refs, nil +} diff --git a/pkg/artifact/local/unmanaged.go b/pkg/artifact/local/unmanaged.go new file mode 100644 index 00000000000..f5934efd206 --- /dev/null +++ b/pkg/artifact/local/unmanaged.go @@ -0,0 +1,75 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "path" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// rejectUnmanagedPrior refuses to retire a directory containing files outside its Record. +func (r *Repository) rejectUnmanagedPrior(t *tree, job *publishJob, plan *publishPlan) error { + if r.kind != artifact.RepositoryKindLocalDir || plan.prior == nil || plan.prior.Location == "" { + return nil + } + record, _, err := t.loadRecord(plan.prior.RecordDigest) + if err != nil { + return err + } + priorRecords := recordsForPrior(record, job, plan) + if err := checkOwnedDirectory(t, plan.prior.Location, priorRecords...); err != nil { + return err + } + locations := []string{job.retiredDir() + "/payload"} + if plan.oldOp != nil { + locations = append(locations, layout.StagingPath(plan.oldOp.OperationID)+"/retired/payload") + } + for _, location := range locations { + if err := checkOwnedDirectory(t, location, record); err != nil { + return err + } + } + return nil +} + +func recordsForPrior(record *artifact.Record, job *publishJob, plan *publishPlan) []*artifact.Record { + if plan.admission.Resuming() && layout.FoldPath(plan.prior.Location) == layout.FoldPath(job.location) { + return []*artifact.Record{record, job.record} + } + return []*artifact.Record{record} +} + +func checkOwnedDirectory(t *tree, location string, records ...*artifact.Record) error { + allowed := map[string]bool{location: true} + add := func(relative string) { + //nolint:forbidigo // io/fs paths are slash-separated on all platforms. + for name := path.Join(location, relative); name != location && name != "."; name = path.Dir(name) { + allowed[name] = true + } + } + for _, record := range records { + for _, entry := range record.Inventory { + add(entry.Path) + } + } + for _, name := range []string{layout.ManifestName, "README.md", "CHANGELOG.md"} { + add(layout.DistributionDir + "/" + name) + } + err := fs.WalkDir(t.FS(), location, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !allowed[name] { + return fmt.Errorf("%w: cannot replace %q: %q is not managed by the artifact", errUtils.ErrArtifactConflict, location, name) + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} diff --git a/pkg/artifact/local/verifier_test.go b/pkg/artifact/local/verifier_test.go new file mode 100644 index 00000000000..0e5c618a01d --- /dev/null +++ b/pkg/artifact/local/verifier_test.go @@ -0,0 +1,169 @@ +package local + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/layout" +) + +// verifierProbe records how many evidence blobs reached the verifier. +type verifierProbe struct{ seen int } + +// mockVerifier returns a generated verifier mock with a canned outcome. Its probe records the +// evidence count of the latest Verify call. +func mockVerifier(t *testing.T, result *artifact.VerificationResult, err error) (*artifact.MockVerifier, *verifierProbe) { + t.Helper() + probe := &verifierProbe{} + verifier := artifact.NewMockVerifier(gomock.NewController(t)) + verifier.EXPECT().Formats().AnyTimes().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}) + verifier.EXPECT().Verify(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes(). + DoAndReturn(func(_ context.Context, _ artifact.SignatureSubject, evidence []artifact.EvidenceBlob, _ artifact.VerificationPolicy) (*artifact.VerificationResult, error) { + probe.seen = len(evidence) + return result, err + }) + return verifier, probe +} + +// mockSigner returns a generated signer mock that always signs data. +func mockSigner(t *testing.T, data []byte) *artifact.MockSigner { + t.Helper() + signer := artifact.NewMockSigner(gomock.NewController(t)) + signer.EXPECT().Formats().AnyTimes().Return([]artifact.EvidenceFormat{artifact.EvidenceSigstoreBundle}) + signer.EXPECT().Sign(gomock.Any(), gomock.Any()).AnyTimes(). + Return(&artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "application/json", Data: data}, nil) + return signer +} + +func signedWith(t *testing.T, verifier artifact.Verifier) (*Repository, artifact.SignatureSubject) { + t.Helper() + repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(verifier)}}) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, mockSigner(t, []byte(`{"bundle":1}`)), artifact.SignOptions{}) + require.NoError(t, err) + return repo, subject +} + +var trustedPolicy = artifact.VerificationPolicy{ + RequireSignature: true, + TrustSets: []artifact.TrustSet{{Identities: []artifact.TrustedIdentity{{Identity: "who", Issuer: "https://issuer.example.com"}}}}, +} + +func TestVerify_TranslatesVerifierOutcomes(t *testing.T) { + tests := []struct { + name string + result *artifact.VerificationResult + err error + wantErr error + wantStatus artifact.VerificationStatus + }{ + {"verified", &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil, nil, artifact.VerificationVerified}, + {"unchecked is not verified", &artifact.VerificationResult{Status: artifact.VerificationUnchecked}, nil, errUtils.ErrArtifactVerificationFailed, artifact.VerificationUnchecked}, + {"failed status without an error still fails", &artifact.VerificationResult{Status: artifact.VerificationFailed}, nil, errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + {"verifier error is wrapped as a verification failure", nil, errors.New("crypto exploded"), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + {"verification sentinel is kept", &artifact.VerificationResult{Status: artifact.VerificationFailed}, fmt.Errorf("%w: policy", errUtils.ErrArtifactVerificationFailed), errUtils.ErrArtifactVerificationFailed, artifact.VerificationFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub, probe := mockVerifier(t, tt.result, tt.err) + repo, subject := signedWith(t, stub) + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + require.NotNil(t, result) + assert.Equal(t, tt.wantStatus, result.Status) + assert.Equal(t, subject, result.Subject) + assert.NotEmpty(t, result.PolicyFingerprint, "the applied policy is always fingerprinted") + assert.Equal(t, 1, probe.seen) + }) + } +} + +func TestVerify_ReportsDiagnosticsAlongsideTheVerifier(t *testing.T) { + stub, _ := mockVerifier(t, &artifact.VerificationResult{ + Status: artifact.VerificationVerified, PolicyFingerprint: "verifier-fingerprint", + Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityInfo, Code: "from-verifier", Message: "ok"}}, + }, nil) + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(artifact.SubjectRecord, subject.Digest))) + require.NoError(t, os.WriteFile(filepath.Join(dir, "note.txt"), []byte("x"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "zz.json"), []byte("{}"), 0o644)) + require.NoError(t, os.Mkdir(filepath.Join(dir, "sub.json"), 0o755)) + + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.NoError(t, err) + assert.Equal(t, "verifier-fingerprint", result.PolicyFingerprint, "the verifier's own fingerprint is kept") + codes := map[string]bool{} + for _, diagnostic := range result.Diagnostics { + codes[diagnostic.Code] = true + } + assert.True(t, codes["from-verifier"]) + assert.True(t, codes["malformed-evidence"], "a file whose name is not an evidence digest is reported, not trusted") +} + +func TestVerify_ReadsAllValidEvidence(t *testing.T) { + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) + repo, subject := signedWith(t, stub) + for index := range maxMalformedEvidence + 5 { + evidence, err := layout.NewEvidence(subject, &artifact.EvidencePayload{Format: artifact.EvidenceSigstoreBundle, MediaType: "x", Data: []byte(fmt.Sprintf("bundle-%d", index))}) + require.NoError(t, err) + raw, err := evidence.Marshal() + require.NoError(t, err) + path := filepath.Join(repo.root, filepath.FromSlash(layout.SignaturePath(artifact.SubjectRecord, subject.Digest, evidence.Digest()))) + require.NoError(t, os.WriteFile(path, raw, 0o644)) + } + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.NoError(t, err) + assert.Equal(t, maxMalformedEvidence+6, probe.seen, "all valid evidence reaches the verifier") + assert.Equal(t, artifact.VerificationVerified, result.Status) +} + +func TestSign_RejectsOversizeEvidence(t *testing.T) { + verifier, _ := mockVerifier(t, nil, nil) + repo := newRepo(t, &repoSpec{opts: []Option{WithVerifier(verifier)}}) + publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}) + subject := recordSubject(t, repo) + _, err := repo.Sign(context.Background(), subject, mockSigner(t, make([]byte, artifact.MaxEvidenceSize+1)), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + _, err = repo.Sign(context.Background(), subject, mockSigner(t, nil), artifact.SignOptions{}) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed, "empty evidence is invalid") +} + +func TestVerifyRejectsUnreadableEvidenceDirectory(t *testing.T) { + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + require.NoError(t, os.RemoveAll(dir)) + require.NoError(t, os.WriteFile(dir, []byte("not a directory"), 0o644)) + result, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.Contains(t, result.Diagnostics[0].Code, "evidence-read-failed") + assert.Zero(t, probe.seen) +} + +func TestVerifyFailsClosedOnTooManyMalformedEvidenceFiles(t *testing.T) { + stub, probe := mockVerifier(t, &artifact.VerificationResult{Status: artifact.VerificationVerified}, nil) + repo, subject := signedWith(t, stub) + dir := filepath.Join(repo.root, filepath.FromSlash(layout.SignatureDir(subject.Kind, subject.Digest))) + for i := range maxMalformedEvidence + 1 { + require.NoError(t, os.WriteFile(filepath.Join(dir, fmt.Sprintf("bad-%03d.json", i)), []byte("bad"), 0o644)) + } + _, err := repo.Verify(context.Background(), subject, trustedPolicy) + require.ErrorIs(t, err, errUtils.ErrArtifactVerificationFailed) + require.ErrorIs(t, err, errUtils.ErrArtifactEvidenceMalformed) + assert.Zero(t, probe.seen) +} diff --git a/pkg/artifact/produce/produce_test.go b/pkg/artifact/produce/produce_test.go index 709701757fd..c71d332d9f4 100644 --- a/pkg/artifact/produce/produce_test.go +++ b/pkg/artifact/produce/produce_test.go @@ -51,6 +51,7 @@ func readBlob(t *testing.T, blob *artifact.StagedBlob) []byte { t.Helper() reader, err := blob.Open(context.Background()) require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) data, err := io.ReadAll(reader) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/pkg/artifact/service/consume_test.go b/pkg/artifact/service/consume_test.go index fa77e9db59c..8ac8fa8e1e0 100644 --- a/pkg/artifact/service/consume_test.go +++ b/pkg/artifact/service/consume_test.go @@ -379,8 +379,12 @@ type listGate struct { peak atomic.Int32 entered chan struct{} release chan struct{} + freeing sync.Once } +// open lets every blocked listing continue. It is safe to call any number of times. +func (g *listGate) open() { g.freeing.Do(func() { close(g.release) }) } + // gatedList returns a generated repository that blocks every List on the gate before delegating. func gatedList(t *testing.T, base *artifacttest.Repository, gate *listGate) artifact.Repository { return delegatingRepository(t, base, repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { @@ -403,6 +407,7 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { seedList(t, e, alias, map[string]string{"app/x:1": alias}, nil) } gate := &listGate{entered: make(chan struct{}, 3), release: make(chan struct{})} + t.Cleanup(gate.open) // A failed assertion must not leave a listing blocked on the gate. for _, alias := range []string{"a", "b", "c"} { e.static.Repositories[alias] = gatedList(t, e.repos[alias], gate) } @@ -422,15 +427,19 @@ func TestList_ConcurrencyIsInjectable(t *testing.T) { select { case <-gate.entered: case <-time.After(2 * time.Second): - close(gate.release) t.Fatal("listing did not start") } select { case <-gate.entered: case <-time.After(200 * time.Millisecond): } - close(gate.release) - outcome := <-done + gate.open() + var outcome listOutcome + select { + case outcome = <-done: + case <-time.After(10 * time.Second): + t.Fatal("the listing did not finish after the gate opened") + } require.NoError(t, outcome.err) assert.Equal(t, 3, outcome.count) assert.Equal(t, int32(1), gate.peak.Load()) diff --git a/pkg/artifact/service/local_sign_test.go b/pkg/artifact/service/local_sign_test.go new file mode 100644 index 00000000000..9d1b17c0e0a --- /dev/null +++ b/pkg/artifact/service/local_sign_test.go @@ -0,0 +1,294 @@ +package service_test + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guard for the schema fields the local repository config uses. +var _ = schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": ""}} + +// gatedSigner signs like [artifacttest.FakeSigner] but announces when it starts and waits to be +// released, so a test can run another operation while a signature is in flight. +type gatedSigner struct { + artifacttest.FakeSigner + started chan struct{} + release chan struct{} + once sync.Once + freeing sync.Once +} + +func newGatedSigner() *gatedSigner { + return &gatedSigner{ + FakeSigner: artifacttest.FakeSigner{Identity: "release@example.com", Issuer: "https://issuer.example.com"}, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +// free lets the signer finish. It is safe to call any number of times. +func (g *gatedSigner) free() { g.freeing.Do(func() { close(g.release) }) } + +func (g *gatedSigner) Sign(ctx context.Context, subject artifact.SignatureSubject) (*artifact.EvidencePayload, error) { + g.once.Do(func() { close(g.started) }) + select { + case <-g.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + return g.FakeSigner.Sign(ctx, subject) +} + +// localSignEnv is a service over a real local directory repository. +type localSignEnv struct { + svc *service.Service + repo *local.Repository + published *service.PublishOutcome + digestRef artifact.Ref +} + +func newLocalSignEnv(t *testing.T, name string, signer artifact.Signer) *localSignEnv { + t.Helper() + cfg := &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindLocalDir, + Options: map[string]any{"path": filepath.Join(t.TempDir(), "repo")}, + } + repo, err := local.NewDir("dst", cfg, local.WithVerifier(&artifacttest.FakeVerifier{}), local.WithOwner("local-test")) + require.NoError(t, err) + static := &service.StaticRepositories{Repositories: map[string]artifact.Repository{"dst": repo}, Info: map[string]service.RepositoryInfo{}} + svc, err := service.New(static, nil, service.WithSigner(signer), service.WithOwner("service-test"), service.WithStagingRoot(t.TempDir())) + require.NoError(t, err) + + src := (&env{t: t}).source("hello") + published, err := svc.PublishDetailed(context.Background(), artifact.PublishRequest{Repository: "dst", Source: src, Name: name, Version: "1.0.0"}) + require.NoError(t, err) + return &localSignEnv{svc: svc, repo: repo, published: published, digestRef: artifact.Ref{Name: name, Digest: published.Operation.RecordDigest}} +} + +func (e *localSignEnv) sign(ctx context.Context) (*artifact.SignResult, error) { + return e.svc.Sign(ctx, artifact.SignRequest{Repository: "dst", Ref: e.digestRef}) +} + +// signerEntryTimeout bounds how long a test waits for the signer to start or an operation to finish. +const signerEntryTimeout = 10 * time.Second + +// awaitSignerEntry waits until the signer has started. When signing ends first, for example because +// the service refused it before calling the signer, it returns that real result instead of waiting +// for a signer that will never run. A signer that never starts within the timeout is an error too. +func awaitSignerEntry(started <-chan struct{}, signed <-chan error, timeout time.Duration) error { + select { + case <-started: + return nil + case err := <-signed: + if err == nil { + return errSignedWithoutTheSigner + } + return fmt.Errorf("signing ended before the signer started: %w", err) + case <-time.After(timeout): + return errSignerNeverStarted + } +} + +var ( + errSignedWithoutTheSigner = errors.New("signing finished without calling the signer") + errSignerNeverStarted = errors.New("the signer did not start in time") +) + +// startSigning signs through the service in the background and waits for the signer to be working. +// Its context is cancelled and the signer freed when the test ends, so a failed assertion cannot +// leave the signing goroutine holding the reference locks. +func startSigning(t *testing.T, e *localSignEnv, signer *gatedSigner) <-chan error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + signer.free() + }) + signed := make(chan error, 1) + go func() { + _, err := e.sign(ctx) + signed <- err + }() + if err := awaitSignerEntry(signer.started, signed, signerEntryTimeout); err != nil { + t.Fatal(err) + } + return signed +} + +// Signing through the service must work against a real local repository: the lock the service +// holds has to be one the repository accepts. Names cover a reference that sorts before and after +// the "subject:" lock resource, because the service takes its locks in canonical order. +func TestSign_ThroughTheServiceStoresEvidenceInALocalRepository(t *testing.T) { + tests := []struct { + name string + ref string + }{ + {name: "reference sorts after the subject lock", ref: "tools/example"}, + {name: "reference sorts before the subject lock", ref: "a/example"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newLocalSignEnv(t, tt.ref, releaseSigner) + + result, err := e.sign(context.Background()) + require.NoError(t, err) + require.Len(t, result.Evidence, 1) + assert.Equal(t, artifact.SubjectRecord, result.Subject.Kind) + assert.Equal(t, e.published.Operation.RecordDigest, result.Subject.Digest) + assert.Equal(t, artifact.LockScopeHost, result.Operation.LockScope) + + inspected, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: artifact.Ref{Name: tt.ref, Version: "1.0.0"}}) + require.NoError(t, err) + require.Len(t, inspected.Inspection.Evidence, 1) + assert.Equal(t, result.Evidence[0].Descriptor.Digest, inspected.Inspection.Evidence[0].Descriptor.Digest) + + verified, err := e.svc.Verify(context.Background(), artifact.VerifyRequest{Repository: "dst", Ref: e.digestRef}) + require.Error(t, err, "no trust configuration is set, so verification reports unchecked rather than green") + assert.Equal(t, artifact.VerificationUnchecked, verified.Verification.Status) + + // Every lock the service took was released: a second signature and a delete both proceed. + _, err = e.sign(context.Background()) + require.NoError(t, err) + _, err = e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: tt.ref, Version: "1.0.0"}}) + require.NoError(t, err) + }) + } +} + +// While a signer is working, a concurrent delete through the service must wait for the signature +// to be stored instead of removing the Record underneath it, and it must not deadlock with the +// signing locks whichever way the reference sorts against the subject lock. +func TestSign_ThroughTheServiceExcludesAConcurrentDelete(t *testing.T) { + for _, name := range []string{"tools/example", "a/example"} { + t.Run(name, func(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, name, signer) + signed := startSigning(t, e, signer) + + deleted := make(chan error, 1) + go func() { + _, err := e.svc.Delete(context.Background(), artifact.DeleteRequest{Repository: "dst", Ref: artifact.Ref{Name: name, Version: "1.0.0"}}) + deleted <- err + }() + + select { + case err := <-deleted: + t.Fatalf("delete finished while the signer was still working: %v", err) + case <-time.After(200 * time.Millisecond): + } + + signer.free() + require.NoError(t, waitFor(t, signed), "signing completes once the signer returns") + require.NoError(t, waitFor(t, deleted), "the delete runs after the signature is stored") + + _, err := e.svc.Inspect(context.Background(), artifact.InspectRequest{Repository: "dst", Ref: artifact.Ref{Name: name, Version: "1.0.0"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + }) + } +} + +// A writer that bypasses the service and deletes through the repository takes the repository's own +// reference lock, which signing through the service must also hold. +func TestSign_ThroughTheServiceExcludesADirectRepositoryDelete(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, "tools/example", signer) + signed := startSigning(t, e, signer) + + pinned, err := e.repo.Resolve(context.Background(), artifact.Ref{Name: "tools/example", Version: "1.0.0"}) + require.NoError(t, err) + deleted := make(chan error, 1) + go func() { + _, deleteErr := e.repo.Delete(context.Background(), pinned, artifact.DeleteOptions{}) + deleted <- deleteErr + }() + + select { + case err := <-deleted: + t.Fatalf("a direct delete finished while the signer was still working: %v", err) + case <-time.After(200 * time.Millisecond): + } + signer.free() + require.NoError(t, waitFor(t, signed)) + require.NoError(t, waitFor(t, deleted)) +} + +func waitFor(t *testing.T, ch <-chan error) error { + t.Helper() + select { + case err := <-ch: + return err + case <-time.After(signerEntryTimeout): + t.Fatal("timed out waiting for the operation to finish") + return nil + } +} + +// A signing that ends before it reaches the signer must surface its own error. Waiting only on the +// signer's start would hang forever, because the signer is never called. +func TestAwaitSignerEntry(t *testing.T) { + boom := errors.New("boom") + closed := make(chan struct{}) + close(closed) + + tests := []struct { + name string + started <-chan struct{} + signed func() <-chan error + wantErr error + }{ + {name: "signer started", started: closed, signed: func() <-chan error { return make(chan error) }}, + {name: "signing failed first", started: make(chan struct{}), signed: func() <-chan error { + ch := make(chan error, 1) + ch <- boom + return ch + }, wantErr: boom}, + {name: "signing returned without the signer", started: make(chan struct{}), signed: func() <-chan error { + ch := make(chan error, 1) + ch <- nil + return ch + }, wantErr: errSignedWithoutTheSigner}, + {name: "nothing happens", started: make(chan struct{}), signed: func() <-chan error { return make(chan error) }, wantErr: errSignerNeverStarted}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := awaitSignerEntry(tt.started, tt.signed(), 50*time.Millisecond) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.wantErr) + }) + } +} + +// Signing through the service with a context that is already cancelled fails before the signer +// runs. The test helper must report that failure rather than wait for the signer to start. +func TestStartSigning_ReportsAnEarlyServiceError(t *testing.T) { + signer := newGatedSigner() + e := newLocalSignEnv(t, "tools/example", signer) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + signed := make(chan error, 1) + go func() { + _, err := e.sign(ctx) + signed <- err + }() + err := awaitSignerEntry(signer.started, signed, signerEntryTimeout) + require.ErrorIs(t, err, context.Canceled, "the real service error is reported") +} diff --git a/pkg/artifact/service/mutate_test.go b/pkg/artifact/service/mutate_test.go index 07436eac227..797abce9b8d 100644 --- a/pkg/artifact/service/mutate_test.go +++ b/pkg/artifact/service/mutate_test.go @@ -145,6 +145,7 @@ func TestLocks_HeldByAnotherOwnerBlocksUntilCanceledAndLeaksNothing(t *testing.T e := newEnv(t, envConfig{}) other, err := e.repos["dst"].Lock(context.Background(), artifact.LockRequest{Resource: exampleRef.String(), Owner: "someone-else"}) require.NoError(t, err) + t.Cleanup(func() { _ = other.Release(context.Background()) }) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) defer cancel() diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go index b126bf60034..afe2b3f483e 100644 --- a/pkg/artifact/service/sign.go +++ b/pkg/artifact/service/sign.go @@ -40,8 +40,8 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig if err := checkSigningCapability(request.Repository, &info, s.signer, subject); err != nil { return err } - resource := subjectLockResource(request.Ref.Name) - locks, err := s.acquireLocks(ctx, op, opSign, []lockTarget{{alias: request.Repository, repo: repo, info: &info, resource: resource}}) + targets, writerResource := signLockTargets(request, repo, &info, inspection) + locks, err := s.acquireLocks(ctx, op, opSign, targets) if err != nil { return err } @@ -50,7 +50,7 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig if err := verifyLockedSubject(ctx, repo, request.Ref, &pinned); err != nil { return err } - evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, resource)}) + evidence, err := repo.Sign(ctx, subject, s.signer, artifact.SignOptions{OperationID: op.id, Lock: locks.handleForResource(request.Repository, writerResource)}) if err != nil { return fmt.Errorf("sign %s in repository %q: %w", request.Ref, request.Repository, err) } @@ -61,6 +61,22 @@ func (s *Service) sign(ctx context.Context, op *operation, request *artifact.Sig return nil } +// signLockTargets returns the locks signing holds and the resource whose handle is passed to the +// repository as the writer lock. Signing takes the same two locks as a delete: the subject lock, +// which serializes every operation on the artifact name, and the reference lock of the Record, +// which publish, delete and a repository's own internal locking all honor. They are acquired in +// canonical order by [Service.acquireLocks], so signing and deleting never deadlock. A subject +// without a Record (an OCI root) has no reference lock to hold and uses the subject lock alone. +func signLockTargets(request *artifact.SignRequest, repo artifact.Repository, info *RepositoryInfo, inspection *artifact.Inspection) (targets []lockTarget, writerResource string) { + subjectResource := subjectLockResource(request.Ref.Name) + targets = []lockTarget{{alias: request.Repository, repo: repo, info: info, resource: subjectResource}} + if inspection.Record == nil { + return targets, subjectResource + } + writerResource = artifact.Ref{Name: inspection.Record.Name, Version: inspection.Record.Version}.String() + return append(targets, lockTarget{alias: request.Repository, repo: repo, info: info, resource: writerResource}), writerResource +} + func verifyLockedSubject(ctx context.Context, repo artifact.Repository, ref artifact.Ref, pinned *artifact.PinnedReference) error { current, err := repo.Resolve(ctx, ref) if err != nil { diff --git a/pkg/artifact/snapshot/snapshot_test.go b/pkg/artifact/snapshot/snapshot_test.go index 6ebf58bef8c..0016d6413f0 100644 --- a/pkg/artifact/snapshot/snapshot_test.go +++ b/pkg/artifact/snapshot/snapshot_test.go @@ -69,6 +69,7 @@ func TestCapture_FilePackageInventory(t *testing.T) { reader, err := snap.Open("nested/deep/file.txt") require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) data, err := io.ReadAll(reader) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/pkg/ci/artifact/local/store.go b/pkg/ci/artifact/local/store.go index 521056a0edf..a6d7989ec25 100644 --- a/pkg/ci/artifact/local/store.go +++ b/pkg/ci/artifact/local/store.go @@ -1,6 +1,7 @@ package local import ( + "bytes" "context" "encoding/json" "fmt" @@ -13,12 +14,17 @@ import ( "time" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobLocal "github.com/cloudposse/atmos/pkg/artifact/blob/local" "github.com/cloudposse/atmos/pkg/ci/artifact" "github.com/cloudposse/atmos/pkg/config/homedir" "github.com/cloudposse/atmos/pkg/perf" ) const ( + // Format that wraps a failure to open the artifact directory with the operation's sentinel. + openDirFormat = "%w: failed to open artifact directory for %s: %w" + storeName = "local/dir" metadataSuffix = ".metadata.json" defaultDirPerms = 0o755 @@ -26,7 +32,9 @@ const ( ) // Store implements the artifact.Backend interface using the local filesystem. -// It stores each artifact as a single file with a metadata JSON sidecar. +// It stores each artifact as a single file with a metadata JSON sidecar. Object I/O is delegated +// to the shared transport in pkg/artifact/blob/local; this type only owns the legacy CI +// concerns: artifact naming, the metadata sidecar format, and query matching. type Store struct { basePath string } @@ -59,9 +67,31 @@ func NewStore(opts artifact.StoreOptions) (artifact.Backend, error) { // Name returns the store type name. func (s *Store) Name() string { + defer perf.Track(nil, "artifact.local.Name")() + return storeName } +// transport returns the shared filesystem blob store rooted at the base path. The store is +// stateless apart from a per-root lock registry, so it is built on demand. +func (s *Store) transport() (*blobLocal.Store, error) { + root := s.basePath + if root == "" { + root = "." + } + return blobLocal.New(root, blobLocal.WithLiteralBackslashes(), blobLocal.WithPlainFileCompatibility()) +} + +// key maps a validated artifact name to the transport key. Names are cleaned the same way the +// filesystem would clean them, so "a/./b" and "a//b" address the same object as before. +func (s *Store) key(name string) string { + rel, err := filepath.Rel(s.basePath, filepath.Join(s.basePath, name)) + if err != nil { + return filepath.ToSlash(name) + } + return filepath.ToSlash(rel) +} + // Upload uploads a single data stream to the local filesystem. // Writes the data as a single file and the metadata as a JSON sidecar. func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size int64, metadata *Metadata) error { @@ -70,25 +100,16 @@ func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size in if err := s.validateName(name); err != nil { return err } - - filePath := filepath.Join(s.basePath, name) - - // Create parent directories for the artifact file. - if err := os.MkdirAll(filepath.Dir(filePath), defaultDirPerms); err != nil { - return fmt.Errorf("%w: failed to create directory for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) - } - - // Write the data stream to a single file. - f, err := os.Create(filePath) + transport, err := s.transport() if err != nil { - return fmt.Errorf("%w: failed to create file %s: %w", errUtils.ErrArtifactUploadFailed, name, err) + return fmt.Errorf(openDirFormat, errUtils.ErrArtifactUploadFailed, name, err) } + key := s.key(name) - if _, err := io.Copy(f, data); err != nil { - f.Close() + // The declared size is advisory for this backend, as it always was. + if _, err := transport.Put(ctx, key, data, blob.PutOptions{Size: blob.UnknownSize}); err != nil { return fmt.Errorf("%w: failed to write file %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } - f.Close() // Write metadata sidecar. if metadata == nil { @@ -97,18 +118,12 @@ func (s *Store) Upload(ctx context.Context, name string, data io.Reader, size in } } - metadataPath := filepath.Join(s.basePath, name+metadataSuffix) - - // Create parent directories for metadata file. - if err := os.MkdirAll(filepath.Dir(metadataPath), defaultDirPerms); err != nil { - return fmt.Errorf("%w: failed to create directory for metadata: %w", errUtils.ErrArtifactUploadFailed, err) - } - metadataJSON, err := json.MarshalIndent(metadata, "", " ") if err != nil { return fmt.Errorf("%w: failed to marshal metadata for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } - if err := os.WriteFile(metadataPath, metadataJSON, defaultFilePerms); err != nil { + _, err = transport.Put(ctx, key+metadataSuffix, bytes.NewReader(metadataJSON), blob.PutOptions{Size: int64(len(metadataJSON))}) + if err != nil { return fmt.Errorf("%w: failed to write metadata for %s: %w", errUtils.ErrArtifactUploadFailed, name, err) } @@ -127,30 +142,33 @@ func (s *Store) Download(ctx context.Context, name string) (io.ReadCloser, *arti if err := s.validateName(name); err != nil { return nil, nil, err } - - filePath := filepath.Join(s.basePath, name) - - // Check if artifact file exists and is not a directory. - info, err := os.Stat(filePath) + transport, err := s.transport() if err != nil { - if os.IsNotExist(err) { - return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) - } - return nil, nil, fmt.Errorf("%w: failed to stat artifact %s: %w", errUtils.ErrArtifactDownloadFailed, name, err) - } - if info.IsDir() { - return nil, nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactDownloadFailed, name) + return nil, nil, fmt.Errorf(openDirFormat, errUtils.ErrArtifactDownloadFailed, name, err) } + key := s.key(name) - f, err := os.Open(filePath) + object, err := transport.Get(ctx, key, blob.GetOptions{SkipETag: true}) if err != nil { + if blob.IsNotFound(err) { + if s.isDirectory(name) { + return nil, nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactDownloadFailed, name) + } + return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) + } return nil, nil, fmt.Errorf("%w: failed to open artifact %s: %w", errUtils.ErrArtifactDownloadFailed, name, err) } // Load metadata. - metadata, _ := s.loadMetadata(filepath.Join(s.basePath, name+metadataSuffix)) + metadata, _ := s.loadMetadata(ctx, transport, key+metadataSuffix) - return f, metadata, nil + return object.Body, metadata, nil +} + +// isDirectory reports whether name addresses a directory, which is not an artifact. +func (s *Store) isDirectory(name string) bool { + info, err := os.Stat(filepath.Join(s.basePath, name)) + return err == nil && info.IsDir() } // Delete deletes an artifact from the local filesystem. @@ -162,78 +180,72 @@ func (s *Store) Delete(ctx context.Context, name string) error { if err := s.validateName(name); err != nil { return err } + transport, err := s.transport() + if err != nil { + return fmt.Errorf(openDirFormat, errUtils.ErrArtifactDeleteFailed, name, err) + } + key := s.key(name) - filePath := filepath.Join(s.basePath, name) - - // Delete the artifact file. - if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + // Delete the artifact file; the transport also removes emptied parent directories. + if err := transport.Delete(ctx, key, blob.DeleteOptions{}); err != nil { return fmt.Errorf("%w: failed to delete artifact %s: %w", errUtils.ErrArtifactDeleteFailed, name, err) } // Delete metadata sidecar (ignore errors if it doesn't exist). - _ = os.Remove(filepath.Join(s.basePath, name+metadataSuffix)) - - // Try to clean up empty parent directories. - s.cleanupEmptyDirs(filepath.Dir(filePath)) + _ = transport.Delete(ctx, key+metadataSuffix, blob.DeleteOptions{}) return nil } // List lists artifacts matching the given query. -// Walks basePath looking for metadata sidecar files with a corresponding artifact file (not dir), +// Finds metadata sidecar files with a corresponding artifact file (not dir), // applies Query filters, and sorts newest first. -func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { +func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { //nolint:gocritic // Query is passed by value to satisfy the Backend interface. defer perf.Track(nil, "artifact.local.List")() - var artifacts []artifact.ArtifactInfo - - // Walk basePath looking for metadata sidecar files. - err := filepath.WalkDir(s.basePath, func(path string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return nil // Skip entries with errors. - } + transport, err := s.transport() + if err != nil { + return nil, fmt.Errorf("%w: failed to open artifact directory: %w", errUtils.ErrArtifactListFailed, err) + } + objects, _, err := blob.ListAll(ctx, transport, blob.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("%w: failed to list artifacts: %w", errUtils.ErrArtifactListFailed, err) + } - // We only care about .metadata.json files. - if d.IsDir() || !strings.HasSuffix(path, metadataSuffix) { - return nil - } + byKey := make(map[string]blob.ObjectInfo, len(objects)) + for _, object := range objects { + byKey[object.Key] = object + } - // Derive artifact name from metadata path. - relPath, err := filepath.Rel(s.basePath, path) - if err != nil { - return nil + var artifacts []artifact.ArtifactInfo + for _, object := range objects { + name, ok := strings.CutSuffix(object.Key, metadataSuffix) + if !ok { + continue } - artifactName := strings.TrimSuffix(relPath, metadataSuffix) - // Verify the artifact file exists and is not a directory. - artifactFile := filepath.Join(s.basePath, artifactName) - fileInfo, err := os.Stat(artifactFile) - if err != nil || fileInfo.IsDir() { - return nil + // Only metadata sidecars with a corresponding artifact file are artifacts. + file, ok := byKey[name] + if !ok { + continue } - // Load metadata. - metadata, err := s.loadMetadata(path) + metadata, err := s.loadMetadata(ctx, transport, object.Key) if err != nil { - return nil + continue } // Apply query filters. if !query.All && !s.matchesQuery(metadata, query) { - return nil + continue } artifacts = append(artifacts, artifact.ArtifactInfo{ - Name: artifactName, - Size: fileInfo.Size(), - LastModified: fileInfo.ModTime(), + Name: filepath.FromSlash(name), + Size: file.Size, + LastModified: file.LastModified, Metadata: metadata, }) - - return nil - }) - if err != nil { - return nil, fmt.Errorf("%w: failed to list artifacts: %w", errUtils.ErrArtifactListFailed, err) } // Sort by last modified (newest first). @@ -251,16 +263,18 @@ func (s *Store) Exists(ctx context.Context, name string) (bool, error) { if err := s.validateName(name); err != nil { return false, err } - - filePath := filepath.Join(s.basePath, name) - info, err := os.Stat(filePath) + transport, err := s.transport() if err != nil { - if os.IsNotExist(err) { + return false, fmt.Errorf("failed to check if artifact %s exists: %w", name, err) + } + + if _, err := transport.Stat(ctx, s.key(name), blob.StatOptions{SkipETag: true}); err != nil { + if blob.IsNotFound(err) { return false, nil } return false, fmt.Errorf("failed to check if artifact %s exists: %w", name, err) } - return !info.IsDir(), nil + return true, nil } // GetMetadata retrieves metadata for an artifact without downloading the content. @@ -270,29 +284,32 @@ func (s *Store) GetMetadata(ctx context.Context, name string) (*artifact.Metadat if err := s.validateName(name); err != nil { return nil, err } + transport, err := s.transport() + if err != nil { + return nil, fmt.Errorf(openDirFormat, errUtils.ErrArtifactMetadataFailed, name, err) + } + key := s.key(name) // Check if the artifact file exists. - filePath := filepath.Join(s.basePath, name) - fileInfo, err := os.Stat(filePath) + info, err := transport.Stat(ctx, key, blob.StatOptions{SkipETag: true}) if err != nil { - if os.IsNotExist(err) { + if blob.IsNotFound(err) { + if s.isDirectory(name) { + return nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactNotFound, name) + } return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, name) } return nil, fmt.Errorf("%w: failed to stat artifact %s: %w", errUtils.ErrArtifactMetadataFailed, name, err) } - if fileInfo.IsDir() { - return nil, fmt.Errorf("%w: %s is a directory, not a file", errUtils.ErrArtifactNotFound, name) - } - metadataPath := filepath.Join(s.basePath, name+metadataSuffix) - metadata, err := s.loadMetadata(metadataPath) + metadata, err := s.loadMetadata(ctx, transport, key+metadataSuffix) if err != nil { return nil, fmt.Errorf("%w: failed to load metadata for %s: %w", errUtils.ErrArtifactMetadataFailed, name, err) } if metadata == nil { // Return minimal metadata from file info. metadata = &artifact.Metadata{ - CreatedAt: fileInfo.ModTime(), + CreatedAt: info.LastModified, } } @@ -311,15 +328,21 @@ func (s *Store) validateName(name string) error { return nil } -// loadMetadata loads metadata from a JSON sidecar file. -func (s *Store) loadMetadata(metadataPath string) (*artifact.Metadata, error) { - data, err := os.ReadFile(metadataPath) +// loadMetadata loads metadata from a JSON sidecar object. A missing sidecar yields nil metadata. +func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, metadataKey string) (*artifact.Metadata, error) { + object, err := transport.Get(ctx, metadataKey, blob.GetOptions{SkipETag: true}) if err != nil { - if os.IsNotExist(err) { + if blob.IsNotFound(err) { return nil, nil } return nil, err } + defer object.Body.Close() + + data, err := io.ReadAll(object.Body) + if err != nil { + return nil, err + } var metadata artifact.Metadata if err := json.Unmarshal(data, &metadata); err != nil { @@ -358,18 +381,6 @@ func (s *Store) matchesQuery(metadata *artifact.Metadata, query artifact.Query) return true } -// cleanupEmptyDirs removes empty parent directories up to the base path. -func (s *Store) cleanupEmptyDirs(dir string) { - for dir != s.basePath && dir != "" { - entries, err := os.ReadDir(dir) - if err != nil || len(entries) > 0 { - return - } - _ = os.Remove(dir) - dir = filepath.Dir(dir) - } -} - func init() { artifact.Register(storeName, NewStore) } diff --git a/pkg/ci/artifact/local/store_test.go b/pkg/ci/artifact/local/store_test.go index 71fcc8f34d1..d762b4a1c3e 100644 --- a/pkg/ci/artifact/local/store_test.go +++ b/pkg/ci/artifact/local/store_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -541,3 +542,55 @@ func TestIntegration_FullLifecycle(t *testing.T) { _, err = os.Stat(filepath.Join(tmpDir, name+metadataSuffix)) assert.True(t, os.IsNotExist(err)) } + +// Replacing an artifact keeps the permission bits of the file already on disk. The previous +// adapter truncated the file in place, so an artifact restricted by hand stayed restricted. +func TestUploadReplaceKeepsDestinationMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + + tests := []struct { + name string + existing os.FileMode + want os.FileMode + }{ + {name: "restricted artifact stays restricted", existing: 0o600, want: 0o600}, + {name: "readable artifact stays readable", existing: 0o644, want: 0o644}, + {name: "group-writable artifact stays group-writable", existing: 0o664, want: 0o664}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + store := &Store{basePath: dir} + path := filepath.Join(dir, "plan.tar") + require.NoError(t, os.WriteFile(path, []byte("old"), tt.existing)) + require.NoError(t, os.Chmod(path, tt.existing)) + + require.NoError(t, store.Upload(context.Background(), "plan.tar", strings.NewReader("new contents"), -1, nil)) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, tt.want, info.Mode().Perm()) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "new contents", string(data)) + }) + } +} + +// A new artifact has no destination to inherit from and keeps the default mode. +func TestUploadNewArtifactUsesDefaultMode(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("POSIX permission bits are not enforced on %s", runtime.GOOS) + } + dir := t.TempDir() + store := &Store{basePath: dir} + + require.NoError(t, store.Upload(context.Background(), "fresh.tar", strings.NewReader("data"), -1, nil)) + + info, err := os.Stat(filepath.Join(dir, "fresh.tar")) + require.NoError(t, err) + assert.NotZero(t, info.Mode().Perm()&0o600) + assert.Zero(t, info.Mode().Perm()&^os.FileMode(0o644), "default mode must not exceed 0644 before umask") +} diff --git a/pkg/ci/artifact/local/transport_parity_test.go b/pkg/ci/artifact/local/transport_parity_test.go new file mode 100644 index 00000000000..d6bc601b67b --- /dev/null +++ b/pkg/ci/artifact/local/transport_parity_test.go @@ -0,0 +1,257 @@ +package local + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +// dirEntries lists every regular file under root as slash-separated relative paths. +func dirEntries(t *testing.T, root string) []string { + t.Helper() + var entries []string + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + require.NoError(t, err) + if !d.IsDir() { + rel, relErr := filepath.Rel(root, path) + require.NoError(t, relErr) + entries = append(entries, filepath.ToSlash(rel)) + } + return nil + })) + sort.Strings(entries) + return entries +} + +// TestTransportKeepsTheLegacyOnDiskLayout pins the wire format: one file per artifact plus a +// ".metadata.json" sidecar holding indented JSON, and nothing else in the directory. +func TestTransportKeepsTheLegacyOnDiskLayout(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + created := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc", CreatedAt: created, Custom: map[string]string{"k": "v"}} + body := []byte("tar-bytes\x00\x01") + + require.NoError(t, store.Upload(context.Background(), "dev/vpc/abc.tar", bytes.NewReader(body), int64(len(body)), metadata)) + + assert.Equal(t, []string{"dev/vpc/abc.tar", "dev/vpc/abc.tar.metadata.json"}, dirEntries(t, root)) + onDisk, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "abc.tar")) + require.NoError(t, err) + assert.Equal(t, body, onDisk) + sidecar, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "abc.tar.metadata.json")) + require.NoError(t, err) + want, err := json.MarshalIndent(metadata, "", " ") + require.NoError(t, err) + assert.Equal(t, want, sidecar, "the sidecar is byte-identical to the legacy encoding") + if info, statErr := os.Stat(filepath.Join(root, "dev", "vpc", "abc.tar")); statErr == nil && filepath.Separator == '/' { + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm()) + } +} + +// TestBundledStoreTarBytesAndChecksumSurviveTheTransport verifies the tar archive round trip and +// the SHA-256 recorded in the sidecar still describe the stored bytes. +func TestBundledStoreTarBytesAndChecksumSurviveTheTransport(t *testing.T) { + root := t.TempDir() + bundled := artifact.NewBundledStore(&Store{basePath: root}) + files := []artifact.FileEntry{ + {Name: "plan.tfplan", Data: strings.NewReader("plan-bytes"), Size: 10}, + {Name: "lock.hcl", Data: strings.NewReader("lock"), Size: 4}, + } + require.NoError(t, bundled.Upload(context.Background(), "dev/vpc/sha.tfplan.tar", files, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "sha"})) + + stored, err := os.ReadFile(filepath.Join(root, "dev", "vpc", "sha.tfplan.tar")) + require.NoError(t, err) + sum := sha256.Sum256(stored) + meta, err := bundled.GetMetadata(context.Background(), "dev/vpc/sha.tfplan.tar") + require.NoError(t, err) + assert.Equal(t, hex.EncodeToString(sum[:]), meta.SHA256) + + got, _, err := bundled.Download(context.Background(), "dev/vpc/sha.tfplan.tar") + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "plan.tfplan", got[0].Name) + assert.Equal(t, "lock.hcl", got[1].Name) + for index, want := range []string{"plan-bytes", "lock"} { + body := got[index].Data + t.Cleanup(func() { _ = body.Close() }) + data, readErr := io.ReadAll(body) + require.NoError(t, readErr) + require.NoError(t, got[index].Data.Close()) + assert.Equal(t, want, string(data)) + } +} + +// TestNamesAreCleanedLikeTheFilesystemDid pins that equivalent spellings address one artifact. +func TestNamesAreCleanedLikeTheFilesystemDid(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a/./b//c.tar", strings.NewReader("x"), 1, nil)) + assert.Equal(t, []string{"a/b/c.tar", "a/b/c.tar.metadata.json"}, dirEntries(t, root)) + + exists, err := store.Exists(ctx, "a/b/c.tar") + require.NoError(t, err) + assert.True(t, exists) + exists, err = store.Exists(ctx, "a/x/../b/c.tar") + require.NoError(t, err) + assert.True(t, exists) +} + +// TestDirectoryNamesKeepTheirLegacyErrors pins error behavior for a name that is a directory. +func TestDirectoryNamesKeepTheirLegacyErrors(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "dev/vpc/a.tar", strings.NewReader("x"), 1, nil)) + + _, _, err := store.Download(ctx, "dev/vpc") + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) + _, err = store.GetMetadata(ctx, "dev/vpc") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + exists, err := store.Exists(ctx, "dev/vpc") + require.NoError(t, err) + assert.False(t, exists) +} + +// TestUploadReplacesAtomically checks a failed overwrite leaves the previous artifact intact. +func TestUploadReplacesAtomically(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a.tar", strings.NewReader("original"), 8, nil)) + + err := store.Upload(ctx, "a.tar", &brokenReader{}, -1, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + + reader, _, err := store.Download(ctx, "a.tar") + require.NoError(t, err) + defer reader.Close() + var buf bytes.Buffer + _, err = buf.ReadFrom(reader) + require.NoError(t, err) + assert.Equal(t, "original", buf.String(), "a failed upload no longer truncates the existing artifact") + for _, entry := range dirEntries(t, root) { + assert.NotContains(t, entry, ".atmos-blob-tmp-", "no temporary files are left behind") + } +} + +type brokenReader struct{ done bool } + +func (b *brokenReader) Read(p []byte) (int, error) { + if !b.done { + b.done = true + return copy(p, "partial"), nil + } + return 0, os.ErrClosed +} + +// TestListSkipsOrphanSidecarsAndFiltersByQuery covers the sidecar-driven listing semantics. +func TestListSkipsOrphanSidecarsAndFiltersByQuery(t *testing.T) { + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "dev/vpc/1.tar", strings.NewReader("a"), 1, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "1"})) + require.NoError(t, store.Upload(ctx, "prod/vpc/2.tar", strings.NewReader("bb"), 2, &artifact.Metadata{Stack: "prod", Component: "vpc", SHA: "2"})) + require.NoError(t, os.WriteFile(filepath.Join(root, "orphan.tar.metadata.json"), []byte(`{"stack":"x"}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "no-sidecar.tar"), []byte("data"), 0o644)) + + all, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + names := make([]string, 0, len(all)) + for _, info := range all { + names = append(names, filepath.ToSlash(info.Name)) + } + sort.Strings(names) + assert.Equal(t, []string{"dev/vpc/1.tar", "prod/vpc/2.tar"}, names, "artifacts need both the file and its sidecar") + + prod, err := store.List(ctx, artifact.Query{Stacks: []string{"prod"}}) + require.NoError(t, err) + require.Len(t, prod, 1) + assert.Equal(t, int64(2), prod[0].Size) + assert.Equal(t, "prod", prod[0].Metadata.Stack) +} + +func TestLegacyLiteralBackslashName(t *testing.T) { + if filepath.Separator == '\\' { + t.Skip("backslash is a path separator") + } + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + name := `literal\name` + require.NoError(t, store.Upload(ctx, name, strings.NewReader("payload"), 7, nil)) + assert.FileExists(t, filepath.Join(root, name)) + reader, _, err := store.Download(ctx, name) + require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "payload", string(data)) + listed, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, name, listed[0].Name) + require.NoError(t, store.Delete(ctx, name)) +} + +// TestAttributeLookingArtifactNamesStayIndependent covers legacy stores that hold both "a.tar" and +// "a.tar.atmos-blob.json": the transport keeps no attribute files in this layout, so neither +// artifact may shadow, parse, or delete the other. +func TestAttributeLookingArtifactNamesStayIndependent(t *testing.T) { + const object, lookalike = "a.tar", "a.tar.atmos-blob.json" + root := t.TempDir() + store := &Store{basePath: root} + ctx := context.Background() + upload := func(name, body string) { + t.Helper() + require.NoError(t, store.Upload(ctx, name, strings.NewReader(body), int64(len(body)), nil), name) + } + download := func(name string) string { + t.Helper() + reader, _, err := store.Download(ctx, name) + require.NoError(t, err, name) + defer reader.Close() + data, err := io.ReadAll(reader) + require.NoError(t, err) + return string(data) + } + + upload(object, "tar-v1") + upload(lookalike, "not an attribute document") + upload(object, "tar-v2") + + assert.Equal(t, "tar-v2", download(object), "a.tar downloads although a.tar.atmos-blob.json exists beside it") + assert.Equal(t, "not an attribute document", download(lookalike), "updating a.tar does not delete the second artifact") + + listed, err := store.List(ctx, artifact.Query{All: true}) + require.NoError(t, err) + names := make([]string, 0, len(listed)) + for _, info := range listed { + names = append(names, info.Name) + } + assert.ElementsMatch(t, []string{object, lookalike}, names, "both legacy artifacts are listed") + + exists, err := store.Exists(ctx, lookalike) + require.NoError(t, err) + assert.True(t, exists) + + require.NoError(t, store.Delete(ctx, object)) + assert.Equal(t, "not an attribute document", download(lookalike), "deleting a.tar does not delete the second artifact") + assert.Equal(t, []string{lookalike, lookalike + ".metadata.json"}, dirEntries(t, root)) +} diff --git a/pkg/ci/artifact/s3/endpoint_request_test.go b/pkg/ci/artifact/s3/endpoint_request_test.go index 8e1dcaf6a86..8bd73858e0e 100644 --- a/pkg/ci/artifact/s3/endpoint_request_test.go +++ b/pkg/ci/artifact/s3/endpoint_request_test.go @@ -18,7 +18,10 @@ import ( func TestIdentityEndpointUsesBucketPath(t *testing.T) { requests := make(chan *http.Request, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests <- r.Clone(r.Context()) + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -40,3 +43,32 @@ func TestIdentityEndpointUsesBucketPath(t *testing.T) { t.Fatal("SDK request did not reach the emulator endpoint") } } + +func TestAdapterIdentityResolverRoutesToEndpoint(t *testing.T) { + requests := make(chan *http.Request, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case requests <- r.Clone(r.Context()): + default: // A retried request must not block the handler behind an unread channel. + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + resolver := &fakeAuthContextResolver{awsResult: &artifact.AWSAuthConfig{Region: "us-east-1", EndpointURL: server.URL}} + backend, err := NewStore(artifact.StoreOptions{Identity: "deploy", Options: map[string]any{"bucket": "artifacts"}}) + require.NoError(t, err) + store := backend.(*Store) + store.SetAuthContext(resolver, "") + exists, err := store.Exists(t.Context(), "path/object") + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, "deploy", resolver.requestedIDName) + select { + case request := <-requests: + require.Equal(t, "/artifacts/path/object", request.URL.Path) + default: + t.Fatal("identity endpoint received no request") + } +} diff --git a/pkg/ci/artifact/s3/store.go b/pkg/ci/artifact/s3/store.go index 44e68567145..c3f61e937c5 100644 --- a/pkg/ci/artifact/s3/store.go +++ b/pkg/ci/artifact/s3/store.go @@ -8,15 +8,16 @@ import ( "fmt" "io" "path" + "strings" "sync" - "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob" + blobs3 "github.com/cloudposse/atmos/pkg/artifact/blob/s3" "github.com/cloudposse/atmos/pkg/ci/artifact" "github.com/cloudposse/atmos/pkg/perf" ) @@ -26,9 +27,18 @@ const ( // Metadata is stored as a JSON file alongside the artifact. metadataSuffix = ".metadata.json" + + // The wrap format joins a static sentinel error with the underlying cause so both stay in the chain. + wrapFormat = "%w: %w" ) // Store implements the artifact.Backend interface using AWS S3. +// +// It is a CI adapter over the shared transport in pkg/artifact/blob/s3: this type owns the legacy +// key pattern (prefix joined with path.Join), the ".metadata.json" sidecar, and the legacy error +// sentinels, while client construction and object I/O live in the transport. Unlike the generic +// transport, it keeps the historical fallback to the default credential chain when an identity is +// configured but no resolver was injected. type Store struct { client *s3.Client bucket string @@ -41,6 +51,11 @@ type Store struct { authResolver artifact.AuthContextResolver initOnce sync.Once initErr error + + // Shared transport over client, built on first use. + transportOnce sync.Once + transportStore *blobs3.Store + transportErr error } // NewStore creates a new S3 backend. @@ -84,6 +99,8 @@ func NewStore(opts artifact.StoreOptions) (artifact.Backend, error) { // registry honors this contract by calling SetAuthContext synchronously // before returning the backend. func (s *Store) SetAuthContext(resolver artifact.AuthContextResolver, identityName string) { + defer perf.Track(nil, "s3.Store.SetAuthContext")() + s.authResolver = resolver if identityName != "" { s.identityName = identityName @@ -92,41 +109,33 @@ func (s *Store) SetAuthContext(resolver artifact.AuthContextResolver, identityNa // initDefaultClient initializes the AWS client using the default credential chain. func (s *Store) initDefaultClient(ctx context.Context) error { - var awsOpts []func(*config.LoadOptions) error - if s.region != "" { - awsOpts = append(awsOpts, config.WithRegion(s.region)) - } - - cfg, err := config.LoadDefaultConfig(ctx, awsOpts...) + client, err := blobs3.NewClient(ctx, &blobs3.ClientConfig{Region: s.region, Legacy: true}) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + return err } - - s.client = s3.NewFromConfig(cfg) + s.client = client return nil } // initIdentityClient initializes the AWS client using identity-based credentials // resolved through the injected AuthContextResolver. func (s *Store) initIdentityClient(ctx context.Context) error { - if s.authResolver == nil { - // No resolver was injected; fall back to the default chain. - return s.initDefaultClient(ctx) - } - - authContext, err := s.authResolver.ResolveAWSAuthContext(ctx, s.identityName) - if err != nil { - return fmt.Errorf("%w: failed to resolve AWS auth context for identity %q: %w", - errUtils.ErrAWSConfigLoadFailed, s.identityName, err) - } - - cfgOpts := s.buildAuthConfigOpts(authContext) - cfg, err := config.LoadDefaultConfig(ctx, cfgOpts...) + // A missing resolver falls back to the default chain: legacy CI behavior that the generic + // transport deliberately does not share. + var resolver blobs3.AuthResolver + if s.authResolver != nil { + resolver = s.authResolver + } + client, err := blobs3.NewClient(ctx, &blobs3.ClientConfig{ + Region: s.region, + Identity: s.identityName, + Resolver: resolver, + Legacy: true, + }) if err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrAWSConfigLoadFailed, err) + return err } - - s.client = s3.NewFromConfig(cfg, buildClientOptFns(authContext)...) + s.client = client return nil } @@ -135,42 +144,12 @@ func (s *Store) initIdentityClient(ctx context.Context) error { // Without this, an identity scoped to an emulator (e.g. Floci) still routes // S3 calls to the real AWS endpoint. func buildClientOptFns(authContext *artifact.AWSAuthConfig) []func(*s3.Options) { - var optFns []func(*s3.Options) - if authContext != nil && authContext.EndpointURL != "" { - optFns = append(optFns, func(o *s3.Options) { - o.BaseEndpoint = aws.String(authContext.EndpointURL) - o.UsePathStyle = true - }) - } - return optFns + return blobs3.BuildClientOptFns(authContext) } // buildAuthConfigOpts constructs AWS SDK config options from the auth context. func (s *Store) buildAuthConfigOpts(authContext *artifact.AWSAuthConfig) []func(*config.LoadOptions) error { - var cfgOpts []func(*config.LoadOptions) error - if authContext == nil { - return cfgOpts - } - if authContext.CredentialsFile != "" { - cfgOpts = append(cfgOpts, config.WithSharedCredentialsFiles([]string{authContext.CredentialsFile})) - } - if authContext.ConfigFile != "" { - cfgOpts = append(cfgOpts, config.WithSharedConfigFiles([]string{authContext.ConfigFile})) - } - if authContext.Profile != "" { - cfgOpts = append(cfgOpts, config.WithSharedConfigProfile(authContext.Profile)) - } - - // Use region from auth context if the store doesn't specify one. - region := s.region - if region == "" && authContext.Region != "" { - region = authContext.Region - } - if region != "" { - cfgOpts = append(cfgOpts, config.WithRegion(region)) - } - - return cfgOpts + return blobs3.BuildAuthConfigOpts(s.region, authContext) } // ensureClient lazily initializes the AWS client on first operation. @@ -207,30 +186,36 @@ func (s *Store) fullKey(key string) string { return path.Join(s.prefix, key) } +// transport returns the shared S3 transport over the lazily initialized client. Keys are passed +// through untouched (fullKey already applied the legacy prefix), so the transport is created +// without a prefix and with opaque-key validation to keep historical key patterns valid. Uploads use +// one PutObject at any size, as the previous adapter did, because multipart uploads need extra +// permissions (kms:Decrypt on SSE-KMS buckets, s3:AbortMultipartUpload) that a write-only CI role +// may not hold. +func (s *Store) transport(ctx context.Context) (*blobs3.Store, error) { + if err := s.ensureClient(ctx); err != nil { + return nil, err + } + s.transportOnce.Do(func() { + s.transportStore, s.transportErr = blobs3.NewFromClient(s.client, s.bucket, blobs3.WithOpaqueKeys(), blobs3.WithSinglePutOnly()) + }) + return s.transportStore, s.transportErr +} + // Upload uploads a single data stream to S3 with a metadata sidecar. func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int64, metadata *artifact.Metadata) error { defer perf.Track(nil, "s3.Upload")() - if err := s.ensureClient(ctx); err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrArtifactUploadFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return fmt.Errorf(wrapFormat, errUtils.ErrArtifactUploadFailed, err) } fullKey := s.fullKey(key) - // Read all data into memory for S3 PutObject (needs content length). - dataBytes, err := io.ReadAll(data) - if err != nil { - return fmt.Errorf("%w: failed to read data: %w", errUtils.ErrArtifactUploadFailed, err) - } - - // Upload the data. - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - Body: bytes.NewReader(dataBytes), - ContentLength: aws.Int64(int64(len(dataBytes))), - }) - if err != nil { + // Upload the data. The declared size is advisory, as it always was; the transport streams the + // body and uses a multipart upload only for bodies larger than one part. + if _, err := transport.Put(ctx, fullKey, data, blob.PutOptions{Size: blob.UnknownSize}); err != nil { return fmt.Errorf("%w: failed to upload artifact to S3: %w", errUtils.ErrArtifactUploadFailed, err) } @@ -242,12 +227,9 @@ func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int return fmt.Errorf("%w: failed to marshal metadata: %w", errUtils.ErrArtifactUploadFailed, err) } - _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(metadataKey), - Body: bytes.NewReader(metadataJSON), - ContentLength: aws.Int64(int64(len(metadataJSON))), - ContentType: aws.String("application/json"), + _, err = transport.Put(ctx, metadataKey, bytes.NewReader(metadataJSON), blob.PutOptions{ + Size: int64(len(metadataJSON)), + ContentType: "application/json", }) if err != nil { return fmt.Errorf("%w: failed to upload metadata to S3: %w", errUtils.ErrArtifactUploadFailed, err) @@ -262,111 +244,92 @@ func (s *Store) Upload(ctx context.Context, key string, data io.Reader, size int func (s *Store) Download(ctx context.Context, key string) (io.ReadCloser, *artifact.Metadata, error) { defer perf.Track(nil, "s3.Download")() - if err := s.ensureClient(ctx); err != nil { - return nil, nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactDownloadFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return nil, nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactDownloadFailed, err) } fullKey := s.fullKey(key) // Download the data. - result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) + object, err := transport.Get(ctx, fullKey, blob.GetOptions{}) if err != nil { - if isNoSuchKeyError(err) { + if isMissing(err) { return nil, nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, key) } return nil, nil, fmt.Errorf("%w: failed to download artifact from S3: %w", errUtils.ErrArtifactDownloadFailed, err) } // Try to load metadata. - metadata, _ := s.loadMetadata(ctx, fullKey) + metadata, _ := s.loadMetadata(ctx, transport, fullKey) - return result.Body, metadata, nil + return object.Body, metadata, nil } // Delete deletes an artifact from S3. func (s *Store) Delete(ctx context.Context, key string) error { defer perf.Track(nil, "s3.Delete")() - if err := s.ensureClient(ctx); err != nil { - return fmt.Errorf("%w: %w", errUtils.ErrArtifactDeleteFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return fmt.Errorf(wrapFormat, errUtils.ErrArtifactDeleteFailed, err) } fullKey := s.fullKey(key) // Delete the artifact. - _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) - if err != nil { + if err := transport.Delete(ctx, fullKey, blob.DeleteOptions{}); err != nil { return fmt.Errorf("%w: failed to delete artifact from S3: %w", errUtils.ErrArtifactDeleteFailed, err) } // Try to delete metadata (ignore errors). - _, _ = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey + metadataSuffix), - }) + _ = transport.Delete(ctx, fullKey+metadataSuffix, blob.DeleteOptions{}) return nil } // List lists artifacts matching the given query. -func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { +func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.ArtifactInfo, error) { //nolint:gocritic // Query is passed by value to satisfy the Backend interface. defer perf.Track(nil, "s3.List")() - if err := s.ensureClient(ctx); err != nil { - return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactListFailed, err) } // Convert query to prefix-based S3 listing. - prefix := s.queryToPrefix(query) + prefix := s.queryToPrefix(&query) fullPrefix := s.fullKey(prefix) + objects, _, err := blob.ListAll(ctx, transport, blob.ListOptions{Prefix: fullPrefix}) + if err != nil { + return nil, fmt.Errorf("%w: failed to list artifacts in S3: %w", errUtils.ErrArtifactListFailed, err) + } + var files []artifact.ArtifactInfo - paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(fullPrefix), - }) + for _, obj := range objects { + key := obj.Key - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("%w: failed to list artifacts in S3: %w", errUtils.ErrArtifactListFailed, err) + // Skip metadata files. + if strings.HasSuffix(key, metadataSuffix) && len(key) > len(metadataSuffix) { + continue } - for _, obj := range page.Contents { - key := aws.ToString(obj.Key) - - // Skip metadata files. - if len(key) > len(metadataSuffix) && key[len(key)-len(metadataSuffix):] == metadataSuffix { - continue - } - - // Remove prefix to get relative key. - relKey := key - if s.prefix != "" && len(key) > len(s.prefix)+1 { - relKey = key[len(s.prefix)+1:] - } - - // Try to load metadata. - metadata, _ := s.loadMetadata(ctx, key) - - var lastModified time.Time - if obj.LastModified != nil { - lastModified = *obj.LastModified - } - - files = append(files, artifact.ArtifactInfo{ - Name: relKey, - Size: aws.ToInt64(obj.Size), - LastModified: lastModified, - Metadata: metadata, - }) + // Remove prefix to get relative key. + relKey := key + if prefix := s.fullKey(""); s.prefix != "" { + relKey = strings.TrimPrefix(key, prefix+"/") } + + // Try to load metadata. + metadata, _ := s.loadMetadata(ctx, transport, key) + + files = append(files, artifact.ArtifactInfo{ + Name: relKey, + Size: obj.Size, + LastModified: obj.LastModified, + Metadata: metadata, + }) } return files, nil @@ -376,18 +339,15 @@ func (s *Store) List(ctx context.Context, query artifact.Query) ([]artifact.Arti func (s *Store) Exists(ctx context.Context, key string) (bool, error) { defer perf.Track(nil, "s3.Exists")() - if err := s.ensureClient(ctx); err != nil { - return false, fmt.Errorf("%w: %w", errUtils.ErrArtifactListFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return false, fmt.Errorf(wrapFormat, errUtils.ErrArtifactListFailed, err) } fullKey := s.fullKey(key) - _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) - if err != nil { - if isNotFoundError(err) { + if _, err := transport.Stat(ctx, fullKey, blob.StatOptions{}); err != nil { + if isMissing(err) { return false, nil } return false, fmt.Errorf("%w: failed to check if %s exists in S3: %w", errUtils.ErrArtifactListFailed, key, err) @@ -400,45 +360,37 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) { func (s *Store) GetMetadata(ctx context.Context, key string) (*artifact.Metadata, error) { defer perf.Track(nil, "s3.GetMetadata")() - if err := s.ensureClient(ctx); err != nil { - return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactMetadataFailed, err) + transport, err := s.transport(ctx) + if err != nil { + return nil, fmt.Errorf(wrapFormat, errUtils.ErrArtifactMetadataFailed, err) } fullKey := s.fullKey(key) // Check if the artifact exists. - headResult, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(fullKey), - }) + info, err := transport.Stat(ctx, fullKey, blob.StatOptions{}) if err != nil { - if isNotFoundError(err) { + if isMissing(err) { return nil, fmt.Errorf("%w: %s", errUtils.ErrArtifactNotFound, key) } return nil, fmt.Errorf("%w: failed to get metadata for %s from S3: %w", errUtils.ErrArtifactMetadataFailed, key, err) } // Try to load metadata from separate file. - metadata, err := s.loadMetadata(ctx, fullKey) + metadata, err := s.loadMetadata(ctx, transport, fullKey) if err != nil || metadata == nil { // Return minimal metadata from S3 object. - metadata = &artifact.Metadata{} - if headResult.LastModified != nil { - metadata.CreatedAt = *headResult.LastModified - } + metadata = &artifact.Metadata{CreatedAt: info.LastModified} } return metadata, nil } // loadMetadata loads metadata from the metadata file in S3. -func (s *Store) loadMetadata(ctx context.Context, artifactKey string) (*artifact.Metadata, error) { +func (s *Store) loadMetadata(ctx context.Context, transport blob.Store, artifactKey string) (*artifact.Metadata, error) { metadataKey := artifactKey + metadataSuffix - result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(metadataKey), - }) + result, err := transport.Get(ctx, metadataKey, blob.GetOptions{}) if err != nil { return nil, fmt.Errorf("%w: failed to get metadata from S3: %w", errUtils.ErrArtifactMetadataFailed, err) } @@ -453,7 +405,7 @@ func (s *Store) loadMetadata(ctx context.Context, artifactKey string) (*artifact } // queryToPrefix converts an artifact.Query to an S3 prefix string. -func (s *Store) queryToPrefix(query artifact.Query) string { +func (s *Store) queryToPrefix(query *artifact.Query) string { if query.All { return "" } @@ -469,6 +421,12 @@ func (s *Store) queryToPrefix(query artifact.Query) string { return prefix } +// isMissing reports whether err means the object does not exist, whether it was classified by the +// shared transport or is a raw SDK error. +func isMissing(err error) bool { + return blob.IsNotFound(err) || isNoSuchKeyError(err) || isNotFoundError(err) +} + // isNoSuchKeyError checks if the error is a NoSuchKey error. func isNoSuchKeyError(err error) bool { var noSuchKey *types.NoSuchKey diff --git a/pkg/ci/artifact/s3/store_test.go b/pkg/ci/artifact/s3/store_test.go index 6541a6c316a..2a58c34804e 100644 --- a/pkg/ci/artifact/s3/store_test.go +++ b/pkg/ci/artifact/s3/store_test.go @@ -502,7 +502,7 @@ func TestStore_QueryToPrefix(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := store.queryToPrefix(tt.query) + result := store.queryToPrefix(&tt.query) assert.Equal(t, tt.expected, result) }) } diff --git a/pkg/ci/artifact/s3/transport_parity_test.go b/pkg/ci/artifact/s3/transport_parity_test.go new file mode 100644 index 00000000000..678f9dc6733 --- /dev/null +++ b/pkg/ci/artifact/s3/transport_parity_test.go @@ -0,0 +1,215 @@ +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +// newFakeStore returns the legacy adapter over a fake S3 endpoint, using the SDK's default +// checksum behavior like production. +func newFakeStore(t *testing.T, prefix string) (*Store, *s3test.Server) { + t.Helper() + fake := s3test.NewServer(t) + return &Store{client: fake.LegacyClient(), bucket: fake.Bucket(), prefix: prefix}, fake +} + +func TestAdapterKeepsKeyPatternAndSidecarFormat(t *testing.T) { + store, fake := newFakeStore(t, "planfiles") + created := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc", CreatedAt: created, Custom: map[string]string{"k": "v"}} + body := []byte("tar-bytes\x00\x01") + + require.NoError(t, store.Upload(context.Background(), "dev/vpc/abc.tar", bytes.NewReader(body), int64(len(body)), metadata)) + + assert.Equal(t, []string{"planfiles/dev/vpc/abc.tar", "planfiles/dev/vpc/abc.tar.metadata.json"}, fake.Keys()) + data, ok := fake.Object("planfiles/dev/vpc/abc.tar") + require.True(t, ok) + assert.Equal(t, body, data.Data) + assert.Empty(t, data.Tags, "the legacy adapter adds no object tags") + + sidecar, ok := fake.Object("planfiles/dev/vpc/abc.tar.metadata.json") + require.True(t, ok) + want, err := json.MarshalIndent(metadata, "", " ") + require.NoError(t, err) + assert.Equal(t, want, sidecar.Data, "the sidecar is byte-identical to the legacy encoding") + assert.Equal(t, "application/json", sidecar.ContentType) + + puts := fake.RequestsFor(s3test.OpPutObject) + require.Len(t, puts, 2, "one request per object, data first and sidecar second") + assert.Equal(t, "planfiles/dev/vpc/abc.tar", puts[0].Key) + assert.Equal(t, "planfiles/dev/vpc/abc.tar.metadata.json", puts[1].Key) + for _, put := range puts { + assert.Empty(t, put.Header.Get("If-None-Match"), "legacy writes stay unconditional") + assert.Empty(t, put.Header.Get("X-Amz-Tagging")) + } +} + +func TestAdapterLegacyKeyQuirksArePreserved(t *testing.T) { + store, fake := newFakeStore(t, "pre/fix/") + require.NoError(t, store.Upload(context.Background(), "a//b/../c.tar", strings.NewReader("x"), 1, nil)) + assert.Equal(t, []string{"pre/fix/a/c.tar"}, fake.Keys(), "path.Join cleaning of prefix and key is unchanged; nil metadata writes no sidecar") + listed, err := store.List(context.Background(), artifact.Query{All: true}) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "a/c.tar", listed[0].Name) +} + +func TestAdapterRoundTripWithoutPrefix(t *testing.T) { + store, fake := newFakeStore(t, "") + ctx := context.Background() + metadata := &artifact.Metadata{Stack: "s", Component: "c", SHA: "1"} + require.NoError(t, store.Upload(ctx, "k.tar", strings.NewReader("payload"), 7, metadata)) + assert.Equal(t, []string{"k.tar", "k.tar.metadata.json"}, fake.Keys()) + + reader, got, err := store.Download(ctx, "k.tar") + require.NoError(t, err) + t.Cleanup(func() { _ = reader.Close() }) + var buf bytes.Buffer + _, err = buf.ReadFrom(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "payload", buf.String()) + require.NotNil(t, got) + assert.Equal(t, "s", got.Stack) + + exists, err := store.Exists(ctx, "k.tar") + require.NoError(t, err) + assert.True(t, exists) + exists, err = store.Exists(ctx, "missing.tar") + require.NoError(t, err) + assert.False(t, exists) + + _, _, err = store.Download(ctx, "missing.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = store.GetMetadata(ctx, "missing.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestAdapterDownloadWithoutSidecarReturnsNilMetadata(t *testing.T) { + store, fake := newFakeStore(t, "") + fake.Seed("bare.tar", []byte("data"), nil) + reader, meta, err := store.Download(context.Background(), "bare.tar") + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Nil(t, meta) + + minimal, err := store.GetMetadata(context.Background(), "bare.tar") + require.NoError(t, err) + assert.False(t, minimal.CreatedAt.IsZero(), "falls back to the object's modification time") +} + +func TestAdapterDeleteRemovesObjectAndSidecarAndIsIdempotent(t *testing.T) { + store, fake := newFakeStore(t, "p") + ctx := context.Background() + require.NoError(t, store.Upload(ctx, "a.tar", strings.NewReader("x"), 1, &artifact.Metadata{})) + require.NoError(t, store.Delete(ctx, "a.tar")) + assert.Empty(t, fake.Keys()) + require.NoError(t, store.Delete(ctx, "a.tar")) +} + +func TestAdapterListHandlesPrefixSidecarsAndPagination(t *testing.T) { + store, fake := newFakeStore(t, "plans") + const total = 1105 + for i := range total { + key := fmt.Sprintf("plans/dev/c%04d.tar", i) + fake.Seed(key, []byte("x"), nil) + fake.Seed(key+metadataSuffix, []byte(fmt.Sprintf(`{"stack":"dev","component":"c%04d","sha":"s"}`, i)), nil) + } + fake.Seed("plans/prod/other.tar", []byte("x"), nil) + fake.Seed("elsewhere/dev/x.tar", []byte("x"), nil) + + infos, err := store.List(context.Background(), artifact.Query{Stacks: []string{"dev"}}) + require.NoError(t, err) + require.Len(t, infos, total, "sidecars are skipped and every page is consumed") + assert.Equal(t, "dev/c0000.tar", infos[0].Name, "names are relative to the prefix") + assert.Equal(t, "dev/c1104.tar", infos[total-1].Name) + require.NotNil(t, infos[0].Metadata) + assert.Equal(t, "c0000", infos[0].Metadata.Component) + assert.Equal(t, int64(1), infos[0].Size) + assert.False(t, infos[0].LastModified.IsZero()) +} + +func TestAdapterBundledStoreChecksumAndTarSurviveTheTransport(t *testing.T) { + store, fake := newFakeStore(t, "ci") + bundled := artifact.NewBundledStore(store) + files := []artifact.FileEntry{{Name: "plan.tfplan", Data: strings.NewReader("plan-bytes"), Size: 10}} + require.NoError(t, bundled.Upload(context.Background(), "dev/vpc/sha.tar", files, &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "sha"})) + + stored, ok := fake.Object("ci/dev/vpc/sha.tar") + require.True(t, ok) + sum := sha256.Sum256(stored.Data) + meta, err := bundled.GetMetadata(context.Background(), "dev/vpc/sha.tar") + require.NoError(t, err) + assert.Equal(t, hex.EncodeToString(sum[:]), meta.SHA256) + + got, _, err := bundled.Download(context.Background(), "dev/vpc/sha.tar") + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "plan.tfplan", got[0].Name) + t.Cleanup(func() { _ = got[0].Data.Close() }) + data, err := io.ReadAll(got[0].Data) + require.NoError(t, err) + require.NoError(t, got[0].Data.Close()) + assert.Equal(t, "plan-bytes", string(data)) +} + +// The adapter sends one PutObject at any size, as the previous adapter did, so a role limited to +// PutObject keeps working. Memory stays bounded because the body is spooled, not buffered. +func TestAdapterLargeUploadIsOnePutObjectWithLegacyClient(t *testing.T) { + store, fake := newFakeStore(t, "") + body := bytes.Repeat([]byte("0123456789abcdef"), (9<<20)/16) + require.NoError(t, store.Upload(context.Background(), "big.tar", bytes.NewReader(body), int64(len(body)), nil)) + + stored, ok := fake.Object("big.tar") + require.True(t, ok) + assert.Equal(t, len(body), len(stored.Data)) + assert.True(t, bytes.Equal(body, stored.Data)) + assert.Equal(t, 0, fake.CompletedUploads()) + assert.Empty(t, fake.OpenUploads()) + require.Len(t, fake.Requests(), 1) + assert.Equal(t, s3test.OpPutObject, fake.Requests()[0].Op) +} + +func TestAdapterWrapsTransportFailuresInLegacySentinels(t *testing.T) { + store, fake := newFakeStore(t, "") + ctx := context.Background() + fake.Inject(s3test.Fault{Op: s3test.OpPutObject, Status: 403, Code: "AccessDenied", Times: -1}) + err := store.Upload(ctx, "k.tar", strings.NewReader("x"), 1, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactUploadFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpGetObject, Status: 403, Code: "AccessDenied", Times: -1}) + _, _, err = store.Download(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactDownloadFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpListObjectsV2, Status: 403, Code: "AccessDenied", Times: -1}) + _, err = store.List(ctx, artifact.Query{All: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactListFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpHeadObject, Status: 403, Code: "Forbidden", Times: -1}) + _, err = store.Exists(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactListFailed) + _, err = store.GetMetadata(ctx, "k.tar") + require.ErrorIs(t, err, errUtils.ErrArtifactMetadataFailed) + + fake.ClearFaults() + fake.Inject(s3test.Fault{Op: s3test.OpDeleteObject, Status: 403, Code: "AccessDenied", Times: -1}) + require.ErrorIs(t, store.Delete(ctx, "k.tar"), errUtils.ErrArtifactDeleteFailed) +} diff --git a/pkg/ci/artifact/s3/upload_permissions_test.go b/pkg/ci/artifact/s3/upload_permissions_test.go new file mode 100644 index 00000000000..f6e06746c61 --- /dev/null +++ b/pkg/ci/artifact/s3/upload_permissions_test.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/blob/s3/s3test" + "github.com/cloudposse/atmos/pkg/ci/artifact" +) + +const ( + partSize = 8 << 20 + denied = 403 +) + +// denyEverythingButPutObject models a role that may only write objects. On an SSE-KMS bucket such +// a role holds s3:PutObject and kms:GenerateDataKey but not kms:Decrypt, so every operation that +// reads an object back, and every multipart operation, is refused. +func denyEverythingButPutObject(server *s3test.Server) { + for _, op := range []string{ + s3test.OpGetObject, + s3test.OpHeadObject, + s3test.OpGetObjectTagging, + s3test.OpListObjectsV2, + s3test.OpDeleteObject, + s3test.OpCreateMultipartUpload, + s3test.OpUploadPart, + s3test.OpCompleteMultipartUpload, + s3test.OpAbortMultipartUpload, + } { + server.Inject(s3test.Fault{Op: op, Times: -1, Action: s3test.FailStatus, Status: denied, Code: "AccessDenied"}) + } +} + +func patterned(n int) []byte { + data := make([]byte, n) + for i := range data { + data[i] = byte(i*7 + i/251) + } + return data +} + +// An upload that worked with the previous single-PutObject adapter must keep working for a role +// that can only put objects, whatever the artifact size, including sizes past the multipart +// threshold. The adapter may only issue PutObject requests. +func TestUploadNeedsOnlyPutObjectPermission(t *testing.T) { + tests := []struct { + name string + size int + }{ + {name: "empty", size: 0}, + {name: "small", size: 1 << 10}, + {name: "exactly one part", size: partSize}, + {name: "one byte past one part", size: partSize + 1}, + {name: "multiple parts", size: partSize*2 + 123}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s3test.NewServer(t) + denyEverythingButPutObject(server) + store := &Store{client: server.Client(), bucket: server.Bucket(), prefix: "planfiles"} + data := patterned(tt.size) + metadata := &artifact.Metadata{Stack: "dev", Component: "vpc", SHA: "abc"} + + err := store.Upload(context.Background(), "dev/vpc/abc.tfplan", bytes.NewReader(data), int64(len(data)), metadata) + + require.NoError(t, err) + stored, ok := server.Object("planfiles/dev/vpc/abc.tfplan") + require.True(t, ok) + assert.True(t, bytes.Equal(data, stored.Data), "stored %d bytes, want %d", len(stored.Data), len(data)) + _, ok = server.Object("planfiles/dev/vpc/abc.tfplan.metadata.json") + assert.True(t, ok, "the metadata sidecar is uploaded") + for _, request := range server.Requests() { + assert.Equal(t, s3test.OpPutObject, request.Op, "request %d", request.Seq) + assert.Empty(t, request.Query.Get("uploadId"), "no multipart request") + } + assert.Empty(t, server.OpenUploads()) + }) + } +} + +// Refusing an upload for lack of PutObject permission is reported, not hidden by a fallback. +func TestUploadReportsPutObjectDenied(t *testing.T) { + server := s3test.NewServer(t) + server.Inject(s3test.Fault{Op: s3test.OpPutObject, Times: -1, Action: s3test.FailStatus, Status: denied, Code: "AccessDenied"}) + store := &Store{client: server.Client(), bucket: server.Bucket()} + + err := store.Upload(context.Background(), "k", bytes.NewReader(patterned(partSize+1)), -1, nil) + + require.Error(t, err) + assert.Empty(t, server.Keys()) +}