diff --git a/.atmos-git-hooks.yaml b/.atmos-git-hooks.yaml index dc8c4dd23f2..8b707979d06 100644 --- a/.atmos-git-hooks.yaml +++ b/.atmos-git-hooks.yaml @@ -21,7 +21,7 @@ checks: - id: go-fumpt handler: gofumpt files: '\.go$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/)' fix: true - id: go-mod-tidy @@ -37,7 +37,7 @@ checks: - id: golangci-lint handler: custom_lint files: '\.go$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/)' - id: gomodcheck handler: module_compatibility @@ -57,18 +57,18 @@ checks: - id: trailing-whitespace handler: trailing_whitespace - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|.*\.svg|website/src/components/Screengrabs/)' fix: true - id: end-of-file-fixer handler: end_of_file - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|.*\.svg|website/src/components/Screengrabs/)' fix: true - id: check-yaml handler: yaml_syntax files: '\.(yaml|yml)$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|tests/fixtures/)' - id: check-added-large-files handler: large_files diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ea3821ae37..e1c73a821ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -311,7 +311,9 @@ jobs: # builds silently drop that list in the aggregate `validate --affected` path. # Exclude README.md: it's machine-generated by cloudposse/build-harness, so its # formatting isn't ours to enforce. - run: atmos validate --affected --exclude 'tests/fixtures/**' --exclude '**/*.go' --exclude 'specs/**' --exclude 'README.md' --format rich + # pkg/artifact/compat/testdata/** holds frozen, digest-pinned fixtures that must stay + # byte-exact (no trailing newline); this mirrors scripts/checks/checks.star. + run: atmos validate --affected --exclude 'tests/fixtures/**' --exclude 'pkg/artifact/compat/testdata/**' --exclude '**/*.go' --exclude 'specs/**' --exclude 'README.md' --format rich # Committed website casts and screengrabs are generated artifacts, but a # change can also edit or hand-record them directly, bypassing the pipeline diff --git a/.gitignore b/.gitignore index 7f52759e452..1cfe9270714 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,8 @@ tools/gomodcheck/.gomodcheck .DS_Store /variant *.tar +# Frozen compatibility fixtures must be committed even when they are archives. +!pkg/artifact/compat/testdata/**/*.tar *.gz /bin diff --git a/errors/errors.go b/errors/errors.go index cc179229494..91e322a1422 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -258,6 +258,7 @@ var ( ErrYamlFuncInvalidArguments = errors.New("invalid number of arguments in the Atmos YAML function") ErrYamlFuncTemplateEmbedding = errors.New("a YAML function value cannot be used in this template expression") ErrYamlFuncMaxResolutionDepth = errors.New("Atmos YAML function resolution exceeded the maximum dependency depth (likely an undetected circular dependency)") + ErrYamlAliasExpansionLimit = errors.New("YAML aliases expand to more values than the configuration loader allows") ErrComponentFuncDuringManifestLoad = errors.New("atmos.Component cannot be evaluated while stack manifests are being loaded") ErrDeferredTemplateContextMissing = errors.New("cannot resolve deferred !template value: template context is unavailable because template processing was disabled for this invocation") ErrAwsGetCallerIdentity = errors.New("failed to get AWS caller identity") @@ -2080,6 +2081,47 @@ var ( ErrPolicyPlan = errors.New("saved plan policy enforcement failed") ) +// Generic artifact domain sentinels. They describe repository-independent outcomes of the +// pkg/artifact contracts; backend adapters wrap them and keep their SDK causes for diagnostics. +// Missing artifacts use ErrArtifactNotFound and integrity mismatches use ErrArtifactIntegrityFailed. +var ( + // ErrArtifactConfigInvalid is returned when artifact repository or verification configuration is invalid. + ErrArtifactConfigInvalid = errors.New("invalid artifact configuration") + + // ErrArtifactRefInvalid is returned when an artifact name, version, digest, or reference is malformed. + ErrArtifactRefInvalid = errors.New("invalid artifact reference") + + // ErrArtifactConflict is returned when a publication, copy, or delete conflicts with existing state. + ErrArtifactConflict = errors.New("artifact conflict") + + // ErrArtifactPublicationUnavailable is returned when a reference exists but cannot currently be + // published to or read, for example while another operation is pending or after an interrupted replacement. + ErrArtifactPublicationUnavailable = errors.New("artifact publication is unavailable") + + // ErrArtifactUnsupportedCapability is returned when a repository does not provide a requested + // guarantee. It is never a successful no-op. + ErrArtifactUnsupportedCapability = errors.New("artifact repository capability is not supported") + + // ErrArtifactDocumentMissing is returned when a requested package document (README or CHANGELOG) does not exist. + ErrArtifactDocumentMissing = errors.New("artifact document not found") + + // ErrArtifactPackagingInvalid is returned when source content cannot be packaged as an artifact. + ErrArtifactPackagingInvalid = errors.New("invalid artifact packaging") + + // ErrArtifactUnsafePath is returned for absolute, escaping, empty-segment, or reserved paths. + ErrArtifactUnsafePath = errors.New("unsafe artifact path") + + // ErrArtifactRecordInvalid is returned when an artifact record fails validation or is not canonical. + ErrArtifactRecordInvalid = errors.New("invalid artifact record") + + // ErrArtifactTooLarge is returned when a record, document, or evidence blob exceeds its bounded-read limit. + ErrArtifactTooLarge = errors.New("artifact data exceeds the size limit") + + // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The + // underlying I/O error is wrapped alongside it for diagnostics. + ErrArtifactStreamFailed = errors.New("artifact stream failed") +) + // ExitCodeError is a typed error that preserves subcommand exit codes. // This allows the root command to exit with the same code as the subcommand. // When Code is 0, it indicates successful completion that should exit cleanly without printing errors. diff --git a/pkg/artifact/boundary_test.go b/pkg/artifact/boundary_test.go new file mode 100644 index 00000000000..63bc5e95542 --- /dev/null +++ b/pkg/artifact/boundary_test.go @@ -0,0 +1,47 @@ +package artifact + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPackageDependencyBoundary keeps the domain package free of CLI, runner, hook, plugin, and CI +// dependencies, as required by the package boundary contract. +func TestPackageDependencyBoundary(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + "github.com/google/go-containerregistry", + "oras.land/oras-go", + "github.com/aws/aws-sdk-go-v2", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/artifact/canonical.go b/pkg/artifact/canonical.go new file mode 100644 index 00000000000..11bbf0cbba8 --- /dev/null +++ b/pkg/artifact/canonical.go @@ -0,0 +1,188 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Validate checks the Record against the V1 schema, including that ContentDigest matches the +// inventory and documents. It does not check size limits of the serialized form; Canonical does. +func (r *Record) Validate() error { + defer perf.Track(nil, "artifact.Record.Validate")() + + if r.SchemaVersion != SchemaVersion { + return fmt.Errorf("%w: unsupported schema_version %d (want %d)", errUtils.ErrArtifactRecordInvalid, r.SchemaVersion, SchemaVersion) + } + if err := ValidateName(r.Name); err != nil { + return err + } + if err := ValidateVersion(r.Version); err != nil { + return err + } + if err := ValidateKind(r.Kind); err != nil { + return err + } + if r.CreatedAt.IsZero() { + return fmt.Errorf("%w: created_at is required", errUtils.ErrArtifactRecordInvalid) + } + if err := r.Metadata.validate(); err != nil { + return err + } + expected, err := ComputeContentDigest(r.Inventory, r.Documents) + if err != nil { + return err + } + if r.ContentDigest != expected { + return fmt.Errorf("%w: content_digest %q does not match the inventory and documents (%s)", errUtils.ErrArtifactRecordInvalid, r.ContentDigest, expected) + } + return nil +} + +func (m Metadata) validate() error { + for key := range m.Labels { + if key == "" { + return fmt.Errorf("%w: label keys must not be empty", errUtils.ErrArtifactRecordInvalid) + } + } + return nil +} + +// Seal sets SchemaVersion (when unset) and ContentDigest from the inventory and documents, then +// validates the Record. Call it after assembling a Record from scratch. +func (r *Record) Seal() error { + defer perf.Track(nil, "artifact.Record.Seal")() + + if r.SchemaVersion == 0 { + r.SchemaVersion = SchemaVersion + } + computed, err := ComputeContentDigest(r.Inventory, r.Documents) + if err != nil { + return err + } + r.ContentDigest = computed + return r.Validate() +} + +// normalized returns a deep-enough copy with sorted arrays, non-nil arrays, normalized time, and +// empty provenance dropped, ready for serialization. The receiver is never mutated. +func (r *Record) normalized() Record { + out := *r + out.CreatedAt = NewTimestamp(r.CreatedAt.Time()) + out.Inventory = sortedInventory(r.Inventory) + out.Documents = sortedDocuments(r.Documents) + out.Metadata.Provenance = r.Metadata.Provenance.normalized() + if len(r.Metadata.Labels) == 0 { + out.Metadata.Labels = nil + } + return out +} + +func (p *Provenance) normalized() *Provenance { + if p == nil { + return nil + } + out := Provenance{} + if p.CI != nil && *p.CI != (CIProvenance{}) { + ci := *p.CI + out.CI = &ci + } + if p.Source != nil && *p.Source != (SourceProvenance{}) { + source := *p.Source + out.Source = &source + } + if out.CI == nil && out.Source == nil { + return nil + } + return &out +} + +// Canonical returns the canonical Record bytes: sorted keys and arrays, UTF-8, integer sizes and +// modes, UTC RFC3339 seconds, no insignificant whitespace, and no trailing newline. The same bytes +// are stored as the OCI config blob and in local and S3 record files. The Record is validated +// first, and records larger than [MaxRecordSize] are rejected. +func (r *Record) Canonical() ([]byte, error) { + defer perf.Track(nil, "artifact.Record.Canonical")() + + if err := r.Validate(); err != nil { + return nil, err + } + canonical, err := canonicalJSON(r.normalized()) + if err != nil { + return nil, err + } + if int64(len(canonical)) > MaxRecordSize { + return nil, fmt.Errorf("%w: record is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(canonical), MaxRecordSize) + } + return canonical, nil +} + +// canonicalJSON marshals v with encoding/json and re-encodes the result canonically. +func canonicalJSON(v any) ([]byte, error) { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return CanonicalizeJSON(buf.Bytes()) +} + +// RecordDigest returns the SHA-256 of the full canonical Record bytes, including created_at. +func (r *Record) RecordDigest() (digest.Digest, error) { + defer perf.Track(nil, "artifact.Record.RecordDigest")() + + canonical, err := r.Canonical() + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} + +// DecodeRecord strictly parses canonical Record bytes. It rejects oversize input, unknown fields, +// invalid records, and any bytes that are not exactly the canonical serialization of the decoded +// Record, so the record digest of the stored bytes is always the digest of the decoded value. +func DecodeRecord(raw []byte) (*Record, error) { + defer perf.Track(nil, "artifact.DecodeRecord")() + + if int64(len(raw)) > MaxRecordSize { + return nil, fmt.Errorf("%w: record is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(raw), MaxRecordSize) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var record Record + if err := decoder.Decode(&record); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + canonical, err := record.Canonical() + if err != nil { + return nil, err + } + if !bytes.Equal(canonical, raw) { + return nil, fmt.Errorf("%w: bytes are not the canonical serialization", errUtils.ErrArtifactRecordInvalid) + } + return &record, nil +} + +// ReadRecord reads and strictly decodes a Record from r, buffering at most [MaxRecordSize] bytes. +// It returns the canonical bytes alongside the Record so callers can compute the record digest +// of exactly what was stored. +func ReadRecord(r io.Reader) (*Record, []byte, error) { + defer perf.Track(nil, "artifact.ReadRecord")() + + raw, err := ReadBounded(r, MaxRecordSize) + if err != nil { + return nil, nil, err + } + record, err := DecodeRecord(raw) + if err != nil { + return nil, nil, err + } + return record, raw, nil +} diff --git a/pkg/artifact/canonicaljson.go b/pkg/artifact/canonicaljson.go new file mode 100644 index 00000000000..72044dd930e --- /dev/null +++ b/pkg/artifact/canonicaljson.go @@ -0,0 +1,262 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + // The nesting limit bounds depth so hostile kind data cannot exhaust the stack. + maxJSONDepth = 64 + + // The numeric base of canonical integers. + decimalBase = 10 + + // The bit width of canonical integers. + int64Bits = 64 + + // The first character that JSON strings need not escape. + firstPrintable = 0x20 +) + +// jsonKind enumerates the node types of the canonicalizer's ordered tree. +type jsonKind int + +const ( + jsonObject jsonKind = iota + jsonArray + jsonString + jsonInteger + jsonBool + jsonNull +) + +// jsonNode is one parsed JSON value. Objects keep their members as an ordered slice rather than a +// map, so no map[string]any or float64 is ever involved in canonicalization. +type jsonNode struct { + kind jsonKind + text string // String value, canonical integer text, or "true"/"false". + members []jsonMember + items []*jsonNode +} + +type jsonMember struct { + key string + value *jsonNode +} + +// CanonicalizeJSON re-encodes JSON text into the canonical form used by Records. +// +// The canonical form has object members sorted by key (bytewise on UTF-8), no insignificant +// whitespace, integers in plain decimal, and strings with only the escapes required by JSON (the +// quote, the backslash, and control characters as lowercase \u00xx); every other character is +// written as literal UTF-8, including <, >, &, U+2028 and U+2029. The result is independent of the +// Go version's encoding/json escaping behavior. The input must be valid UTF-8, contain no +// duplicate object keys, no floating-point numbers, no integers outside the int64 range, no +// U+FFFD replacement characters (which indicate invalid input), and no trailing data. +func CanonicalizeJSON(raw []byte) ([]byte, error) { + defer perf.Track(nil, "artifact.CanonicalizeJSON")() + + if !utf8.Valid(raw) { + return nil, fmt.Errorf("%w: JSON is not valid UTF-8", errUtils.ErrArtifactRecordInvalid) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + root, err := parseJSONNode(decoder, 0) + if err != nil { + return nil, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%w: unexpected data after the JSON value", errUtils.ErrArtifactRecordInvalid) + } + var out bytes.Buffer + root.write(&out) + return out.Bytes(), nil +} + +// parseJSONNode reads one complete JSON value from the decoder's token stream. +func parseJSONNode(decoder *json.Decoder, depth int) (*jsonNode, error) { + if depth > maxJSONDepth { + return nil, fmt.Errorf("%w: JSON nesting exceeds %d levels", errUtils.ErrArtifactRecordInvalid, maxJSONDepth) + } + token, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("%w: malformed JSON: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return parseJSONToken(decoder, token, depth) +} + +// parseJSONToken converts one token (and, for containers, the tokens that follow) into a node. +func parseJSONToken(decoder *json.Decoder, token json.Token, depth int) (*jsonNode, error) { + switch value := token.(type) { + case json.Delim: + if value == '{' { + return parseJSONObject(decoder, depth) + } + return parseJSONArray(decoder, depth) + case string: + return parseJSONStringNode(value) + case json.Number: + return parseJSONInteger(value) + case bool: + return &jsonNode{kind: jsonBool, text: strconv.FormatBool(value)}, nil + case nil: + return &jsonNode{kind: jsonNull}, nil + default: + return nil, fmt.Errorf("%w: unsupported JSON token %v", errUtils.ErrArtifactRecordInvalid, token) + } +} + +func parseJSONStringNode(value string) (*jsonNode, error) { + if err := checkJSONString(value); err != nil { + return nil, err + } + return &jsonNode{kind: jsonString, text: value}, nil +} + +func parseJSONObject(decoder *json.Decoder, depth int) (*jsonNode, error) { + node := &jsonNode{kind: jsonObject} + seen := make(map[string]struct{}) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("%w: malformed JSON object: %w", errUtils.ErrArtifactRecordInvalid, err) + } + key, ok := keyToken.(string) + if !ok { + return nil, fmt.Errorf("%w: JSON object key is not a string", errUtils.ErrArtifactRecordInvalid) + } + if err := checkJSONString(key); err != nil { + return nil, err + } + if _, duplicate := seen[key]; duplicate { + return nil, fmt.Errorf("%w: duplicate JSON object key %q", errUtils.ErrArtifactRecordInvalid, key) + } + seen[key] = struct{}{} + value, err := parseJSONNode(decoder, depth+1) + if err != nil { + return nil, err + } + node.members = append(node.members, jsonMember{key: key, value: value}) + } + if _, err := decoder.Token(); err != nil { // Consume '}'. + return nil, fmt.Errorf("%w: malformed JSON object: %w", errUtils.ErrArtifactRecordInvalid, err) + } + sort.Slice(node.members, func(i, j int) bool { return node.members[i].key < node.members[j].key }) + return node, nil +} + +func parseJSONArray(decoder *json.Decoder, depth int) (*jsonNode, error) { + node := &jsonNode{kind: jsonArray} + for decoder.More() { + item, err := parseJSONNode(decoder, depth+1) + if err != nil { + return nil, err + } + node.items = append(node.items, item) + } + if _, err := decoder.Token(); err != nil { // Consume ']'. + return nil, fmt.Errorf("%w: malformed JSON array: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return node, nil +} + +// parseJSONInteger accepts only plain decimal integers that fit in an int64. +func parseJSONInteger(number json.Number) (*jsonNode, error) { + value, err := strconv.ParseInt(number.String(), decimalBase, int64Bits) + if err != nil { + return nil, fmt.Errorf("%w: number %s is not an int64 integer; floats are not allowed", errUtils.ErrArtifactRecordInvalid, number) + } + return &jsonNode{kind: jsonInteger, text: strconv.FormatInt(value, decimalBase)}, nil +} + +// checkJSONString rejects U+FFFD, which is how encoding/json surfaces invalid UTF-8 and lone +// surrogate escapes. Accepting it would silently corrupt data. +func checkJSONString(s string) error { + if strings.ContainsRune(s, utf8.RuneError) { + return fmt.Errorf("%w: string contains an invalid UTF-8 sequence or U+FFFD", errUtils.ErrArtifactRecordInvalid) + } + return nil +} + +func (n *jsonNode) write(out *bytes.Buffer) { + switch n.kind { + case jsonObject: + out.WriteByte('{') + for i, member := range n.members { + if i > 0 { + out.WriteByte(',') + } + writeJSONString(out, member.key) + out.WriteByte(':') + member.value.write(out) + } + out.WriteByte('}') + case jsonArray: + out.WriteByte('[') + for i, item := range n.items { + if i > 0 { + out.WriteByte(',') + } + item.write(out) + } + out.WriteByte(']') + case jsonString: + writeJSONString(out, n.text) + case jsonInteger, jsonBool: + out.WriteString(n.text) + case jsonNull: + out.WriteString("null") + } +} + +const hexDigits = "0123456789abcdef" + +// writeJSONString writes s with the minimal, Go-version-independent escape set. +func writeJSONString(out *bytes.Buffer, s string) { + out.WriteByte('"') + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '"' || c == '\\': + out.WriteByte('\\') + out.WriteByte(c) + case c < firstPrintable: + out.WriteString(`\u00`) + out.WriteByte(hexDigits[c>>4]) + out.WriteByte(hexDigits[c&0x0f]) + default: + out.WriteByte(c) + } + } + out.WriteByte('"') +} + +// ReadBounded reads at most limit bytes from r and fails with ErrArtifactTooLarge when the stream +// is longer, so callers never buffer unbounded input. +func ReadBounded(r io.Reader, limit int64) ([]byte, error) { + defer perf.Track(nil, "artifact.ReadBounded")() + + if limit < 0 || limit == math.MaxInt64 { + return nil, fmt.Errorf("%w: invalid read limit %d", errUtils.ErrArtifactTooLarge, limit) + } + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, fmt.Errorf("%w: reading: %w", errUtils.ErrArtifactStreamFailed, err) + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("%w: more than %d bytes", errUtils.ErrArtifactTooLarge, limit) + } + return data, nil +} diff --git a/pkg/artifact/canonicaljson_test.go b/pkg/artifact/canonicaljson_test.go new file mode 100644 index 00000000000..c17dbf46062 --- /dev/null +++ b/pkg/artifact/canonicaljson_test.go @@ -0,0 +1,110 @@ +package artifact + +import ( + "bytes" + "errors" + "io" + "math" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestCanonicalizeJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"sorts keys recursively", `{"b":{"z":1,"a":2},"a":[3,{"y":true,"x":null}]}`, `{"a":[3,{"x":null,"y":true}],"b":{"a":2,"z":1}}`}, + {"strips whitespace", " {\n \"a\" : [ 1 , 2 ] }\n", `{"a":[1,2]}`}, + {"minimal escapes", `{"s":"<>&
\u007f"}`, "{\"s\":\"<>&
\u007f\"}"}, + {"control characters use lowercase u00xx", `{"s":"\n\t\u001f"}`, `{"s":"\u000a\u0009\u001f"}`}, + {"quote and backslash", `{"s":"\"\\"}`, `{"s":"\"\\"}`}, + {"unicode literal", `{"s":"é é"}`, `{"s":"é é"}`}, + {"integers normalized", `[0,-0,-12,9223372036854775807]`, `[0,0,-12,9223372036854775807]`}, + {"scalar", `true`, `true`}, + {"empty containers", `{"a":{},"b":[]}`, `{"a":{},"b":[]}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CanonicalizeJSON([]byte(tt.input)) + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + again, err := CanonicalizeJSON(got) + require.NoError(t, err) + assert.Equal(t, got, again, "canonicalization is idempotent") + }) + } +} + +func TestCanonicalizeJSONRejects(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"float", `{"a":1.5}`}, + {"exponent", `{"a":1e3}`}, + {"decimal zero", `{"a":1.0}`}, + {"integer overflow", `{"a":9223372036854775808}`}, + {"duplicate key", `{"a":1,"a":2}`}, + {"nested duplicate key", `{"x":{"a":1,"a":1}}`}, + {"invalid utf8", "{\"a\":\"\xff\"}"}, + {"lone surrogate escape", `{"a":"\ud800"}`}, + {"replacement character", "{\"a\":\"�\"}"}, + {"trailing data", `{"a":1} {"b":2}`}, + {"trailing garbage", `{"a":1} x`}, + {"truncated", `{"a":`}, + {"empty", ``}, + {"deep nesting", strings.Repeat("[", maxJSONDepth+2) + strings.Repeat("]", maxJSONDepth+2)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := CanonicalizeJSON([]byte(tt.input)) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) + } +} + +func TestReadBounded(t *testing.T) { + _, limitErr := ReadBounded(strings.NewReader("x"), math.MaxInt64) + require.Error(t, limitErr) + data, err := ReadBounded(strings.NewReader("12345"), 5) + require.NoError(t, err) + assert.Equal(t, "12345", string(data)) + + _, err = ReadBounded(strings.NewReader("123456"), 5) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + boom := errors.New("boom") + _, err = ReadBounded(io.MultiReader(strings.NewReader("1"), errReader{boom}), 5) + require.ErrorIs(t, err, boom) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "reader failures carry the artifact sentinel") + assert.NotErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + _, _, err = ReadRecord(io.MultiReader(strings.NewReader("{"), errReader{boom})) + require.ErrorIs(t, err, boom) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "ReadRecord surfaces the sentinel") +} + +type errReader struct{ err error } + +func (e errReader) Read([]byte) (int, error) { return 0, e.err } + +func TestReadRecord(t *testing.T) { + record, raw, err := ReadRecord(strings.NewReader(vectorACanonical)) + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(raw)) + assert.Equal(t, "components/terraform/vpc", record.Name) + + _, _, err = ReadRecord(bytes.NewReader(make([]byte, MaxRecordSize+1))) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + _, _, err = ReadRecord(strings.NewReader("{}")) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) +} diff --git a/pkg/artifact/capabilities.go b/pkg/artifact/capabilities.go new file mode 100644 index 00000000000..ddd00f13639 --- /dev/null +++ b/pkg/artifact/capabilities.go @@ -0,0 +1,95 @@ +package artifact + +import ( + "slices" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// DeleteSemantics describes what deleting a reference removes. +type DeleteSemantics string + +// Delete semantics a repository can advertise. +const ( + // DeleteNone means the repository cannot delete. + DeleteNone DeleteSemantics = "none" + + // DeleteReference removes only the named reference (tag-safe). + DeleteReference DeleteSemantics = "reference" + + // DeleteManifest removes a manifest by digest, affecting every reference to it. + DeleteManifest DeleteSemantics = "manifest" + + // DeleteObject removes the stored record and payload objects. + DeleteObject DeleteSemantics = "object" +) + +// Capabilities describes what a repository can do. It is computed without network access. +type Capabilities struct { + // Read reports that Resolve, Inspect, OpenDocument, and Read are supported. + Read bool `json:"read"` + + // Write reports that Publish is supported. + Write bool `json:"write"` + + // EnumerateNamed reports that listing versions of a known name is supported. + EnumerateNamed bool `json:"enumerate_named"` + + // EnumerateCollection reports that listing every name is supported (catalog or native). + EnumerateCollection bool `json:"enumerate_collection"` + + // Delete describes delete semantics. + Delete DeleteSemantics `json:"delete"` + + // ConditionalUpdate reports conditional (compare-and-swap) reference publication. + ConditionalUpdate bool `json:"conditional_update"` + + // Packagings lists supported file payload packaging modes. + Packagings []Packaging `json:"packagings,omitempty"` + + // PreservesModes reports that ordinary permission bits survive a round trip. + PreservesModes bool `json:"preserves_modes"` + + // SubjectKinds lists the immutable signature subjects the repository can bind evidence to. + SubjectKinds []SubjectKind `json:"subject_kinds,omitempty"` + + // EvidenceFormats lists the signature evidence formats the repository can store and verify. + EvidenceFormats []EvidenceFormat `json:"evidence_formats,omitempty"` + + // LockScopes lists the mutation lock scopes the repository can provide. + LockScopes []LockScope `json:"lock_scopes,omitempty"` +} + +// HasPackaging reports whether the repository supports the packaging mode. +func (c *Capabilities) HasPackaging(p Packaging) bool { + defer perf.Track(nil, "artifact.Capabilities.HasPackaging")() + + return slices.Contains(c.Packagings, p) +} + +// HasEvidenceFormat reports whether the repository supports the evidence format. +func (c *Capabilities) HasEvidenceFormat(f EvidenceFormat) bool { + defer perf.Track(nil, "artifact.Capabilities.HasEvidenceFormat")() + + return slices.Contains(c.EvidenceFormats, f) +} + +// HasSubjectKind reports whether the repository supports the signature subject kind. +func (c *Capabilities) HasSubjectKind(k SubjectKind) bool { + defer perf.Track(nil, "artifact.Capabilities.HasSubjectKind")() + + return slices.Contains(c.SubjectKinds, k) +} + +// HasLockScope reports whether the repository can provide at least the requested lock scope. +// A repository-scope lock satisfies a host-scope request. +func (c *Capabilities) HasLockScope(s LockScope) bool { + defer perf.Track(nil, "artifact.Capabilities.HasLockScope")() + + for _, have := range c.LockScopes { + if have.Satisfies(s) { + return true + } + } + return false +} diff --git a/pkg/artifact/compat/compat_test.go b/pkg/artifact/compat/compat_test.go new file mode 100644 index 00000000000..ec81d680c89 --- /dev/null +++ b/pkg/artifact/compat/compat_test.go @@ -0,0 +1,236 @@ +package compat + +import ( + "archive/tar" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/ci/artifact" + "github.com/cloudposse/atmos/pkg/ci/artifact/local" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" + "github.com/cloudposse/atmos/pkg/plugin/oci/ocitest" +) + +// Frozen compatibility constants. They were recorded from the code that existed before the +// artifact work and must not be edited to make a test pass: a failure here means a legacy CI +// planfile archive or a plugin release manifest changed identity. +const ( + // The legacy tar SHA-256 of the bundled planfile tar and the sha256 field of its sidecar. + legacyTarSHA256 = "cfb77a4fd8064a9e743ec84d6ed64c6101a5684fc1b7d1fc18a2f6789051118d" + + // The plugin release index digest. + pluginIndexDigest = "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + + // The per-platform manifest digests of the plugin release. + pluginManifestDigestLinuxAMD64 = "sha256:e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee" + pluginManifestDigestDarwinARM64 = "sha256:6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380" + + // The digest of plugin.yaml recorded in the release annotations. + pluginDefinitionDigest = "sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090" +) + +// pluginBlobDigests lists every blob in the plugin release layout (index, two manifests, two +// configs, two bundle layers), sorted. +var pluginBlobDigests = []string{ + "192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485", + "2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79", + "6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380", + "9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f", + "af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59", + "b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e", + "e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee", +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func legacyArchivePath(root string) string { + return filepath.Join(root, filepath.FromSlash(legacyKey)) +} + +func TestLegacyCIArchiveDigestIsStable(t *testing.T) { + committed, err := os.ReadFile(legacyArchivePath(legacyFixtureDir())) + require.NoError(t, err) + require.Positive(t, len(committed), "fixture must not be empty") + assert.Equal(t, legacyTarSHA256, sha256Hex(committed), "committed archive bytes changed") + + t.Run("current code reproduces the committed fixture byte for byte", func(t *testing.T) { + rebuilt := t.TempDir() + buildLegacyStore(t, rebuilt) + got := treeBytes(t, rebuilt) + want := treeBytes(t, legacyFixtureDir()) + require.Len(t, want, 2, "fixture holds the archive and its metadata sidecar") + require.Equal(t, len(want), len(got)) + for name, content := range want { + assert.Equal(t, content, got[name], "%s differs from what the current CreateTarArchive/BundledStore/local backend produce", name) + } + }) + + t.Run("sidecar records the archive checksum", func(t *testing.T) { + raw, err := os.ReadFile(legacyArchivePath(legacyFixtureDir()) + ".metadata.json") + require.NoError(t, err) + assert.Contains(t, string(raw), `"sha256": "`+legacyTarSHA256+`"`) + }) + + t.Run("legacy encoding is mode 0644 with no timestamps in file order", func(t *testing.T) { + raw, err := os.ReadFile(legacyArchivePath(legacyFixtureDir())) + require.NoError(t, err) + reader := tar.NewReader(strings.NewReader(string(raw))) + var names []string + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, header.Name) + assert.Equal(t, int64(0o644), header.Mode, header.Name) + assert.True(t, header.ModTime.IsZero() || header.ModTime.Unix() == 0, "%s has a modification time", header.Name) + } + assert.Equal(t, []string{"plan.tfplan", ".terraform.lock.hcl"}, names) + }) +} + +func TestLegacyCIFixtureStillReadsThroughCurrentStore(t *testing.T) { + copyRoot := t.TempDir() + for name, content := range treeBytes(t, legacyFixtureDir()) { + target := filepath.Join(copyRoot, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.WriteFile(target, content, 0o644)) + } + backend, err := local.NewStore(artifact.StoreOptions{Type: "local", Options: map[string]any{"path": copyRoot}}) + require.NoError(t, err) + store := artifact.NewBundledStore(backend) + + files, metadata, err := store.Download(t.Context(), legacyKey) + require.NoError(t, err) + require.Len(t, files, 2) + bodies := map[string]string{} + for _, file := range files { + body, readErr := io.ReadAll(file.Data) + require.NoError(t, readErr) + require.NoError(t, file.Data.Close()) + bodies[file.Name] = string(body) + } + assert.Equal(t, map[string]string{"plan.tfplan": legacyPlanBody, ".terraform.lock.hcl": legacyLockBody}, bodies) + assert.Equal(t, legacyMetadata().Stack, metadata.Stack) + assert.Equal(t, legacyMetadata().SHA, metadata.SHA) + assert.Equal(t, legacyMetadata().Component, metadata.Component) + assert.Equal(t, legacyMetadata().BaseSHA, metadata.BaseSHA) + assert.Equal(t, legacyMetadata().Branch, metadata.Branch) + assert.Equal(t, legacyMetadata().RunID, metadata.RunID) + assert.Equal(t, legacyMetadata().Repository, metadata.Repository) + assert.Equal(t, legacyMetadata().AtmosVersion, metadata.AtmosVersion) + assert.True(t, legacyMetadata().CreatedAt.Equal(metadata.CreatedAt)) + assert.Equal(t, 42, metadata.PRNumber) + assert.Equal(t, legacyTarSHA256, metadata.SHA256) + assert.Equal(t, map[string]string{"env": "dev", "zone": "a"}, metadata.Custom) + require.NoError(t, metadata.Validate(), "the legacy Metadata.Validate contract is unchanged") + + t.Run("a changed archive fails the integrity check", func(t *testing.T) { + archive := legacyArchivePath(copyRoot) + content, err := os.ReadFile(archive) + require.NoError(t, err) + content[0] ^= 0xff + require.NoError(t, os.WriteFile(archive, content, 0o644)) + _, _, err = store.Download(t.Context(), legacyKey) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +func TestPluginReleaseFixtureDigestsAreStable(t *testing.T) { + root := pluginFixtureDir() + + t.Run("layout holds exactly the recorded content-addressed blobs", func(t *testing.T) { + tree := treeBytes(t, root) + var blobs []string + for name, content := range tree { + digestHex, found := strings.CutPrefix(name, "blobs/sha256/") + if !found { + continue + } + blobs = append(blobs, digestHex) + assert.Equal(t, digestHex, sha256Hex(content), "blob %s does not hash to its name", digestHex) + } + slices.Sort(blobs) + assert.Equal(t, pluginBlobDigests, blobs) + require.Contains(t, tree, "index.json") + require.Contains(t, tree, "oci-layout") + }) + + t.Run("release inspects to the recorded manifest digests", func(t *testing.T) { + index, err := atmosoci.LoadIndex(root, pluginIndexDigest) + require.NoError(t, err) + release, err := pluginoci.Inspect(index, pluginIndexDigest) + require.NoError(t, err) + assert.Equal(t, pluginIndexDigest, release.Digest) + assert.Equal(t, "example/fixture", release.Name) + assert.Equal(t, "1.2.3", release.Version) + require.Len(t, release.Platforms, 2) + assert.Equal(t, pluginManifestDigestLinuxAMD64, release.Platforms["linux/amd64"].Digest.String()) + assert.Equal(t, pluginManifestDigestDarwinARM64, release.Platforms["darwin/arm64"].Digest.String()) + assert.Equal(t, pluginDefinitionDigest, release.Platforms["linux/amd64"].Annotations["dev.atmos.plugin.manifest.digest"]) + }) + + t.Run("platform payloads extract to the original executables", func(t *testing.T) { + index, err := atmosoci.LoadIndex(root, pluginIndexDigest) + require.NoError(t, err) + release, err := pluginoci.Inspect(index, pluginIndexDigest) + require.NoError(t, err) + for platform, want := range map[string][]byte{ + "linux/amd64": ocitest.Binary("linux", "amd64"), + "darwin/arm64": ocitest.Binary("darwin", "arm64"), + } { + destination := t.TempDir() + require.NoError(t, release.Extract(platform, destination), platform) + got, err := os.ReadFile(filepath.Join(destination, "bin", "plugin")) + require.NoError(t, err) + assert.Equal(t, want, got, platform) + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(destination, "bin", "plugin")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + } + } + }) +} + +func TestPluginReleaseRebuildMatchesFixture(t *testing.T) { + rebuilt := filepath.Join(t.TempDir(), "layout") + packPlugin(t, "1.2.3", rebuilt) + + got := treeBytes(t, rebuilt) + want := treeBytes(t, pluginFixtureDir()) + require.Positive(t, len(want)) + require.Equal(t, len(want), len(got), "layout file set changed") + for name, content := range want { + assert.Equal(t, content, got[name], "%s differs from what the current plugin packer produces", name) + } +} + +// TestPluginDigestChangeIsDetected proves the constants above would fail if the packer's output +// changed: a one-field change in the packed release moves the index digest. +func TestPluginDigestChangeIsDetected(t *testing.T) { + changed := filepath.Join(t.TempDir(), "layout") + packPlugin(t, "1.2.4", changed) + + index, err := atmosoci.LoadIndex(changed, "") + require.NoError(t, err) + digest, err := index.Digest() + require.NoError(t, err) + assert.NotEqual(t, pluginIndexDigest, digest.String()) +} diff --git a/pkg/artifact/compat/doc.go b/pkg/artifact/compat/doc.go new file mode 100644 index 00000000000..2ca7f71c3db --- /dev/null +++ b/pkg/artifact/compat/doc.go @@ -0,0 +1,8 @@ +// Package compat holds test-only compatibility fixtures that freeze the byte-level behavior of the +// existing CI planfile archive format and the existing plugin OCI release format before the +// artifact implementation replaces any of their code paths. +// +// It lives apart from pkg/artifact so the core domain package never imports pkg/ci or pkg/plugin. +// Fixture bytes under testdata are committed and must never be regenerated to hide an +// incompatible change; see fixtures_test.go for the generation rules. +package compat diff --git a/pkg/artifact/compat/fixtures_test.go b/pkg/artifact/compat/fixtures_test.go new file mode 100644 index 00000000000..e731a19f6e8 --- /dev/null +++ b/pkg/artifact/compat/fixtures_test.go @@ -0,0 +1,152 @@ +package compat + +import ( + "flag" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ci/artifact" + "github.com/cloudposse/atmos/pkg/ci/artifact/local" + "github.com/cloudposse/atmos/pkg/plugin/extension" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" + "github.com/cloudposse/atmos/pkg/plugin/oci/ocitest" +) + +// generateFixtures writes missing fixtures under testdata. It never overwrites an existing +// fixture: to refresh one on purpose, a human deletes it and reviews the resulting digest change, +// because a changed legacy archive or plugin manifest digest is an incompatibility, not a refresh. +var generateFixtures = flag.Bool("generate-fixtures", false, "write missing fixtures to testdata (never overwrites)") + +const ( + // The legacy planfile key follows the default key pattern {{ .Stack }}/{{ .Component }}/{{ .SHA }}.tfplan.tar. + legacyKey = "plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar" + + legacyPlanBody = "legacy-planfile-bytes\n" + legacyLockBody = "# legacy lock file\nprovider \"registry.terraform.io/hashicorp/aws\" {\n version = \"5.0.0\"\n}\n" +) + +// legacyFixtureDir is the committed pre-change CI planfile store (local/dir backend layout). +func legacyFixtureDir() string { return filepath.Join("testdata", "legacy-ci") } + +// pluginFixtureDir is the committed plugin release OCI layout. +func pluginFixtureDir() string { return filepath.Join("testdata", "plugin-release") } + +// legacyMetadata is fully fixed so the archive and sidecar are byte-deterministic. +func legacyMetadata() *artifact.Metadata { + return &artifact.Metadata{ + Stack: "plat-ue2-dev", + Component: "vpc", + SHA: "0123456789abcdef0123456789abcdef01234567", + BaseSHA: "89abcdef0123456789abcdef0123456789abcdef", + Branch: "feature/fixture", + PRNumber: 42, + RunID: "run-1", + Repository: "acme/infra", + CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + AtmosVersion: "0.0.0-fixture", + Custom: map[string]string{"zone": "a", "env": "dev"}, + } +} + +// buildLegacyStore writes the legacy planfile through the CURRENT BundledStore and local backend +// into root and returns the relative slash paths of every file produced. +func buildLegacyStore(t *testing.T, root string) { + t.Helper() + backend, err := local.NewStore(artifact.StoreOptions{Type: "local", Options: map[string]any{"path": root}}) + require.NoError(t, err) + store := artifact.NewBundledStore(backend) + files := []artifact.FileEntry{ + {Name: "plan.tfplan", Data: bytesReader(legacyPlanBody), Size: -1}, + {Name: ".terraform.lock.hcl", Data: bytesReader(legacyLockBody), Size: -1}, + } + require.NoError(t, store.Upload(t.Context(), legacyKey, files, legacyMetadata())) +} + +// pluginPackageDir assembles the plugin source tree from fixed inputs. Executables are valid +// but never-executed native headers, so the build needs no toolchain. +func pluginPackageDir(t *testing.T, version string) (string, *extension.Registry) { + t.Helper() + directory := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(directory, "bin"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "bin", "linux"), ocitest.Binary("linux", "amd64"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "bin", "mac"), ocitest.Binary("darwin", "arm64"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "plugin.yaml"), []byte(pluginManifest(version)), 0o644)) + descriptors := extension.New() + require.NoError(t, descriptors.Register(extension.Descriptor{ + Capability: "fixture", Path: []string{"fixtures"}, Version: 1, + Validate: func(extension.Binding) error { return nil }, + })) + return directory, descriptors +} + +func pluginManifest(version string) string { + return `apiVersion: atmos/v1 +kind: AtmosPluginConfig +metadata: + name: example/fixture + version: ` + version + ` +spec: + runtime: + type: native + protocol: 1 + executable: bin/plugin + compatibility: + protocol: ">=1 <2" + platforms: + linux/amd64: + executable: bin/linux + darwin/arm64: + executable: bin/mac + capabilities: + fixtures: + example/fixture: {} +` +} + +// packPlugin builds the plugin OCI layout at output (which must not exist) with the CURRENT packer. +func packPlugin(t *testing.T, version, output string) { + t.Helper() + directory, descriptors := pluginPackageDir(t, version) + require.NoError(t, pluginoci.Pack(directory, output, descriptors)) +} + +// treeBytes returns every regular file below root keyed by slash-separated relative path. +func treeBytes(t *testing.T, root string) map[string][]byte { + t.Helper() + tree := make(map[string][]byte) + require.NoError(t, filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + tree[filepath.ToSlash(relative)] = content + return nil + })) + return tree +} + +// TestGenerateFixtures is skipped unless -generate-fixtures is passed. It only fills in missing +// fixtures and fails if one already exists, so a regeneration can never mask a change. +func TestGenerateFixtures(t *testing.T) { + if !*generateFixtures { + t.Skipf("pass -generate-fixtures to write missing fixtures") + } + for _, target := range []string{legacyFixtureDir(), pluginFixtureDir()} { + _, err := os.Lstat(target) + require.ErrorIs(t, err, fs.ErrNotExist, "%s already exists; fixtures are frozen. Delete it deliberately and review the digest change", target) + } + buildLegacyStore(t, legacyFixtureDir()) + packPlugin(t, "1.2.3", pluginFixtureDir()) +} diff --git a/pkg/artifact/compat/helpers_test.go b/pkg/artifact/compat/helpers_test.go new file mode 100644 index 00000000000..317f713022a --- /dev/null +++ b/pkg/artifact/compat/helpers_test.go @@ -0,0 +1,8 @@ +package compat + +import ( + "bytes" + "io" +) + +func bytesReader(s string) io.Reader { return bytes.NewReader([]byte(s)) } diff --git a/pkg/artifact/compat/testdata/.gitattributes b/pkg/artifact/compat/testdata/.gitattributes new file mode 100644 index 00000000000..425ff5530e0 --- /dev/null +++ b/pkg/artifact/compat/testdata/.gitattributes @@ -0,0 +1,2 @@ +# Frozen compatibility fixtures are byte-exact; never normalize line endings or treat them as text. +* -text diff --git a/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar new file mode 100644 index 00000000000..22022820a52 Binary files /dev/null and b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar differ diff --git a/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json new file mode 100644 index 00000000000..a10a8071294 --- /dev/null +++ b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json @@ -0,0 +1,17 @@ +{ + "stack": "plat-ue2-dev", + "component": "vpc", + "sha": "0123456789abcdef0123456789abcdef01234567", + "base_sha": "89abcdef0123456789abcdef0123456789abcdef", + "branch": "feature/fixture", + "pr_number": 42, + "run_id": "run-1", + "repository": "acme/infra", + "created_at": "2026-01-02T03:04:05Z", + "sha256": "cfb77a4fd8064a9e743ec84d6ed64c6101a5684fc1b7d1fc18a2f6789051118d", + "atmos_version": "0.0.0-fixture", + "custom": { + "env": "dev", + "zone": "a" + } +} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 new file mode 100644 index 00000000000..cc666b04eeb Binary files /dev/null and b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 differ diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 new file mode 100644 index 00000000000..cb7ec038172 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 @@ -0,0 +1 @@ +{"architecture":"arm64","created":"0001-01-01T00:00:00Z","history":[{"created":"0001-01-01T00:00:00Z"}],"os":"darwin","rootfs":{"type":"layers","diff_ids":["sha256:192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485"]},"config":{"Labels":{"dev.atmos.plugin.manifest":"apiVersion: atmos/v1\nkind: AtmosPluginConfig\nmetadata:\n name: example/fixture\n version: 1.2.3\nspec:\n runtime:\n type: native\n protocol: 1\n executable: bin/plugin\n compatibility:\n protocol: \"\u003e=1 \u003c2\"\n platforms:\n linux/amd64:\n executable: bin/linux\n darwin/arm64:\n executable: bin/mac\n capabilities:\n fixtures:\n example/fixture: {}\n"}}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 new file mode 100644 index 00000000000..2774d4bd44e --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.atmos.plugin.config.v1+json","size":688,"digest":"sha256:2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79"},"layers":[{"mediaType":"application/vnd.atmos.plugin.bundle.v1+tar","size":3072,"digest":"sha256:192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f new file mode 100644 index 00000000000..812e6f2e133 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f @@ -0,0 +1 @@ +{"architecture":"amd64","created":"0001-01-01T00:00:00Z","history":[{"created":"0001-01-01T00:00:00Z"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e"]},"config":{"Labels":{"dev.atmos.plugin.manifest":"apiVersion: atmos/v1\nkind: AtmosPluginConfig\nmetadata:\n name: example/fixture\n version: 1.2.3\nspec:\n runtime:\n type: native\n protocol: 1\n executable: bin/plugin\n compatibility:\n protocol: \"\u003e=1 \u003c2\"\n platforms:\n linux/amd64:\n executable: bin/linux\n darwin/arm64:\n executable: bin/mac\n capabilities:\n fixtures:\n example/fixture: {}\n"}}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 new file mode 100755 index 00000000000..bd6d4cd17b7 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","size":606,"digest":"sha256:6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380","annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"},"platform":{"architecture":"arm64","os":"darwin"},"artifactType":"application/vnd.atmos.plugin.config.v1+json"},{"mediaType":"application/vnd.oci.image.manifest.v1+json","size":606,"digest":"sha256:e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee","annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"},"platform":{"architecture":"amd64","os":"linux"},"artifactType":"application/vnd.atmos.plugin.config.v1+json"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e new file mode 100644 index 00000000000..14dd56846e1 Binary files /dev/null and b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e differ diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee new file mode 100644 index 00000000000..623c41d20c8 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.atmos.plugin.config.v1+json","size":687,"digest":"sha256:9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f"},"layers":[{"mediaType":"application/vnd.atmos.plugin.bundle.v1+tar","size":3072,"digest":"sha256:b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/index.json b/pkg/artifact/compat/testdata/plugin-release/index.json new file mode 100755 index 00000000000..837db573436 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/index.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": 1220, + "digest": "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + } + ] +} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/oci-layout b/pkg/artifact/compat/testdata/plugin-release/oci-layout new file mode 100755 index 00000000000..224a8698125 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} \ No newline at end of file diff --git a/pkg/artifact/config/catalog.go b/pkg/artifact/config/catalog.go new file mode 100644 index 00000000000..6ab0728fc5a --- /dev/null +++ b/pkg/artifact/config/catalog.go @@ -0,0 +1,70 @@ +package config + +import ( + "fmt" + "net/url" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// validateCatalog checks the optional catalog reference. A catalog belongs directly to its +// repository: it must be an oci:// reference with a tag or digest (there is no implicit latest), +// carry no credentials, and name the repository's own registry host, because the repository's +// identity must never authenticate a different host. +func validateCatalog(desc *artifact.KindDescriptor, repo *schema.ArtifactRepositoryConfig) error { + if repo.Catalog == "" { + return nil + } + if !desc.AllowsCatalog { + return fmt.Errorf("%w: kind %q does not support a catalog; direct named operations and managed-record enumeration work without one", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + parsed, err := parseCatalogURI(repo.Catalog) + if err != nil { + return err + } + registry, _ := repo.Options["registry"].(string) + if registry != "" && !strings.EqualFold(registry, parsed.Host) { + return fmt.Errorf("%w: catalog host %q does not match the repository registry %q; a catalog cannot direct the repository identity to another host", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(parsed.Host), artifact.RedactURL(registry)) + } + return nil +} + +// parseCatalogURI parses and checks the shape of an oci:// catalog reference. +func parseCatalogURI(raw string) (*url.URL, error) { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "oci" || parsed.Host == "" { + return nil, fmt.Errorf("%w: catalog %q must be an oci:///: reference", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw)) + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return nil, fmt.Errorf("%w: catalog %q must not contain credentials, a query, or a fragment", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw)) + } + if err := validateCatalogReference(strings.TrimPrefix(parsed.Path, "/")); err != nil { + return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw), artifact.RedactError(err, raw)) + } + return parsed, nil +} + +// validateCatalogReference checks ":" or "@sha256:". +func validateCatalogReference(reference string) error { + path, tag, dgst := reference, "", "" + if before, after, found := strings.Cut(reference, "@"); found { + path, dgst = before, after + } else if index := strings.LastIndex(reference, ":"); index >= 0 { + path, tag = reference[:index], reference[index+1:] + } + if tag == "" && dgst == "" { + return fmt.Errorf("%w: a tag or digest is required (there is no implicit latest)", errUtils.ErrArtifactRefInvalid) + } + if err := artifact.ValidateName(path); err != nil { + return err + } + if dgst != "" { + return artifact.ValidateDigest(digest.Digest(dgst)) + } + return artifact.ValidateVersion(tag) +} diff --git a/pkg/artifact/config/decode.go b/pkg/artifact/config/decode.go new file mode 100644 index 00000000000..2962209755a --- /dev/null +++ b/pkg/artifact/config/decode.go @@ -0,0 +1,123 @@ +package config + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/go-viper/mapstructure/v2" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + keyRepositories = "repositories" + keyPermissions = "permissions" +) + +// Decode strictly decodes a raw `artifacts` section (as parsed from YAML, with original key +// spelling) into the schema type. Unknown fields, mistyped values, and non-string permissions are +// errors, so `permissions: 0600` (which YAML reads as the integer 384) never silently becomes the +// wrong mode. A nil or empty section decodes to the zero value. +func Decode(raw map[string]any) (schema.ArtifactConfig, error) { + defer perf.Track(nil, "artifactconfig.Decode")() + + var cfg schema.ArtifactConfig + if len(raw) == 0 { + return cfg, nil + } + if err := checkPermissionTypes(raw); err != nil { + return cfg, err + } + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &cfg, + TagName: "mapstructure", + ErrorUnused: true, + }) + if err != nil { + return cfg, fmt.Errorf("%w: decoder: %w", errUtils.ErrArtifactConfigInvalid, err) + } + if err := decoder.Decode(raw); err != nil { + return cfg, fmt.Errorf("%w: artifacts: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactDecodeError(err, raw)) + } + normalize(&cfg) + return cfg, nil +} + +// normalize turns empty collections into nil so that a value decoded from `trusted_keys: []` +// is identical to the same value after a YAML, JSON, or mapstructure round trip, all of which omit +// empty collections. +func normalize(cfg *schema.ArtifactConfig) { + if len(cfg.Repositories) == 0 { + cfg.Repositories = nil + } + normalizeVerification(cfg.Verification) + for alias := range cfg.Repositories { + repo := cfg.Repositories[alias] + if len(repo.Options) == 0 { + repo.Options = nil + } + if repo.Defaults != nil && repo.Defaults.Metadata != nil && len(repo.Defaults.Metadata.Labels) == 0 { + repo.Defaults.Metadata.Labels = nil + } + normalizeVerification(repo.Verification) + cfg.Repositories[alias] = repo + } +} + +func normalizeVerification(verification *schema.ArtifactVerificationConfig) { + if verification == nil { + return + } + if len(verification.TrustedKeys) == 0 { + verification.TrustedKeys = nil + } + if len(verification.TrustedIdentities) == 0 { + verification.TrustedIdentities = nil + } +} + +// checkPermissionTypes rejects any repository whose permissions value is not a string. +func checkPermissionTypes(raw map[string]any) error { + repos, ok := lookupKey(raw, keyRepositories).(map[string]any) + if !ok { + return nil + } + aliases := make([]string, 0, len(repos)) + for alias := range repos { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + var errs []error + for _, alias := range aliases { + repo, ok := repos[alias].(map[string]any) + if !ok { + continue + } + value := lookupKey(repo, keyPermissions) + if value == nil { + continue + } + if _, isString := value.(string); !isString { + errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), value)) + } + } + return errors.Join(errs...) +} + +// lookupKey finds a key case-insensitively, matching the loader's case-insensitive key handling. +func lookupKey(m map[string]any, key string) any { + if value, ok := m[key]; ok { + return value + } + for candidate, value := range m { + if strings.EqualFold(candidate, key) { + return value + } + } + return nil +} diff --git a/pkg/artifact/config/decode_test.go b/pkg/artifact/config/decode_test.go new file mode 100644 index 00000000000..46028d87933 --- /dev/null +++ b/pkg/artifact/config/decode_test.go @@ -0,0 +1,258 @@ +package config + +import ( + "encoding/json" + "path/filepath" + "strings" + "testing" + + "github.com/go-viper/mapstructure/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// prdDocument is every YAML example from the artifacts PRD that sits under `artifacts:`, merged +// into one document: the public-configuration repositories plus the catalog, verification, and +// locking examples. +const prdDocument = ` +verification: + require_signature: true + trusted_identities: + - identity: https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main + issuer: https://token.actions.githubusercontent.com + trusted_keys: [] +repositories: + shared: + kind: oci/registry + identity: registry-publisher + options: + registry: ghcr.io + prefix: acme/atmos + defaults: + metadata: + description: Shared platform artifacts + labels: + team: platform + catalog: oci://ghcr.io/acme/atmos/catalog:stable + locking: + required_scope: repository + repository: coordination + backups: + kind: aws/s3 + identity: backup-writer + packaging: tgz + options: + bucket: acme-artifacts + prefix: atmos + region: us-east-2 + tags: + purpose: backup + local: + kind: local/dir + options: + path: .atmos/artifacts + archives: + kind: local/archive + packaging: tgz + permissions: "0600" + options: + path: .atmos/backups + key_pattern: '{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz' + offline: + kind: oci/layout + options: + path: .atmos/oci + coordination: + kind: aws/s3 + identity: lock-writer + options: + bucket: acme-artifact-coordination + verification: + require_signature: true + trusted_keys: + - /etc/atmos/cosign.pub +` + +func decodeYAMLDocument(t *testing.T, document string) (schema.ArtifactConfig, map[string]any) { + t.Helper() + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(document), &raw)) + cfg, err := Decode(raw) + require.NoError(t, err) + return cfg, raw +} + +func TestDecode_PRDDocumentLoadsUnchanged(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, prdDocument) + + require.Len(t, cfg.Repositories, 6) + archives := cfg.Repositories["archives"] + assert.Equal(t, "0600", archives.Permissions) + assert.Equal(t, `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, archives.Options["key_pattern"], "key_pattern survives decoding verbatim") + assert.Equal(t, "Shared platform artifacts", cfg.Repositories["shared"].Defaults.Metadata.Description) + assert.Equal(t, []string{"/etc/atmos/cosign.pub"}, cfg.Repositories["coordination"].Verification.TrustedKeys) + assert.Empty(t, cfg.Verification.TrustedKeys) + require.Len(t, cfg.Verification.TrustedIdentities, 1) + assert.Equal(t, "https://token.actions.githubusercontent.com", cfg.Verification.TrustedIdentities[0].Issuer) +} + +// TestDecode_YAMLJSONAndMapstructureAgree decodes the same document through YAML, JSON, and a +// plain mapstructure round trip and requires identical values. +func TestDecode_YAMLJSONAndMapstructureAgree(t *testing.T) { + fromYAML, raw := decodeYAMLDocument(t, prdDocument) + + // YAML round trip: marshal the typed value and decode it again. + yamlBytes, err := yaml.Marshal(fromYAML) + require.NoError(t, err) + var yamlRoundTrip schema.ArtifactConfig + require.NoError(t, yaml.Unmarshal(yamlBytes, &yamlRoundTrip)) + assert.Equal(t, fromYAML, yamlRoundTrip) + + // JSON round trip. + jsonBytes, err := json.Marshal(fromYAML) + require.NoError(t, err) + var jsonRoundTrip schema.ArtifactConfig + require.NoError(t, json.Unmarshal(jsonBytes, &jsonRoundTrip)) + assert.Equal(t, fromYAML, jsonRoundTrip) + + // The raw map re-encoded as JSON, then decoded strictly, matches too. + var rawFromJSON map[string]any + require.NoError(t, json.Unmarshal(mustJSON(t, raw), &rawFromJSON)) + fromJSONMap, err := Decode(rawFromJSON) + require.NoError(t, err) + assert.Equal(t, fromYAML, fromJSONMap) + + // mapstructure struct -> map -> struct round trip with weak typing disabled. + var asMap map[string]any + require.NoError(t, mapstructure.Decode(fromYAML, &asMap)) + var mapstructureRoundTrip schema.ArtifactConfig + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &mapstructureRoundTrip, TagName: "mapstructure"}) + require.NoError(t, err) + require.NoError(t, decoder.Decode(asMap)) + assert.Equal(t, fromYAML, mapstructureRoundTrip) +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + out, err := json.Marshal(value) + require.NoError(t, err) + return out +} + +func TestDecode_OriginsAreNeverSerialized(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, prdDocument) + repo := cfg.Repositories["local"] + repo.PathOrigin = NewOrigin("/etc/atmos/atmos.yaml", true) + cfg.Repositories["local"] = repo + cfg.Verification.KeysOrigin = NewOrigin("/etc/atmos/atmos.yaml", false) + + jsonBytes := mustJSON(t, cfg) + yamlBytes, err := yaml.Marshal(cfg) + require.NoError(t, err) + for _, encoded := range []string{string(jsonBytes), string(yamlBytes)} { + assert.NotContains(t, encoded, "/etc/atmos/atmos.yaml") + assert.NotContains(t, encoded, "origin") + } +} + +func TestDecode_RejectsUnquotedNumericPermissions(t *testing.T) { + tests := []struct { + name string + document string + }{ + {name: "octal literal", document: "repositories:\n a:\n kind: local/archive\n permissions: 0600\n"}, + {name: "decimal literal", document: "repositories:\n a:\n kind: local/archive\n permissions: 384\n"}, + {name: "float", document: "repositories:\n a:\n kind: local/archive\n permissions: 6.0\n"}, + {name: "bool", document: "repositories:\n a:\n kind: local/archive\n permissions: true\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(tt.document), &raw)) + _, err := Decode(raw) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "quoted octal string") + assert.Contains(t, err.Error(), `"a"`) + }) + } + + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte("repositories:\n a:\n kind: local/archive\n permissions: \"0600\"\n"), &raw)) + cfg, err := Decode(raw) + require.NoError(t, err) + assert.Equal(t, "0600", cfg.Repositories["a"].Permissions) +} + +func TestDecode_StrictAboutUnknownAndMistypedFields(t *testing.T) { + tests := []struct { + name string + document string + wantErr string + }{ + {name: "unknown repository field", document: "repositories:\n a:\n kind: local/dir\n pakaging: none\n", wantErr: "pakaging"}, + {name: "unknown top level field", document: "repositorys: {}\n", wantErr: "repositorys"}, + {name: "unknown verification field", document: "verification:\n require_sig: true\n", wantErr: "require_sig"}, + {name: "non-boolean require_signature", document: "verification:\n require_signature: sometimes\n", wantErr: "require_signature"}, + {name: "non-string label value", document: "repositories:\n a:\n kind: local/dir\n defaults:\n metadata:\n labels:\n cost: 1234\n", wantErr: "cost"}, + {name: "repositories not a map", document: "repositories: [a]\n", wantErr: "repositories"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(tt.document), &raw)) + _, err := Decode(raw) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestDecode_EmptySectionIsTheZeroValue(t *testing.T) { + cfg, err := Decode(nil) + require.NoError(t, err) + assert.Equal(t, schema.ArtifactConfig{}, cfg) + cfg, err = Decode(map[string]any{}) + require.NoError(t, err) + assert.Empty(t, cfg.Repositories) +} + +func TestDecode_KeysAreCaseSensitiveForAliasesLabelsAndTags(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, ` +repositories: + Prod.US: + kind: aws/s3 + options: + bucket: acme-artifacts + tags: + Purpose: Backup + defaults: + metadata: + labels: + Team.Name: Platform +`) + repo, ok := cfg.Repositories["Prod.US"] + require.True(t, ok) + assert.Equal(t, map[string]string{"Team.Name": "Platform"}, repo.Defaults.Metadata.Labels) + assert.Equal(t, map[string]any{"Purpose": "Backup"}, repo.Options["tags"]) +} + +func TestDecodedPRDDocumentValidatesWithAnchoredPaths(t *testing.T) { + // A rooted Unix path such as /etc/atmos has no volume on Windows, so it is not absolute there. + // Build the trusted key path and the declaring file from a platform-absolute directory instead. + dir := t.TempDir() + keyPath := filepath.ToSlash(filepath.Join(dir, "cosign.pub")) + document := strings.Replace(prdDocument, "/etc/atmos/cosign.pub", keyPath, 1) + require.NotEqual(t, prdDocument, document, "the PRD document must contain the trusted key path being replaced") + + cfg, _ := decodeYAMLDocument(t, document) + origin := NewOrigin(filepath.Join(dir, "atmos.yaml"), false) + for alias, repo := range cfg.Repositories { + repo.PathOrigin = origin + cfg.Repositories[alias] = repo + } + require.NoError(t, Validate(&cfg)) +} diff --git a/pkg/artifact/config/doc.go b/pkg/artifact/config/doc.go new file mode 100644 index 00000000000..99400dd6941 --- /dev/null +++ b/pkg/artifact/config/doc.go @@ -0,0 +1,25 @@ +// Package config decodes, validates, and interprets the `artifacts` section of atmos.yaml. +// +// It is separate from pkg/artifact so that the domain package stays free of configuration policy, +// while pkg/config (the loader) and the artifact service share one implementation of: +// +// - strict decoding of the raw section into schema.ArtifactConfig, including rejecting +// unquoted numeric permissions that YAML would silently turn into the wrong octal value; +// - whole-configuration validation (kinds, per-kind options, packaging, permissions, catalogs, +// identities, labels, verification policy, and delegated lock chains); +// - the effective verification policy model, where a global requirement can be strengthened but +// never weakened by a repository and both trust sets must be satisfied; +// - anchoring relative local paths to the configuration file that declared them. +// +// Validation performs no network access, resolves no credentials, and creates no directories. +// An `identity` is only checked for shape (and, when the caller supplies a lookup, existence); +// it is resolved lazily by the operation that needs credentials. +// +// # Repository kinds +// +// Kind names are checked against artifact.RepositoryRegistry, which returns a registered backend's +// descriptor when one exists and otherwise the documented static set of V1 kinds (local/dir, +// local/archive, aws/s3, oci/registry, oci/layout). Validation therefore works before backend +// packages register, and a backend that registers a kind refines the descriptor rather than +// changing the accepted names. +package config diff --git a/pkg/artifact/config/locking.go b/pkg/artifact/config/locking.go new file mode 100644 index 00000000000..f925ce0d3a9 --- /dev/null +++ b/pkg/artifact/config/locking.go @@ -0,0 +1,115 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// LockPlan is the resolved lock coordination of one repository. +type LockPlan struct { + // Provider is the alias of the repository that provides the lock. It is the repository itself + // unless locking is delegated. + Provider string + + // Required is the minimum scope mutations need. + Required artifact.LockScope + + // Chain lists the aliases visited from the repository to the provider, in order. + Chain []string +} + +// ResolveLockPlan follows `locking.repository` delegation from alias to the repository that +// provides the lock and reports the required scope. It fails on an unknown delegate, on a cycle, +// and on an invalid scope. It does not check the provider's capability; [Validator.Validate] does. +func ResolveLockPlan(cfg *schema.ArtifactConfig, alias string) (LockPlan, error) { + defer perf.Track(nil, "artifactconfig.ResolveLockPlan")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return LockPlan{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias)) + } + required, err := requiredScope(repo.Locking) + if err != nil { + return LockPlan{}, err + } + plan := LockPlan{Provider: alias, Required: required, Chain: []string{alias}} + visited := map[string]bool{alias: true} + for { + current := cfg.Repositories[plan.Provider] + if current.Locking == nil || current.Locking.Repository == "" { + return plan, nil + } + next := current.Locking.Repository + if _, exists := cfg.Repositories[next]; !exists { + return LockPlan{}, fmt.Errorf("%w: repository %q delegates locking to %q, which is not configured", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(plan.Provider), artifact.RedactURL(next)) + } + if visited[next] { + return LockPlan{}, fmt.Errorf("%w: lock delegation cycle: %s -> %s", errUtils.ErrArtifactConfigInvalid, redactedChain(plan.Chain), artifact.RedactURL(next)) + } + visited[next] = true + plan.Chain = append(plan.Chain, next) + plan.Provider = next + } +} + +// requiredScope returns the configured scope, defaulting to host. +func requiredScope(locking *schema.ArtifactLockingConfig) (artifact.LockScope, error) { + if locking == nil || locking.RequiredScope == "" { + return artifact.LockScopeHost, nil + } + switch scope := artifact.LockScope(locking.RequiredScope); scope { + case artifact.LockScopeHost, artifact.LockScopeRepository: + return scope, nil + default: + return "", fmt.Errorf("%w: locking.required_scope %q must be %q or %q", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(locking.RequiredScope), artifact.LockScopeHost, artifact.LockScopeRepository) + } +} + +// validateLocking checks the lock configuration: scope value, delegate existence, delegation +// cycles, and that the repository that finally provides the lock advertises the required scope. +func validateLocking(v *Validator, cfg *schema.ArtifactConfig, alias string, repo *schema.ArtifactRepositoryConfig) error { + if repo.Locking == nil { + return nil + } + plan, err := ResolveLockPlan(cfg, alias) + if err != nil { + return err + } + provider := cfg.Repositories[plan.Provider] + desc, ok := v.registry.Lookup(provider.Kind) + if !ok { + // The provider's own unknown kind is reported when that repository is validated. + return nil + } + if desc.Capabilities.HasLockScope(plan.Required) { + return nil + } + return errors.Join( + errUtils.ErrArtifactConfigInvalid, + fmt.Errorf("%w: locking.required_scope %q is not provided by %q (kind %q advertises %v)%s", + errUtils.ErrArtifactUnsupportedCapability, plan.Required, artifact.RedactURL(plan.Provider), artifact.RedactURL(provider.Kind), desc.Capabilities.LockScopes, delegateHint(plan)), + ) +} + +// redactedChain joins the aliases of a delegation chain, each redacted: an alias is a name the user +// typed, and it may be a pasted credential-bearing URL. +func redactedChain(chain []string) string { + shown := make([]string, len(chain)) + for i, alias := range chain { + shown[i] = artifact.RedactURL(alias) + } + return strings.Join(shown, " -> ") +} + +func delegateHint(plan LockPlan) string { + if len(plan.Chain) > 1 { + return "" + } + return "; set locking.repository to a repository (for example an aws/s3 coordinator) that advertises repository scope" +} diff --git a/pkg/artifact/config/locking_test.go b/pkg/artifact/config/locking_test.go new file mode 100644 index 00000000000..4422fc52c20 --- /dev/null +++ b/pkg/artifact/config/locking_test.go @@ -0,0 +1,169 @@ +package config + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +func ptr[T any](v T) *T { return &v } + +func withLocking(repo *schema.ArtifactRepositoryConfig, scope, delegate string) schema.ArtifactRepositoryConfig { + out := *repo + out.Locking = &schema.ArtifactLockingConfig{RequiredScope: scope, Repository: delegate} + return out +} + +func TestValidate_PRDLockingExample(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": withLocking(ptr(registryRepo()), "repository", "coordination"), + "coordination": s3Repo(), + }} + require.NoError(t, Validate(cfg)) + + plan, err := ResolveLockPlan(cfg, "shared") + require.NoError(t, err) + assert.Equal(t, LockPlan{Provider: "coordination", Required: artifact.LockScopeRepository, Chain: []string{"shared", "coordination"}}, plan) +} + +func TestValidate_LockingScopes(t *testing.T) { + tests := []struct { + name string + repos map[string]schema.ArtifactRepositoryConfig + alias string + wantErr string + wantCap bool + }{ + { + name: "host scope is the default and always available", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "", "")}, + alias: "r", + }, + { + name: "registry cannot provide repository scope itself", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "repository", "")}, + alias: "r", + wantErr: "not provided by", + wantCap: true, + }, + { + name: "local directory cannot provide repository scope", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(localRepo(t.TempDir())), "repository", "")}, + alias: "r", + wantErr: "not provided by", + wantCap: true, + }, + { + name: "s3 provides repository scope itself", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(s3Repo()), "repository", "")}, + alias: "r", + }, + { + name: "delegate lacking repository scope is insufficient", + repos: map[string]schema.ArtifactRepositoryConfig{ + "r": withLocking(ptr(registryRepo()), "repository", "other"), + "other": func() schema.ArtifactRepositoryConfig { + repo := registryRepo() + repo.Options["prefix"] = "other" + return repo + }(), + }, + alias: "r", + wantErr: `not provided by "other"`, + wantCap: true, + }, + { + name: "unknown scope", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(s3Repo()), "global", "")}, + alias: "r", + wantErr: "required_scope", + }, + { + name: "unknown delegate", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "repository", "missing")}, + alias: "r", + wantErr: `delegates locking to "missing", which is not configured`, + }, + { + name: "host scope with a stronger delegate is fine", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "host", "c"), "c": s3Repo()}, + alias: "r", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{Repositories: tt.repos}, tt.alias) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantCap, errors.Is(err, errUtils.ErrArtifactUnsupportedCapability)) + }) + } +} + +func TestValidate_DelegatedLockCyclesAreRejected(t *testing.T) { + tests := []struct { + name string + repos map[string]schema.ArtifactRepositoryConfig + want string + }{ + { + name: "self delegation", + repos: map[string]schema.ArtifactRepositoryConfig{"a": withLocking(ptr(s3Repo()), "repository", "a")}, + want: "a -> a", + }, + { + name: "two repositories", + repos: map[string]schema.ArtifactRepositoryConfig{ + "a": withLocking(ptr(s3Repo()), "repository", "b"), + "b": withLocking(ptr(s3Repo()), "repository", "a"), + }, + want: "a -> b -> a", + }, + { + name: "three repositories", + repos: map[string]schema.ArtifactRepositoryConfig{ + "a": withLocking(ptr(s3Repo()), "repository", "b"), + "b": withLocking(ptr(s3Repo()), "repository", "c"), + "c": withLocking(ptr(s3Repo()), "repository", "a"), + }, + want: "a -> b -> c -> a", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Validate(&schema.ArtifactConfig{Repositories: tt.repos}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "lock delegation cycle") + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestResolveLockPlan_FollowsChainToTheProvider(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "registry": withLocking(ptr(registryRepo()), "repository", "middle"), + "middle": withLocking(ptr(s3Repo()), "repository", "final"), + "final": s3Repo(), + }} + plan, err := ResolveLockPlan(cfg, "registry") + require.NoError(t, err) + assert.Equal(t, "final", plan.Provider) + assert.Equal(t, []string{"registry", "middle", "final"}, plan.Chain) + + plain, err := ResolveLockPlan(cfg, "final") + require.NoError(t, err) + assert.Equal(t, LockPlan{Provider: "final", Required: artifact.LockScopeHost, Chain: []string{"final"}}, plain) + + _, err = ResolveLockPlan(cfg, "nope") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} diff --git a/pkg/artifact/config/origin.go b/pkg/artifact/config/origin.go new file mode 100644 index 00000000000..5b8baade3b7 --- /dev/null +++ b/pkg/artifact/config/origin.go @@ -0,0 +1,47 @@ +package config + +import ( + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// AnchorPath returns the absolute, cleaned form of p. An absolute path is only cleaned. A relative +// path resolves against the directory of the configuration file that declared it, so the result +// does not depend on the working directory or on which file was merged last. +// +// A relative path declared by a remote import cannot be anchored: its local copy lives in an +// ephemeral download directory that must never become a write root. A relative path with no known +// declaring file is likewise rejected. In both cases the caller must provide an absolute path or +// override the value in a locally declared configuration file. +func AnchorPath(origin schema.ArtifactConfigOrigin, p string) (string, error) { + defer perf.Track(nil, "artifactconfig.AnchorPath")() + + if filepath.IsAbs(p) { + return filepath.Clean(p), nil + } + switch { + case origin.Remote: + return "", fmt.Errorf("%w: %w: relative path %q is declared only by a remote import (%s); declare it in a local configuration file or use an absolute path", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, p, origin.File) + case origin.Dir == "": + return "", fmt.Errorf("%w: %w: relative path %q has no declaring configuration file to resolve against; use an absolute path", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, p) + } + return filepath.Join(origin.Dir, p), nil +} + +// NewOrigin builds the origin of a value declared in file. Remote marks a downloaded import +// whose local path is ephemeral. +func NewOrigin(file string, remote bool) schema.ArtifactConfigOrigin { + defer perf.Track(nil, "artifactconfig.NewOrigin")() + + if file == "" { + return schema.ArtifactConfigOrigin{Remote: remote} + } + if abs, err := filepath.Abs(file); err == nil { + file = abs + } + return schema.ArtifactConfigOrigin{File: file, Dir: filepath.Dir(file), Remote: remote} +} diff --git a/pkg/artifact/config/redact_values_test.go b/pkg/artifact/config/redact_values_test.go new file mode 100644 index 00000000000..1c50130ab33 --- /dev/null +++ b/pkg/artifact/config/redact_values_test.go @@ -0,0 +1,87 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// TestEchoedNames_DoNotEchoCredentials checks that a repository alias, kind, lock scope or lock +// delegate that is a pasted credential-bearing URL never reaches a validation error. +func TestEchoedNames_DoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + local := schema.ArtifactRepositoryConfig{Kind: "local/archive"} + delegating := func(target string) *schema.ArtifactConfig { + return &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: target}}, + }} + } + _, missingAlias := ResolveLockPlan(&schema.ArtifactConfig{}, credential) + _, missingDelegate := ResolveLockPlan(delegating(credential), "a") + _, scope := ResolveLockPlan(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{RequiredScope: credential}}, + }}, "a") + _, cycle := ResolveLockPlan(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: credential}}, + credential: {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: "a"}}, + }}, "a") + withAlias := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{credential: local}} + + tests := []struct { + name string + err error + }{ + {name: "lock plan of an unknown alias", err: missingAlias}, + {name: "lock delegate that is not configured", err: missingDelegate}, + {name: "lock scope", err: scope}, + {name: "lock delegation cycle", err: cycle}, + {name: "alias that is not valid", err: NewValidator().ValidateRepository(withAlias, credential)}, + {name: "configured aliases listed beside an unknown one", err: NewValidator().ValidateRepository(withAlias, "nope")}, + {name: "unknown kind", err: NewValidator().ValidateRepository(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{"a": {Kind: credential}}}, "a")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.ErrorIs(t, tt.err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, tt.err.Error(), "S3cr3tA") + assert.NotContains(t, tt.err.Error(), "usr7") + }) + } +} + +// TestEchoedNames_KeepNamesWithoutCredentials is the negative path of the test above: a plain alias +// is still named in the error. +func TestEchoedNames_KeepNamesWithoutCredentials(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{}, "prod-eu") + require.Error(t, err) + assert.Contains(t, err.Error(), `repository "prod-eu" is not configured`) +} + +// TestDecode_ErrorsDoNotEchoCredentials checks that the keys and values a decode error quotes are +// scrubbed, and that a non-string permissions value is shown without its credential. +func TestDecode_ErrorsDoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + tests := []struct { + name string + raw map[string]any + }{ + {name: "unknown repository field", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"kind": "local/archive", credential: "x"}}}}, + {name: "unknown top-level field", raw: map[string]any{credential: "x"}}, + {name: "credential-like unknown field", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"usr7:S3cr3tA@password": "x"}}}}, + {name: "permissions holding a credential", raw: map[string]any{"repositories": map[string]any{credential: map[string]any{"kind": "local/archive", "permissions": map[string]any{"k": credential}}}}}, + {name: "permissions holding a scheme-less credential", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"kind": "local/archive", "permissions": map[string]any{"k": "usr7:S3cr3tA@example.com"}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Decode(tt.raw) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go new file mode 100644 index 00000000000..4a6bea4546b --- /dev/null +++ b/pkg/artifact/config/validate.go @@ -0,0 +1,208 @@ +package config + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" + "unicode" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + // The longest auth identity reference. + maxIdentityLength = 256 + + // The longest default artifact description. + maxDescriptionLength = 4096 +) + +// aliasPattern is the shape of a repository alias, which appears in `--repository` and step fields. +var aliasPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) + +// Validator validates artifact configuration. The zero value is not usable; use [NewValidator]. +type Validator struct { + registry *artifact.RepositoryRegistry + identityExists func(name string) bool +} + +// Option configures a Validator. +type Option func(*Validator) + +// WithRegistry validates kinds against a specific registry instead of the process-wide one. +func WithRegistry(registry *artifact.RepositoryRegistry) Option { + defer perf.Track(nil, "artifactconfig.WithRegistry")() + + return func(v *Validator) { v.registry = registry } +} + +// WithIdentityLookup also checks that each referenced identity exists. Without it only the shape +// of the identity name is checked: identities are resolved lazily and may be provisioned later. +func WithIdentityLookup(exists func(name string) bool) Option { + defer perf.Track(nil, "artifactconfig.WithIdentityLookup")() + + return func(v *Validator) { v.identityExists = exists } +} + +// NewValidator returns a Validator. +func NewValidator(opts ...Option) *Validator { + defer perf.Track(nil, "artifactconfig.NewValidator")() + + v := &Validator{registry: artifact.DefaultRepositoryRegistry()} + for _, opt := range opts { + if opt != nil { + opt(v) + } + } + return v +} + +// Validate validates a whole artifact configuration with a default Validator. +func Validate(cfg *schema.ArtifactConfig, opts ...Option) error { + defer perf.Track(nil, "artifactconfig.Validate")() + + return NewValidator(opts...).Validate(cfg) +} + +// Validate checks the global verification policy and every repository, returning all problems +// joined in a deterministic order. Every error wraps errUtils.ErrArtifactConfigInvalid. It +// performs no network access, credential resolution, or directory creation. +func (v *Validator) Validate(cfg *schema.ArtifactConfig) error { + defer perf.Track(nil, "artifactconfig.Validator.Validate")() + + if cfg == nil { + return nil + } + var errs []error + if err := validateVerification("artifacts.verification", cfg.Verification); err != nil { + errs = append(errs, err) + } + for _, alias := range sortedAliases(cfg) { + repo := cfg.Repositories[alias] + if err := v.validateRepository(cfg, alias, &repo); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// ValidateRepository validates one repository in the context of the whole configuration, which +// the lock delegation and verification checks need. +func (v *Validator) ValidateRepository(cfg *schema.ArtifactConfig, alias string) error { + defer perf.Track(nil, "artifactconfig.Validator.ValidateRepository")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return fmt.Errorf("%w: repository %q is not configured (configured: %v)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), redactedAliases(cfg)) + } + return v.validateRepository(cfg, alias, &repo) +} + +func (v *Validator) validateRepository(cfg *schema.ArtifactConfig, alias string, repo *schema.ArtifactRepositoryConfig) error { + prefix := fmt.Sprintf("repository %q", artifact.RedactURL(alias)) + if !aliasPattern.MatchString(alias) { + return fmt.Errorf("%w: %s: alias must start with a letter or digit and contain only letters, digits, '_', '.' and '-'", errUtils.ErrArtifactConfigInvalid, prefix) + } + desc, err := v.resolveKind(repo.Kind) + if err != nil { + return fmt.Errorf("%s: %w", prefix, err) + } + errs := []error{ + v.validateIdentity(repo.Identity), + artifact.ValidateRepositoryConfig(desc, repo), + validatePathOptions(desc, repo), + validateCatalog(desc, repo), + validateDefaults(repo.Defaults), + validateVerification("verification", repo.Verification), + validateWeakening(cfg.Verification, repo.Verification), + validateLocking(v, cfg, alias, repo), + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("%s: %w", prefix, err) + } + return nil +} + +// resolveKind finds the descriptor for a kind, rejecting empty and unknown kinds. +func (v *Validator) resolveKind(kind string) (*artifact.KindDescriptor, error) { + if kind == "" { + return nil, fmt.Errorf("%w: kind is required (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, v.registry.Kinds()) + } + desc, ok := v.registry.Lookup(kind) + if !ok { + return nil, fmt.Errorf("%w: unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(kind), v.registry.Kinds()) + } + return desc, nil +} + +// validateIdentity checks the shape of an auth identity reference. Resolution is lazy. +func (v *Validator) validateIdentity(identity string) error { + if identity == "" { + return nil + } + switch { + case utf8.RuneCountInString(identity) > maxIdentityLength: + return fmt.Errorf("%w: identity exceeds %d characters", errUtils.ErrArtifactConfigInvalid, maxIdentityLength) + case strings.IndexFunc(identity, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) >= 0: + return fmt.Errorf("%w: identity must be an auth identity name without whitespace or control characters", errUtils.ErrArtifactConfigInvalid) + case strings.Contains(identity, "://"): + return fmt.Errorf("%w: identity looks like a credential or URL; reference an Atmos auth identity name", errUtils.ErrArtifactConfigInvalid) + case v.identityExists != nil && !v.identityExists(identity): + return fmt.Errorf("%w: identity %q is not defined under auth.identities", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(identity)) + } + return nil +} + +// validatePathOptions rejects local path options that cannot be anchored to a local file. +func validatePathOptions(desc *artifact.KindDescriptor, repo *schema.ArtifactRepositoryConfig) error { + var errs []error + for _, option := range desc.PathOptions { + value, ok := repo.Options[option].(string) + if !ok || value == "" { + continue + } + if _, err := AnchorPath(repo.PathOrigin, value); err != nil { + errs = append(errs, fmt.Errorf("options.%s: %w", option, err)) + } + } + return errors.Join(errs...) +} + +// validateDefaults checks the default artifact metadata. +func validateDefaults(defaults *schema.ArtifactRepositoryDefaults) error { + if defaults == nil || defaults.Metadata == nil { + return nil + } + var errs []error + if len(defaults.Metadata.Description) > maxDescriptionLength { + errs = append(errs, fmt.Errorf("%w: defaults.metadata.description exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, maxDescriptionLength)) + } + if err := artifact.ValidateLabels(defaults.Metadata.Labels); err != nil { + errs = append(errs, fmt.Errorf("defaults.metadata.labels: %w", err)) + } + return errors.Join(errs...) +} + +// redactedAliases returns the configured aliases in order, each redacted, for an error that lists them. +func redactedAliases(cfg *schema.ArtifactConfig) []string { + aliases := sortedAliases(cfg) + for i, alias := range aliases { + aliases[i] = artifact.RedactURL(alias) + } + return aliases +} + +func sortedAliases(cfg *schema.ArtifactConfig) []string { + aliases := make([]string, 0, len(cfg.Repositories)) + for alias := range cfg.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/config/validate_test.go b/pkg/artifact/config/validate_test.go new file mode 100644 index 00000000000..aee0505e74b --- /dev/null +++ b/pkg/artifact/config/validate_test.go @@ -0,0 +1,249 @@ +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinels: renaming a schema field these tests rely on must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: "", Identity: "", Packaging: "", Permissions: "", Catalog: ""} + +var _ = schema.ArtifactLockingConfig{RequiredScope: "", Repository: ""} + +func localRepo(path string) schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": path}} +} + +func s3Repo() schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifact-coordination"}} +} + +func registryRepo() schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "ghcr.io", "prefix": "acme/atmos"}} +} + +func validateOne(t *testing.T, alias string, repo *schema.ArtifactRepositoryConfig, opts ...Option) error { + t.Helper() + return Validate(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{alias: *repo}}, opts...) +} + +func TestValidate_NilAndEmptyConfigurationAreValid(t *testing.T) { + require.NoError(t, Validate(nil)) + require.NoError(t, Validate(&schema.ArtifactConfig{})) +} + +func TestValidate_RepositoryBasics(t *testing.T) { + root := t.TempDir() + tests := []struct { + name string + alias string + mutate func(*schema.ArtifactRepositoryConfig) + wantErr string + }{ + {name: "valid local", alias: "local"}, + {name: "alias with dots and case", alias: "Prod.US-1"}, + {name: "alias with slash", alias: "a/b", wantErr: "alias must start"}, + {name: "alias leading dash", alias: "-x", wantErr: "alias must start"}, + {name: "missing kind", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Kind = "" }, wantErr: "kind is required"}, + {name: "unknown kind", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Kind = "ftp/server" }, wantErr: `unknown kind "ftp/server"`}, + {name: "missing required option", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options = nil }, wantErr: `"path" is required`}, + {name: "unknown option", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["bucket"] = "b" }, wantErr: "bucket"}, + {name: "credentials in options", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["access_key"] = "AKIA" }, wantErr: "auth identity"}, + {name: "identity with whitespace", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "a b" }, wantErr: "without whitespace"}, + {name: "identity that is a URL", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "https://u:p@host" }, wantErr: "credential or URL"}, + {name: "identity at the character limit", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = strings.Repeat("é", 256) }}, + {name: "identity over the character limit", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = strings.Repeat("é", 257) }, wantErr: "exceeds 256 characters"}, + {name: "identity with whitespace and a secret", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "user:hunter2@host extra" }, wantErr: "without whitespace"}, + {name: "permissions on a directory", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Permissions = "0600" }, wantErr: "does not accept permissions"}, + {name: "packaging outside the matrix", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Packaging = "tgz" }, wantErr: "does not support packaging"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := localRepo(root) + if tt.mutate != nil { + tt.mutate(&repo) + } + err := validateOne(t, tt.alias, &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "credential-looking values are never echoed") + assert.NotContains(t, err.Error(), "u:p@", "credential-looking values are never echoed") + }) + } +} + +func TestValidate_JoinsErrorsAcrossRepositoriesInAliasOrder(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "zeta": {Kind: "bad/one"}, + "alpha": {Kind: "bad/two"}, + }} + err := Validate(cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + message := err.Error() + assert.Less(t, strings.Index(message, `"alpha"`), strings.Index(message, `"zeta"`)) +} + +func TestValidate_IdentityLookupIsOptional(t *testing.T) { + repo := localRepo(t.TempDir()) + repo.Identity = "registry-publisher" + require.NoError(t, validateOne(t, "x", &repo), "identity is resolved lazily; shape only by default") + + err := validateOne(t, "x", &repo, WithIdentityLookup(func(string) bool { return false })) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "not defined under auth.identities") + + require.NoError(t, validateOne(t, "x", &repo, WithIdentityLookup(func(name string) bool { return name == "registry-publisher" }))) +} + +func TestValidate_UsesInjectedRegistryDescriptor(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, registry.Register(&artifact.KindDescriptor{Kind: "gcp/gcs", Capabilities: artifact.Capabilities{Packagings: []artifact.Packaging{artifact.PackagingTgz}}, DefaultPackaging: artifact.PackagingTgz}, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...artifact.RepositoryOption) (artifact.Repository, error) { + return nil, nil + })) + require.NoError(t, validateOne(t, "x", &schema.ArtifactRepositoryConfig{Kind: "gcp/gcs"}, WithRegistry(registry))) + require.Error(t, validateOne(t, "x", &schema.ArtifactRepositoryConfig{Kind: "gcp/gcs"}, WithRegistry(artifact.NewRepositoryRegistry()))) +} + +func TestValidateRepository_UnknownAlias(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{}, "nope") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestValidate_RelativePathsNeedALocalDeclaringFile(t *testing.T) { + dir := t.TempDir() + tests := []struct { + name string + origin schema.ArtifactConfigOrigin + path string + wantErr string + }{ + {name: "relative with local origin", origin: NewOrigin(filepath.Join(dir, "atmos.yaml"), false), path: "store"}, + {name: "absolute needs no origin", path: filepath.Join(dir, "store")}, + {name: "relative from remote import", origin: NewOrigin(filepath.Join(dir, "tmp", "remote.yaml"), true), path: "store", wantErr: "remote import"}, + {name: "relative with unknown origin", path: "store", wantErr: "no declaring configuration file"}, + {name: "absolute from remote import is allowed", origin: NewOrigin(filepath.Join(dir, "tmp", "remote.yaml"), true), path: filepath.Join(dir, "store")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := localRepo(tt.path) + repo.PathOrigin = tt.origin + err := validateOne(t, "x", &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestAnchorPath(t *testing.T) { + dir := t.TempDir() + origin := NewOrigin(filepath.Join(dir, "conf", "atmos.yaml"), false) + assert.Equal(t, filepath.Join(dir, "conf"), origin.Dir) + + got, err := AnchorPath(origin, filepath.Join("..", "shared", "store")) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "shared", "store"), got) + + absolute := filepath.Join(dir, "abs", "..", "abs2") + got, err = AnchorPath(origin, absolute) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "abs2"), got) + + assert.True(t, NewOrigin("", true).Remote) + assert.True(t, NewOrigin("", false).IsZero()) +} + +func TestValidate_DefaultsMetadata(t *testing.T) { + repo := localRepo(t.TempDir()) + repo.Defaults = &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{ + Description: "backups", + Labels: map[string]string{"team": "platform", "bad=key": "x", "also": "bad,value"}, + }} + err := validateOne(t, "x", &repo) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "defaults.metadata.labels") + assert.Contains(t, err.Error(), "bad=key") + assert.Contains(t, err.Error(), `"also"`) + + repo.Defaults.Metadata.Labels = map[string]string{"team": "platform"} + require.NoError(t, validateOne(t, "x", &repo)) + + repo.Defaults.Metadata.Description = strings.Repeat("d", 4097) + require.ErrorIs(t, validateOne(t, "x", &repo), errUtils.ErrArtifactConfigInvalid) +} + +func TestValidate_CatalogReferences(t *testing.T) { + tests := []struct { + name string + mutate func(*schema.ArtifactRepositoryConfig) + wantErr string + }{ + {name: "PRD example", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/atmos/catalog:stable" }}, + {name: "digest reference", mutate: func(r *schema.ArtifactRepositoryConfig) { + r.Catalog = "oci://ghcr.io/acme/atmos/catalog@sha256:" + strings.Repeat("a", 64) + }}, + {name: "host compared case-insensitively", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://GHCR.io/acme/catalog:stable" }}, + {name: "another registry host", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://evil.example/acme/catalog:stable" }, wantErr: "does not match the repository registry"}, + {name: "wrong scheme", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "https://ghcr.io/acme/catalog:stable" }, wantErr: "oci://"}, + {name: "no tag or digest", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog" }, wantErr: "no implicit latest"}, + {name: "credentials", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/acme/catalog:stable" }, wantErr: "credentials"}, + {name: "query", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog:stable?token=x" }, wantErr: "credentials, a query"}, + {name: "bad tag", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog:-bad" }, wantErr: "tag syntax"}, + {name: "bad digest", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog@sha256:xyz" }, wantErr: "digest"}, + {name: "uppercase path", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/Acme/catalog:stable" }, wantErr: "lowercase"}, + {name: "malformed escape with credentials", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/acme/%zz:stable" }, wantErr: "must be an oci://"}, + {name: "bad path with credentials is rejected before the path check", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/Acme/catalog:stable" }, wantErr: "credentials"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := registryRepo() + tt.mutate(&repo) + err := validateOne(t, "shared", &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "credentials are never echoed") + }) + } + + s3 := s3Repo() + s3.Catalog = "oci://ghcr.io/acme/catalog:stable" + err := validateOne(t, "backups", &s3) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "does not support a catalog") +} + +func TestValidate_PerformsNoFilesystemSideEffects(t *testing.T) { + root := t.TempDir() + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "local": localRepo(filepath.Join(root, "dest", "local")), + "archives": {Kind: artifact.RepositoryKindLocalArchive, Permissions: "0600", Options: map[string]any{"path": filepath.Join(root, "dest", "archives"), "key_pattern": "{{ .Name }}.tgz"}}, + "layout": {Kind: artifact.RepositoryKindOCILayout, Options: map[string]any{"path": filepath.Join(root, "dest", "layout")}}, + }} + require.NoError(t, Validate(cfg)) + + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries, "validating or describing configuration must not create destination directories") +} diff --git a/pkg/artifact/config/verification.go b/pkg/artifact/config/verification.go new file mode 100644 index 00000000000..428afa4c7fa --- /dev/null +++ b/pkg/artifact/config/verification.go @@ -0,0 +1,115 @@ +package config + +import ( + "errors" + "fmt" + "net/url" + "strings" + "unicode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// validateVerification checks one verification policy level (global or repository). +func validateVerification(scope string, policy *schema.ArtifactVerificationConfig) error { + if policy == nil { + return nil + } + var errs []error + for index, trusted := range policy.TrustedIdentities { + if err := validateTrustedIdentity(trusted); err != nil { + errs = append(errs, fmt.Errorf("trusted_identities[%d]: %w", index, err)) + } + } + for index, key := range policy.TrustedKeys { + if err := validateTrustedKey(policy, key); err != nil { + errs = append(errs, fmt.Errorf("trusted_keys[%d]: %w", index, err)) + } + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("%s: %w", scope, err) + } + return nil +} + +func validateTrustedIdentity(trusted schema.ArtifactTrustedIdentity) error { + var errs []error + if strings.TrimSpace(trusted.Identity) == "" || strings.IndexFunc(trusted.Identity, unicode.IsControl) >= 0 { + errs = append(errs, fmt.Errorf("%w: identity is required and must not contain control characters", errUtils.ErrArtifactConfigInvalid)) + } + issuer, err := url.Parse(trusted.Issuer) + if trusted.Issuer == "" || err != nil || issuer.Scheme != "https" || issuer.Host == "" || issuer.User != nil { + errs = append(errs, fmt.Errorf("%w: issuer %q must be an https URL without credentials", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(trusted.Issuer))) + } + return errors.Join(errs...) +} + +func validateTrustedKey(policy *schema.ArtifactVerificationConfig, key string) error { + if key == "" || strings.ContainsRune(key, 0) { + return fmt.Errorf("%w: key path must be a non-empty file path", errUtils.ErrArtifactConfigInvalid) + } + if _, err := AnchorPath(policy.KeysOrigin, key); err != nil { + return err + } + return nil +} + +// validateWeakening rejects a repository that tries to relax the global policy. +func validateWeakening(global, repo *schema.ArtifactVerificationConfig) error { + if global == nil || repo == nil || global.RequireSignature == nil || repo.RequireSignature == nil { + return nil + } + if *global.RequireSignature && !*repo.RequireSignature { + return fmt.Errorf("%w: verification.require_signature: false cannot weaken the global artifacts.verification.require_signature: true", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// EffectivePolicy returns the verification policy that applies to a repository. +// +// A global requirement can be strengthened but never weakened: RequireSignature is true when +// either level requires it. Within one trust set any configured signer satisfies it; when both +// the global and the repository define trust (identities or keys), the policy carries two trust +// sets and both must be satisfied, so a repository can narrow trust but never widen it. The +// global trust set comes first. +func EffectivePolicy(cfg *schema.ArtifactConfig, alias string) (artifact.VerificationPolicy, error) { + defer perf.Track(nil, "artifactconfig.EffectivePolicy")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return artifact.VerificationPolicy{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + return CombinePolicy(cfg.Verification, repo.Verification), nil +} + +// CombinePolicy combines a global and a repository policy level. Either may be nil. +func CombinePolicy(global, repo *schema.ArtifactVerificationConfig) artifact.VerificationPolicy { + defer perf.Track(nil, "artifactconfig.CombinePolicy")() + + var policy artifact.VerificationPolicy + for _, level := range []*schema.ArtifactVerificationConfig{global, repo} { + if level == nil { + continue + } + if level.RequireSignature != nil && *level.RequireSignature { + policy.RequireSignature = true + } + if set := trustSetOf(level); len(set.Identities) > 0 || len(set.Keys) > 0 { + policy.TrustSets = append(policy.TrustSets, set) + } + } + return policy +} + +// trustSetOf converts one policy level to a trust set, copying slices so the policy never aliases configuration. +func trustSetOf(level *schema.ArtifactVerificationConfig) artifact.TrustSet { + var set artifact.TrustSet + for _, trusted := range level.TrustedIdentities { + set.Identities = append(set.Identities, artifact.TrustedIdentity{Identity: trusted.Identity, Issuer: trusted.Issuer}) + } + set.Keys = append(set.Keys, level.TrustedKeys...) + return set +} diff --git a/pkg/artifact/config/verification_test.go b/pkg/artifact/config/verification_test.go new file mode 100644 index 00000000000..8815f0c4571 --- /dev/null +++ b/pkg/artifact/config/verification_test.go @@ -0,0 +1,154 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + workflowIdentity = "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main" + githubIssuer = "https://token.actions.githubusercontent.com" +) + +func boolPtr(b bool) *bool { return &b } + +func identityPolicy(requireSignature *bool, identity string) *schema.ArtifactVerificationConfig { + return &schema.ArtifactVerificationConfig{ + RequireSignature: requireSignature, + TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: identity, Issuer: githubIssuer}}, + } +} + +func TestValidate_VerificationShape(t *testing.T) { + keyDir := t.TempDir() + tests := []struct { + name string + policy *schema.ArtifactVerificationConfig + wantErr string + }{ + {name: "PRD example", policy: identityPolicy(boolPtr(true), workflowIdentity)}, + {name: "empty key list is allowed", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{}}}, + {name: "missing identity", policy: identityPolicy(nil, ""), wantErr: "identity is required"}, + {name: "issuer without https", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "http://issuer.example"}}}, wantErr: "https URL"}, + {name: "missing issuer", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me"}}}, wantErr: "https URL"}, + {name: "issuer with credentials", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "https://u:p@issuer.example"}}}, wantErr: "https URL"}, + {name: "issuer with credentials and a malformed escape", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "https://u:hunter2@issuer.example/%zz"}}}, wantErr: "https URL"}, + {name: "absolute key", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{filepath.Join(keyDir, "cosign.pub")}}}, + {name: "empty key path", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{""}}, wantErr: "non-empty file path"}, + {name: "relative key without a declaring file", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}}, wantErr: "no declaring configuration file"}, + {name: "relative key from a remote import", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}, KeysOrigin: NewOrigin(filepath.Join(keyDir, "r.yaml"), true)}, wantErr: "remote import"}, + {name: "relative key from a local file", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}, KeysOrigin: NewOrigin(filepath.Join(keyDir, "atmos.yaml"), false)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Validate(&schema.ArtifactConfig{Verification: tt.policy}) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "u:p@", "rejected issuer credentials are never echoed") + assert.NotContains(t, err.Error(), "hunter2", "rejected issuer credentials are never echoed") + }) + } +} + +func TestValidate_RepositoryCannotWeakenGlobalSignatureRequirement(t *testing.T) { + repo := registryRepo() + repo.Verification = &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(false)} + cfg := &schema.ArtifactConfig{ + Verification: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true)}, + Repositories: map[string]schema.ArtifactRepositoryConfig{"shared": repo}, + } + err := Validate(cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "cannot weaken") + + // Strengthening, or leaving it unset, is fine. + repo.Verification.RequireSignature = boolPtr(true) + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) + repo.Verification.RequireSignature = nil + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) + + // A repository may be stricter than a permissive global policy. + cfg.Verification.RequireSignature = boolPtr(false) + repo.Verification.RequireSignature = boolPtr(true) + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) +} + +func TestEffectivePolicy(t *testing.T) { + repoIdentity := "https://github.com/acme/other/.github/workflows/release.yml@refs/heads/main" + tests := []struct { + name string + global, repo *schema.ArtifactVerificationConfig + want artifact.VerificationPolicy + }{ + {name: "nothing configured", want: artifact.VerificationPolicy{}}, + { + name: "global only", + global: identityPolicy(boolPtr(true), workflowIdentity), + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: workflowIdentity, Issuer: githubIssuer}}}, + }}, + }, + { + name: "repository only", + repo: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true), TrustedKeys: []string{"/keys/a.pub"}}, + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: []string{"/keys/a.pub"}}}}, + }, + { + name: "both levels keep two trust sets, global first, never unioned", + global: identityPolicy(boolPtr(true), workflowIdentity), + repo: identityPolicy(nil, repoIdentity), + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: workflowIdentity, Issuer: githubIssuer}}}, + {Identities: []artifact.TrustedIdentity{{Identity: repoIdentity, Issuer: githubIssuer}}}, + }}, + }, + { + name: "repository false cannot lower a global requirement", + global: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true)}, + repo: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(false)}, + want: artifact.VerificationPolicy{RequireSignature: true}, + }, + { + name: "empty trust level adds no trust set", + global: &schema.ArtifactVerificationConfig{TrustedKeys: []string{}}, + want: artifact.VerificationPolicy{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := registryRepo() + repo.Verification = tt.repo + cfg := &schema.ArtifactConfig{Verification: tt.global, Repositories: map[string]schema.ArtifactRepositoryConfig{"shared": repo}} + got, err := EffectivePolicy(cfg, "shared") + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + _, err := EffectivePolicy(&schema.ArtifactConfig{}, "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestCombinePolicy_DoesNotAliasConfiguration(t *testing.T) { + global := identityPolicy(boolPtr(true), workflowIdentity) + global.TrustedKeys = []string{"/keys/a.pub"} + policy := CombinePolicy(global, nil) + policy.TrustSets[0].Keys[0] = "/mutated" + policy.TrustSets[0].Identities[0].Identity = "mutated" + assert.Equal(t, []string{"/keys/a.pub"}, global.TrustedKeys) + assert.Equal(t, workflowIdentity, global.TrustedIdentities[0].Identity) +} diff --git a/pkg/artifact/constants.go b/pkg/artifact/constants.go new file mode 100644 index 00000000000..88f1be98568 --- /dev/null +++ b/pkg/artifact/constants.go @@ -0,0 +1,77 @@ +package artifact + +// Serialization and bounded-read limits shared by every repository adapter. +const ( + // SchemaVersion is the only Record schema version defined by V1. + SchemaVersion = 1 + + // MaxRecordSize is the largest canonical Record (and catalog JSON) read or written, in bytes. + MaxRecordSize int64 = 8 << 20 + + // MaxDocumentSize is the largest README or CHANGELOG document, in bytes. + MaxDocumentSize int64 = 4 << 20 + + // MaxEvidenceSize is the largest single signature evidence blob read into memory, in bytes. + MaxEvidenceSize int64 = 1 << 20 + + // MaxNameLength is the longest artifact name, matching the OCI repository name limit. + MaxNameLength = 255 + + // MaxVersionLength is the longest artifact version, matching the OCI tag limit. + MaxVersionLength = 128 + + // MaxKindLength is the longest artifact kind. + MaxKindLength = 128 + + // DefaultConcurrency is the default transfer and discovery concurrency. + DefaultConcurrency = 4 + + // ReservedNamespace is the managed-record directory below a local or S3 root. Payload + // names and key patterns must never enter it. + ReservedNamespace = ".atmos-artifacts" + + // DocumentMediaType is the default media type of README and CHANGELOG documents. + DocumentMediaType = "text/markdown" +) + +// OCI media types from the artifact PRD. Existing plugin and foreign OCI graphs keep their own types. +const ( + // MediaTypeArtifact is the generic artifact type. + MediaTypeArtifact = "application/vnd.atmos.artifact.v1" + + // MediaTypeScaffold is the scaffold artifact type. + MediaTypeScaffold = "application/vnd.atmos.scaffold.v1" + + // MediaTypeComponent is the component artifact type. + MediaTypeComponent = "application/vnd.atmos.component.v1" + + // MediaTypeToolchainPackage is the toolchain package artifact type. + MediaTypeToolchainPackage = "application/vnd.atmos.toolchain.package.v1" + + // MediaTypeCatalog is the artifact type of a catalog Record. + MediaTypeCatalog = "application/vnd.atmos.artifact.catalog.v1" + + // MediaTypeCatalogPayload is the media type of a catalog's direct JSON payload layer. + MediaTypeCatalogPayload = "application/vnd.atmos.artifact.catalog.v1+json" + + // MediaTypeConfig is the media type of the OCI config blob holding the canonical Record bytes. + MediaTypeConfig = "application/vnd.atmos.artifact.config.v1+json" + + // MediaTypeTar is the media type of an uncompressed tar payload. + MediaTypeTar = "application/vnd.atmos.artifact.content.v1.tar" + + // MediaTypeTarGzip is the media type of a gzip-compressed tar payload. + MediaTypeTarGzip = "application/vnd.atmos.artifact.content.v1.tar+gzip" + + // MediaTypeZip is the media type of a zip payload. + MediaTypeZip = "application/vnd.atmos.artifact.content.v1.zip" +) + +// Well-known artifact kinds. +const ( + // KindFiles is the default kind for generic file packages. + KindFiles = "files" + + // KindCatalog is the kind of an artifact catalog Record. + KindCatalog = "catalog" +) diff --git a/pkg/artifact/contracts_test.go b/pkg/artifact/contracts_test.go new file mode 100644 index 00000000000..8c6d8c4381d --- /dev/null +++ b/pkg/artifact/contracts_test.go @@ -0,0 +1,154 @@ +package artifact + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Compile-time proof that every generated mock satisfies its interface. +var ( + _ Repository = (*MockRepository)(nil) + _ LockProvider = (*MockLockProvider)(nil) + _ LockProvider = Repository(nil) + _ LockHandle = (*MockLockHandle)(nil) + _ Producer = (*MockProducer)(nil) + _ Signer = (*MockSigner)(nil) + _ Verifier = (*MockVerifier)(nil) + _ Copier = (*MockCopier)(nil) + _ Service = (*MockService)(nil) + _ GraphSource = (*MockGraphSource)(nil) +) + +func TestLockScopeSatisfies(t *testing.T) { + tests := []struct { + have, need LockScope + want bool + }{ + {LockScopeHost, LockScopeHost, true}, + {LockScopeHost, "", true}, + {LockScopeRepository, LockScopeHost, true}, + {LockScopeRepository, LockScopeRepository, true}, + {LockScopeHost, LockScopeRepository, false}, + {"", LockScopeHost, false}, + {LockScopeRepository, "cluster", false}, + } + for _, tt := range tests { + t.Run(string(tt.have)+"->"+string(tt.need), func(t *testing.T) { + assert.Equal(t, tt.want, tt.have.Satisfies(tt.need)) + }) + } +} + +func TestCapabilitiesLookups(t *testing.T) { + capabilities := Capabilities{ + Packagings: []Packaging{PackagingTgz, PackagingZip}, + EvidenceFormats: []EvidenceFormat{EvidenceSigstoreBundle}, + SubjectKinds: []SubjectKind{SubjectRecord}, + LockScopes: []LockScope{LockScopeHost}, + } + assert.True(t, capabilities.HasPackaging(PackagingTgz)) + assert.False(t, capabilities.HasPackaging(PackagingNone)) + assert.True(t, capabilities.HasEvidenceFormat(EvidenceSigstoreBundle)) + assert.False(t, capabilities.HasEvidenceFormat(EvidencePublicKeyBundle), "a missing format is not silently supported") + assert.True(t, capabilities.HasSubjectKind(SubjectRecord)) + assert.False(t, capabilities.HasSubjectKind(SubjectOCIRoot)) + assert.True(t, capabilities.HasLockScope(LockScopeHost)) + assert.False(t, capabilities.HasLockScope(LockScopeRepository)) + + capabilities.LockScopes = []LockScope{LockScopeRepository} + assert.True(t, capabilities.HasLockScope(LockScopeHost), "repository scope satisfies a host request") +} + +func TestPreparedArtifactCloseRunsCleanupOnce(t *testing.T) { + calls := 0 + boom := errors.New("cleanup failed") + prepared := &PreparedArtifact{Cleanup: func() error { calls++; return boom }} + require.ErrorIs(t, prepared.Close(), boom) + require.ErrorIs(t, prepared.Close(), errUtils.ErrArtifactStreamFailed, "cleanup failures carry the artifact sentinel") + require.ErrorIs(t, prepared.Close(), boom, "later calls return the first result") + assert.Equal(t, 1, calls) + + assert.NoError(t, (&PreparedArtifact{}).Close(), "nil cleanup is fine") +} + +type trackingCloser struct { + io.Reader + closed *int + err error +} + +func (c trackingCloser) Close() error { *c.closed++; return c.err } + +func TestContentCloseClosesEveryStream(t *testing.T) { + closed := 0 + first := errors.New("first") + content := &Content{Streams: []ContentStream{ + {Reader: trackingCloser{Reader: strings.NewReader("a"), closed: &closed, err: first}}, + {Reader: nil}, + {Reader: trackingCloser{Reader: strings.NewReader("b"), closed: &closed, err: errors.New("second")}}, + }} + err := content.Close() + require.ErrorIs(t, err, first) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "close failures carry the artifact sentinel") + assert.Equal(t, 2, closed, "an early failure does not leak later streams") +} + +func TestSignatureSubjectValidate(t *testing.T) { + good := SignatureSubject{Kind: SubjectRecord, Digest: digest.FromString("r"), MediaType: MediaTypeConfig, Size: 10} + require.NoError(t, good.Validate()) + + bad := good + bad.Kind = "archive" + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) + bad = good + bad.Digest = "sha256:short" + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) + bad = good + bad.Size = -1 + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) +} + +// TestRepositoryContractThroughMocks drives the interfaces the way the service will: ask a +// repository for a guarantee it lacks and observe a typed error, not a no-op. +func TestRepositoryContractThroughMocks(t *testing.T) { + ctrl := gomock.NewController(t) + repository := NewMockRepository(ctrl) + + repository.EXPECT().Capabilities().Return(Capabilities{LockScopes: []LockScope{LockScopeHost}}) + repository.EXPECT(). + Lock(gomock.Any(), LockRequest{Resource: "vpc:1.0", Owner: "host-1", Required: LockScopeRepository}). + Return(nil, NewUnsupportedCapability("shared", "lock", "lock/repository", "configure locking.repository")) + + request := LockRequest{Resource: "vpc:1.0", Owner: "host-1", Required: LockScopeRepository} + var repo Repository = repository + capabilities := repo.Capabilities() + require.False(t, capabilities.HasLockScope(request.Required)) + + handle, err := repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Nil(t, handle) +} + +func TestOptionalCopierIsDetectedByTypeAssertion(t *testing.T) { + ctrl := gomock.NewController(t) + var plain Repository = NewMockRepository(ctrl) + _, ok := plain.(Copier) + assert.False(t, ok, "a repository without native copy falls back to staged copy") + + var native Repository = struct { + *MockRepository + *MockCopier + }{NewMockRepository(ctrl), NewMockCopier(ctrl)} + _, ok = native.(Copier) + assert.True(t, ok) +} diff --git a/pkg/artifact/copier.go b/pkg/artifact/copier.go new file mode 100644 index 00000000000..fc4a537983f --- /dev/null +++ b/pkg/artifact/copier.go @@ -0,0 +1,29 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_copier.go github.com/cloudposse/atmos/pkg/artifact Copier + +// CopyFromOptions control a native transfer. +type CopyFromOptions struct { + // ExpectedPrior is the destination reference observed under the writer guard; nil means none. + ExpectedPrior *PinnedReference + + // Force permits replacing a conflicting destination. It never bypasses verification. + Force bool + + // Lock is the destination lock already held by the service. + Lock LockHandle +} + +// Copier is an optional Repository extension for a compatible native transfer (for example an OCI +// registry-to-registry graph copy that preserves every original descriptor). A repository that does +// not implement it, or whose CanCopyFrom returns false, uses the shared staging fallback. +type Copier interface { + // CanCopyFrom reports whether source is compatible with a native transfer into this repository. + CanCopyFrom(source Repository) bool + + // CopyFrom transfers the pinned artifact from source into this repository, preserving Record + // bytes and content identity. + CopyFrom(ctx context.Context, source Repository, pinned PinnedReference, options CopyFromOptions) (*PublishResult, error) +} diff --git a/pkg/artifact/digest.go b/pkg/artifact/digest.go new file mode 100644 index 00000000000..9d7f1f2f9c3 --- /dev/null +++ b/pkg/artifact/digest.go @@ -0,0 +1,81 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// contentView is the exact document hashed for content_digest. +type contentView struct { + Documents []Document `json:"documents"` + Inventory []Entry `json:"inventory"` + SchemaVersion int `json:"schema_version"` +} + +// userMetadata is the part of Metadata that participates in intent identity. +type userMetadata struct { + Description string `json:"description,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// intentView is the exact document hashed for intent_digest. +type intentView struct { + ContentDigest digest.Digest `json:"content_digest"` + Kind string `json:"kind"` + KindData KindData `json:"kind_data,omitempty"` + Metadata userMetadata `json:"metadata"` + Name string `json:"name"` + SchemaVersion int `json:"schema_version"` + Version string `json:"version"` +} + +// ComputeContentDigest returns the content_digest: the SHA-256 of the canonical JSON object +// {"documents":[...],"inventory":[...],"schema_version":1} built from the sorted inventory and +// document descriptors, including paths, modes, and link targets. It excludes storage locators +// and generated operation data. Inputs are validated first. +func ComputeContentDigest(inventory []Entry, documents []Document) (digest.Digest, error) { + defer perf.Track(nil, "artifact.ComputeContentDigest")() + + if err := validateInventory(inventory); err != nil { + return "", err + } + if err := validateDocuments(documents); err != nil { + return "", err + } + canonical, err := canonicalJSON(contentView{ + Documents: sortedDocuments(documents), + Inventory: sortedInventory(inventory), + SchemaVersion: SchemaVersion, + }) + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} + +// IntentDigest returns the intent_digest: the SHA-256 of the canonical JSON object holding +// name, version, kind, content_digest, the effective user metadata (description and labels), and +// the kind data. It excludes created_at and generated provenance so that a retry of the same +// intent yields the same digest and a no-op can be detected. +func (r *Record) IntentDigest() (digest.Digest, error) { + defer perf.Track(nil, "artifact.Record.IntentDigest")() + + if err := r.Validate(); err != nil { + return "", err + } + normalized := r.normalized() + canonical, err := canonicalJSON(intentView{ + ContentDigest: normalized.ContentDigest, + Kind: normalized.Kind, + KindData: normalized.KindData, + Metadata: userMetadata{Description: normalized.Metadata.Description, Labels: normalized.Metadata.Labels}, + Name: normalized.Name, + SchemaVersion: normalized.SchemaVersion, + Version: normalized.Version, + }) + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} diff --git a/pkg/artifact/doc.go b/pkg/artifact/doc.go new file mode 100644 index 00000000000..4e6e7839b69 --- /dev/null +++ b/pkg/artifact/doc.go @@ -0,0 +1,19 @@ +// Package artifact defines the generic, repository-independent domain contracts for Atmos +// artifacts: references, portable records, canonical serialization and digests, the repository, +// producer, signer, verifier, and lock-provider interfaces, and the typed service requests and +// results. +// +// The package deliberately imports no CLI, runner, hook, plugin, or CI packages. Backends and +// producers register through composition roots and implement the interfaces defined here. +// Backend SDK types (AWS clients, go-containerregistry images, ORAS remote repositories) never +// appear in this API; OCI seams use opencontainers image-spec descriptors and go-digest values. +// +// # Canonical records +// +// A [Record] serializes to one canonical byte sequence (see [Record.Canonical]): typed JSON with +// a fixed field order, sorted inventory and document arrays, sorted map keys, UTF-8 strings, +// integer sizes and modes, UTC RFC3339 timestamps at second precision, no insignificant +// whitespace, no HTML escaping, and no trailing newline. A Record never contains its own hash. +// The content, intent, and record digests are SHA-256 digests over canonical byte sequences +// defined in this package and frozen by test vectors. +package artifact diff --git a/pkg/artifact/entry.go b/pkg/artifact/entry.go new file mode 100644 index 00000000000..31917c69c2a --- /dev/null +++ b/pkg/artifact/entry.go @@ -0,0 +1,206 @@ +package artifact + +import ( + "fmt" + "path" + "sort" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxPathLength bounds one inventory path, in bytes. +const maxPathLength = 4096 + +// firstPrintableRune is the first rune that is not an ASCII control character. +const firstPrintableRune = 0x20 + +// deleteRune is the ASCII DEL control character. +const deleteRune = 0x7f + +// permissionMask holds the ordinary permission bits an Entry may carry. +const permissionMask uint32 = 0o777 + +// ValidatePayloadPath checks a relative, slash-separated path inside a package. It rejects empty, +// absolute, backslash, control-character, "." / ".." segments, invalid UTF-8, and any path that +// enters the managed [ReservedNamespace]. Paths are never rewritten. +func ValidatePayloadPath(p string) error { + defer perf.Track(nil, "artifact.ValidatePayloadPath")() + + if err := validateRelativePath(p, false); err != nil { + return err + } + if first, _, _ := strings.Cut(p, "/"); first == ReservedNamespace { + return fmt.Errorf("%w: %q enters the reserved %s namespace", errUtils.ErrArtifactUnsafePath, p, ReservedNamespace) + } + return nil +} + +// validateRelativePath checks a relative slash-separated path. Link targets pass allowDots because +// "." and ".." segments are legitimate there as long as the resolved target stays inside the package. +func validateRelativePath(p string, allowDots bool) error { + if err := validatePathShape(p); err != nil { + return err + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || (!allowDots && (segment == "." || segment == "..")) { + return fmt.Errorf("%w: path %q has an empty, '.', or '..' segment", errUtils.ErrArtifactUnsafePath, p) + } + } + return nil +} + +// validatePathShape checks length, encoding, rootedness, separators, and control characters. +func validatePathShape(p string) error { + if p == "" || len(p) > maxPathLength { + return fmt.Errorf("%w: path must be 1-%d bytes", errUtils.ErrArtifactUnsafePath, maxPathLength) + } + if !utf8.ValidString(p) { + return fmt.Errorf("%w: path %q is not valid UTF-8", errUtils.ErrArtifactUnsafePath, p) + } + if strings.HasPrefix(p, "/") || strings.Contains(p, `\`) || hasControlRune(p) { + return fmt.Errorf("%w: path %q must be relative, slash-separated, and free of control characters", errUtils.ErrArtifactUnsafePath, p) + } + return nil +} + +func hasControlRune(p string) bool { + return strings.IndexFunc(p, func(r rune) bool { return r < firstPrintableRune || r == deleteRune }) >= 0 +} + +// Validate checks one inventory entry in isolation. +func (e *Entry) Validate() error { + defer perf.Track(nil, "artifact.Entry.Validate")() + + if err := ValidatePayloadPath(e.Path); err != nil { + return err + } + if e.Mode&^permissionMask != 0 { + return fmt.Errorf("%w: entry %q mode %#o has bits outside 0777; privileged bits are cleared", errUtils.ErrArtifactRecordInvalid, e.Path, e.Mode) + } + switch e.Type { + case EntryFile: + return e.validateFile() + case EntryDirectory: + return e.validateDirectory() + case EntrySymlink: + return e.validateSymlink() + default: + return fmt.Errorf("%w: entry %q has unsupported type %q", errUtils.ErrArtifactPackagingInvalid, e.Path, e.Type) + } +} + +func (e *Entry) validateFile() error { + if e.Size < 0 || e.Target != "" { + return fmt.Errorf("%w: file %q needs a non-negative size and no link target", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if e.Digest == "" { + return fmt.Errorf("%w: file %q needs a digest", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if err := ValidateDigest(e.Digest); err != nil { + return fmt.Errorf("%w: file %q: %w", errUtils.ErrArtifactRecordInvalid, e.Path, err) + } + return nil +} + +func (e *Entry) validateDirectory() error { + if e.Size != 0 || e.Digest != "" || e.Target != "" { + return fmt.Errorf("%w: directory %q must have no size, digest, or target", errUtils.ErrArtifactRecordInvalid, e.Path) + } + return nil +} + +func (e *Entry) validateSymlink() error { + if e.Size != 0 || e.Mode != 0 || e.Digest != "" { + return fmt.Errorf("%w: symlink %q must have no size, mode, or digest", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if err := validateRelativePath(e.Target, true); err != nil { + return fmt.Errorf("%w: symlink %q target: %w", errUtils.ErrArtifactUnsafePath, e.Path, err) + } + resolved := path.Clean(path.Dir(e.Path) + "/" + e.Target) + if resolved == ".." || strings.HasPrefix(resolved, "../") { + return fmt.Errorf("%w: symlink %q target %q escapes the package", errUtils.ErrArtifactUnsafePath, e.Path, e.Target) + } + return nil +} + +// validateInventory checks every entry, rejects duplicate paths, rejects entries nested under +// a file or symlink entry, and resolves every symlink target through the in-package links so a +// target cannot escape the package or loop. +func validateInventory(inventory []Entry) error { + types := make(map[string]EntryType, len(inventory)) + for _, entry := range inventory { + if err := entry.Validate(); err != nil { + return err + } + if _, duplicate := types[entry.Path]; duplicate { + return fmt.Errorf("%w: duplicate inventory path %q", errUtils.ErrArtifactPackagingInvalid, entry.Path) + } + types[entry.Path] = entry.Type + } + for _, entry := range inventory { + for parent := path.Dir(entry.Path); parent != "."; parent = path.Dir(parent) { + if parentType, found := types[parent]; found && parentType != EntryDirectory { + return fmt.Errorf("%w: %q is nested under non-directory %q", errUtils.ErrArtifactPackagingInvalid, entry.Path, parent) + } + } + } + return validateSymlinkTargets(inventory) +} + +// Validate checks one document descriptor in isolation. +func (d Document) Validate() error { + defer perf.Track(nil, "artifact.Document.Validate")() + + if d.Role != DocumentReadme && d.Role != DocumentChangelog { + return fmt.Errorf("%w: unsupported document role %q", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if err := validateRelativePath(d.SourcePath, false); err != nil { + return fmt.Errorf("%w: %s source path: %w", errUtils.ErrArtifactRecordInvalid, d.Role, err) + } + if d.MediaType == "" { + return fmt.Errorf("%w: %s needs a media type", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if d.Size < 0 { + return fmt.Errorf("%w: %s has a negative size", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if d.Size > MaxDocumentSize { + return fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, d.Role, d.Size, MaxDocumentSize) + } + if err := ValidateDigest(d.Digest); err != nil { + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactRecordInvalid, d.Role, err) + } + return nil +} + +func validateDocuments(documents []Document) error { + seen := make(map[DocumentRole]struct{}, len(documents)) + for _, document := range documents { + if err := document.Validate(); err != nil { + return err + } + if _, duplicate := seen[document.Role]; duplicate { + return fmt.Errorf("%w: duplicate %s document", errUtils.ErrArtifactRecordInvalid, document.Role) + } + seen[document.Role] = struct{}{} + } + return nil +} + +// sortedInventory returns a copy of inventory sorted by path (bytewise). +func sortedInventory(inventory []Entry) []Entry { + out := make([]Entry, len(inventory)) + copy(out, inventory) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// sortedDocuments returns a copy of documents sorted by role. +func sortedDocuments(documents []Document) []Document { + out := make([]Document, len(documents)) + copy(out, documents) + sort.Slice(out, func(i, j int) bool { return out[i].Role < out[j].Role }) + return out +} diff --git a/pkg/artifact/entry_symlink.go b/pkg/artifact/entry_symlink.go new file mode 100644 index 00000000000..262fdb108f0 --- /dev/null +++ b/pkg/artifact/entry_symlink.go @@ -0,0 +1,79 @@ +package artifact + +import ( + "fmt" + "path" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// maxSymlinkExpansions bounds how many in-package links one target may traverse. It matches the +// limit that common operating systems apply before reporting a symlink loop. +const maxSymlinkExpansions = 40 + +// validateSymlinkTargets resolves every symlink target against the inventory, so a target that is +// lexically contained but passes through an in-package link cannot leave the package. For example +// "link -> dir/up/../outside" cleans to "dir/outside", yet when "dir/up -> .." is also in the +// inventory, the filesystem follows "dir/up" to the package root before applying "..". +// +// Resolution follows the way a filesystem walks a path: component by component, replacing a link +// with its own target, and rejecting a ".." that would pop above the package root. Paths that name +// nothing in the inventory resolve literally, so dangling targets stay valid. The expansion limit +// rejects link cycles of any shape. +func validateSymlinkTargets(inventory []Entry) error { + links := make(map[string]string) + for _, entry := range inventory { + if entry.Type == EntrySymlink { + links[entry.Path] = entry.Target + } + } + if len(links) == 0 { + return nil + } + for index := range inventory { + if inventory[index].Type != EntrySymlink { + continue + } + if err := resolveSymlink(links, &inventory[index]); err != nil { + return err + } + } + return nil +} + +// resolveSymlink walks the target of one link through the in-package links. +func resolveSymlink(links map[string]string, link *Entry) error { + var resolved []string + if dir := path.Dir(link.Path); dir != "." { + resolved = strings.Split(dir, "/") + } + pending := strings.Split(link.Target, "/") + for expansions := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case "", ".": + continue + case "..": + if len(resolved) == 0 { + return fmt.Errorf("%w: symlink %q target %q escapes the package", errUtils.ErrArtifactUnsafePath, link.Path, link.Target) + } + resolved = resolved[:len(resolved)-1] + continue + } + candidate := strings.Join(append(resolved[:len(resolved):len(resolved)], component), "/") + target, isLink := links[candidate] + if !isLink { + resolved = append(resolved, component) + continue + } + expansions++ + if expansions > maxSymlinkExpansions { + return fmt.Errorf("%w: symlink %q target %q loops or traverses more than %d links", errUtils.ErrArtifactUnsafePath, link.Path, link.Target, maxSymlinkExpansions) + } + // A link is replaced by its target, resolved from the directory that holds the link. + pending = append(strings.Split(target, "/"), pending...) + } + return nil +} diff --git a/pkg/artifact/errors.go b/pkg/artifact/errors.go new file mode 100644 index 00000000000..65cfa7c42fb --- /dev/null +++ b/pkg/artifact/errors.go @@ -0,0 +1,107 @@ +package artifact + +import ( + "errors" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// CapabilityError reports an operation that a repository cannot honor. It is never a successful +// no-op: callers receive it instead of silently degraded behavior. It wraps a sentinel +// (ErrArtifactUnsupportedCapability by default, or ErrArtifactPublicationUnavailable) and an +// optional cause, so errors.Is works against both. +type CapabilityError struct { + // Repository is the repository alias or kind that lacks the capability. + Repository string + + // Operation is the attempted operation, for example "lock" or "sign". + Operation string + + // Capability names the missing guarantee, for example "lock/repository" or "enumerate/collection". + Capability string + + // Reason explains the limitation and, where useful, the supported alternative. + Reason string + + // Sentinel is the wrapped sentinel; nil means ErrArtifactUnsupportedCapability. + Sentinel error + + // Cause is an optional underlying error kept for diagnostics. + Cause error +} + +// NewUnsupportedCapability returns a CapabilityError wrapping ErrArtifactUnsupportedCapability. +func NewUnsupportedCapability(repository, operation, capability, reason string) *CapabilityError { + defer perf.Track(nil, "artifact.NewUnsupportedCapability")() + + return &CapabilityError{ + Repository: repository, + Operation: operation, + Capability: capability, + Reason: reason, + Sentinel: errUtils.ErrArtifactUnsupportedCapability, + } +} + +// NewPublicationUnavailable returns a CapabilityError wrapping ErrArtifactPublicationUnavailable. +func NewPublicationUnavailable(repository, operation, reason string, cause error) *CapabilityError { + defer perf.Track(nil, "artifact.NewPublicationUnavailable")() + + return &CapabilityError{ + Repository: repository, + Operation: operation, + Reason: reason, + Sentinel: errUtils.ErrArtifactPublicationUnavailable, + Cause: cause, + } +} + +// Error implements error. +func (e *CapabilityError) Error() string { + defer perf.Track(nil, "artifact.CapabilityError.Error")() + + message := e.sentinel().Error() + if e.Repository != "" { + message = fmt.Sprintf("%s: repository %q", message, RedactURL(e.Repository)) + } + if e.Operation != "" { + message = fmt.Sprintf("%s: %s", message, e.Operation) + } + if e.Capability != "" { + message = fmt.Sprintf("%s: capability %s", message, RedactURL(e.Capability)) + } + if e.Reason != "" { + message = fmt.Sprintf("%s: %s", message, e.Reason) + } + return message +} + +// Unwrap exposes the sentinel and the cause to errors.Is and errors.As. +func (e *CapabilityError) Unwrap() []error { + defer perf.Track(nil, "artifact.CapabilityError.Unwrap")() + + if e.Cause == nil { + return []error{e.sentinel()} + } + return []error{e.sentinel(), e.Cause} +} + +func (e *CapabilityError) sentinel() error { + if e.Sentinel != nil { + return e.Sentinel + } + return errUtils.ErrArtifactUnsupportedCapability +} + +// AsCapabilityError extracts a *CapabilityError from err's chain. +func AsCapabilityError(err error) (*CapabilityError, bool) { + defer perf.Track(nil, "artifact.AsCapabilityError")() + + var target *CapabilityError + if errors.As(err, &target) { + return target, true + } + return nil, false +} diff --git a/pkg/artifact/errors_test.go b/pkg/artifact/errors_test.go new file mode 100644 index 00000000000..2eee7f8ed90 --- /dev/null +++ b/pkg/artifact/errors_test.go @@ -0,0 +1,69 @@ +package artifact + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// TestSentinelsAreDistinct guards against two sentinels aliasing each other, which would make +// errors.Is match the wrong failure class. +func TestSentinelsAreDistinct(t *testing.T) { + sentinels := []error{ + errUtils.ErrArtifactConfigInvalid, + errUtils.ErrArtifactRefInvalid, + errUtils.ErrArtifactConflict, + errUtils.ErrArtifactNotFound, + errUtils.ErrArtifactPublicationUnavailable, + errUtils.ErrArtifactUnsupportedCapability, + errUtils.ErrArtifactIntegrityFailed, + errUtils.ErrArtifactDocumentMissing, + errUtils.ErrArtifactPackagingInvalid, + errUtils.ErrArtifactUnsafePath, + errUtils.ErrArtifactRecordInvalid, + errUtils.ErrArtifactTooLarge, + } + for i, sentinel := range sentinels { + wrapped := fmt.Errorf("%w: context", sentinel) + for j, other := range sentinels { + assert.Equal(t, i == j, errors.Is(wrapped, other), "%v vs %v", sentinel, other) + } + } +} + +func TestCapabilityError(t *testing.T) { + t.Run("unsupported capability wraps the sentinel", func(t *testing.T) { + err := NewUnsupportedCapability("shared", "lock", "lock/repository", "OCI tags cannot provide distributed exclusion") + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.NotErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + assert.Contains(t, err.Error(), "shared") + assert.Contains(t, err.Error(), "lock/repository") + }) + + t.Run("survives further wrapping and is extractable", func(t *testing.T) { + wrapped := fmt.Errorf("%w: publishing vpc:1.0", NewUnsupportedCapability("shared", "lock", "lock/repository", "")) + require.ErrorIs(t, wrapped, errUtils.ErrArtifactUnsupportedCapability) + capability, ok := AsCapabilityError(wrapped) + require.True(t, ok) + assert.Equal(t, "lock/repository", capability.Capability) + _, ok = AsCapabilityError(errors.New("plain")) + assert.False(t, ok) + }) + + t.Run("publication unavailable keeps its cause", func(t *testing.T) { + cause := errors.New("sdk: 503") + err := NewPublicationUnavailable("backups", "publish", "pending operation op-1", cause) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + require.ErrorIs(t, err, cause, "backend causes stay available for diagnostics") + assert.NotErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + + t.Run("zero value defaults to unsupported", func(t *testing.T) { + require.ErrorIs(t, &CapabilityError{}, errUtils.ErrArtifactUnsupportedCapability) + }) +} diff --git a/pkg/artifact/kinddata.go b/pkg/artifact/kinddata.go new file mode 100644 index 00000000000..db2f1b1a363 --- /dev/null +++ b/pkg/artifact/kinddata.go @@ -0,0 +1,77 @@ +package artifact + +import ( + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// KindData is optional, kind-specific record data held as canonical JSON bytes. +// +// Design: a Record cannot carry map[string]any or float64 in its canonical path, yet kinds (for +// example legacy CI planfiles) need typed extra data. KindData is therefore a typed raw-JSON +// holder. [NewKindData] marshals a caller-owned typed value and immediately re-encodes it with +// [CanonicalizeJSON], which sorts object keys, writes stable string escapes, and rejects floats, +// duplicate keys, and invalid UTF-8. A KindData value is always canonical, so it embeds into the +// Record bytes unchanged and participates in the intent and record digests byte-for-byte. +// Consumers read it back into their own typed struct with [KindData.Decode]. +type KindData []byte + +// NewKindData canonicalizes v, which should be a typed struct, into KindData. +func NewKindData(v any) (KindData, error) { + defer perf.Track(nil, "artifact.NewKindData")() + + raw, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("%w: kind data: %w", errUtils.ErrArtifactRecordInvalid, err) + } + if string(raw) == "null" { + return nil, nil + } + canonical, err := CanonicalizeJSON(raw) + if err != nil { + return nil, err + } + return KindData(canonical), nil +} + +// Decode unmarshals the kind data into v, a pointer to the kind's typed struct. +func (k KindData) Decode(v any) error { + defer perf.Track(nil, "artifact.KindData.Decode")() + + if len(k) == 0 { + return fmt.Errorf("%w: record has no kind data", errUtils.ErrArtifactRecordInvalid) + } + if err := json.Unmarshal(k, v); err != nil { + return fmt.Errorf("%w: decoding kind data: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return nil +} + +// MarshalJSON implements json.Marshaler, emitting the canonical bytes. +func (k KindData) MarshalJSON() ([]byte, error) { + defer perf.Track(nil, "artifact.KindData.MarshalJSON")() + + if len(k) == 0 { + return []byte("null"), nil + } + return CanonicalizeJSON(k) +} + +// UnmarshalJSON implements json.Unmarshaler, canonicalizing the input. +func (k *KindData) UnmarshalJSON(data []byte) error { + defer perf.Track(nil, "artifact.KindData.UnmarshalJSON")() + + canonical, err := CanonicalizeJSON(data) + if err != nil { + return err + } + if string(canonical) == "null" { + *k = nil + return nil + } + *k = KindData(canonical) + return nil +} diff --git a/pkg/artifact/kinddata_test.go b/pkg/artifact/kinddata_test.go new file mode 100644 index 00000000000..febd318823d --- /dev/null +++ b/pkg/artifact/kinddata_test.go @@ -0,0 +1,75 @@ +package artifact + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestKindDataJSONBehavior(t *testing.T) { + data, err := NewKindData(nil) + require.NoError(t, err) + assert.Empty(t, data) + record := vectorRecordB(t) + record.KindData = data + raw, err := record.Canonical() + require.NoError(t, err) + decoded, err := DecodeRecord(raw) + require.NoError(t, err) + assert.Empty(t, decoded.KindData) + t.Run("empty kind data marshals to null and cannot be decoded", func(t *testing.T) { + raw, err := KindData(nil).MarshalJSON() + require.NoError(t, err) + assert.Equal(t, "null", string(raw)) + require.ErrorIs(t, KindData(nil).Decode(&struct{}{}), errUtils.ErrArtifactRecordInvalid) + }) + + t.Run("unmarshal canonicalizes and treats null as absent", func(t *testing.T) { + var data KindData + require.NoError(t, json.Unmarshal([]byte(` {"b": 1, "a": 2} `), &data)) + assert.Equal(t, `{"a":2,"b":1}`, string(data)) + require.NoError(t, data.UnmarshalJSON([]byte("null"))) + assert.Empty(t, data) + }) + + t.Run("unmarshal rejects floats", func(t *testing.T) { + var data KindData + require.ErrorIs(t, json.Unmarshal([]byte(`{"a":1.5}`), &data), errUtils.ErrArtifactRecordInvalid) + }) + + t.Run("decode failures carry the artifact sentinel", func(t *testing.T) { + type target struct { + Count int `json:"count"` + } + tests := []struct { + name string + data KindData + want int + }{ + {"string where an integer is required", KindData(`{"count":"x"}`), 0}, + {"array where an object is required", KindData(`[1]`), 0}, + {"matching data decodes", KindData(`{"count":7}`), 7}, + } + for _, tt := range tests { + var got target + err := tt.data.Decode(&got) + if tt.want == 0 { + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, tt.name) + var typeErr *json.UnmarshalTypeError + require.ErrorAs(t, err, &typeErr, "the cause is preserved") + continue + } + require.NoError(t, err, tt.name) + assert.Equal(t, tt.want, got.Count) + } + }) + + t.Run("marshal failure surfaces as an invalid record", func(t *testing.T) { + _, err := NewKindData(make(chan int)) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) +} diff --git a/pkg/artifact/kinds.go b/pkg/artifact/kinds.go new file mode 100644 index 00000000000..950d1bbad2c --- /dev/null +++ b/pkg/artifact/kinds.go @@ -0,0 +1,211 @@ +package artifact + +import ( + "errors" + "fmt" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Repository kinds defined by V1. They name repository backends and are independent of artifact +// kinds such as "files" or "component/terraform". +const ( + // RepositoryKindLocalDir stores copied files below a local directory. + RepositoryKindLocalDir = "local/dir" + + // RepositoryKindLocalArchive stores one archive per artifact below a local directory. + RepositoryKindLocalArchive = "local/archive" + + // RepositoryKindS3 stores objects in an S3 bucket. + RepositoryKindS3 = "aws/s3" + + // RepositoryKindOCIRegistry stores artifacts in an OCI registry. + RepositoryKindOCIRegistry = "oci/registry" + + // RepositoryKindOCILayout stores artifacts in an OCI image layout directory. + RepositoryKindOCILayout = "oci/layout" +) + +// KindDescriptor is the static, network-free description of a repository kind. Configuration +// validation, path anchoring, and lock delegation checks use it before any backend exists or +// any credential is resolved. +// +// The V1 kinds below have built-in descriptors so that validation works before backend packages +// register. A backend that registers its kind supplies its own descriptor, which then takes +// precedence (typically a copy of the built-in one with refinements). +type KindDescriptor struct { + // Kind is the repository kind name, for example "aws/s3". + Kind string + + // Capabilities are the capabilities known from configuration alone (supported packagings and + // lock scopes). A constructed repository's own Capabilities refine them. + Capabilities Capabilities + + // DefaultPackaging is the packaging used when the configuration omits it. + DefaultPackaging Packaging + + // PathOptions names the option keys that are local destination paths. Relative values resolve + // against the declaring configuration file when the configuration is loaded. + PathOptions []string + + // AllowsPermissions reports that the kind accepts the `permissions` field. + AllowsPermissions bool + + // AllowsCatalog reports that the kind accepts the `catalog` field. + AllowsCatalog bool + + // NewOptions returns a fresh typed options struct for strict decoding and validation. + NewOptions func() Options +} + +// UsesLocalPath reports whether the named option of the kind is a local destination path. +func (d *KindDescriptor) UsesLocalPath(option string) bool { + defer perf.Track(nil, "artifact.KindDescriptor.UsesLocalPath")() + + for _, name := range d.PathOptions { + if name == option { + return true + } + } + return false +} + +// EffectivePackaging returns the configured packaging, or the kind's default when it is omitted. +func (d *KindDescriptor) EffectivePackaging(configured string) Packaging { + defer perf.Track(nil, "artifact.KindDescriptor.EffectivePackaging")() + + if configured == "" { + return d.DefaultPackaging + } + return Packaging(configured) +} + +// builtinKinds returns the documented static set of V1 repository kinds. +func builtinKinds() map[string]KindDescriptor { + archive := []Packaging{PackagingTar, PackagingTgz, PackagingZip} + return map[string]KindDescriptor{ + RepositoryKindLocalDir: { + Kind: RepositoryKindLocalDir, + Capabilities: Capabilities{Read: true, Write: true, Packagings: []Packaging{PackagingNone}, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingNone, + PathOptions: []string{OptionPath}, + NewOptions: func() Options { return &LocalDirOptions{} }, + }, + RepositoryKindLocalArchive: { + Kind: RepositoryKindLocalArchive, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + PathOptions: []string{OptionPath}, + AllowsPermissions: true, + NewOptions: func() Options { return &LocalArchiveOptions{} }, + }, + RepositoryKindS3: { + Kind: RepositoryKindS3, + Capabilities: Capabilities{Read: true, Write: true, Packagings: append([]Packaging{PackagingNone}, archive...), LockScopes: []LockScope{LockScopeHost, LockScopeRepository}}, + DefaultPackaging: PackagingTgz, + NewOptions: func() Options { return &S3Options{} }, + }, + RepositoryKindOCIRegistry: { + Kind: RepositoryKindOCIRegistry, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + AllowsCatalog: true, + NewOptions: func() Options { return &OCIRegistryOptions{} }, + }, + RepositoryKindOCILayout: { + Kind: RepositoryKindOCILayout, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + PathOptions: []string{OptionPath}, + NewOptions: func() Options { return &OCILayoutOptions{} }, + }, + } +} + +// BuiltinKind returns the built-in descriptor of a V1 repository kind. +func BuiltinKind(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.BuiltinKind")() + + desc, ok := builtinKinds()[kind] + if !ok { + return nil, false + } + return &desc, true +} + +// BuiltinKinds returns the sorted names of the V1 repository kinds. +func BuiltinKinds() []string { + defer perf.Track(nil, "artifact.BuiltinKinds")() + + kinds := builtinKinds() + names := make([]string, 0, len(kinds)) + for name := range kinds { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// ValidateRepositoryConfig checks one repository configuration against its kind descriptor: +// the packaging matrix, permissions (accepted only where the kind allows them, and only as a +// quoted octal string), and strict typed options with their required values. It performs no +// network access, credential resolution, or directory creation. +func ValidateRepositoryConfig(desc *KindDescriptor, cfg *schema.ArtifactRepositoryConfig) error { + defer perf.Track(nil, "artifact.ValidateRepositoryConfig")() + + if desc == nil || cfg == nil { + return fmt.Errorf("%w: repository configuration is nil", errUtils.ErrArtifactConfigInvalid) + } + var errs []error + packaging, err := validatePackagingFor(desc, cfg.Packaging) + errs = append(errs, err) + errs = append(errs, validatePermissionsFor(desc, cfg.Permissions)) + errs = append(errs, validateOptionsFor(desc, cfg, packaging)) + return errors.Join(errs...) +} + +// validatePackagingFor checks the packaging against the kind's matrix and returns the effective packaging. +func validatePackagingFor(desc *KindDescriptor, configured string) (Packaging, error) { + packaging := desc.EffectivePackaging(configured) + if configured == "" && packaging == "" { + // A descriptor without a default has nothing to check until the user chooses one. + return packaging, nil + } + if !desc.Capabilities.HasPackaging(packaging) { + source := "packaging" + if configured == "" { + source = "default packaging" + } + return packaging, fmt.Errorf("%w: %w: kind %q does not support %s %q (supported: %v)", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, source, packaging, desc.Capabilities.Packagings) + } + return packaging, nil +} + +func validatePermissionsFor(desc *KindDescriptor, permissions string) error { + if permissions == "" { + return nil + } + if !desc.AllowsPermissions { + return fmt.Errorf("%w: kind %q does not accept permissions; they apply only to local archive files", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + _, err := ParsePermissions(permissions) + return err +} + +func validateOptionsFor(desc *KindDescriptor, cfg *schema.ArtifactRepositoryConfig, packaging Packaging) error { + if desc.NewOptions == nil { + return nil + } + options := desc.NewOptions() + if err := DecodeOptions(cfg.Options, options); err != nil { + return err + } + errs := []error{options.Validate()} + if aware, ok := options.(PackagingValidator); ok { + errs = append(errs, aware.ValidateForPackaging(packaging)) + } + return errors.Join(errs...) +} diff --git a/pkg/artifact/kinds_test.go b/pkg/artifact/kinds_test.go new file mode 100644 index 00000000000..b503ffcd3e8 --- /dev/null +++ b/pkg/artifact/kinds_test.go @@ -0,0 +1,183 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestValidateRepositoryConfig_PackagingMatrix(t *testing.T) { + tests := []struct { + kind string + options map[string]any + packaging string + valid bool + }{ + {kind: RepositoryKindLocalDir, packaging: "", valid: true}, + {kind: RepositoryKindLocalDir, packaging: "none", valid: true}, + {kind: RepositoryKindLocalDir, packaging: "tar"}, + {kind: RepositoryKindLocalDir, packaging: "tgz"}, + {kind: RepositoryKindLocalDir, packaging: "zip"}, + {kind: RepositoryKindLocalArchive, packaging: "", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "none"}, + {kind: RepositoryKindLocalArchive, packaging: "tar", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "tgz", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "zip", valid: true}, + {kind: RepositoryKindS3, packaging: "", valid: true}, + {kind: RepositoryKindS3, packaging: "none", valid: true}, + {kind: RepositoryKindS3, packaging: "tar", valid: true}, + {kind: RepositoryKindS3, packaging: "tgz", valid: true}, + {kind: RepositoryKindS3, packaging: "zip", valid: true}, + {kind: RepositoryKindS3, packaging: "rar"}, + {kind: RepositoryKindOCIRegistry, packaging: "", valid: true}, + {kind: RepositoryKindOCIRegistry, packaging: "none"}, + {kind: RepositoryKindOCIRegistry, packaging: "tar", valid: true}, + {kind: RepositoryKindOCILayout, packaging: "none"}, + {kind: RepositoryKindOCILayout, packaging: "zip", valid: true}, + } + for _, tt := range tests { + t.Run(tt.kind+"/"+tt.packaging, func(t *testing.T) { + desc, ok := BuiltinKind(tt.kind) + require.True(t, ok) + cfg := &schema.ArtifactRepositoryConfig{Kind: tt.kind, Packaging: tt.packaging, Options: validOptionsFor(tt.kind)} + err := ValidateRepositoryConfig(desc, cfg) + if tt.valid { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) + } +} + +// TestValidateRepositoryConfig_DefaultPackagingMustBeSupported covers a registered descriptor whose +// default packaging is outside its own capability matrix: an omitted packaging must be rejected +// exactly like the same value written explicitly. +func TestValidateRepositoryConfig_DefaultPackagingMustBeSupported(t *testing.T) { + builtin, ok := BuiltinKind(RepositoryKindLocalArchive) + require.True(t, ok) + zipOnly := *builtin + zipOnly.Capabilities.Packagings = []Packaging{PackagingZip} + zipOnly.DefaultPackaging = PackagingTgz + + noDefault := zipOnly + noDefault.DefaultPackaging = "" + + tests := []struct { + name string + desc KindDescriptor + packaging string + valid bool + }{ + {name: "default outside the matrix is rejected", desc: zipOnly, packaging: ""}, + {name: "explicit value outside the matrix is rejected", desc: zipOnly, packaging: "tgz"}, + {name: "explicit supported value overrides a bad default", desc: zipOnly, packaging: "zip", valid: true}, + {name: "descriptor without a default accepts an omitted packaging", desc: noDefault, packaging: "", valid: true}, + {name: "built-in default is supported", desc: *builtin, packaging: "", valid: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &schema.ArtifactRepositoryConfig{Kind: tt.desc.Kind, Packaging: tt.packaging, Options: validOptionsFor(tt.desc.Kind)} + err := ValidateRepositoryConfig(&tt.desc, cfg) + if tt.valid { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + } +} + +func validOptionsFor(kind string) map[string]any { + switch kind { + case RepositoryKindS3: + return map[string]any{"bucket": "acme-artifacts"} + case RepositoryKindOCIRegistry: + return map[string]any{"registry": "ghcr.io"} + default: + return map[string]any{"path": "out"} + } +} + +func TestValidateRepositoryConfig_Permissions(t *testing.T) { + tests := []struct { + name string + kind string + permissions string + wantErr bool + }{ + {name: "archive accepts quoted octal", kind: RepositoryKindLocalArchive, permissions: "0600"}, + {name: "archive accepts three digits", kind: RepositoryKindLocalArchive, permissions: "640"}, + {name: "archive rejects preserve", kind: RepositoryKindLocalArchive, permissions: "preserve", wantErr: true}, + {name: "archive rejects privileged bits", kind: RepositoryKindLocalArchive, permissions: "4755", wantErr: true}, + {name: "archive rejects 0o prefix", kind: RepositoryKindLocalArchive, permissions: "0o600", wantErr: true}, + {name: "archive rejects decimal-looking 384", kind: RepositoryKindLocalArchive, permissions: "384", wantErr: true}, + {name: "directory rejects permissions", kind: RepositoryKindLocalDir, permissions: "0600", wantErr: true}, + {name: "s3 rejects permissions", kind: RepositoryKindS3, permissions: "0600", wantErr: true}, + {name: "registry rejects permissions", kind: RepositoryKindOCIRegistry, permissions: "0600", wantErr: true}, + {name: "layout rejects permissions", kind: RepositoryKindOCILayout, permissions: "0600", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + desc, _ := BuiltinKind(tt.kind) + err := ValidateRepositoryConfig(desc, &schema.ArtifactRepositoryConfig{Kind: tt.kind, Permissions: tt.permissions, Options: validOptionsFor(tt.kind)}) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + return + } + require.NoError(t, err) + }) + } +} + +func TestValidateRepositoryConfig_JoinsEveryProblem(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + err := ValidateRepositoryConfig(desc, &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, Packaging: "none", Permissions: "preserve", Options: map[string]any{"pth": "x"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "preserve") + assert.Contains(t, err.Error(), "pth") +} + +func TestValidateRepositoryConfig_KeyPatternAgreesWithEffectivePackaging(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + cfg := &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, Packaging: "zip", + Options: map[string]any{"path": "out", "key_pattern": "{{ .Name }}/{{ .Version }}.tgz"}, + } + require.ErrorIs(t, ValidateRepositoryConfig(desc, cfg), errUtils.ErrArtifactConfigInvalid) + + cfg.Packaging = "" + require.NoError(t, ValidateRepositoryConfig(desc, cfg), "the default packaging for local/archive is tgz") +} + +func TestValidateRepositoryConfig_NilConfig(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.ErrorIs(t, ValidateRepositoryConfig(desc, nil), errUtils.ErrArtifactConfigInvalid) +} + +func TestKindDescriptorHelpers(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + assert.True(t, desc.UsesLocalPath("path")) + assert.False(t, desc.UsesLocalPath("key_pattern")) + assert.Equal(t, PackagingTgz, desc.EffectivePackaging("")) + assert.Equal(t, PackagingZip, desc.EffectivePackaging("zip")) + + _, ok := BuiltinKind("nope") + assert.False(t, ok) + assert.Equal(t, []string{"aws/s3", "local/archive", "local/dir", "oci/layout", "oci/registry"}, BuiltinKinds()) +} + +func TestBuiltinKindsAreIndependentCopies(t *testing.T) { + first, _ := BuiltinKind(RepositoryKindS3) + first.Capabilities.Packagings[0] = PackagingZip + second, _ := BuiltinKind(RepositoryKindS3) + assert.Equal(t, PackagingNone, second.Capabilities.Packagings[0], "mutating a returned descriptor must not change the built-in set") +} diff --git a/pkg/artifact/labels.go b/pkg/artifact/labels.go new file mode 100644 index 00000000000..f9a426d0773 --- /dev/null +++ b/pkg/artifact/labels.go @@ -0,0 +1,75 @@ +package artifact + +import ( + "errors" + "fmt" + "sort" + "strings" + "unicode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Label limits. Labels must be expressible and selectable with `--labels 'key=value,key2=value2'`, +// which splits pairs on ',' and each pair on the first '=' or ':'. +const ( + // MaxLabelKeyLength is the longest label key, in bytes. + MaxLabelKeyLength = 128 + + // MaxLabelValueLength is the longest label value, in bytes. + MaxLabelValueLength = 1024 + + // MaxLabels is the largest number of labels on one artifact version or repository default. + MaxLabels = 64 +) + +// ValidateLabel checks one exact key/value label. The key must be non-empty, free of control +// characters and surrounding whitespace, and free of '=', ':' and ',' so that it can be selected +// with `--labels`; the value must be free of control characters and ','. +func ValidateLabel(key, value string) error { + defer perf.Track(nil, "artifact.ValidateLabel")() + + // The key is echoed in every message, and a key may be a pasted credential-bearing URL. + shown := RedactURL(key) + switch { + case key == "": + return fmt.Errorf("%w: label key must not be empty", errUtils.ErrArtifactConfigInvalid) + case len(key) > MaxLabelKeyLength: + return fmt.Errorf("%w: label key %q exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, shown, MaxLabelKeyLength) + case strings.TrimSpace(key) != key: + return fmt.Errorf("%w: label key %q must not have leading or trailing whitespace", errUtils.ErrArtifactConfigInvalid, shown) + case strings.ContainsAny(key, "=:,"): + return fmt.Errorf("%w: label key %q must not contain '=', ':' or ',' (they delimit --labels selectors)", errUtils.ErrArtifactConfigInvalid, shown) + case strings.IndexFunc(key, unicode.IsControl) >= 0: + return fmt.Errorf("%w: label key %q must not contain control characters", errUtils.ErrArtifactConfigInvalid, shown) + case len(value) > MaxLabelValueLength: + return fmt.Errorf("%w: label %q value exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, shown, MaxLabelValueLength) + case strings.Contains(value, ","): + return fmt.Errorf("%w: label %q value must not contain ',' (it delimits --labels selectors)", errUtils.ErrArtifactConfigInvalid, shown) + case strings.IndexFunc(value, unicode.IsControl) >= 0: + return fmt.Errorf("%w: label %q value must not contain control characters", errUtils.ErrArtifactConfigInvalid, shown) + } + return nil +} + +// ValidateLabels checks a label set, returning every malformed label joined in key order. +func ValidateLabels(labels map[string]string) error { + defer perf.Track(nil, "artifact.ValidateLabels")() + + var errs []error + if len(labels) > MaxLabels { + errs = append(errs, fmt.Errorf("%w: %d labels exceed the limit of %d", errUtils.ErrArtifactConfigInvalid, len(labels), MaxLabels)) + } + keys := make([]string, 0, len(labels)) + for key := range labels { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if err := ValidateLabel(key, labels[key]); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/pkg/artifact/labels_test.go b/pkg/artifact/labels_test.go new file mode 100644 index 00000000000..c28bfde40c0 --- /dev/null +++ b/pkg/artifact/labels_test.go @@ -0,0 +1,61 @@ +package artifact + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestValidateLabel(t *testing.T) { + tests := []struct { + name string + key, value string + wantErr string + }{ + {name: "plain", key: "team", value: "platform"}, + {name: "empty value is allowed", key: "team", value: ""}, + {name: "dots and dashes in key", key: "cost-center.id", value: "1234"}, + {name: "empty key", key: "", value: "x", wantErr: "must not be empty"}, + {name: "space-padded key", key: " team", value: "x", wantErr: "whitespace"}, + {name: "equals in key", key: "a=b", value: "x", wantErr: "--labels"}, + {name: "colon in key", key: "a:b", value: "x", wantErr: "--labels"}, + {name: "comma in key", key: "a,b", value: "x", wantErr: "--labels"}, + {name: "control char in key", key: "a\nb", value: "x", wantErr: "control"}, + {name: "oversized key", key: strings.Repeat("k", 129), value: "x", wantErr: "exceeds"}, + {name: "comma in value", key: "team", value: "a,b", wantErr: "','"}, + {name: "control char in value", key: "team", value: "a\tb", wantErr: "control"}, + {name: "oversized value", key: "team", value: strings.Repeat("v", 1025), wantErr: "exceeds"}, + {name: "colon in value is selectable", key: "ref", value: "refs/heads/main:abc"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateLabel(tt.key, tt.value) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestValidateLabels_JoinsInKeyOrderAndBoundsCount(t *testing.T) { + err := ValidateLabels(map[string]string{"b,": "x", "a=": "y", "ok": "fine"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + message := err.Error() + assert.Less(t, strings.Index(message, `"a="`), strings.Index(message, `"b,"`), "errors follow sorted key order") + assert.NotContains(t, message, `"ok"`) + + require.NoError(t, ValidateLabels(nil)) + + many := make(map[string]string) + for i := range MaxLabels + 1 { + many[strings.Repeat("k", 1)+string(rune('a'+i%26))+strings.Repeat("z", i/26)] = "v" + } + require.Error(t, ValidateLabels(many)) +} diff --git a/pkg/artifact/lock.go b/pkg/artifact/lock.go new file mode 100644 index 00000000000..24d70712f6b --- /dev/null +++ b/pkg/artifact/lock.go @@ -0,0 +1,82 @@ +package artifact + +import ( + "context" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// LockScope is the strength of mutation coordination a lock provides. +type LockScope string + +// Lock scopes. No S3 Object Lock retention, OCI tag, or local-cache lock is a distributed lease. +const ( + // LockScopeHost serializes writers on one host. It is the default for single-host use. + LockScopeHost LockScope = "host" + + // LockScopeRepository serializes writers across hosts through the repository's own + // coordination (for example S3 conditional owner claims or a configured lock repository). + LockScopeRepository LockScope = "repository" +) + +// Satisfies reports whether a lock of scope s meets a request for scope required. A repository +// scope lock satisfies a host scope request; an empty requirement is always satisfied. +func (s LockScope) Satisfies(required LockScope) bool { + defer perf.Track(nil, "artifact.LockScope.Satisfies")() + + switch required { + case "", LockScopeHost: + return s == LockScopeHost || s == LockScopeRepository + case LockScopeRepository: + return s == LockScopeRepository + default: + return false + } +} + +// LockRequest asks a repository to coordinate a mutation of one resource. Multi-resource work +// acquires locks in canonical repository/resource order to avoid deadlocks. +type LockRequest struct { + // Resource identifies what is being mutated, for example a canonical reference key. + Resource string + + // Owner identifies the writer, for example host and process. + Owner string + + // OperationID is the logical operation identifier recorded in the claim. + OperationID string + + // Required is the minimum guarantee the caller needs. Empty means host scope. + Required LockScope + + // Lease is the requested lease duration for providers that expire claims. V1 claims do not + // expire automatically, so zero means no expiry and providers without fencing must reject + // a non-zero lease with an unsupported-capability error. + Lease time.Duration +} + +// LockHandle is an acquired, ownership-checked lock. Release verifies the ownership token and +// never releases another operation's claim. Providers never steal a lock automatically. +type LockHandle interface { + // Resource returns the locked resource. + Resource() string + + // Owner returns the owner recorded in the claim. + Owner() string + + // Scope returns the actual scope provided, which results report. + Scope() LockScope + + // Token returns the ownership token checked on renew and release. + Token() string + + // ExpiresAt returns the lease expiry and whether the lock expires at all. + ExpiresAt() (time.Time, bool) + + // Renew extends a leased lock; providers without leases return an unsupported-capability error. + Renew(ctx context.Context, lease time.Duration) error + + // Release relinquishes the lock after checking the ownership token. + Release(ctx context.Context) error +} diff --git a/pkg/artifact/lockprovider.go b/pkg/artifact/lockprovider.go new file mode 100644 index 00000000000..636101f676c --- /dev/null +++ b/pkg/artifact/lockprovider.go @@ -0,0 +1,14 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_lockprovider.go github.com/cloudposse/atmos/pkg/artifact LockProvider,LockHandle + +// LockProvider coordinates artifact mutations. Repositories implement it directly or by composing +// a shared provider (filesystem locks, S3 conditional owner claims, a delegated lock repository). +type LockProvider interface { + // Lock acquires an ownership-checked lock satisfying request.Required, or returns a + // *CapabilityError wrapping ErrArtifactUnsupportedCapability when the guarantee is unavailable. + // It never steals another operation's lock and never silently downgrades the scope. + Lock(ctx context.Context, request LockRequest) (LockHandle, error) +} diff --git a/pkg/artifact/mock_copier.go b/pkg/artifact/mock_copier.go new file mode 100644 index 00000000000..9e3519a0648 --- /dev/null +++ b/pkg/artifact/mock_copier.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Copier) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_copier.go github.com/cloudposse/atmos/pkg/artifact Copier +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockCopier is a mock of Copier interface. +type MockCopier struct { + ctrl *gomock.Controller + recorder *MockCopierMockRecorder + isgomock struct{} +} + +// MockCopierMockRecorder is the mock recorder for MockCopier. +type MockCopierMockRecorder struct { + mock *MockCopier +} + +// NewMockCopier creates a new mock instance. +func NewMockCopier(ctrl *gomock.Controller) *MockCopier { + mock := &MockCopier{ctrl: ctrl} + mock.recorder = &MockCopierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCopier) EXPECT() *MockCopierMockRecorder { + return m.recorder +} + +// CanCopyFrom mocks base method. +func (m *MockCopier) CanCopyFrom(source Repository) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanCopyFrom", source) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanCopyFrom indicates an expected call of CanCopyFrom. +func (mr *MockCopierMockRecorder) CanCopyFrom(source any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanCopyFrom", reflect.TypeOf((*MockCopier)(nil).CanCopyFrom), source) +} + +// CopyFrom mocks base method. +func (m *MockCopier) CopyFrom(ctx context.Context, source Repository, pinned PinnedReference, options CopyFromOptions) (*PublishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CopyFrom", ctx, source, pinned, options) + ret0, _ := ret[0].(*PublishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CopyFrom indicates an expected call of CopyFrom. +func (mr *MockCopierMockRecorder) CopyFrom(ctx, source, pinned, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFrom", reflect.TypeOf((*MockCopier)(nil).CopyFrom), ctx, source, pinned, options) +} diff --git a/pkg/artifact/mock_lockprovider.go b/pkg/artifact/mock_lockprovider.go new file mode 100644 index 00000000000..ff9096387c9 --- /dev/null +++ b/pkg/artifact/mock_lockprovider.go @@ -0,0 +1,180 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: LockProvider,LockHandle) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_lockprovider.go github.com/cloudposse/atmos/pkg/artifact LockProvider,LockHandle +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + time "time" + + gomock "go.uber.org/mock/gomock" +) + +// MockLockProvider is a mock of LockProvider interface. +type MockLockProvider struct { + ctrl *gomock.Controller + recorder *MockLockProviderMockRecorder + isgomock struct{} +} + +// MockLockProviderMockRecorder is the mock recorder for MockLockProvider. +type MockLockProviderMockRecorder struct { + mock *MockLockProvider +} + +// NewMockLockProvider creates a new mock instance. +func NewMockLockProvider(ctrl *gomock.Controller) *MockLockProvider { + mock := &MockLockProvider{ctrl: ctrl} + mock.recorder = &MockLockProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLockProvider) EXPECT() *MockLockProviderMockRecorder { + return m.recorder +} + +// Lock mocks base method. +func (m *MockLockProvider) Lock(ctx context.Context, request LockRequest) (LockHandle, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lock", ctx, request) + ret0, _ := ret[0].(LockHandle) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Lock indicates an expected call of Lock. +func (mr *MockLockProviderMockRecorder) Lock(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockLockProvider)(nil).Lock), ctx, request) +} + +// MockLockHandle is a mock of LockHandle interface. +type MockLockHandle struct { + ctrl *gomock.Controller + recorder *MockLockHandleMockRecorder + isgomock struct{} +} + +// MockLockHandleMockRecorder is the mock recorder for MockLockHandle. +type MockLockHandleMockRecorder struct { + mock *MockLockHandle +} + +// NewMockLockHandle creates a new mock instance. +func NewMockLockHandle(ctrl *gomock.Controller) *MockLockHandle { + mock := &MockLockHandle{ctrl: ctrl} + mock.recorder = &MockLockHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLockHandle) EXPECT() *MockLockHandleMockRecorder { + return m.recorder +} + +// ExpiresAt mocks base method. +func (m *MockLockHandle) ExpiresAt() (time.Time, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExpiresAt") + ret0, _ := ret[0].(time.Time) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// ExpiresAt indicates an expected call of ExpiresAt. +func (mr *MockLockHandleMockRecorder) ExpiresAt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpiresAt", reflect.TypeOf((*MockLockHandle)(nil).ExpiresAt)) +} + +// Owner mocks base method. +func (m *MockLockHandle) Owner() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Owner") + ret0, _ := ret[0].(string) + return ret0 +} + +// Owner indicates an expected call of Owner. +func (mr *MockLockHandleMockRecorder) Owner() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Owner", reflect.TypeOf((*MockLockHandle)(nil).Owner)) +} + +// Release mocks base method. +func (m *MockLockHandle) Release(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Release", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Release indicates an expected call of Release. +func (mr *MockLockHandleMockRecorder) Release(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockLockHandle)(nil).Release), ctx) +} + +// Renew mocks base method. +func (m *MockLockHandle) Renew(ctx context.Context, lease time.Duration) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Renew", ctx, lease) + ret0, _ := ret[0].(error) + return ret0 +} + +// Renew indicates an expected call of Renew. +func (mr *MockLockHandleMockRecorder) Renew(ctx, lease any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Renew", reflect.TypeOf((*MockLockHandle)(nil).Renew), ctx, lease) +} + +// Resource mocks base method. +func (m *MockLockHandle) Resource() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resource") + ret0, _ := ret[0].(string) + return ret0 +} + +// Resource indicates an expected call of Resource. +func (mr *MockLockHandleMockRecorder) Resource() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resource", reflect.TypeOf((*MockLockHandle)(nil).Resource)) +} + +// Scope mocks base method. +func (m *MockLockHandle) Scope() LockScope { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Scope") + ret0, _ := ret[0].(LockScope) + return ret0 +} + +// Scope indicates an expected call of Scope. +func (mr *MockLockHandleMockRecorder) Scope() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Scope", reflect.TypeOf((*MockLockHandle)(nil).Scope)) +} + +// Token mocks base method. +func (m *MockLockHandle) Token() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Token") + ret0, _ := ret[0].(string) + return ret0 +} + +// Token indicates an expected call of Token. +func (mr *MockLockHandleMockRecorder) Token() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockLockHandle)(nil).Token)) +} diff --git a/pkg/artifact/mock_producer.go b/pkg/artifact/mock_producer.go new file mode 100644 index 00000000000..d98247d3332 --- /dev/null +++ b/pkg/artifact/mock_producer.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Producer) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_producer.go github.com/cloudposse/atmos/pkg/artifact Producer +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockProducer is a mock of Producer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder + isgomock struct{} +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// Kind mocks base method. +func (m *MockProducer) Kind() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Kind") + ret0, _ := ret[0].(string) + return ret0 +} + +// Kind indicates an expected call of Kind. +func (mr *MockProducerMockRecorder) Kind() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kind", reflect.TypeOf((*MockProducer)(nil).Kind)) +} + +// Prepare mocks base method. +func (m *MockProducer) Prepare(ctx context.Context, request PrepareRequest) (*PreparedArtifact, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Prepare", ctx, request) + ret0, _ := ret[0].(*PreparedArtifact) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Prepare indicates an expected call of Prepare. +func (mr *MockProducerMockRecorder) Prepare(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prepare", reflect.TypeOf((*MockProducer)(nil).Prepare), ctx, request) +} diff --git a/pkg/artifact/mock_repository.go b/pkg/artifact/mock_repository.go new file mode 100644 index 00000000000..ac160ce6daf --- /dev/null +++ b/pkg/artifact/mock_repository.go @@ -0,0 +1,290 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Repository,GraphSource) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_repository.go github.com/cloudposse/atmos/pkg/artifact Repository,GraphSource +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + io "io" + reflect "reflect" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockRepository is a mock of Repository interface. +type MockRepository struct { + ctrl *gomock.Controller + recorder *MockRepositoryMockRecorder + isgomock struct{} +} + +// MockRepositoryMockRecorder is the mock recorder for MockRepository. +type MockRepositoryMockRecorder struct { + mock *MockRepository +} + +// NewMockRepository creates a new mock instance. +func NewMockRepository(ctrl *gomock.Controller) *MockRepository { + mock := &MockRepository{ctrl: ctrl} + mock.recorder = &MockRepositoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepository) EXPECT() *MockRepositoryMockRecorder { + return m.recorder +} + +// Capabilities mocks base method. +func (m *MockRepository) Capabilities() Capabilities { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Capabilities") + ret0, _ := ret[0].(Capabilities) + return ret0 +} + +// Capabilities indicates an expected call of Capabilities. +func (mr *MockRepositoryMockRecorder) Capabilities() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Capabilities", reflect.TypeOf((*MockRepository)(nil).Capabilities)) +} + +// Delete mocks base method. +func (m *MockRepository) Delete(ctx context.Context, pinned PinnedReference, options DeleteOptions) (*DeleteResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, pinned, options) + ret0, _ := ret[0].(*DeleteResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockRepositoryMockRecorder) Delete(ctx, pinned, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRepository)(nil).Delete), ctx, pinned, options) +} + +// Inspect mocks base method. +func (m *MockRepository) Inspect(ctx context.Context, pinned PinnedReference) (*Inspection, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inspect", ctx, pinned) + ret0, _ := ret[0].(*Inspection) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Inspect indicates an expected call of Inspect. +func (mr *MockRepositoryMockRecorder) Inspect(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inspect", reflect.TypeOf((*MockRepository)(nil).Inspect), ctx, pinned) +} + +// List mocks base method. +func (m *MockRepository) List(ctx context.Context, query ListQuery) (*ListPage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, query) + ret0, _ := ret[0].(*ListPage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockRepositoryMockRecorder) List(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRepository)(nil).List), ctx, query) +} + +// Lock mocks base method. +func (m *MockRepository) Lock(ctx context.Context, request LockRequest) (LockHandle, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lock", ctx, request) + ret0, _ := ret[0].(LockHandle) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Lock indicates an expected call of Lock. +func (mr *MockRepositoryMockRecorder) Lock(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockRepository)(nil).Lock), ctx, request) +} + +// OpenDocument mocks base method. +func (m *MockRepository) OpenDocument(ctx context.Context, pinned PinnedReference, role DocumentRole) (*DocumentStream, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OpenDocument", ctx, pinned, role) + ret0, _ := ret[0].(*DocumentStream) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OpenDocument indicates an expected call of OpenDocument. +func (mr *MockRepositoryMockRecorder) OpenDocument(ctx, pinned, role any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenDocument", reflect.TypeOf((*MockRepository)(nil).OpenDocument), ctx, pinned, role) +} + +// Pin mocks base method. +func (m *MockRepository) Pin(ctx context.Context, ref Ref) (PinnedReference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pin", ctx, ref) + ret0, _ := ret[0].(PinnedReference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pin indicates an expected call of Pin. +func (mr *MockRepositoryMockRecorder) Pin(ctx, ref any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pin", reflect.TypeOf((*MockRepository)(nil).Pin), ctx, ref) +} + +// Publish mocks base method. +func (m *MockRepository) Publish(ctx context.Context, prepared *PreparedArtifact, options PublishOptions) (*PublishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, prepared, options) + ret0, _ := ret[0].(*PublishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Publish indicates an expected call of Publish. +func (mr *MockRepositoryMockRecorder) Publish(ctx, prepared, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockRepository)(nil).Publish), ctx, prepared, options) +} + +// Read mocks base method. +func (m *MockRepository) Read(ctx context.Context, pinned PinnedReference) (*Content, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Read", ctx, pinned) + ret0, _ := ret[0].(*Content) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Read indicates an expected call of Read. +func (mr *MockRepositoryMockRecorder) Read(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockRepository)(nil).Read), ctx, pinned) +} + +// Resolve mocks base method. +func (m *MockRepository) Resolve(ctx context.Context, ref Ref) (PinnedReference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, ref) + ret0, _ := ret[0].(PinnedReference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockRepositoryMockRecorder) Resolve(ctx, ref any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockRepository)(nil).Resolve), ctx, ref) +} + +// Sign mocks base method. +func (m *MockRepository) Sign(ctx context.Context, subject SignatureSubject, signer Signer, options SignOptions) ([]SignatureEvidence, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, subject, signer, options) + ret0, _ := ret[0].([]SignatureEvidence) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockRepositoryMockRecorder) Sign(ctx, subject, signer, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockRepository)(nil).Sign), ctx, subject, signer, options) +} + +// Verify mocks base method. +func (m *MockRepository) Verify(ctx context.Context, subject SignatureSubject, policy VerificationPolicy) (*VerificationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, subject, policy) + ret0, _ := ret[0].(*VerificationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockRepositoryMockRecorder) Verify(ctx, subject, policy any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockRepository)(nil).Verify), ctx, subject, policy) +} + +// MockGraphSource is a mock of GraphSource interface. +type MockGraphSource struct { + ctrl *gomock.Controller + recorder *MockGraphSourceMockRecorder + isgomock struct{} +} + +// MockGraphSourceMockRecorder is the mock recorder for MockGraphSource. +type MockGraphSourceMockRecorder struct { + mock *MockGraphSource +} + +// NewMockGraphSource creates a new mock instance. +func NewMockGraphSource(ctrl *gomock.Controller) *MockGraphSource { + mock := &MockGraphSource{ctrl: ctrl} + mock.recorder = &MockGraphSourceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGraphSource) EXPECT() *MockGraphSourceMockRecorder { + return m.recorder +} + +// Fetch mocks base method. +func (m *MockGraphSource) Fetch(ctx context.Context, descriptor v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, descriptor) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockGraphSourceMockRecorder) Fetch(ctx, descriptor any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockGraphSource)(nil).Fetch), ctx, descriptor) +} + +// Root mocks base method. +func (m *MockGraphSource) Root() v1.Descriptor { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Root") + ret0, _ := ret[0].(v1.Descriptor) + return ret0 +} + +// Root indicates an expected call of Root. +func (mr *MockGraphSourceMockRecorder) Root() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockGraphSource)(nil).Root)) +} + +// Successors mocks base method. +func (m *MockGraphSource) Successors(ctx context.Context, descriptor v1.Descriptor) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Successors", ctx, descriptor) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Successors indicates an expected call of Successors. +func (mr *MockGraphSourceMockRecorder) Successors(ctx, descriptor any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Successors", reflect.TypeOf((*MockGraphSource)(nil).Successors), ctx, descriptor) +} diff --git a/pkg/artifact/mock_service.go b/pkg/artifact/mock_service.go new file mode 100644 index 00000000000..cd99d9cda7d --- /dev/null +++ b/pkg/artifact/mock_service.go @@ -0,0 +1,206 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Service) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_service.go github.com/cloudposse/atmos/pkg/artifact Service +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockService is a mock of Service interface. +type MockService struct { + ctrl *gomock.Controller + recorder *MockServiceMockRecorder + isgomock struct{} +} + +// MockServiceMockRecorder is the mock recorder for MockService. +type MockServiceMockRecorder struct { + mock *MockService +} + +// NewMockService creates a new mock instance. +func NewMockService(ctrl *gomock.Controller) *MockService { + mock := &MockService{ctrl: ctrl} + mock.recorder = &MockServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockService) EXPECT() *MockServiceMockRecorder { + return m.recorder +} + +// Copy mocks base method. +func (m *MockService) Copy(ctx context.Context, request CopyRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Copy", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Copy indicates an expected call of Copy. +func (mr *MockServiceMockRecorder) Copy(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockService)(nil).Copy), ctx, request) +} + +// Delete mocks base method. +func (m *MockService) Delete(ctx context.Context, request DeleteRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockServiceMockRecorder) Delete(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockService)(nil).Delete), ctx, request) +} + +// Inspect mocks base method. +func (m *MockService) Inspect(ctx context.Context, request InspectRequest) (*InspectResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inspect", ctx, request) + ret0, _ := ret[0].(*InspectResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Inspect indicates an expected call of Inspect. +func (mr *MockServiceMockRecorder) Inspect(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inspect", reflect.TypeOf((*MockService)(nil).Inspect), ctx, request) +} + +// List mocks base method. +func (m *MockService) List(ctx context.Context, request ListRequest) (*ListResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, request) + ret0, _ := ret[0].(*ListResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockServiceMockRecorder) List(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockService)(nil).List), ctx, request) +} + +// Pin mocks base method. +func (m *MockService) Pin(ctx context.Context, request PinRequest) (*PinResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pin", ctx, request) + ret0, _ := ret[0].(*PinResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pin indicates an expected call of Pin. +func (mr *MockServiceMockRecorder) Pin(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pin", reflect.TypeOf((*MockService)(nil).Pin), ctx, request) +} + +// Publish mocks base method. +func (m *MockService) Publish(ctx context.Context, request PublishRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Publish indicates an expected call of Publish. +func (mr *MockServiceMockRecorder) Publish(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockService)(nil).Publish), ctx, request) +} + +// Pull mocks base method. +func (m *MockService) Pull(ctx context.Context, request PullRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pull", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pull indicates an expected call of Pull. +func (mr *MockServiceMockRecorder) Pull(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pull", reflect.TypeOf((*MockService)(nil).Pull), ctx, request) +} + +// Search mocks base method. +func (m *MockService) Search(ctx context.Context, request SearchRequest) (*ListResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, request) + ret0, _ := ret[0].(*ListResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockServiceMockRecorder) Search(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockService)(nil).Search), ctx, request) +} + +// ShowDocument mocks base method. +func (m *MockService) ShowDocument(ctx context.Context, request ShowDocumentRequest) (*DocumentResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ShowDocument", ctx, request) + ret0, _ := ret[0].(*DocumentResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ShowDocument indicates an expected call of ShowDocument. +func (mr *MockServiceMockRecorder) ShowDocument(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShowDocument", reflect.TypeOf((*MockService)(nil).ShowDocument), ctx, request) +} + +// Sign mocks base method. +func (m *MockService) Sign(ctx context.Context, request SignRequest) (*SignResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, request) + ret0, _ := ret[0].(*SignResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockServiceMockRecorder) Sign(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockService)(nil).Sign), ctx, request) +} + +// Verify mocks base method. +func (m *MockService) Verify(ctx context.Context, request VerifyRequest) (*VerifyResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, request) + ret0, _ := ret[0].(*VerifyResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockServiceMockRecorder) Verify(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockService)(nil).Verify), ctx, request) +} diff --git a/pkg/artifact/mock_signer.go b/pkg/artifact/mock_signer.go new file mode 100644 index 00000000000..85ac8a921ba --- /dev/null +++ b/pkg/artifact/mock_signer.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Signer) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_signer.go github.com/cloudposse/atmos/pkg/artifact Signer +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockSigner is a mock of Signer interface. +type MockSigner struct { + ctrl *gomock.Controller + recorder *MockSignerMockRecorder + isgomock struct{} +} + +// MockSignerMockRecorder is the mock recorder for MockSigner. +type MockSignerMockRecorder struct { + mock *MockSigner +} + +// NewMockSigner creates a new mock instance. +func NewMockSigner(ctrl *gomock.Controller) *MockSigner { + mock := &MockSigner{ctrl: ctrl} + mock.recorder = &MockSignerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSigner) EXPECT() *MockSignerMockRecorder { + return m.recorder +} + +// Formats mocks base method. +func (m *MockSigner) Formats() []EvidenceFormat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Formats") + ret0, _ := ret[0].([]EvidenceFormat) + return ret0 +} + +// Formats indicates an expected call of Formats. +func (mr *MockSignerMockRecorder) Formats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Formats", reflect.TypeOf((*MockSigner)(nil).Formats)) +} + +// Sign mocks base method. +func (m *MockSigner) Sign(ctx context.Context, subject SignatureSubject) (*EvidencePayload, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, subject) + ret0, _ := ret[0].(*EvidencePayload) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockSignerMockRecorder) Sign(ctx, subject any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockSigner)(nil).Sign), ctx, subject) +} diff --git a/pkg/artifact/mock_verifier.go b/pkg/artifact/mock_verifier.go new file mode 100644 index 00000000000..cfcc9f1c384 --- /dev/null +++ b/pkg/artifact/mock_verifier.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Verifier) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_verifier.go github.com/cloudposse/atmos/pkg/artifact Verifier +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockVerifier is a mock of Verifier interface. +type MockVerifier struct { + ctrl *gomock.Controller + recorder *MockVerifierMockRecorder + isgomock struct{} +} + +// MockVerifierMockRecorder is the mock recorder for MockVerifier. +type MockVerifierMockRecorder struct { + mock *MockVerifier +} + +// NewMockVerifier creates a new mock instance. +func NewMockVerifier(ctrl *gomock.Controller) *MockVerifier { + mock := &MockVerifier{ctrl: ctrl} + mock.recorder = &MockVerifierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVerifier) EXPECT() *MockVerifierMockRecorder { + return m.recorder +} + +// Formats mocks base method. +func (m *MockVerifier) Formats() []EvidenceFormat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Formats") + ret0, _ := ret[0].([]EvidenceFormat) + return ret0 +} + +// Formats indicates an expected call of Formats. +func (mr *MockVerifierMockRecorder) Formats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Formats", reflect.TypeOf((*MockVerifier)(nil).Formats)) +} + +// Verify mocks base method. +func (m *MockVerifier) Verify(ctx context.Context, subject SignatureSubject, evidence []EvidenceBlob, policy VerificationPolicy) (*VerificationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, subject, evidence, policy) + ret0, _ := ret[0].(*VerificationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockVerifierMockRecorder) Verify(ctx, subject, evidence, policy any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockVerifier)(nil).Verify), ctx, subject, evidence, policy) +} diff --git a/pkg/artifact/operation.go b/pkg/artifact/operation.go new file mode 100644 index 00000000000..165c7c8936b --- /dev/null +++ b/pkg/artifact/operation.go @@ -0,0 +1,124 @@ +package artifact + +import ( + "time" + + "github.com/opencontainers/go-digest" +) + +// Action names an artifact operation. +type Action string + +// Artifact operations. +const ( + ActionPublish Action = "publish" + ActionPull Action = "pull" + ActionCopy Action = "copy" + ActionInspect Action = "inspect" + ActionList Action = "list" + ActionSearch Action = "search" + ActionDelete Action = "delete" + ActionShow Action = "show" + ActionSign Action = "sign" + ActionVerify Action = "verify" + ActionPin Action = "pin" +) + +// Outcome is the result classification of an operation. +type Outcome string + +// Operation outcomes. A no-op returns the existing Record unchanged. +const ( + // OutcomeCreated means new state was written. + OutcomeCreated Outcome = "created" + + // OutcomeUnchanged means the same intent already existed and nothing was written. + OutcomeUnchanged Outcome = "unchanged" + + // OutcomeReplaced means existing state was replaced under force. + OutcomeReplaced Outcome = "replaced" + + // OutcomeDeleted means state was removed. + OutcomeDeleted Outcome = "deleted" + + // OutcomeRead means state was read or materialized without mutation. + OutcomeRead Outcome = "read" + + // OutcomeDryRun means the operation was validated and planned but nothing was written. + OutcomeDryRun Outcome = "dry-run" +) + +// Severity classifies a diagnostic. +type Severity string + +// Diagnostic severities. +const ( + SeverityInfo Severity = "info" + SeverityWarning Severity = "warning" + SeverityError Severity = "error" +) + +// Diagnostic is a structured, non-fatal message attached to a result. +type Diagnostic struct { + // Severity is info, warning, or error. + Severity Severity `json:"severity"` + + // Code is a stable machine-readable identifier. + Code string `json:"code,omitempty"` + + // Message is the human-readable text. It never contains credentials. + Message string `json:"message"` +} + +// OperationResult is the typed result of one operation. Its JSON field names match the named step +// outputs and structured CLI output; optional values are omitted rather than filled with +// misleading empty strings. +type OperationResult struct { + // OperationID is the logical operation identifier. + OperationID string `json:"operation_id"` + + // Action is the operation performed. + Action Action `json:"action"` + + // Outcome classifies the result. + Outcome Outcome `json:"outcome"` + + // Repository is the configured repository alias the operation targeted. + Repository string `json:"repository,omitempty"` + + // Requested is the reference as requested by the caller. + Requested string `json:"requested_ref,omitempty"` + + // Ref is the resolved reference. + Ref string `json:"ref,omitempty"` + + // Location is the primary resolved storage location. + Location string `json:"location,omitempty"` + + // Aliases lists additional locations or tags that resolve to the same content. + Aliases []string `json:"aliases,omitempty"` + + // ResolvedDigest is the OCI manifest/index digest, or the record digest for local and S3. + ResolvedDigest digest.Digest `json:"resolved_digest,omitempty"` + + // RecordDigest is the digest of the canonical Record bytes. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // ContentDigest is the portable content digest from the Record. + ContentDigest digest.Digest `json:"content_digest,omitempty"` + + // RepresentationDigest is the checksum of the transferred payload representation. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` + + // Size is the transferred or described payload size in bytes. + Size int64 `json:"size,omitempty"` + + // Duration is the wall-clock duration of the operation. + Duration time.Duration `json:"duration_ns,omitempty"` + + // LockScope is the actual lock scope that coordinated a mutation. + LockScope LockScope `json:"lock_scope,omitempty"` + + // Diagnostics are non-fatal messages. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go new file mode 100644 index 00000000000..e6f44c02a8f --- /dev/null +++ b/pkg/artifact/options.go @@ -0,0 +1,460 @@ +package artifact + +import ( + "errors" + "fmt" + "net" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "github.com/go-viper/mapstructure/v2" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Options is a typed, validated view of one repository kind's `options` map. The configuration +// schema keeps `options` a generic map so it never depends on a backend; each kind decodes it into +// its own struct with [DecodeOptions] and checks it with Validate. Validate never performs +// network access, credential resolution, or directory creation. +type Options interface { + // Validate checks required options and value shapes. + Validate() error +} + +// PackagingValidator is implemented by Options that depend on the effective packaging, for example +// to check that a key_pattern extension agrees with an archive format. +type PackagingValidator interface { + // ValidateForPackaging checks the options against the effective packaging. + ValidateForPackaging(packaging Packaging) error +} + +// Option keys shared by several kinds. +const ( + // OptionPath is the local destination root of local/dir, local/archive, and oci/layout. + OptionPath = "path" + + // OptionKeyPattern overrides the path below the stable root for local and S3 payloads. + OptionKeyPattern = "key_pattern" +) + +// S3 object tag limits from the S3 documentation. +const ( + maxS3Tags = 10 + maxS3TagKeyLen = 128 + maxS3TagValLen = 256 + reservedTagHead = "aws:" +) + +var ( + // Flags option keys that look like credentials. Credentials never belong + // in repository options; the `identity` field references an Atmos auth identity instead. + credentialOptionPattern = regexp.MustCompile(`(?i)secret|password|passwd|token|credential|access[_-]?key|api[_-]?key`) + + // The S3 general-purpose bucket naming rule (3-63 lowercase letters, digits, dots, hyphens). + s3BucketPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) + + // The shape of a cloud region name such as us-east-2. + regionPattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) + + // A registry host[:port] without a scheme, path, or credentials. + registryHostPattern = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]{1,5})?$`) + + // Matches one Go template action. + templateActionPattern = regexp.MustCompile(`(?s)\{\{.*?\}\}`) + + // Matches Go template string literals so field references are not read from them. + templateStringPattern = regexp.MustCompile("\"(?:[^\"\\\\]|\\\\.)*\"|`[^`]*`") + + // Matches a field reference such as .Name. + templateFieldPattern = regexp.MustCompile(`\.([A-Za-z_][A-Za-z0-9_]*)`) +) + +// Path segments that would escape or empty a path. +const ( + currentDirSegment = "." + parentDirSegment = ".." +) + +// keyPatternFields are the fields a key_pattern may reference. +var keyPatternFields = map[string]bool{"Name": true, "Version": true, "Kind": true, "CreatedAt": true} + +// LocalDirOptions are the options of a local/dir repository. +type LocalDirOptions struct { + // Path is the destination root directory. + Path string `mapstructure:"path"` + + // KeyPattern overrides the payload directory path below the root. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *LocalDirOptions) Validate() error { + defer perf.Track(nil, "artifact.LocalDirOptions.Validate")() + + return errors.Join(validateLocalPath(o.Path), validateKeyPattern(o.KeyPattern, "")) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *LocalDirOptions) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.LocalDirOptions.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// LocalArchiveOptions are the options of a local/archive repository. +type LocalArchiveOptions struct { + // Path is the destination root directory. + Path string `mapstructure:"path"` + + // KeyPattern overrides the archive file path below the root. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *LocalArchiveOptions) Validate() error { + defer perf.Track(nil, "artifact.LocalArchiveOptions.Validate")() + + return errors.Join(validateLocalPath(o.Path), validateKeyPattern(o.KeyPattern, "")) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *LocalArchiveOptions) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.LocalArchiveOptions.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// S3Options are the options of an aws/s3 repository. +type S3Options struct { + // Bucket is the S3 bucket name. + Bucket string `mapstructure:"bucket"` + + // Prefix is the collection root below the bucket. + Prefix string `mapstructure:"prefix"` + + // Region is the bucket region. + Region string `mapstructure:"region"` + + // Tags are S3 object tags applied to every object of a publication. They are not artifact labels. + Tags map[string]string `mapstructure:"tags"` + + // KeyPattern overrides the object key below the prefix. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *S3Options) Validate() error { + defer perf.Track(nil, "artifact.S3Options.Validate")() + + var errs []error + if o.Bucket == "" { + errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "bucket")) + } else if !s3BucketPattern.MatchString(o.Bucket) || net.ParseIP(o.Bucket) != nil { + errs = append(errs, fmt.Errorf("%w: bucket %q is not a valid S3 bucket name", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Bucket))) + } + if o.Region != "" && !regionPattern.MatchString(o.Region) { + errs = append(errs, fmt.Errorf("%w: region %q is not a valid region name", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Region))) + } + errs = append(errs, validatePrefix(o.Prefix), validateS3Tags(o.Tags), validateKeyPattern(o.KeyPattern, "")) + return errors.Join(errs...) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *S3Options) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.S3Options.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// OCIRegistryOptions are the options of an oci/registry repository. The prefix is an Atmos +// collection-root convention, not an ORAS namespace field: Atmos maps a collection and artifact +// name onto an OCI repository path below it. +type OCIRegistryOptions struct { + // Registry is the registry host, with an optional port, for example ghcr.io. + Registry string `mapstructure:"registry"` + + // Prefix is the collection root path below the registry, for example acme/atmos. + Prefix string `mapstructure:"prefix"` +} + +// Validate implements Options. +func (o *OCIRegistryOptions) Validate() error { + defer perf.Track(nil, "artifact.OCIRegistryOptions.Validate")() + + var errs []error + if o.Registry == "" { + errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) + } else if !validRegistryHost(o.Registry) { + errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Registry))) + } + errs = append(errs, validatePrefix(o.Prefix)) + if o.Prefix != "" { + for _, segment := range strings.Split(o.Prefix, "/") { + if err := ValidateName(segment); err != nil { + errs = append(errs, fmt.Errorf("%w: prefix %q is not a valid OCI repository path: %w", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Prefix), RedactError(err, o.Prefix))) + break + } + } + } + return errors.Join(errs...) +} + +const registryPortBits = 16 + +func validRegistryHost(host string) bool { + if !registryHostPattern.MatchString(host) { + return false + } + _, port, found := strings.Cut(host, ":") + if !found { + return true + } + _, err := strconv.ParseUint(port, decimalBase, registryPortBits) + return err == nil +} + +// OCILayoutOptions are the options of an oci/layout repository. +type OCILayoutOptions struct { + // Path is the OCI image layout directory. + Path string `mapstructure:"path"` +} + +// Validate implements Options. +func (o *OCILayoutOptions) Validate() error { + defer perf.Track(nil, "artifact.OCILayoutOptions.Validate")() + + return validateLocalPath(o.Path) +} + +// DecodeOptions decodes a generic options map into a typed options struct. Decoding is strict: +// unknown keys and mistyped values are errors, so a typo such as `buckt` never silently becomes a +// missing option. Keys that look like credentials get a pointer to the `identity` field. +func DecodeOptions(raw map[string]any, out any) error { + defer perf.Track(nil, "artifact.DecodeOptions")() + + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: out, + TagName: "mapstructure", + ErrorUnused: true, + }) + if err != nil { + return fmt.Errorf("%w: options decoder: %w", errUtils.ErrArtifactConfigInvalid, err) + } + if err := decoder.Decode(raw); err != nil { + // A decode error may quote a value, such as a mistyped number that is really a credential, + // so every string key and value is a credential candidate when the message is scrubbed. + return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, RedactDecodeError(err, raw), credentialHint(raw)) + } + return nil +} + +// stringValues returns every string held by raw, keys included, at any depth, in no particular order. +func stringValues(raw any) []string { + var out []string + switch v := raw.(type) { + case string: + out = append(out, v) + case map[string]any: + for key, item := range v { + out = append(out, key) + out = append(out, stringValues(item)...) + } + case []any: + for _, item := range v { + out = append(out, stringValues(item)...) + } + } + return out +} + +// credentialHint explains where credentials belong when an option key looks like one. +func credentialHint(raw map[string]any) string { + keys := make([]string, 0, len(raw)) + for key := range raw { + if credentialOptionPattern.MatchString(key) { + keys = append(keys, RedactURL(key)) + } + } + if len(keys) == 0 { + return "" + } + sort.Strings(keys) + return fmt.Sprintf(" (option %s looks like a credential; credentials never belong in repository options, reference an auth identity with `identity` instead)", strings.Join(keys, ", ")) +} + +// validateLocalPath checks a local destination root. A relative path is legitimate (it resolves +// against the declaring configuration file), so only emptiness and NUL bytes are rejected here. +func validateLocalPath(path string) error { + switch { + case path == "": + return fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, OptionPath) + case strings.ContainsRune(path, 0): + return fmt.Errorf("%w: %w: path contains a NUL byte", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath) + } + return nil +} + +// ValidatePrefix checks a collection-root prefix such as acme/atmos below a bucket or registry. It +// must be a relative, slash-separated path with no empty, "." or ".." segments, so the prefix +// cannot escape its root, and it must not enter the reserved managed-record namespace. +func ValidatePrefix(prefix string) error { + defer perf.Track(nil, "artifact.ValidatePrefix")() + + return validatePrefix(prefix) +} + +func validatePrefix(prefix string) error { + if prefix == "" { + return nil + } + if strings.HasPrefix(prefix, "/") || strings.Contains(prefix, `\`) || strings.ContainsRune(prefix, 0) { + return fmt.Errorf("%w: %w: prefix %q must be relative and slash-separated", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix)) + } + for _, segment := range strings.Split(prefix, "/") { + switch segment { + case "", currentDirSegment, parentDirSegment: + return fmt.Errorf("%w: %w: prefix %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix), currentDirSegment, parentDirSegment) + case ReservedNamespace: + return fmt.Errorf("%w: %w: prefix %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix), ReservedNamespace) + } + } + return nil +} + +// ValidateKeyPattern statically checks a key_pattern template against the effective packaging +// without expanding it. Expansion needs the artifact name, version, kind, and creation time, so +// only the literal structure is checked here: balanced template delimiters, referenced fields +// limited to .Name, .Version, .Kind and .CreatedAt, no literal root escape or empty segment, no +// entry into the reserved namespace, and an archive extension that agrees with the packaging. +// The pattern is never rendered during configuration loading. +func ValidateKeyPattern(pattern string, packaging Packaging) error { + defer perf.Track(nil, "artifact.ValidateKeyPattern")() + + return validateKeyPattern(pattern, packaging) +} + +func validateKeyPattern(pattern string, packaging Packaging) error { + if pattern == "" { + return nil + } + if strings.ContainsRune(pattern, 0) || strings.Contains(pattern, `\`) { + return fmt.Errorf("%w: %w: key_pattern %q must not contain NUL bytes or backslashes", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern)) + } + actions := templateActionPattern.FindAllString(pattern, -1) + literal := templateActionPattern.ReplaceAllString(pattern, "\x01") + if strings.Contains(literal, "{{") || strings.Contains(literal, "}}") { + return fmt.Errorf("%w: key_pattern %q has unbalanced template delimiters", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern)) + } + if err := validateKeyPatternFields(pattern, actions); err != nil { + return err + } + if err := validateKeyPatternSegments(pattern, literal); err != nil { + return err + } + return validateKeyPatternExtension(pattern, literal, packaging) +} + +func validateKeyPatternFields(pattern string, actions []string) error { + for _, action := range actions { + stripped := templateStringPattern.ReplaceAllString(action, `""`) + for _, match := range templateFieldPattern.FindAllStringSubmatch(stripped, -1) { + if !keyPatternFields[match[1]] { + return unknownKeyPatternField(pattern, match[1]) + } + } + } + return nil +} + +// unknownKeyPatternField reports a template field that key_pattern may not reference. The field +// name is a fragment of the pattern, and a pattern that carries a credential may hold it in the +// userinfo, as in "https://user:{{.secret}}@host/", where scrubbing the pattern does not scrub the +// fragment. The name is therefore withheld whenever the redaction changes the pattern. +func unknownKeyPatternField(pattern, field string) error { + redacted := RedactURL(pattern) + if redacted != pattern { + return fmt.Errorf("%w: key_pattern %q references a field that is not allowed (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, redacted) + } + return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, redacted, field) +} + +func validateKeyPatternSegments(pattern, literal string) error { + if strings.HasPrefix(literal, "/") { + return fmt.Errorf("%w: %w: key_pattern %q must be relative to the repository root", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern)) + } + for _, segment := range strings.Split(literal, "/") { + switch segment { + case "", currentDirSegment, parentDirSegment: + return fmt.Errorf("%w: %w: key_pattern %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern), currentDirSegment, parentDirSegment) + case ReservedNamespace: + return fmt.Errorf("%w: %w: key_pattern %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern), ReservedNamespace) + } + } + return nil +} + +// archiveExtensions maps the literal trailing extension of a key_pattern to the packaging it implies. +var archiveExtensions = []struct { + suffix string + packaging Packaging +}{ + {".tar.gz", PackagingTgz}, + {".tgz", PackagingTgz}, + {".tar", PackagingTar}, + {".zip", PackagingZip}, +} + +func validateKeyPatternExtension(pattern, literal string, packaging Packaging) error { + // Only a literal tail (after the last template action) has a known extension. + tail := literal[strings.LastIndex(literal, "\x01")+1:] + for _, candidate := range archiveExtensions { + if !strings.HasSuffix(tail, candidate.suffix) { + continue + } + if packaging != "" && candidate.packaging != packaging { + return fmt.Errorf("%w: key_pattern %q ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern), candidate.suffix, packaging) + } + return nil + } + return nil +} + +func validateS3Tags(tags map[string]string) error { + if len(tags) > maxS3Tags { + return fmt.Errorf("%w: %d S3 tags exceed the S3 limit of %d", errUtils.ErrArtifactConfigInvalid, len(tags), maxS3Tags) + } + keys := make([]string, 0, len(tags)) + for key := range tags { + keys = append(keys, key) + } + sort.Strings(keys) + var errs []error + for _, key := range keys { + switch { + case key == "" || utf8.RuneCountInString(key) > maxS3TagKeyLen: + errs = append(errs, fmt.Errorf("%w: S3 tag key %q must be 1-%d characters", errUtils.ErrArtifactConfigInvalid, RedactURL(key), maxS3TagKeyLen)) + case strings.HasPrefix(strings.ToLower(key), reservedTagHead): + errs = append(errs, fmt.Errorf("%w: S3 tag key %q uses the reserved %q prefix", errUtils.ErrArtifactConfigInvalid, RedactURL(key), reservedTagHead)) + case utf8.RuneCountInString(tags[key]) > maxS3TagValLen: + errs = append(errs, fmt.Errorf("%w: S3 tag %q value exceeds %d characters", errUtils.ErrArtifactConfigInvalid, RedactURL(key), maxS3TagValLen)) + } + } + return errors.Join(errs...) +} + +// RegistryHostOf returns the host of an oci:// URI without credentials, query, or path, and +// whether the URI is well formed. It is used to bind a catalog to its repository's registry. +func RegistryHostOf(uri string) (string, bool) { + defer perf.Track(nil, "artifact.RegistryHostOf")() + + parsed, err := url.Parse(uri) + if err != nil || parsed.Scheme != "oci" || parsed.Host == "" || parsed.User != nil { + return "", false + } + return strings.ToLower(parsed.Host), true +} diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go new file mode 100644 index 00000000000..7225967d632 --- /dev/null +++ b/pkg/artifact/options_test.go @@ -0,0 +1,213 @@ +package artifact + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinels: a rename of these schema fields must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: "", Options: nil, Packaging: "", Permissions: "", Catalog: ""} + +func TestDecodeOptions_IsStrict(t *testing.T) { + tests := []struct { + name string + raw map[string]any + wantErr string + wantHint bool + }{ + {name: "valid", raw: map[string]any{"bucket": "acme-artifacts", "region": "us-east-2"}}, + {name: "unknown key is a typo", raw: map[string]any{"buckt": "acme-artifacts"}, wantErr: "buckt"}, + {name: "wrong type", raw: map[string]any{"bucket": 42}, wantErr: "bucket"}, + {name: "credential-looking key gets a hint", raw: map[string]any{"bucket": "acme-artifacts", "secret_key": "x"}, wantErr: "secret_key", wantHint: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var options S3Options + err := DecodeOptions(tt.raw, &options) + if tt.wantErr == "" { + require.NoError(t, err) + assert.Equal(t, "acme-artifacts", options.Bucket) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantHint, strings.Contains(err.Error(), "auth identity")) + }) + } +} + +func TestOptionsValidate(t *testing.T) { + tests := []struct { + name string + options Options + wantErr string + }{ + {name: "local dir ok", options: &LocalDirOptions{Path: ".atmos/artifacts"}}, + {name: "local dir missing path", options: &LocalDirOptions{}, wantErr: `"path" is required`}, + {name: "local dir NUL", options: &LocalDirOptions{Path: "a\x00b"}, wantErr: "NUL"}, + {name: "relative path with traversal is a legitimate destination", options: &LocalDirOptions{Path: "../shared/artifacts"}}, + {name: "archive missing path", options: &LocalArchiveOptions{}, wantErr: `"path" is required`}, + {name: "layout missing path", options: &OCILayoutOptions{}, wantErr: `"path" is required`}, + {name: "s3 ok", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos", Region: "us-east-2", Tags: map[string]string{"purpose": "backup"}}}, + {name: "s3 missing bucket", options: &S3Options{}, wantErr: `"bucket" is required`}, + {name: "s3 IP bucket", options: &S3Options{Bucket: "192.168.0.1"}, wantErr: "not a valid S3 bucket name"}, + {name: "s3 bad bucket", options: &S3Options{Bucket: "Bad_Bucket"}, wantErr: "not a valid S3 bucket name"}, + {name: "s3 bad region", options: &S3Options{Bucket: "acme-artifacts", Region: "US East"}, wantErr: "not a valid region"}, + {name: "s3 prefix traversal", options: &S3Options{Bucket: "acme-artifacts", Prefix: "a/../b"}, wantErr: "prefix"}, + {name: "s3 absolute prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "/atmos"}, wantErr: "prefix"}, + {name: "s3 trailing slash prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos/"}, wantErr: "prefix"}, + {name: "s3 reserved prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos/.atmos-artifacts"}, wantErr: "reserved"}, + {name: "s3 reserved tag prefix", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"aws:foo": "x"}}, wantErr: "reserved"}, + {name: "s3 oversized tag value", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("x", 257)}}, wantErr: "value exceeds"}, + {name: "s3 tag value at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("x", 256)}}}, + {name: "s3 multibyte tag value at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("é", 256)}}}, + {name: "s3 multibyte tag value over the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("é", 257)}}, wantErr: "value exceeds"}, + {name: "s3 tag key at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("k", 128): "v"}}}, + {name: "s3 multibyte tag key at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("é", 128): "v"}}}, + {name: "s3 multibyte tag key over the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("é", 129): "v"}}, wantErr: "must be 1-128 characters"}, + {name: "s3 empty tag key", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"": "v"}}, wantErr: "must be 1-128 characters"}, + {name: "s3 too many tags", options: &S3Options{Bucket: "acme-artifacts", Tags: elevenTags()}, wantErr: "exceed the S3 limit"}, + {name: "s3 bad key pattern", options: &S3Options{Bucket: "acme-artifacts", KeyPattern: "../x"}, wantErr: "segment"}, + {name: "registry ok", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/atmos"}}, + {name: "registry with port", options: &OCIRegistryOptions{Registry: "localhost:5000"}}, + {name: "registry max port", options: &OCIRegistryOptions{Registry: "localhost:65535"}}, + {name: "registry port overflow", options: &OCIRegistryOptions{Registry: "localhost:65536"}, wantErr: "without a scheme"}, + {name: "registry missing", options: &OCIRegistryOptions{}, wantErr: `"registry" is required`}, + {name: "registry with scheme", options: &OCIRegistryOptions{Registry: "https://ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry with credentials", options: &OCIRegistryOptions{Registry: "user:pw@ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry with credentials and a scheme", options: &OCIRegistryOptions{Registry: "https://user:hunter2@ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry uppercase prefix", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "Acme/Atmos"}, wantErr: "not a valid OCI repository path"}, + {name: "registry prefix traversal", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/../x"}, wantErr: "prefix"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "rejected registry credentials are never echoed") + assert.NotContains(t, err.Error(), "user:pw", "rejected registry credentials are never echoed") + }) + } +} + +func elevenTags() map[string]string { + tags := make(map[string]string) + for _, key := range strings.Split("a b c d e f g h i j k", " ") { + tags[key] = "v" + } + return tags +} + +func TestValidateKeyPattern(t *testing.T) { + prd := `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz` + tests := []struct { + name string + pattern string + packaging Packaging + wantErr string + wantUnsafe bool + }{ + {name: "empty is fine", pattern: ""}, + {name: "PRD example", pattern: prd, packaging: PackagingTgz}, + {name: "tar.gz extension for tgz", pattern: "{{ .Name }}/{{ .Version }}.tar.gz", packaging: PackagingTgz}, + {name: "extension conflicts with packaging", pattern: "{{ .Name }}/{{ .Version }}.zip", packaging: PackagingTgz, wantErr: "does not match packaging"}, + {name: "archive extension with directory packaging", pattern: "{{ .Name }}/{{ .Version }}.tgz", packaging: PackagingNone, wantErr: "does not match packaging"}, + {name: "extension not checked without packaging", pattern: "{{ .Name }}.zip"}, + {name: "date layout slashes inside action are not segments", pattern: `{{ .CreatedAt | date "2006/01/02" }}/x`}, + {name: "unknown field", pattern: "{{ .Owner }}/x", wantErr: "unknown field .Owner"}, + {name: "unbalanced open", pattern: "{{ .Name /x", wantErr: "unbalanced"}, + {name: "unbalanced close", pattern: ".Name }}/x", wantErr: "unbalanced"}, + {name: "absolute", pattern: "/{{ .Name }}", wantErr: "relative", wantUnsafe: true}, + {name: "traversal", pattern: "{{ .Name }}/../x", wantErr: "segment", wantUnsafe: true}, + {name: "empty segment", pattern: "a//b", wantErr: "segment", wantUnsafe: true}, + {name: "reserved namespace", pattern: ".atmos-artifacts/{{ .Name }}", wantErr: "reserved", wantUnsafe: true}, + {name: "backslash", pattern: `a\b`, wantErr: "backslash", wantUnsafe: true}, + {name: "field name inside string literal is not a field", pattern: `{{ printf ".Owner" }}/{{ .Name }}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateKeyPattern(tt.pattern, tt.packaging) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantUnsafe, strings.Contains(err.Error(), errUtils.ErrArtifactUnsafePath.Error())) + }) + } +} + +func TestValidatePrefix(t *testing.T) { + require.NoError(t, ValidatePrefix("")) + require.NoError(t, ValidatePrefix("acme/atmos")) + for _, bad := range []string{"/abs", "a/./b", "a//b", "..", `a\b`, "a/.atmos-artifacts"} { + err := ValidatePrefix(bad) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, bad) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, bad) + } +} + +func TestRegistryHostOf(t *testing.T) { + host, ok := RegistryHostOf("oci://GHCR.io/acme/atmos/catalog:stable") + require.True(t, ok) + assert.Equal(t, "ghcr.io", host) + for _, bad := range []string{"https://ghcr.io/x", "oci:///x", "oci://user:pw@ghcr.io/x", "::"} { + _, ok := RegistryHostOf(bad) + assert.False(t, ok, bad) + } +} + +// TestBuiltinKindsMatchTypedOptions keeps each descriptor's path options and typed options struct in step. +func TestBuiltinKindsMatchTypedOptions(t *testing.T) { + require.NotEmpty(t, BuiltinKinds()) + for _, kind := range BuiltinKinds() { + desc, ok := BuiltinKind(kind) + require.True(t, ok, kind) + require.NotNil(t, desc.NewOptions, kind) + tags := mapstructureTags(desc.NewOptions()) + for _, path := range desc.PathOptions { + assert.Contains(t, tags, path, "%s declares path option %q that its typed options do not decode", kind, path) + } + assert.True(t, desc.Capabilities.HasPackaging(desc.DefaultPackaging), "%s default packaging must be supported", kind) + assert.NotEmpty(t, desc.Capabilities.LockScopes, kind) + } +} + +func mapstructureTags(options Options) []string { + rt := reflect.TypeOf(options).Elem() + tags := make([]string, 0, rt.NumField()) + for i := range rt.NumField() { + tags = append(tags, rt.Field(i).Tag.Get("mapstructure")) + } + return tags +} + +// TestValidationCreatesNothing proves validation is free of filesystem side effects. +func TestValidationCreatesNothing(t *testing.T) { + root := t.TempDir() + cfg := &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, + Permissions: "0600", + Options: map[string]any{"path": filepath.Join(root, "not", "created", "yet"), "key_pattern": "{{ .Name }}/{{ .Version }}.tgz"}, + } + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + require.NoError(t, ValidateRepositoryConfig(desc, cfg)) + + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries, "validation must not create destination directories") +} diff --git a/pkg/artifact/packaging.go b/pkg/artifact/packaging.go new file mode 100644 index 00000000000..fd06445137b --- /dev/null +++ b/pkg/artifact/packaging.go @@ -0,0 +1,22 @@ +package artifact + +// Packaging identifies how a file payload is stored in a repository. +type Packaging string + +// Payload packaging modes. +const ( + // PackagingNone stores copied files (local directories, S3 object-per-file). + PackagingNone Packaging = "none" + + // PackagingTar stores one uncompressed tar archive. + PackagingTar Packaging = "tar" + + // PackagingTgz stores one gzip-compressed tar archive. + PackagingTgz Packaging = "tgz" + + // PackagingZip stores one zip archive. + PackagingZip Packaging = "zip" + + // PackagingOCIGraph identifies an existing native OCI graph carried unchanged. + PackagingOCIGraph Packaging = "oci-graph" +) diff --git a/pkg/artifact/permissions.go b/pkg/artifact/permissions.go new file mode 100644 index 00000000000..f1f5acf017e --- /dev/null +++ b/pkg/artifact/permissions.go @@ -0,0 +1,49 @@ +package artifact + +import ( + "fmt" + "io/fs" + "regexp" + "strconv" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultArchivePermissions is the mode of a local archive file when permissions are not configured. +const DefaultArchivePermissions fs.FileMode = 0o600 + +const ( + // The numeric base of permission strings. + octalBase = 8 + + // The bit size of a parsed permission value. + permissionBitSize = 32 + + // The largest accepted mode: ordinary permission bits without setuid, setgid, or sticky. + maxPermissionBits = 0o777 +) + +// permissionsPattern accepts three or four octal digits, for example "600" or "0600". +var permissionsPattern = regexp.MustCompile(`^[0-7]{3,4}$`) + +// ParsePermissions parses a quoted octal permission string such as "0600". Only ordinary +// permission bits (at most 0777) are accepted: privileged bits (setuid, setgid, sticky) are +// rejected, as are "preserve" and non-octal forms such as "0o600". +func ParsePermissions(value string) (fs.FileMode, error) { + defer perf.Track(nil, "artifact.ParsePermissions")() + + // The value is echoed, and it may be a pasted credential-bearing URL. + shown := RedactURL(value) + if !permissionsPattern.MatchString(value) { + return 0, fmt.Errorf("%w: permissions %q must be a quoted octal mode such as \"0600\"", errUtils.ErrArtifactConfigInvalid, shown) + } + parsed, err := strconv.ParseUint(value, octalBase, permissionBitSize) + if err != nil { + return 0, fmt.Errorf("%w: permissions %q: %w", errUtils.ErrArtifactConfigInvalid, shown, err) + } + if parsed > maxPermissionBits { + return 0, fmt.Errorf("%w: permissions %q must not set privileged mode bits (maximum \"0777\")", errUtils.ErrArtifactConfigInvalid, shown) + } + return fs.FileMode(parsed), nil +} diff --git a/pkg/artifact/permissions_test.go b/pkg/artifact/permissions_test.go new file mode 100644 index 00000000000..d7c602e7ac8 --- /dev/null +++ b/pkg/artifact/permissions_test.go @@ -0,0 +1,46 @@ +package artifact + +import ( + "io/fs" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestParsePermissions(t *testing.T) { + tests := []struct { + in string + want fs.FileMode + wantErr bool + }{ + {in: "0600", want: 0o600}, + {in: "600", want: 0o600}, + {in: "0644", want: 0o644}, + {in: "0777", want: 0o777}, + {in: "0000", want: 0}, + {in: "", wantErr: true}, + {in: "preserve", wantErr: true}, + {in: "0o600", wantErr: true}, + {in: "0900", wantErr: true}, + {in: "4755", wantErr: true}, + {in: "1777", wantErr: true}, + {in: "06000", wantErr: true}, + {in: "-0600", wantErr: true}, + {in: " 0600", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got, err := ParsePermissions(tt.in) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + assert.Equal(t, fs.FileMode(0o600), DefaultArchivePermissions) +} diff --git a/pkg/artifact/pinned.go b/pkg/artifact/pinned.go new file mode 100644 index 00000000000..015a8cdd5bc --- /dev/null +++ b/pkg/artifact/pinned.go @@ -0,0 +1,32 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// PinnedReference is an immutable reference suitable for consumer lockfiles. It is not a write +// lock and not a substitute for signature verification; trust policy is reevaluated on use. +type PinnedReference struct { + // Ref is the reference that was pinned, as requested. + Ref Ref `json:"ref"` + + // Repository is the repository locator (a registry path, bucket and prefix, or directory), + // never credentials. + Repository string `json:"repository"` + + // ResolvedDigest is the OCI manifest/index digest, or the record digest for local and S3. + ResolvedDigest digest.Digest `json:"resolved_digest"` + + // RecordDigest is the digest of the canonical Record bytes, when a Record is bound. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // ContentDigest is the portable content digest from the Record, when a Record is bound. + ContentDigest digest.Digest `json:"content_digest,omitempty"` + + // Evidence lists the signature evidence descriptors known at pin time. + Evidence []ocispec.Descriptor `json:"evidence,omitempty"` + + // Platform is the selected platform when a multi-platform index was narrowed. + Platform *ocispec.Platform `json:"platform,omitempty"` +} diff --git a/pkg/artifact/policy.go b/pkg/artifact/policy.go new file mode 100644 index 00000000000..3a129e53228 --- /dev/null +++ b/pkg/artifact/policy.go @@ -0,0 +1,98 @@ +package artifact + +import ( + "fmt" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// TrustedIdentity is a certificate identity and issuer pair that may satisfy a trust set. +type TrustedIdentity struct { + // Identity is the expected certificate identity. + Identity string `json:"identity"` + + // Issuer is the expected OIDC issuer. + Issuer string `json:"issuer"` +} + +// TrustSet is one set of acceptable signers. Within a trust set any one signer satisfies it. +type TrustSet struct { + // Identities are trusted certificate identities. + Identities []TrustedIdentity `json:"identities,omitempty"` + + // Keys are operator-managed public-key file paths. + Keys []string `json:"keys,omitempty"` +} + +// VerificationPolicy is the effective policy passed to Verify. When a global and a repository +// trust restriction both exist, the policy lists both TrustSets and every one must be satisfied; +// they are never unioned into a weaker policy. +type VerificationPolicy struct { + // RequireSignature makes a missing or invalid signature a failure. + RequireSignature bool `json:"require_signature"` + + // TrustSets are the sets that must each be satisfied by at least one valid permitted signature. + TrustSets []TrustSet `json:"trust_sets,omitempty"` + + // Offline forbids network access; only preserved evidence and provisioned trust material are used. + Offline bool `json:"offline"` +} + +// HasTrustConstraints reports whether the policy supplies any trust material. Without it a +// verifier reports unchecked rather than verified. +func (p VerificationPolicy) HasTrustConstraints() bool { + defer perf.Track(nil, "artifact.VerificationPolicy.HasTrustConstraints")() + + for _, set := range p.TrustSets { + if len(set.Identities) > 0 || len(set.Keys) > 0 { + return true + } + } + return false +} + +// Fingerprint returns a stable SHA-256 fingerprint of the policy, independent of the order of +// trust sets, identities, and keys, for recording in verification results. +func (p VerificationPolicy) Fingerprint() (string, error) { + defer perf.Track(nil, "artifact.VerificationPolicy.Fingerprint")() + + normalized := VerificationPolicy{RequireSignature: p.RequireSignature, Offline: p.Offline} + for _, set := range p.TrustSets { + next := TrustSet{ + Identities: append([]TrustedIdentity(nil), set.Identities...), + Keys: append([]string(nil), set.Keys...), + } + sort.Slice(next.Identities, func(i, j int) bool { + if next.Identities[i].Identity != next.Identities[j].Identity { + return next.Identities[i].Identity < next.Identities[j].Identity + } + return next.Identities[i].Issuer < next.Identities[j].Issuer + }) + sort.Strings(next.Keys) + normalized.TrustSets = append(normalized.TrustSets, next) + } + sort.Slice(normalized.TrustSets, func(i, j int) bool { + return trustSetKey(normalized.TrustSets[i]) < trustSetKey(normalized.TrustSets[j]) + }) + canonical, err := canonicalJSON(normalized) + if err != nil { + return "", fmt.Errorf("%w: policy fingerprint: %w", errUtils.ErrArtifactConfigInvalid, err) + } + return digest.FromBytes(canonical).String(), nil +} + +// trustSetKey is a deterministic sort key for a trust set. +func trustSetKey(set TrustSet) string { + key := "" + for _, identity := range set.Identities { + key += identity.Identity + "\x00" + identity.Issuer + "\x00" + } + for _, path := range set.Keys { + key += path + "\x01" + } + return key +} diff --git a/pkg/artifact/policy_test.go b/pkg/artifact/policy_test.go new file mode 100644 index 00000000000..318adc66d15 --- /dev/null +++ b/pkg/artifact/policy_test.go @@ -0,0 +1,70 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVerificationPolicyFingerprint(t *testing.T) { + identityA := TrustedIdentity{Identity: "https://github.com/acme/a/.github/workflows/r.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + identityB := TrustedIdentity{Identity: "https://github.com/acme/b/.github/workflows/r.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + + base := VerificationPolicy{ + RequireSignature: true, + TrustSets: []TrustSet{ + {Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"b.pub", "a.pub"}}, + {Keys: []string{"c.pub"}}, + }, + } + fingerprint, err := base.Fingerprint() + require.NoError(t, err) + assert.Regexp(t, `^sha256:[0-9a-f]{64}$`, fingerprint) + + reordered := VerificationPolicy{ + RequireSignature: true, + TrustSets: []TrustSet{ + {Keys: []string{"c.pub"}}, + {Identities: []TrustedIdentity{identityB, identityA}, Keys: []string{"a.pub", "b.pub"}}, + }, + } + same, err := reordered.Fingerprint() + require.NoError(t, err) + assert.Equal(t, fingerprint, same, "order of trust sets, identities, and keys does not matter") + assert.Equal(t, "b.pub", base.TrustSets[0].Keys[0], "fingerprinting does not reorder the caller's policy") + + tests := []struct { + name string + mutate func(p *VerificationPolicy) + }{ + {"require signature", func(p *VerificationPolicy) { p.RequireSignature = false }}, + {"offline", func(p *VerificationPolicy) { p.Offline = true }}, + {"dropped trust set", func(p *VerificationPolicy) { p.TrustSets = p.TrustSets[:1] }}, + {"different key", func(p *VerificationPolicy) { + p.TrustSets = []TrustSet{{Keys: []string{"x.pub"}}, {Keys: []string{"c.pub"}}} + }}, + {"unioned instead of separate sets", func(p *VerificationPolicy) { + p.TrustSets = []TrustSet{{Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"a.pub", "b.pub", "c.pub"}}} + }}, + } + for _, tt := range tests { + t.Run("differs with "+tt.name, func(t *testing.T) { + policy := VerificationPolicy{RequireSignature: true, TrustSets: []TrustSet{ + {Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"a.pub", "b.pub"}}, + {Keys: []string{"c.pub"}}, + }} + tt.mutate(&policy) + other, err := policy.Fingerprint() + require.NoError(t, err) + assert.NotEqual(t, fingerprint, other) + }) + } +} + +func TestHasTrustConstraints(t *testing.T) { + assert.False(t, VerificationPolicy{}.HasTrustConstraints()) + assert.False(t, VerificationPolicy{RequireSignature: true, TrustSets: []TrustSet{{}}}.HasTrustConstraints()) + assert.True(t, VerificationPolicy{TrustSets: []TrustSet{{Keys: []string{"a.pub"}}}}.HasTrustConstraints()) + assert.True(t, VerificationPolicy{TrustSets: []TrustSet{{Identities: []TrustedIdentity{{Identity: "i", Issuer: "o"}}}}}.HasTrustConstraints()) +} diff --git a/pkg/artifact/prepared.go b/pkg/artifact/prepared.go new file mode 100644 index 00000000000..741d5e30417 --- /dev/null +++ b/pkg/artifact/prepared.go @@ -0,0 +1,92 @@ +package artifact + +import ( + "context" + "fmt" + "io" + "sync" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// BlobRole identifies what a staged blob contains. +type BlobRole string + +// Staged blob roles. +const ( + // BlobPayload is payload content (an archive or one file). + BlobPayload BlobRole = "payload" + + // BlobDocument is a README or CHANGELOG document. + BlobDocument BlobRole = "document" +) + +// StagedBlob is one blob ready to be uploaded. Open returns a fresh reader each time, so a retry +// within one invocation reuses the same prepared content. +type StagedBlob struct { + // Descriptor identifies the blob by media type, digest, and size. + Descriptor ocispec.Descriptor + + // Role is payload or document. + Role BlobRole + + // DocumentRole is the document role when Role is [BlobDocument]. + DocumentRole DocumentRole + + // Path is the inventory path for per-file payloads; empty for archives. + Path string + + // Open returns a new reader positioned at the start of the blob. The caller closes it. + Open func(ctx context.Context) (io.ReadCloser, error) +} + +// PreparedArtifact is the output of a [Producer]: a sealed Record plus staged blobs. The preparer +// captured one UTC preparation time and one logical operation ID before the first attempt. +// +// A PreparedArtifact owns temporary resources. Whoever finally consumes it must call Close, which +// runs the cleanup function exactly once. It must not be copied after first use. +type PreparedArtifact struct { + // Record is the sealed Record to publish. + Record Record + + // IntentDigest is the Record's intent digest. + IntentDigest digest.Digest + + // OperationID is the logical operation identifier, stable across retries. + OperationID string + + // PreparedAt is the single UTC preparation time. + PreparedAt time.Time + + // Packaging is the packaging mode of the staged payload. + Packaging Packaging + + // Blobs are the staged payload and document blobs. + Blobs []StagedBlob + + // Cleanup releases temporary resources owned by this operation; may be nil. + Cleanup func() error + + closeOnce sync.Once + closeErr error +} + +// Close releases the temporary resources owned by the PreparedArtifact. It is idempotent: the +// cleanup function runs at most once and later calls return the first result. +func (p *PreparedArtifact) Close() error { + defer perf.Track(nil, "artifact.PreparedArtifact.Close")() + + p.closeOnce.Do(func() { + if p.Cleanup != nil { + if err := p.Cleanup(); err != nil { + p.closeErr = fmt.Errorf("%w: cleanup: %w", errUtils.ErrArtifactStreamFailed, err) + } + } + }) + return p.closeErr +} diff --git a/pkg/artifact/producer.go b/pkg/artifact/producer.go new file mode 100644 index 00000000000..f55863b9ea5 --- /dev/null +++ b/pkg/artifact/producer.go @@ -0,0 +1,53 @@ +package artifact + +import ( + "context" + "time" +) + +//go:generate mockgen -package artifact -destination mock_producer.go github.com/cloudposse/atmos/pkg/artifact Producer + +// PrepareRequest carries everything a Producer needs. The service captures PreparedAt (UTC) and +// OperationID once, before the first attempt, and reuses them on every retry. +type PrepareRequest struct { + // Source is the local source file or directory. + Source string + + // Name is the explicit artifact name; empty lets the producer supply one. + Name string + + // Version is the explicit version; empty lets the producer supply one or the operation fails. + Version string + + // Kind is the artifact kind; empty means [KindFiles]. + Kind string + + // Metadata is the effective user metadata (description, labels) merged by the service. + Metadata Metadata + + // Readme overrides the README source path, relative to the source directory. + Readme string + + // Changelog overrides the CHANGELOG source path, relative to the source directory. + Changelog string + + // Packaging is the target repository's effective packaging mode. + Packaging Packaging + + // OperationID is the logical operation identifier. + OperationID string + + // PreparedAt is the single UTC preparation time. + PreparedAt time.Time +} + +// Producer turns a source into a sealed Record plus staged blobs. Producers walk sources with +// lstat, never follow links outside the source, and never mutate the user's source tree. The +// caller owns the returned PreparedArtifact and must Close it. +type Producer interface { + // Kind returns the artifact kind this producer supplies, for example "files". + Kind() string + + // Prepare reads the source and returns a stable, sealed PreparedArtifact. + Prepare(ctx context.Context, request PrepareRequest) (*PreparedArtifact, error) +} diff --git a/pkg/artifact/record.go b/pkg/artifact/record.go new file mode 100644 index 00000000000..abca49d080c --- /dev/null +++ b/pkg/artifact/record.go @@ -0,0 +1,159 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" +) + +// Record is the portable, canonical description of one published artifact. It is independent of +// the repository alias, credentials, resolved storage paths, upload duration, and archive or OCI +// envelope digests; those belong to [Representation] and [OperationResult]. +// +// A Record never contains its own hash. Serialize it with [Record.Canonical]; the struct's field +// order below is the canonical field order and is frozen by test vectors. +type Record struct { + // SchemaVersion is always [SchemaVersion] (1). + SchemaVersion int `json:"schema_version"` + + // Name is the artifact name, e.g. "components/terraform/vpc". + Name string `json:"name"` + + // Version is the explicit version label. + Version string `json:"version"` + + // Kind is the artifact kind, e.g. "files" or "component/terraform". + Kind string `json:"kind"` + + // CreatedAt is the creation time, captured once per publication in UTC. + CreatedAt Timestamp `json:"created_at"` + + // Metadata is the user-visible description, labels, and optional provenance. + Metadata Metadata `json:"metadata"` + + // Inventory lists every payload entry, sorted by path in the canonical form. + Inventory []Entry `json:"inventory"` + + // Documents lists the README and CHANGELOG descriptors, sorted by role in the canonical form. + Documents []Document `json:"documents"` + + // ContentDigest is the SHA-256 of the canonical inventory and document descriptors. + ContentDigest digest.Digest `json:"content_digest"` + + // KindData is optional kind-specific data. It is held as canonical JSON; see [KindData]. + KindData KindData `json:"kind_data,omitempty"` +} + +// Metadata is the descriptive metadata of a Record. No stack, component, or SHA is required. +type Metadata struct { + // Description is a short human-readable description. + Description string `json:"description,omitempty"` + + // Labels are opaque string key/value pairs set by the user. + Labels map[string]string `json:"labels,omitempty"` + + // Provenance is generated CI and source context. It is part of the record digest but excluded + // from the intent digest so that retries and no-op detection are not defeated by run IDs. + Provenance *Provenance `json:"provenance,omitempty"` +} + +// Provenance carries optional CI and source context captured at publication time. +type Provenance struct { + // CI describes the CI run that produced the artifact. + CI *CIProvenance `json:"ci,omitempty"` + + // Source describes the source tree the artifact was built from. + Source *SourceProvenance `json:"source,omitempty"` +} + +// CIProvenance identifies the CI run that published an artifact. +type CIProvenance struct { + // Provider is the CI provider name, e.g. "github". + Provider string `json:"provider,omitempty"` + + // RunID is the provider's run identifier. + RunID string `json:"run_id,omitempty"` + + // RunURL links to the run. + RunURL string `json:"run_url,omitempty"` + + // Job is the job or step name. + Job string `json:"job,omitempty"` +} + +// SourceProvenance identifies the source revision an artifact was built from. +type SourceProvenance struct { + // Repository is the source repository identifier or URL (without credentials). + Repository string `json:"repository,omitempty"` + + // Commit is the source commit SHA. + Commit string `json:"commit,omitempty"` + + // Ref is the branch or tag name. + Ref string `json:"ref,omitempty"` +} + +// EntryType is the type of an inventory entry. +type EntryType string + +// Inventory entry types. Devices, sockets, and other special files are never representable. +const ( + // EntryFile is a regular file. + EntryFile EntryType = "file" + + // EntryDirectory is a directory. + EntryDirectory EntryType = "directory" + + // EntrySymlink is a symbolic link whose target stays inside the package. + EntrySymlink EntryType = "symlink" +) + +// Entry is one inventory item. Hard links are represented as independent regular files. +type Entry struct { + // Path is the relative, slash-separated path inside the package. + Path string `json:"path"` + + // Type is file, directory, or symlink. + Type EntryType `json:"type"` + + // Size is the file size in bytes; zero for directories and symlinks. + Size int64 `json:"size"` + + // Mode holds ordinary permission bits (0o777 mask); privileged bits are always cleared. + // Symlinks have mode zero so equivalent trees share one canonical form. + Mode uint32 `json:"mode"` + + // Digest is the SHA-256 of a regular file's bytes; empty for other types. + Digest digest.Digest `json:"digest,omitempty"` + + // Target is a symlink's relative target; empty for other types. + Target string `json:"target,omitempty"` +} + +// DocumentRole is the role of a package document. +type DocumentRole string + +// Package document roles. +const ( + // DocumentReadme is the package README. + DocumentReadme DocumentRole = "readme" + + // DocumentChangelog is the package CHANGELOG. + DocumentChangelog DocumentRole = "changelog" +) + +// Document describes one distribution document. Catalogs carry descriptors, never full Markdown. +type Document struct { + // Role is readme or changelog. At most one document exists per role. + Role DocumentRole `json:"role"` + + // SourcePath is the original source path, relative to the publish source. + SourcePath string `json:"source_path"` + + // MediaType is the document media type, normally [DocumentMediaType]. + MediaType string `json:"media_type"` + + // Size is the document size in bytes (at most [MaxDocumentSize]). + Size int64 `json:"size"` + + // Digest is the SHA-256 of the document bytes. + Digest digest.Digest `json:"digest"` +} diff --git a/pkg/artifact/record_test.go b/pkg/artifact/record_test.go new file mode 100644 index 00000000000..93100003def --- /dev/null +++ b/pkg/artifact/record_test.go @@ -0,0 +1,260 @@ +package artifact + +import ( + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestEntryValidate(t *testing.T) { + fileDigest := digest.FromString("x") + tests := []struct { + name string + entry Entry + want error + }{ + {"file", Entry{Path: "a/b.txt", Type: EntryFile, Size: 1, Mode: 0o644, Digest: fileDigest}, nil}, + {"executable file", Entry{Path: "run.sh", Type: EntryFile, Mode: 0o755, Digest: fileDigest}, nil}, + {"directory", Entry{Path: "a", Type: EntryDirectory, Mode: 0o755}, nil}, + {"contained symlink", Entry{Path: "a/link", Type: EntrySymlink, Target: "../b.txt"}, nil}, + {"symlink to own directory", Entry{Path: "a/link", Type: EntrySymlink, Target: "."}, nil}, + {"symlink empty segment", Entry{Path: "a/link", Type: EntrySymlink, Target: "b//c"}, errUtils.ErrArtifactUnsafePath}, + {"symlink escaping the package", Entry{Path: "a/link", Type: EntrySymlink, Target: "../../b"}, errUtils.ErrArtifactUnsafePath}, + {"symlink at root escaping", Entry{Path: "link", Type: EntrySymlink, Target: "../x"}, errUtils.ErrArtifactUnsafePath}, + {"absolute symlink target", Entry{Path: "link", Type: EntrySymlink, Target: "/etc/passwd"}, errUtils.ErrArtifactUnsafePath}, + {"empty symlink target", Entry{Path: "link", Type: EntrySymlink}, errUtils.ErrArtifactUnsafePath}, + {"symlink with mode", Entry{Path: "link", Type: EntrySymlink, Target: "a", Mode: 0o777}, errUtils.ErrArtifactRecordInvalid}, + {"setuid bit", Entry{Path: "su", Type: EntryFile, Mode: 0o4755, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"file without digest", Entry{Path: "a", Type: EntryFile}, errUtils.ErrArtifactRecordInvalid}, + {"file with bad digest", Entry{Path: "a", Type: EntryFile, Digest: "sha256:zz"}, errUtils.ErrArtifactRecordInvalid}, + {"file with negative size", Entry{Path: "a", Type: EntryFile, Size: -1, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"directory with digest", Entry{Path: "a", Type: EntryDirectory, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"device type", Entry{Path: "dev", Type: "device"}, errUtils.ErrArtifactPackagingInvalid}, + {"absolute path", Entry{Path: "/a", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"traversal path", Entry{Path: "a/../b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"empty segment", Entry{Path: "a//b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"backslash path", Entry{Path: `a\b`, Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"control character", Entry{Path: "a\x00b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"invalid UTF-8", Entry{Path: "a\xffb", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"reserved namespace", Entry{Path: ReservedNamespace + "/v1/refs/x.json", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"reserved namespace prefix only as first segment", Entry{Path: "a/" + ReservedNamespace, Type: EntryDirectory}, nil}, + {"unicode name", Entry{Path: "ünï/名前.txt", Type: EntryFile, Digest: fileDigest}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.entry.Validate() + if tt.want == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.want) + }) + } +} + +// TestValidateInventory_SymlinkResolution resolves every link target through the in-package links, +// component by component, so a ".." after a symlinked directory cannot leave the package. +func TestValidateInventory_SymlinkResolution(t *testing.T) { + file := func(p string) Entry { return Entry{Path: p, Type: EntryFile, Digest: digest.FromString(p)} } + dir := func(p string) Entry { return Entry{Path: p, Type: EntryDirectory} } + link := func(p, target string) Entry { return Entry{Path: p, Type: EntrySymlink, Target: target} } + + tests := []struct { + name string + inventory []Entry + wantErr string + }{ + {name: "no symlinks", inventory: []Entry{file("a"), dir("d"), file("d/b")}}, + {name: "dangling target is allowed", inventory: []Entry{link("l", "missing/file")}}, + {name: "chain of links to a file", inventory: []Entry{file("lib.so.1.2.3"), link("lib.so.1", "lib.so.1.2.3"), link("lib.so", "lib.so.1")}}, + {name: "link to a directory link", inventory: []Entry{dir("real"), file("real/x"), link("alias", "real"), link("via", "alias/x")}}, + {name: "dotdot after a symlinked directory stays inside", inventory: []Entry{dir("dir"), link("dir/up", ".."), link("l", "dir/up/inside")}}, + {name: "dotdot through a link that resolves deeper stays inside", inventory: []Entry{dir("a"), dir("a/b"), link("deep", "a/b"), link("l", "deep/../x")}}, + {name: "link to its own directory", inventory: []Entry{dir("d"), link("d/l", ".")}}, + { + name: "dotdot after a link to the parent escapes", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("link", "dir/up/../outside")}, + wantErr: "escapes the package", + }, + { + name: "escape is independent of entry order", + inventory: []Entry{link("link", "dir/up/../outside"), link("dir/up", ".."), dir("dir")}, + wantErr: "escapes the package", + }, + { + name: "escape through a chain of two links", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("l1", "dir/up"), link("l2", "l1/../out")}, + wantErr: "escapes the package", + }, + { + name: "escape from a deep link through a link to the parent", + inventory: []Entry{dir("a"), dir("a/b"), link("a/b/up", "../.."), link("a/b/l", "up/../out")}, + wantErr: "escapes the package", + }, + { + name: "link resolving to a link that escapes", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("a/l", "../dir/up/../x")}, + wantErr: "escapes the package", + }, + {name: "absolute target", inventory: []Entry{link("l", "/etc/passwd")}, wantErr: "relative"}, + {name: "lexical escape", inventory: []Entry{link("l", "../x")}, wantErr: "escapes the package"}, + {name: "self loop", inventory: []Entry{link("l", "l")}, wantErr: "loop"}, + {name: "two link loop", inventory: []Entry{link("a", "b"), link("b", "a")}, wantErr: "loop"}, + {name: "loop through a directory component", inventory: []Entry{dir("d"), link("d/l", "../d/l")}, wantErr: "loop"}, + {name: "loop reached through a longer path", inventory: []Entry{link("a", "b/x"), link("b", "a")}, wantErr: "loop"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateInventory(tt.inventory) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), tt.wantErr) + + _, digestErr := ComputeContentDigest(tt.inventory, nil) + require.ErrorIs(t, digestErr, errUtils.ErrArtifactUnsafePath, "the digest admission gate applies the same check") + }) + } +} + +func TestValidateInventoryStructure(t *testing.T) { + file := func(p string) Entry { return Entry{Path: p, Type: EntryFile, Digest: digest.FromString(p)} } + dir := func(p string) Entry { return Entry{Path: p, Type: EntryDirectory} } + + _, err := ComputeContentDigest([]Entry{file("a"), file("a")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "duplicate path") + + _, err = ComputeContentDigest([]Entry{file("a"), file("a/b")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "entry nested under a file") + + _, err = ComputeContentDigest([]Entry{{Path: "l", Type: EntrySymlink, Target: "x"}, file("l/b")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "entry nested under a symlink") + + _, err = ComputeContentDigest([]Entry{dir("a"), file("a/b")}, nil) + require.NoError(t, err) + + // The digest is order independent for the same logical inventory. + first, err := ComputeContentDigest([]Entry{file("b"), file("a")}, nil) + require.NoError(t, err) + second, err := ComputeContentDigest([]Entry{file("a"), file("b")}, nil) + require.NoError(t, err) + assert.Equal(t, first, second) +} + +func TestDocumentValidate(t *testing.T) { + good := Document{Role: DocumentReadme, SourcePath: "README.md", MediaType: DocumentMediaType, Size: 10, Digest: digest.FromString("r")} + require.NoError(t, good.Validate()) + + tests := []struct { + name string + mutate func(d *Document) + want error + }{ + {"unknown role", func(d *Document) { d.Role = "license" }, errUtils.ErrArtifactRecordInvalid}, + {"absolute source", func(d *Document) { d.SourcePath = "/README.md" }, errUtils.ErrArtifactRecordInvalid}, + {"traversing source", func(d *Document) { d.SourcePath = "../README.md" }, errUtils.ErrArtifactRecordInvalid}, + {"no media type", func(d *Document) { d.MediaType = "" }, errUtils.ErrArtifactRecordInvalid}, + {"negative size", func(d *Document) { d.Size = -1 }, errUtils.ErrArtifactRecordInvalid}, + {"oversize document", func(d *Document) { d.Size = MaxDocumentSize + 1 }, errUtils.ErrArtifactTooLarge}, + {"bad digest", func(d *Document) { d.Digest = "nope" }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + doc := good + tt.mutate(&doc) + require.ErrorIs(t, doc.Validate(), tt.want) + }) + } + + boundary := good + boundary.Size = MaxDocumentSize + require.NoError(t, boundary.Validate(), "exactly the limit is allowed") + + _, err := ComputeContentDigest(nil, []Document{good, good}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, "duplicate role") +} + +func TestRecordValidate(t *testing.T) { + tests := []struct { + name string + mutate func(r *Record) + want error + }{ + {"valid", func(*Record) {}, nil}, + {"wrong schema version", func(r *Record) { r.SchemaVersion = 2 }, errUtils.ErrArtifactRecordInvalid}, + {"bad name", func(r *Record) { r.Name = "../x" }, errUtils.ErrArtifactUnsafePath}, + {"bad version", func(r *Record) { r.Version = "1+x" }, errUtils.ErrArtifactRefInvalid}, + {"bad kind", func(r *Record) { r.Kind = "Files" }, errUtils.ErrArtifactRecordInvalid}, + {"missing created_at", func(r *Record) { r.CreatedAt = Timestamp{} }, errUtils.ErrArtifactRecordInvalid}, + {"empty label key", func(r *Record) { r.Metadata.Labels = map[string]string{"": "v"} }, errUtils.ErrArtifactRecordInvalid}, + {"stale content digest", func(r *Record) { r.ContentDigest = digest.FromString("other") }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + record := vectorRecordB(t) + tt.mutate(&record) + err := record.Validate() + if tt.want == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.want) + _, canonicalErr := record.Canonical() + require.ErrorIs(t, canonicalErr, tt.want, "an invalid record is never serialized") + }) + } +} + +func TestRecordOversizeRejected(t *testing.T) { + record := vectorRecordB(t) + record.Metadata.Description = strings.Repeat("d", int(MaxRecordSize)) + _, err := record.Canonical() + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + _, err = record.RecordDigest() + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestEmptyProvenanceNormalizesAway(t *testing.T) { + bare := vectorRecordB(t) + withEmpty := vectorRecordB(t) + withEmpty.Metadata.Provenance = &Provenance{CI: &CIProvenance{}, Source: &SourceProvenance{}} + withEmpty.Metadata.Labels = map[string]string{} + + first, err := bare.Canonical() + require.NoError(t, err) + second, err := withEmpty.Canonical() + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) +} + +func TestSealSetsSchemaVersionAndContentDigest(t *testing.T) { + record := Record{Name: "a", Version: "1", Kind: KindFiles, CreatedAt: NewTimestamp(time.Unix(1, 0))} + require.NoError(t, record.Seal()) + assert.Equal(t, SchemaVersion, record.SchemaVersion) + assert.NotEmpty(t, record.ContentDigest) +} + +func TestTimestamp(t *testing.T) { + zone := time.FixedZone("x", 5*3600) + stamp := NewTimestamp(time.Date(2026, 10, 30, 17, 0, 0, 999, zone)) + assert.Equal(t, "2026-10-30T12:00:00Z", stamp.String()) + assert.Equal(t, time.UTC, stamp.Time().Location()) + + var parsed Timestamp + require.NoError(t, parsed.UnmarshalJSON([]byte(`"2026-10-30T12:00:00Z"`))) + assert.Equal(t, stamp.Time(), parsed.Time()) + + for _, bad := range []string{`"2026-10-30T12:00:00+00:00"`, `"2026-10-30T12:00:00.5Z"`, `"2026-10-30 12:00:00"`, `12`, `""`} { + var ts Timestamp + require.ErrorIs(t, ts.UnmarshalJSON([]byte(bad)), errUtils.ErrArtifactRecordInvalid, bad) + } + assert.True(t, Timestamp{}.IsZero()) +} diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go new file mode 100644 index 00000000000..09a498cd3dd --- /dev/null +++ b/pkg/artifact/redact.go @@ -0,0 +1,184 @@ +package artifact + +import ( + "net/url" + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// RedactedValue replaces a value that must not be echoed because it may carry credentials. +const RedactedValue = "" + +// RedactURL returns raw with any userinfo removed, so that a mistakenly pasted credential is not +// repeated in an error message or a log line. It is the one implementation every caller shares. +// +// RedactURL is fail-closed. A value without an "@" cannot hold userinfo and is returned as it is. +// A value whose "@" signs all sit in the authority has its userinfo cut at the last one, as +// RFC 3986 requires, and is returned when what is left is a well-formed host. A value that cannot +// be proven free of a credential is replaced by [RedactedValue] as a whole: an "@" after the +// authority that does not introduce a complete digest (a password that held "/", "?", "#" or "\" +// may have closed there), scheme-less userinfo such as "user:secret@registry.example", a host that +// is not shaped like a host, an address that does not parse, or an address with more "@" signs +// than a real one holds or longer than any real one (see [maxBoundaries] and [maxAddressLength]), +// which is withheld before any work that grows with it starts. The diagnostic value of an error +// drops for such inputs by design; a credential is never echoed. A forced-getter prefix such as +// "git::" is kept. +func RedactURL(raw string) string { + defer perf.Track(nil, "artifact.RedactURL")() + + if !strings.Contains(raw, "@") { + return raw + } + a := parseAddress(raw) + if a.overflow || len(a.stray) > 0 { + return RedactedValue + } + if len(a.inside) == 0 { + // Only digest introducers: the address carries no userinfo, but it is still checked. + if a.wellFormed(0) { + return raw + } + return RedactedValue + } + // Scheme-less userinfo cannot be told from a host and a path, so it is never cut out. + if a.scheme == "" { + return RedactedValue + } + start := a.inside[len(a.inside)-1] + 1 + if !a.wellFormed(start) { + return RedactedValue + } + return a.prefix + a.scheme + a.rest[start:] +} + +// wellFormed reports whether the address from offset start of its remainder on has a safe host and +// parses as a URL. +func (a *address) wellFormed(start int) bool { + if !a.hostIsSafe(start) { + return false + } + if a.scheme == "" { + return true + } + _, err := url.Parse(headOf(a.scheme + a.rest[start:])) + return err == nil +} + +// RedactCredentials removes credentials from text, an error message or log line that may quote +// raw, its userinfo, its password on its own, or any other URL. It is fail-closed, in three steps: +// +// 1. The credential carried by raw is cut out of text wherever it is quoted: the userinfo with its +// trailing "@", written as in a URL, decoded, or escaped for a quote, and the password on its +// own together with every fragment of it that a delimiter cut loose. The password is the part +// of the userinfo as written after its first ":", decoded separately. A user name without a +// password is not replaced when it stands alone, since it is usually a service account such as +// "git". +// 2. Every other URL-shaped word of text that still holds an "@" which cannot be proven to +// introduce a digest is replaced by [RedactedValue] as a whole, and so is every word that still +// holds an "@" when raw carried a credential. +// 3. If any part of the password is still present, the whole text is replaced by [RedactedValue]. +// +// A raw address with more "@" signs than a real one holds, or longer than any real one, has no +// credential that can be listed at a bounded cost, so the whole text is replaced by [RedactedValue] +// without taking any of the steps above. +func RedactCredentials(text, raw string) string { + defer perf.Track(nil, "artifact.RedactCredentials")() + + a := parseAddress(raw) + if a.overflow { + // The credential of an address that is past the bounds cannot be listed, so nothing in the + // text can be proven free of it. + return RedactedValue + } + secret := a.credential() + if !secret.empty() { + text = secret.scrub(text) + } + text = redactWords(text, !secret.empty()) + if secret.survives(text) { + return RedactedValue + } + return text +} + +// RedactError returns err with its message scrubbed of the credentials of every address in raws +// and of any other URL-shaped word that carries userinfo (see [RedactCredentials]). The result keeps +// err in its chain, so errors.Is and errors.As still see the cause, and it is err itself when the +// message needs no change. +func RedactError(err error, raws ...string) error { + defer perf.Track(nil, "artifact.RedactError")() + + if err == nil { + return nil + } + message := err.Error() + scrubbed := message + for _, raw := range raws { + scrubbed = RedactCredentials(scrubbed, raw) + } + if len(raws) == 0 { + scrubbed = RedactCredentials(scrubbed, "") + } + if scrubbed == message { + return err + } + return &redactedError{message: scrubbed, err: err} +} + +// RedactDecodeError is [RedactError] for the error of decoding raw, a configuration value as it was +// parsed: a decode error quotes the keys and values it rejects, so every string held by raw, keys +// included and at any depth, is a credential candidate when the message is scrubbed. +func RedactDecodeError(err error, raw any) error { + defer perf.Track(nil, "artifact.RedactDecodeError")() + + if err == nil { + return nil + } + return RedactError(err, credentialCandidates(stringValues(raw))...) +} + +// credentialCandidates returns the distinct strings of values that hold an "@", the only ones that +// can carry userinfo, so a large configuration costs one scrub per distinct candidate and not one +// per string it holds. +func credentialCandidates(values []string) []string { + seen := make(map[string]bool, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + if strings.Contains(value, "@") && !seen[value] { + seen[value] = true + out = append(out, value) + } + } + return out +} + +// redactedError is an error whose text has had a credential removed. It keeps the original error in +// its chain so errors.Is and errors.As still see the cause. +type redactedError struct { + message string + err error +} + +//nolint:lintroller // Trivial getter with no I/O. +func (e *redactedError) Error() string { return e.message } + +//nolint:lintroller // Trivial getter with no I/O. +func (e *redactedError) Unwrap() error { return e.err } + +// IsSCPAddress reports whether s is an SCP-style git address such as "git@github.com:org/repo.git". +// Its user carries no password, so it holds no credential. +func IsSCPAddress(s string) bool { + defer perf.Track(nil, "artifact.IsSCPAddress")() + + return scpPattern.MatchString(s) +} + +// SplitGetterPrefix separates a forced-getter prefix ("git::") from the URL that follows it. The +// prefix is recognized only when a URL scheme follows it, so a "::" inside userinfo is never taken +// for one. The prefix is empty, and body is raw, when there is none. +func SplitGetterPrefix(raw string) (prefix, body string) { + defer perf.Track(nil, "artifact.SplitGetterPrefix")() + + return splitGetterPrefix(raw) +} diff --git a/pkg/artifact/redact_bounds_test.go b/pkg/artifact/redact_bounds_test.go new file mode 100644 index 00000000000..bc38b306918 --- /dev/null +++ b/pkg/artifact/redact_bounds_test.go @@ -0,0 +1,126 @@ +package artifact + +import ( + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // The hostile boundaries count is far above any cap a legitimate address needs, so the tests below hold + // whatever the cap is. + hostileBoundaries = 4096 + // The allocation budget is the memory a single hostile input may cost the whole set of entry points. + // The expansion this guards against grows with the square of the input and costs hundreds of + // megabytes at the input size used here, so the budget is generous and still far below it. + allocationBudget = 32 << 20 + // The generous deadline only catches a run that never finishes. It is not a performance assertion. + generousDeadline = 60 * time.Second +) + +// hostileAddress returns an address whose userinfo holds count "@" signs. +func hostileAddress(count int) string { + return "https://usr7:S3cr3tA" + strings.Repeat("@x", count) + "@example.com/a" +} + +// TestRedact_ManyBoundariesFailClosed checks that an address with more "@" signs than any real +// address holds is withheld whole, by every entry point, without expanding a credential candidate +// for each of them. +func TestRedact_ManyBoundariesFailClosed(t *testing.T) { + raw := hostileAddress(hostileBoundaries) + word := "https://usr7:S3cr3tA" + strings.Repeat("@x", hostileBoundaries) + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + start := time.Now() + + assert.Equal(t, RedactedValue, RedactURL(raw), "RedactURL withholds the address") + assert.Equal(t, RedactedValue, RedactCredentials("fetch "+raw+" failed", raw), "RedactCredentials withholds the text") + assert.Equal(t, RedactedValue, RedactCredentials("unrelated text", raw), "the text is withheld even when it does not quote the address") + err := RedactError(errSentinelf(raw), raw) + assert.Equal(t, RedactedValue, err.Error(), "RedactError withholds the message") + assert.ErrorIs(t, err, errSentinel, "the cause stays in the chain") + assert.Equal(t, "see "+RedactedValue+" ok", RedactCredentials("see "+word+" ok", ""), "a word with too many boundaries is withheld on its own") + assert.NotContains(t, RedactCredentials("see "+word+" ok", ""), "S3cr3tA") + + runtime.ReadMemStats(&after) + assert.Less(t, time.Since(start), generousDeadline) + assert.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(allocationBudget), "the work is bounded by the cap, not by the input") +} + +// errSentinelf returns an error that wraps errSentinel and quotes text. +func errSentinelf(text string) error { + return "edError{text: text} +} + +type quotedError struct{ text string } + +func (e *quotedError) Error() string { return "fetch " + e.text + " failed" } +func (e *quotedError) Unwrap() error { return errSentinel } + +// TestRedact_AddressBoundsKeepRealAddresses checks that the bounds do not touch an address that is +// within them, so the credential is still cut out, not the whole address withheld. +func TestRedact_AddressBoundsKeepRealAddresses(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + {name: "user and password", raw: "https://usr7:S3cr3tA@example.com/a", want: "https://example.com/a"}, + {name: "password with a few at signs", raw: hostileAddress(3), want: "https://example.com/a"}, + {name: "digest reference", raw: "oci://ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, RedactURL(tt.raw)) + scrubbed := RedactCredentials("fetch "+tt.raw+" failed", tt.raw) + assert.NotContains(t, scrubbed, "S3cr3tA") + assert.NotEqual(t, RedactedValue, scrubbed, "text within the bounds is scrubbed, not withheld") + }) + } + require.Positive(t, hostileBoundaries) +} + +// TestRedact_AddressBoundsEdges checks the two bounds exactly where they are crossed: an address +// at a bound is redacted in parts, and one past it is withheld whole. +func TestRedact_AddressBoundsEdges(t *testing.T) { + longPath := strings.Repeat("a", maxAddressLength) + tests := []struct { + name string + raw string + wantURL string + wantWithheld bool + }{ + {name: "at the boundary cap", raw: hostileAddress(maxBoundaries - 1), wantURL: "https://example.com/a"}, + {name: "past the boundary cap", raw: hostileAddress(maxBoundaries), wantWithheld: true}, + {name: "past the length cap with a credential", raw: "https://usr7:S3cr3tA@example.com/" + longPath, wantWithheld: true}, + {name: "long address without an at sign is not a credential", raw: "https://example.com/" + longPath, wantURL: "https://example.com/" + longPath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scrubbed := RedactCredentials("fetch failed for S3cr3tA at "+tt.raw[:min(len(tt.raw), 40)], tt.raw) + if strings.Contains(tt.raw, "S3cr3tA") { + assert.NotContains(t, scrubbed, "S3cr3tA") + } + if tt.wantWithheld { + assert.Equal(t, RedactedValue, RedactURL(tt.raw)) + assert.Equal(t, RedactedValue, scrubbed) + return + } + assert.Equal(t, tt.wantURL, RedactURL(tt.raw)) + assert.NotEqual(t, RedactedValue, scrubbed) + }) + } +} + +// TestCredentialCandidates checks that only distinct strings that can carry userinfo are scrubbed +// for, in the order they were first seen. +func TestCredentialCandidates(t *testing.T) { + got := credentialCandidates([]string{"plain", "a@b", "a@b", "", "c:d@e", "plain"}) + assert.Equal(t, []string{"a@b", "c:d@e"}, got) + assert.Empty(t, credentialCandidates(nil)) +} diff --git a/pkg/artifact/redact_grammar.go b/pkg/artifact/redact_grammar.go new file mode 100644 index 00000000000..3818f7eb014 --- /dev/null +++ b/pkg/artifact/redact_grammar.go @@ -0,0 +1,227 @@ +package artifact + +import ( + "regexp" + "strings" + "unicode" +) + +// The redaction helpers in this package are fail-closed. They pass a value through only when it +// matches the small grammar below, which they can prove carries no credential, and replace +// everything else, either the whole value or the whole suspicious token, with [RedactedValue]. +// +// An address is "[getter::][scheme://|//]authority[/path][?query][#fragment]". The authority ends +// at the first "/", "?", "#" or "\". Every "@" in the authority ends a userinfo, and the last one +// wins. An "@" after the authority is safe only as the introducer of a complete digest +// ("@sha256:<64 hex digits>") that follows a host and a lower-case repository path, as in +// "ghcr.io/acme/catalog@sha256:...": any other one may close a password that held a "/", "?", "#" +// or "\", so the value cannot be proven free of a credential. +var ( + // The getter prefix pattern matches the forced-getter prefix of a go-getter address, such as + // "git::" in "git::https://example.com/repo.git". It only matches when a URL scheme follows, so + // a "::" inside scheme-less userinfo ("user::secret@host") is never mistaken for a prefix. + getterPrefixPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*::[A-Za-z][A-Za-z0-9+.-]*://`) + + // The scheme pattern matches the "scheme://" that starts an address. + schemePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*://`) + + // The authority pattern is the only shape of host that survives redaction: a name, an address, + // or a bracketed IPv6 literal, with an optional port. Anything else may be a password fragment. + authorityPattern = regexp.MustCompile(`^(?:[A-Za-z0-9_.-]+|\[[0-9A-Za-z:.%]+\])(?::[0-9]*)?$`) + + // The repository path pattern matches the path in front of the "@" of a digest reference: the + // lower-case path of an OCI repository with an optional tag. A path that holds anything else + // might be a password that held a "/". + repositoryPathPattern = regexp.MustCompile(`^/?[a-z0-9]+(?:[._/-]+[a-z0-9]+)*(?::[A-Za-z0-9_][A-Za-z0-9._-]*)?$`) + + // The SCP pattern matches an SCP-style git address, "user@host:path", whose user carries no + // password: a ":" before the "@" would make it a credential. + scpPattern = regexp.MustCompile(`^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:[^@\s]*$`) +) + +const ( + // The authority terminators end the authority of an address. + authorityTerminators = `/?#\` + + // The minimum fragment length is the shortest password fragment that is removed on its own from free + // text. Shorter fragments would mangle ordinary words, and they identify nothing. + minFragmentLength = 4 + + // The maximum boundaries is the most "@" signs an address may hold and still be redacted in + // parts. A real address holds a handful: a userinfo, perhaps a password with an "@" in it, and a + // digest. Every "@" is a candidate end of a credential, and each candidate expands into every + // form an error may quote, so the work grows with the number of them. An address with more is + // withheld whole. + maxBoundaries = 64 + + // The maximum address length is the longest address, in bytes, that is redacted in parts when it + // holds an "@". The expansion of a credential candidate is as long as the prefix it came from, so + // the length bounds what each of the [maxBoundaries] candidates costs. + maxAddressLength = 8 << 10 + + // The scheme separator separates a scheme from its authority. + schemeSeparator = "://" + + // The file scheme is the only scheme whose authority may be empty. + fileScheme = "file://" +) + +// maxDigestAlgorithmLength is the length of the longest name in [digestHexLengths]. +const maxDigestAlgorithmLength = len("sha512") + +// digestHexLengths holds the length in hex digits of each digest algorithm that may follow an "@". +// Only these well-known algorithms qualify, so a host named like an algorithm is never mistaken for +// one. +var digestHexLengths = map[string]int{ + "sha1": 40, + "sha224": 56, + "sha256": 64, + "sha384": 96, + "sha512": 128, + "md5": 32, + "blake3": 64, +} + +// address is an address split at the places that decide whether it carries a credential. +type address struct { + // prefix is the forced-getter prefix, "git::" for example, or empty. + prefix string + // scheme is "scheme://", "//", or empty when the address has none. + scheme string + // rest is the address after the scheme. + rest string + // authorityEnd is the offset in rest where the authority ends. + authorityEnd int + // inside are the offsets in rest of every "@" in the authority. + inside []int + // stray are the offsets in rest of every "@" after the authority that does not introduce a + // digest. + stray []int + // overflow is set when the address holds more "@" signs than [maxBoundaries] or is longer than + // [maxAddressLength]. The other fields are empty then, and the address must be withheld whole: + // nothing in it can be proven free of a credential at a cost that is bounded. + overflow bool +} + +// parseAddress splits raw at the getter prefix, the scheme, the authority, and every "@". An +// address that is past the bounds of the grammar is not split at all: it is marked as an overflow +// before any work that grows with the number of its "@" signs starts. +func parseAddress(raw string) address { + a := address{} + if exceedsBounds(raw) { + a.overflow = true + return a + } + a.prefix, raw = splitGetterPrefix(raw) + if loc := schemePattern.FindStringIndex(raw); loc != nil { + a.scheme, a.rest = raw[:loc[1]], raw[loc[1]:] + } else if strings.HasPrefix(raw, "//") { + a.scheme, a.rest = "//", raw[2:] + } else { + a.rest = raw + } + a.authorityEnd = strings.IndexAny(a.rest, authorityTerminators) + if a.authorityEnd < 0 { + a.authorityEnd = len(a.rest) + } + a.findBoundaries() + return a +} + +// exceedsBounds reports whether raw holds more "@" signs than [maxBoundaries], or is longer than +// [maxAddressLength] while holding one, so it is withheld without being split. +func exceedsBounds(raw string) bool { + ats := strings.Count(raw, "@") + return ats > maxBoundaries || (ats > 0 && len(raw) > maxAddressLength) +} + +// findBoundaries records the offset of every "@" of the remainder: those in the authority, and +// those after it that do not introduce a digest. +func (a *address) findBoundaries() { + for i := 0; i < a.authorityEnd; i++ { + if a.rest[i] == '@' { + a.inside = append(a.inside, i) + } + } + for i := a.authorityEnd; i < len(a.rest); i++ { + if a.rest[i] == '@' && !a.introducesDigest(i) { + a.stray = append(a.stray, i) + } + } +} + +// introducesDigest reports whether the "@" at offset i of the remainder, after the authority, +// introduces a digest: a complete digest follows it, the host in front is shaped like a host, and +// the path in front is the lower-case path of a repository. +func (a *address) introducesDigest(i int) bool { + hostStart := 0 + if len(a.inside) > 0 { + hostStart = a.inside[len(a.inside)-1] + 1 + } + return digestFollows(a.rest[i+1:]) && a.hostIsSafe(hostStart) && repositoryPathPattern.MatchString(a.rest[a.authorityEnd:i]) +} + +// boundaries returns the offset of every "@" that may end a userinfo, in order. +func (a *address) boundaries() []int { + out := make([]int, 0, len(a.inside)+len(a.stray)) + out = append(out, a.inside...) + return append(out, a.stray...) +} + +// digestFollows reports whether s, the text after an "@", starts with a complete digest such as +// "sha256:<64 hex digits>" that is not continued by further letters or digits. +func digestFollows(s string) bool { + // The algorithm name ends at the first ":", which is searched for in the few bytes a name can + // take, not in all of s, so the check costs the same however long s is. + colon := strings.IndexByte(s[:min(len(s), maxDigestAlgorithmLength+1)], ':') + if colon < 0 { + return false + } + algorithm, rest := s[:colon], s[colon+1:] + length, known := digestHexLengths[algorithm] + if !known || len(rest) < length { + return false + } + for i := 0; i < length; i++ { + if !isHexDigit(rest[i]) { + return false + } + } + return len(rest) == length || !isAlphanumeric(rest[length]) +} + +func isHexDigit(c byte) bool { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' +} + +func isAlphanumeric(c byte) bool { + return isHexDigit(c) || c >= 'g' && c <= 'z' || c >= 'G' && c <= 'Z' +} + +// hostIsSafe reports whether the host part of the address, the authority after any userinfo, has +// the shape of a host name or address. A host that has not is not echoed: it may be a password +// fragment that a delimiter cut loose. +func (a *address) hostIsSafe(start int) bool { + host := a.rest[start:a.authorityEnd] + if host == "" { + return a.scheme == fileScheme + } + return authorityPattern.MatchString(host) +} + +// headOf returns s up to its first white space, the part of an include argument that is a URL. +func headOf(s string) string { + if i := strings.IndexFunc(s, unicode.IsSpace); i >= 0 { + return s[:i] + } + return s +} + +// splitGetterPrefix separates a forced-getter prefix ("git::") from the URL that follows it. +func splitGetterPrefix(raw string) (prefix, body string) { + if !getterPrefixPattern.MatchString(raw) { + return "", raw + } + cut := strings.Index(raw, "::") + len("::") + return raw[:cut], raw[cut:] +} diff --git a/pkg/artifact/redact_scrub.go b/pkg/artifact/redact_scrub.go new file mode 100644 index 00000000000..d89545098b1 --- /dev/null +++ b/pkg/artifact/redact_scrub.go @@ -0,0 +1,213 @@ +package artifact + +import ( + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "unicode" +) + +const ( + // The hex base is the base of the digits of a percent escape. + hexBase = 16 + // The byte bits are the size of the byte a percent escape encodes. + byteBits = 8 + // The escape length is the length of "%XX". + escapeLength = 3 +) + +// wordPattern matches a white-space delimited word of free text. +var wordPattern = regexp.MustCompile(`\S+`) + +// accountPattern matches a plain account name, the only user name that may stand alone in text. +var accountPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{0,64}$`) + +// credentialSet is everything one address can leak into free text. +type credentialSet struct { + // cuts are the userinfos, each with its trailing "@", that are removed from text. + cuts []string + // passwords are the passwords, in every form an error may quote them, and the fragments a + // delimiter inside a password cuts loose. They are replaced by [RedactedValue]. + passwords []string +} + +// empty reports whether the address carried no credential. +func (c credentialSet) empty() bool { + return len(c.cuts) == 0 +} + +// credential collects the credential of the address. Every "@" that may end a userinfo is +// considered, so a password holding "@", "/", "?", "#" or "\" is found whole. +func (a *address) credential() credentialSet { + var set credentialSet + for _, boundary := range a.boundaries() { + userinfo := a.rest[:boundary] + for _, form := range textForms(userinfo) { + set.cuts = append(set.cuts, form+"@") + } + // The userinfo is split at its literal colon first, while it is still encoded, and only the + // password is decoded afterwards: an escaped colon belongs to the user name. + user, password, found := strings.Cut(userinfo, ":") + if !found || password == "" { + // A plain account name such as "git" is not a secret when it stands alone. Anything else + // in the user position is a token, and a delimiter inside it may cut it into fragments. + if !accountPattern.MatchString(user) { + set.passwords = append(set.passwords, fragmentsOf(user)...) + } + continue + } + // A parser may quote the user name followed by the start of the password, with the "@" cut + // off by a delimiter inside the password, so "user:" goes with its password. + for _, form := range textForms(user) { + set.cuts = append(set.cuts, form+":") + } + // Once a password is known the user name is withheld too, wherever it is quoted: it names + // the account the password opens. + set.passwords = append(set.passwords, fragmentsOf(user)...) + set.passwords = append(set.passwords, fragmentsOf(password)...) + } + set.cuts = unique(set.cuts) + set.passwords = unique(set.passwords) + return set +} + +// fragmentsOf returns s in every form an error may quote it, and the fragments of each form that a +// delimiter inside s cuts loose. +func fragmentsOf(s string) []string { + var out []string + for _, form := range textForms(s) { + out = append(out, form) + for _, fragment := range strings.FieldsFunc(form, isDelimiter) { + if len(fragment) >= minFragmentLength { + out = append(out, fragment) + } + } + } + return out +} + +// isDelimiter reports whether r separates the fragments of a password. +func isDelimiter(r rune) bool { + return unicode.IsSpace(r) || strings.ContainsRune(`:@/?#\[]"'`, r) +} + +// textForms returns s as an error message may quote it: as written, decoded, quoted, and escaped. +func textForms(s string) []string { + bases := []string{s, lenientUnescape(s)} + if decoded, err := url.PathUnescape(s); err == nil { + bases = append(bases, decoded) + } + if decoded, err := url.QueryUnescape(s); err == nil { + bases = append(bases, decoded) + } + forms := make([]string, 0, len(bases)*4) + for _, base := range bases { + quoted := strconv.Quote(base) + forms = append(forms, base, quoted[1:len(quoted)-1], url.PathEscape(base), url.QueryEscape(base)) + } + return unique(forms) +} + +// lenientUnescape decodes every valid "%XX" escape of s and leaves anything else as it is, so a +// stray "%" never keeps a decoded form from being tried. A parser may decode what a strict decoder +// rejects, and an error message quotes whatever the parser saw. +func lenientUnescape(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+escapeLength-1 < len(s) && isHexDigit(s[i+1]) && isHexDigit(s[i+2]) { + high, _ := strconv.ParseUint(s[i+1:i+escapeLength], hexBase, byteBits) + b.WriteByte(byte(high)) + i += escapeLength - 1 + continue + } + b.WriteByte(s[i]) + } + return b.String() +} + +// unique drops empty and repeated strings, longest first, so that a longer form is always tried +// before a shorter form it contains. +func unique(in []string) []string { + seen := make(map[string]bool, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if s != "" && !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) }) + return out +} + +// scrub removes the credential from text in a single pass, longest forms first. +func (c credentialSet) scrub(text string) string { + type pair struct{ old, replacement string } + pairs := make([]pair, 0, len(c.cuts)+len(c.passwords)) + for _, cut := range c.cuts { + pairs = append(pairs, pair{cut, ""}) + } + for _, password := range c.passwords { + pairs = append(pairs, pair{password, RedactedValue}) + } + sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].old) > len(pairs[j].old) }) + args := make([]string, 0, len(pairs)*2) + for _, p := range pairs { + args = append(args, p.old, p.replacement) + } + return strings.NewReplacer(args...).Replace(text) +} + +// survives reports whether any part of the credential is still in text, which means it was not +// removed and the whole text must be withheld. The marker that replaced a credential is ignored. +func (c credentialSet) survives(text string) bool { + if c.empty() { + return false + } + text = strings.ReplaceAll(text, RedactedValue, "") + for _, password := range c.passwords { + if strings.Contains(text, password) { + return true + } + } + for _, cut := range c.cuts { + if strings.Contains(text, cut) { + return true + } + } + return false +} + +// redactWords replaces every URL-shaped word of text that holds an "@" which cannot be proven to +// introduce a digest. When bearing is true, because an address that carried a credential was +// scrubbed from the text, any word with such an "@" is replaced, URL-shaped or not. +func redactWords(text string, bearing bool) string { + if !strings.Contains(text, "@") { + return text + } + return wordPattern.ReplaceAllStringFunc(text, func(word string) string { + if !strings.Contains(word, "@") { + return word + } + body := word + if i := strings.Index(word, schemeSeparator); i >= 0 { + for i > 0 && isSchemeByte(word[i-1]) { + i-- + } + body = word[i:] + } else if !bearing { + return word + } + if a := parseAddress(body); a.overflow || len(a.boundaries()) > 0 { + return RedactedValue + } + return word + }) +} + +// isSchemeByte reports whether c may appear in a URL scheme. +func isSchemeByte(c byte) bool { + return isAlphanumeric(c) || c == '+' || c == '.' || c == '-' +} diff --git a/pkg/artifact/redact_sites_test.go b/pkg/artifact/redact_sites_test.go new file mode 100644 index 00000000000..9d25a7fa366 --- /dev/null +++ b/pkg/artifact/redact_sites_test.go @@ -0,0 +1,96 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// TestDecodeOptions_ErrorDoesNotEchoCredential checks that an option key or value that holds a +// credential never reaches the decode error. +func TestDecodeOptions_ErrorDoesNotEchoCredential(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com/x" + tests := []struct { + name string + raw map[string]any + }{ + {name: "unknown key", raw: map[string]any{"registry": "ghcr.io", credential: "x"}}, + {name: "credential-like unknown key", raw: map[string]any{"registry": "ghcr.io", "usr7:S3cr3tA@password": "x"}}, + {name: "mistyped value", raw: map[string]any{"registry": map[string]any{"host": credential}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var options OCIRegistryOptions + err := DecodeOptions(tt.raw, &options) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} + +// TestValidationErrors_DoNotEchoCredentials checks the validators that echo a user-supplied +// registry, prefix, key pattern, or reference never repeat a credential in it. +func TestValidationErrors_DoNotEchoCredentials(t *testing.T) { + const credential = "usr7:S3cr3tA@example.com" + errs := map[string]error{ + "registry": (&OCIRegistryOptions{Registry: credential}).Validate(), + "prefix": ValidatePrefix(credential + "/../x"), + "key pattern": ValidateKeyPattern("/"+credential+"/{{.Name}}", PackagingTar), + "reference": func() error { _, err := ParseRef(credential); return err }(), + "name": ValidateName(credential), + "version": ValidateVersion(credential), + } + for name, err := range errs { + t.Run(name, func(t *testing.T) { + require.Error(t, err) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} + +// TestValidateKeyPattern_UnknownFieldDoesNotEchoCredential checks that the field name extracted from +// a template action is withheld when the pattern carries a credential: a field name is a fragment +// of the pattern, so scrubbing the pattern alone does not scrub it. +func TestValidateKeyPattern_UnknownFieldDoesNotEchoCredential(t *testing.T) { + tests := []struct { + name string + pattern string + // secrets are what the error must not hold. A field name in the path of the pattern is + // echoed with the rest of the redacted pattern, so only the userinfo is a secret there. + secrets []string + }{ + {name: "password is the field", pattern: "https://usr7:{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t", "usr7"}}, + {name: "token is the field", pattern: "https://{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t"}}, + {name: "scheme-less userinfo", pattern: "usr7:{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t", "usr7"}}, + {name: "field after a literal credential", pattern: "https://usr7:S3cr3tA@example.com/{{.Bogus}}.tar", secrets: []string{"S3cr3tA", "usr7"}}, + {name: "second action holds the field", pattern: "https://usr7:S3cr3tA@example.com/{{.Name}}/{{.Bogus}}.tar", secrets: []string{"S3cr3tA", "usr7"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateKeyPattern(tt.pattern, PackagingTar) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + for _, secret := range tt.secrets { + assert.NotContains(t, err.Error(), secret) + } + assert.NotContains(t, err.Error(), "unknown field .", "the field name is withheld when the pattern carries a credential") + }) + } +} + +// TestValidateKeyPattern_UnknownFieldIsNamedWithoutCredential is the negative path of the test +// above: a pattern that carries no credential still names the unknown field, because the name is +// what lets the user fix the pattern. +func TestValidateKeyPattern_UnknownFieldIsNamedWithoutCredential(t *testing.T) { + err := ValidateKeyPattern("builds/{{.Bogus}}/{{.Name}}.tar", PackagingTar) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "unknown field .Bogus") + assert.Contains(t, err.Error(), "builds/{{.Bogus}}/{{.Name}}.tar") +} diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go new file mode 100644 index 00000000000..dc0dae78518 --- /dev/null +++ b/pkg/artifact/redact_test.go @@ -0,0 +1,151 @@ +package artifact + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" +) + +func TestRedactURL(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + {name: "empty", raw: "", want: ""}, + {name: "no userinfo", raw: "oci://ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, + {name: "digest reference keeps its at sign", raw: "oci://ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + {name: "a digest that is not complete is not a digest", raw: "oci://ghcr.io/acme/catalog@sha256:abc", want: RedactedValue}, + {name: "unknown algorithm is not a digest", raw: "oci://ghcr.io/acme/catalog@sha999:" + strings.Repeat("0f", 32), want: RedactedValue}, + {name: "host named like a digest algorithm", raw: "https://user:hunter2@sha256:443/a", want: "https://sha256:443/a"}, + {name: "scheme-less digest reference", raw: "ghcr.io/acme/catalog@" + sha256Digest, want: "ghcr.io/acme/catalog@" + sha256Digest}, + {name: "scheme-less digest of a bare name", raw: "catalog@" + sha256Digest, want: RedactedValue}, + {name: "at sign after the authority is not a digest", raw: "https://user:hunter2/x@example.com/a", want: RedactedValue}, + {name: "at sign in the query", raw: "https://example.com/a?mail=user@example.com", want: RedactedValue}, + {name: "host that is not shaped like a host", raw: "https://user:hunter2@exa mple.com/a", want: RedactedValue}, + {name: "user and password", raw: "oci://user:hunter2@ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, + {name: "token as user", raw: "https://hunter2@issuer.example", want: "https://issuer.example"}, + {name: "userinfo and digest", raw: "oci://user:hunter2@ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + {name: "forced getter prefix is kept", raw: "git::https://user:hunter2@example.com/r.git", want: "git::https://example.com/r.git"}, + {name: "scheme-relative userinfo", raw: "//user:hunter2@example.com/a", want: "//example.com/a"}, + {name: "password with an at sign", raw: "https://user:pa@ss@example.com/a", want: "https://example.com/a"}, + {name: "malformed escape with userinfo", raw: "oci://user:hunter2@ghcr.io/acme/%zz:stable", want: RedactedValue}, + {name: "scheme-less userinfo", raw: "user:hunter2@ghcr.io", want: RedactedValue}, + {name: "scheme-less userinfo and port", raw: "user:hunter2@ghcr.io:5000", want: RedactedValue}, + {name: "malformed without an at sign is not credential bearing", raw: "oci://ghcr.io/%zz", want: "oci://ghcr.io/%zz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := RedactURL(tt.raw) + assert.Equal(t, tt.want, got) + assert.NotContains(t, got, "hunter2") + }) + } +} + +// TestRedact_SharedCredentialCases feeds the shared table to every redaction entry point of this +// package and checks the one security property they share: no user or password of a +// credential-bearing input survives, whichever way an error quotes it. +func TestRedact_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + got := RedactURL(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "RedactURL of %q", tc.Raw) + } + messages := []string{"fetch " + tc.Raw + " failed: denied", fmt.Sprintf("fetch %q failed", tc.Raw), "bad: " + got} + for _, quoted := range tc.Quoted { + messages = append(messages, "authentication failed for "+quoted) + messages = append(messages, "could not parse reference: "+tc.Raw+" (password "+quoted+")") + messages = append(messages, fmt.Sprintf("password %q rejected", quoted)) + } + for _, message := range messages { + scrubbed := RedactCredentials(message, tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, scrubbed, secret, "RedactCredentials of %q", message) + } + err := RedactError(fmt.Errorf("%w: %s", errSentinel, message), tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, err.Error(), secret, "RedactError of %q", message) + } + assert.ErrorIs(t, err, errSentinel, "the cause stays in the chain") + } + }) + } +} + +var errSentinel = errors.New("fetch failed") + +// TestRedact_GeneratedInputs runs the generated inputs, built from every delimiter of the shared +// table, through every entry point. +func TestRedact_GeneratedInputs(t *testing.T) { + for _, separator := range redacttest.Seeds() { + secret := redacttest.SecretOf("A", separator, "B") + for _, in := range redacttest.Generate(secret) { + checkEntryPoints(t, in) + } + } +} + +// checkEntryPoints runs one generated input through every entry point of this package. +func checkEntryPoints(t testing.TB, in redacttest.Input) { + t.Helper() + redacttest.RequireNoLeak(t, "RedactURL", in, RedactURL(in.Raw)) + for _, message := range in.Messages() { + redacttest.RequireNoLeak(t, "RedactCredentials", in, RedactCredentials(message, in.Raw)) + redacttest.RequireNoLeak(t, "RedactError", in, RedactError(fmt.Errorf("%w: %s", errSentinel, message), in.Raw).Error()) + } +} + +// FuzzRedact embeds a random secret in many address templates and checks that no entry point +// leaks the user name or any fragment of the secret. The seeds are the delimiters of the shared +// table, so every input of the table is a starting point. +func FuzzRedact(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + f.Add("x"+separator+"y", separator, "z"+separator) + } + // Seeds past the bounds of the grammar: more "@" signs than a real address holds, and an address + // longer than any real one. They must be withheld whole, never expanded. + f.Add("A", strings.Repeat("@", maxBoundaries+8), "B") + f.Add("A", strings.Repeat("@x", hostileBoundaries), "B") + f.Add(strings.Repeat("a", maxAddressLength), "@", "B") + f.Fuzz(func(t *testing.T, first, separator, second string) { + secret := redacttest.SecretOf(first, separator, second) + for _, in := range redacttest.Generate(secret) { + checkEntryPoints(t, in) + } + }) +} + +func TestRedactCredentials(t *testing.T) { + tests := []struct { + name string + text string + raw string + want string + }{ + {name: "no credential", text: "boom at example.com", raw: "https://example.com/a.yaml", want: "boom at example.com"}, + {name: "userinfo is cut out", text: "could not parse reference: user:hunter2@example.com/a", raw: "https://user:hunter2@example.com/a", want: "could not parse reference: example.com/a"}, + {name: "password quoted on its own", text: "auth failed for hunter2", raw: "https://user:hunter2@example.com/a", want: "auth failed for "}, + {name: "encoded password is also removed decoded", text: "bad password p@ss", raw: "https://user:p%40ss@example.com/a", want: "bad password "}, + {name: "double colon is part of the password", text: "bad password pa::ss for https://user:pa::ss@example.com/a", raw: "https://user:pa::ss@example.com/a", want: "bad password for https://example.com/a"}, + {name: "last at sign ends the userinfo", text: "failed: https://user:pa@ss@second@example.com/a", raw: "https://user:pa@ss@second@example.com/a", want: "failed: https://example.com/a"}, + {name: "forced getter prefix", text: "clone https://user:hunter2@example.com/r.git failed", raw: "git::https://user:hunter2@example.com/r.git", want: "clone https://example.com/r.git failed"}, + {name: "user name alone is not a password", text: "git failed", raw: "ssh://git@example.com/r.git", want: "git failed"}, + {name: "digest at sign is not userinfo", text: "pull ghcr.io/a/b@" + sha256Digest + " failed", raw: "oci://ghcr.io/a/b@" + sha256Digest, want: "pull ghcr.io/a/b@" + sha256Digest + " failed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, RedactCredentials(tt.text, tt.raw)) + }) + } +} + +var sha256Digest = "sha256:" + strings.Repeat("0f", 32) diff --git a/pkg/artifact/redact_values_test.go b/pkg/artifact/redact_values_test.go new file mode 100644 index 00000000000..bdd5f376937 --- /dev/null +++ b/pkg/artifact/redact_values_test.go @@ -0,0 +1,75 @@ +package artifact + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinel: renaming the schema field the registry test sets must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: ""} + +// TestEchoedValues_DoNotEchoCredentials checks every validator and error that quotes a configured +// name or value, other than the registry, prefix, key pattern and reference covered above, when +// that value is a pasted credential-bearing URL. +func TestEchoedValues_DoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + registry := NewRepositoryRegistry() + _, registryErr := registry.New(context.Background(), credential, &schema.ArtifactRepositoryConfig{Kind: credential}) + _, nilConfigErr := registry.New(context.Background(), credential, nil) + _, permissionsErr := ParsePermissions(credential) + + tests := []struct { + name string + err error + }{ + {name: "s3 bucket", err: (&S3Options{Bucket: credential}).Validate()}, + {name: "s3 region", err: (&S3Options{Bucket: "bucket", Region: credential}).Validate()}, + {name: "s3 reserved tag key", err: validateS3Tags(map[string]string{"aws:" + credential: "v"})}, + {name: "s3 oversized tag value", err: validateS3Tags(map[string]string{credential: strings.Repeat("v", maxS3TagValLen+1)})}, + {name: "label key", err: ValidateLabel(credential, "v")}, + {name: "label key whitespace", err: ValidateLabel(" "+credential, "v")}, + {name: "label value", err: ValidateLabel(credential, strings.Repeat("v", MaxLabelValueLength+1))}, + {name: "label set", err: ValidateLabels(map[string]string{credential: "v"})}, + {name: "permissions", err: permissionsErr}, + {name: "registry unknown kind", err: registryErr}, + {name: "registry nil configuration", err: nilConfigErr}, + {name: "capability repository", err: NewUnsupportedCapability(credential, "push", "kind/"+credential, "not available")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.NotContains(t, tt.err.Error(), "S3cr3tA") + assert.NotContains(t, tt.err.Error(), "usr7") + }) + } +} + +// TestEchoedValues_KeepNamesWithoutCredentials is the negative path of the test above: a plain +// name is still echoed, because the name is what lets the user find the mistake. +func TestEchoedValues_KeepNamesWithoutCredentials(t *testing.T) { + _, registryErr := NewRepositoryRegistry().New(context.Background(), "prod", &schema.ArtifactRepositoryConfig{Kind: "ftp/server"}) + tests := []struct { + name string + err error + want string + }{ + {name: "s3 bucket", err: (&S3Options{Bucket: "Not_A_Bucket"}).Validate(), want: `bucket "Not_A_Bucket"`}, + {name: "s3 region", err: (&S3Options{Bucket: "bucket", Region: "Not A Region"}).Validate(), want: `region "Not A Region"`}, + {name: "label key", err: ValidateLabel("a=b", "v"), want: `label key "a=b"`}, + {name: "registry unknown kind", err: registryErr, want: `repository "prod" has unknown kind "ftp/server"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.ErrorIs(t, tt.err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, tt.err.Error(), tt.want) + }) + } +} diff --git a/pkg/artifact/redacttest/cases.go b/pkg/artifact/redacttest/cases.go new file mode 100644 index 00000000000..c43e4245411 --- /dev/null +++ b/pkg/artifact/redacttest/cases.go @@ -0,0 +1,90 @@ +// Package redacttest holds the credential-bearing inputs that every URL redaction helper must +// handle. The helpers in pkg/artifact and pkg/utils share one security property, so they share one +// table: a test feeds the same cases to each helper and asserts that no secret survives. +package redacttest + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// digestBytes is the number of repeated bytes in the sample digest of the OCI cases. +const digestBytes = 32 + +const ( + // User is a distinctive user name, so a test can tell it apart from any host or path. + User = "usr7" + // SecretA and SecretB are distinctive password fragments. + SecretA = "S3cr3tA" + // SecretB is the second password fragment, used when a delimiter splits a password in two. + SecretB = "S3cr3tB" + // SecretC is the third password fragment, used when two delimiters split a password in three. + SecretC = "S3cr3tC" + // Marker starts every secret the generated inputs carry, so a leak of any fragment of a secret, + // however a delimiter cut it, is a leak of the marker. + Marker = "S3cr3t" +) + +// Case is one credential-bearing input. +type Case struct { + // Name describes the shape of the input. + Name string + // Raw is the URL, or URL-like address, that carries the credential. + Raw string + // Secrets are the strings that must never appear in the redacted form of Raw: the user, the + // password, and every fragment a delimiter inside the password could leave behind. + Secrets []string + // Quoted are the passwords an error may quote on their own, in any form a parser can decode. + Quoted []string +} + +// Cases returns the shared table of nasty inputs. Each call returns a fresh slice, so a caller may +// not alias or mutate another caller's table. +func Cases() []Case { + defer perf.Track(nil, "redacttest.Cases")() + + digest := "sha256:" + strings.Repeat("ab", digestBytes) + return []Case{ + {Name: "user and password", Raw: "https://usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a colon", Raw: "https://usr7:S3cr3tA:S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA:S3cr3tB"}}, + {Name: "password with a double colon", Raw: "https://usr7:S3cr3tA::S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + {Name: "password with an at sign", Raw: "https://usr7:S3cr3tA@S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA@S3cr3tB"}}, + {Name: "password with two at signs", Raw: "https://usr7:S3cr3tA@S3cr3tB@second@example.com/a", Secrets: []string{User, SecretA, SecretB, "second"}, Quoted: []string{"S3cr3tA@S3cr3tB@second"}}, + {Name: "password with a hash", Raw: "https://usr7:S3cr3tA#S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA#S3cr3tB"}}, + {Name: "password with a question mark", Raw: "https://usr7:S3cr3tA?S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA?S3cr3tB"}}, + {Name: "password with a slash", Raw: "https://usr7:S3cr3tA/S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA/S3cr3tB"}}, + {Name: "numeric password with a hash", Raw: "https://usr7:12345#S3cr3tB@example.com/a", Secrets: []string{User, "12345", SecretB}, Quoted: []string{"12345#S3cr3tB"}}, + {Name: "numeric password with a slash", Raw: "https://usr7:12345/S3cr3tB@example.com/a", Secrets: []string{User, "12345", SecretB}, Quoted: []string{"12345/S3cr3tB"}}, + {Name: "percent-escaped password", Raw: "https://usr7:S3cr3t%40Z@example.com/a", Secrets: []string{User, "S3cr3t%40Z", "S3cr3t@Z"}, Quoted: []string{"S3cr3t%40Z", "S3cr3t@Z"}}, + {Name: "malformed percent escape", Raw: "https://usr7:S3cr3t%zzZ@example.com/a", Secrets: []string{User, "S3cr3t%zzZ"}, Quoted: []string{"S3cr3t%zzZ"}}, + {Name: "empty user", Raw: "https://:S3cr3tA@example.com/a", Secrets: []string{SecretA}, Quoted: []string{SecretA}}, + {Name: "empty password", Raw: "https://usr7:@example.com/a", Secrets: []string{User}}, + {Name: "empty user and password", Raw: "https://:@example.com/a"}, + {Name: "token as user", Raw: "https://S3cr3tA@example.com/a", Secrets: []string{SecretA}}, + {Name: "ipv6 host", Raw: "https://usr7:S3cr3tA@[::1]:8443/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "userinfo with a space", Raw: "https://usr7:S3cr3tA S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA S3cr3tB"}}, + {Name: "upper-case scheme", Raw: "HTTPS://usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "scheme-relative", Raw: "//usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "no scheme", Raw: "usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter", Raw: "git::https://usr7:S3cr3tA@example.com/r.git", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter with subdirectory and query", Raw: "git::https://usr7:S3cr3tA@example.com/r.git//sub?ref=v1", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter with a double colon password", Raw: "git::https://usr7:S3cr3tA::S3cr3tB@example.com/r.git", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + {Name: "password with an at sign and a slash", Raw: "https://usr7:S3cr3tA@S3cr3tB/S3cr3tC@example.com/a", Secrets: []string{User, SecretA, SecretB, SecretC}, Quoted: []string{"S3cr3tA@S3cr3tB/S3cr3tC"}}, + {Name: "escaped colon in the user name", Raw: "https://usr7%3AS3cr3tB:S3cr3tA%40S3cr3tC@example.com/a", Secrets: []string{User, SecretA, SecretB, SecretC, "S3cr3tA%40S3cr3tC"}, Quoted: []string{"S3cr3tA%40S3cr3tC", "S3cr3tA@S3cr3tC"}}, + {Name: "escaped colon in the user name of a short password", Raw: "https://usr%3Aname:p%40ss@example.com/a", Secrets: []string{"usr%3Aname", "usr:name", "p%40ss", "p@ss"}, Quoted: []string{"p%40ss", "p@ss"}}, + {Name: "host that looks like a digest algorithm and a port", Raw: "https://usr7:S3cr3tA@sha256:443/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "host that looks like a digest", Raw: "https://usr7:S3cr3tA@" + digest + "/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password that ends in a digest", Raw: "https://usr7:S3cr3tA@" + digest + "@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a slash and a digest after it", Raw: "https://usr7:S3cr3tA/S3cr3tB@" + digest + "@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA/S3cr3tB"}}, + {Name: "credential followed by a digest", Raw: "https://usr7:S3cr3tA@example.com/a@" + digest, Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a backslash", Raw: "https://usr7:S3cr3tA\\S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\\S3cr3tB"}}, + {Name: "password with a bracket", Raw: "https://usr7:S3cr3tA[S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA[S3cr3tB"}}, + {Name: "password with a tab", Raw: "https://usr7:S3cr3tA\tS3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\tS3cr3tB"}}, + {Name: "password with unicode", Raw: "https://usr7:S3cr3tA\u2713S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\u2713S3cr3tB"}}, + {Name: "credential repeated in the query", Raw: "https://usr7:S3cr3tA@example.com/a?next=https://usr7:S3cr3tB@other.example/b", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{SecretA}}, + {Name: "credential in the fragment", Raw: "https://usr7:S3cr3tA@example.com/a#https://usr7:S3cr3tB@other.example/b", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{SecretA}}, + {Name: "oci reference with a digest", Raw: "oci://usr7:S3cr3tA@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "oci reference with a double colon password and a digest", Raw: "oci://usr7:S3cr3tA::S3cr3tB@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + } +} diff --git a/pkg/artifact/redacttest/generate.go b/pkg/artifact/redacttest/generate.go new file mode 100644 index 00000000000..cebb1359a69 --- /dev/null +++ b/pkg/artifact/redacttest/generate.go @@ -0,0 +1,132 @@ +package redacttest + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Input is one generated credential-bearing address and the ways an error may quote it. +type Input struct { + // Name describes the template the address was built from. + Name string + // Raw is the address. + Raw string + // Password is the password as it was typed into Raw, or empty when Raw has none. + Password string +} + +// Messages returns error texts that quote the input the way fetch, parse, and decode errors do: +// verbatim, quoted with %q, stripped of its scheme, as the password on its own, and as the text of +// a URL parse error. +func (in Input) Messages() []string { + defer perf.Track(nil, "redacttest.Input.Messages")() + + _, withoutScheme, _ := strings.Cut(in.Raw, "://") + messages := []string{ + "fetch " + in.Raw + " failed: denied", + fmt.Sprintf("fetch %q failed: denied", in.Raw), + "could not parse reference: " + withoutScheme, + fmt.Sprintf("Get %q: dial tcp: lookup failed", in.Raw), + } + if _, err := url.Parse(in.Raw); err != nil { + messages = append(messages, err.Error()) + } + if in.Password != "" { + messages = append(messages, "authentication failed for "+in.Password, fmt.Sprintf("password %q rejected", in.Password)) + } + return messages +} + +// Leaks returns what survives in out of the user name and of the secret: every fragment of a +// secret starts with [Marker], so a delimiter inside the secret cannot hide a leak. +func Leaks(out string) []string { + defer perf.Track(nil, "redacttest.Leaks")() + + var leaks []string + for _, needle := range []string{Marker, User} { + if strings.Contains(out, needle) { + leaks = append(leaks, needle) + } + } + return leaks +} + +// RequireNoLeak fails the test when out, the redacted output of the entry point named entry for +// in, still holds the user name or any fragment of the secret. +func RequireNoLeak(t testing.TB, entry string, in Input, out string) { + defer perf.Track(nil, "redacttest.RequireNoLeak")() + + t.Helper() + + if leaks := Leaks(out); len(leaks) > 0 { + t.Fatalf("%s leaked %q for %s input %q: %q", entry, leaks, in.Name, in.Raw, out) + } +} + +// Seeds returns the separators a fuzz run starts from: one for every delimiter the shared table +// splits a password at, so that the corpus covers every input of [Cases] as a generated secret. +func Seeds() []string { + defer perf.Track(nil, "redacttest.Seeds")() + + return []string{"", ":", "::", "@", "@@", "#", "?", "/", "%40", "%zz", " ", "\t", "[", "]", "\\", "\u2713", "@" + digestHex, "://", "@sha256:443/"} +} + +// digestHex is a complete sha256 digest, so a template can place a real digest next to a credential. +var digestHex = "sha256:" + strings.Repeat("0f", digestBytes) + +// Generate embeds secret as the password of many address templates: user information of every +// scheme, with the secret before and after other delimiters, in an escaped form, repeated in the +// query and the fragment, and with hosts named like digests. Each address ends its user +// information with an "@" of its own, so the secret is always a credential and never a host. +// The secret may hold any delimiter, so the same template covers a password that closes the +// authority early. +func Generate(secret string) []Input { + defer perf.Track(nil, "redacttest.Generate")() + + escaped := url.PathEscape(secret) + const ( + https = "https://" + colon = ":" + at = "@" + host = "@example.com/a" + ) + credential := User + colon + secret + templates := []struct{ name, raw, password string }{ + {"https", https + credential + host, secret}, + {"upper-case scheme with a port, a query and a fragment", "HTTPS://" + credential + "@example.com:8443/a?x=1#f", secret}, + {"scheme-relative", "//" + credential + host, secret}, + {"scheme-less", credential + host, secret}, + {"forced getter", "git::" + https + credential + "@example.com/r.git//sub?ref=v1", secret}, + {"oci with a digest", "oci://" + credential + "@ghcr.io/acme/catalog@" + digestHex, secret}, + {"host named like an algorithm with a port", https + credential + "@sha256:443/a", secret}, + {"host named like a digest", https + credential + at + digestHex + "/a", secret}, + {"digest in the user information", https + credential + at + digestHex + host, secret}, + {"ipv6 host", https + credential + "@[::1]:8443/a", secret}, + {"token as the user", https + secret + host, ""}, + {"credential then a digest", https + credential + "@example.com/a@" + digestHex, secret}, + {"credential repeated in the query", https + credential + "@example.com/a?next=" + https + credential + "@other.example/b", secret}, + {"credential repeated in the fragment", https + credential + "@example.com/a#" + https + credential + "@other.example/b", secret}, + {"escaped password", https + User + colon + escaped + host, secret}, + {"escaped colon in the user", https + User + "%3A" + secret + colon + escaped + host, secret}, + {"two userinfos", https + credential + at + secret + host, secret}, + {"ssh", "ssh://" + credential + "@example.com:22/r.git", secret}, + {"no host", https + credential + "@", secret}, + } + inputs := make([]Input, 0, len(templates)) + for _, tmpl := range templates { + inputs = append(inputs, Input{Name: tmpl.name, Raw: tmpl.raw, Password: tmpl.password}) + } + return inputs +} + +// SecretOf builds a secret that carries [Marker] twice, joined by an arbitrary separator, so a +// delimiter inside the secret splits it into fragments that each start with the marker. +func SecretOf(first, separator, second string) string { + defer perf.Track(nil, "redacttest.SecretOf")() + + return Marker + first + separator + Marker + second +} diff --git a/pkg/artifact/ref.go b/pkg/artifact/ref.go new file mode 100644 index 00000000000..04d3bd78d31 --- /dev/null +++ b/pkg/artifact/ref.go @@ -0,0 +1,167 @@ +package artifact + +import ( + "fmt" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +var ( + // Name segments use the OCI distribution path-component grammar. It is lowercase-only so + // that a name valid here is valid on every backend without case folding or other rewriting. + nameSegmentPattern = regexp.MustCompile(`^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$`) + + // Versions use the OCI tag grammar. Versions are opaque labels, not required to be SemVer. + versionPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) + + // Kind segments are lowercase tokens, for example "component" in "component/terraform". + kindSegmentPattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*$`) +) + +// Ref addresses an artifact by name and exactly one of version or digest. The repository alias is +// a separate request field and never part of a Ref. +// +// The textual forms are "name:version" and "name@sha256:". There is no implicit "latest". +type Ref struct { + // Name is the slash-separated relative artifact name, e.g. "components/terraform/vpc". + Name string `json:"name"` + + // Version is the explicit version label. Mutually exclusive with Digest. + Version string `json:"version,omitempty"` + + // Digest is an immutable record or manifest digest. Mutually exclusive with Version. + Digest digest.Digest `json:"digest,omitempty"` +} + +// ParseRef parses "name:version" or "name@sha256:" and validates the result. +func ParseRef(s string) (Ref, error) { + defer perf.Track(nil, "artifact.ParseRef")() + + if s == "" { + return Ref{}, fmt.Errorf("%w: reference is empty", errUtils.ErrArtifactRefInvalid) + } + var ref Ref + if name, dgst, found := strings.Cut(s, "@"); found { + ref = Ref{Name: name, Digest: digest.Digest(dgst)} + } else { + index := strings.LastIndex(s, ":") + if index < 0 { + return Ref{}, fmt.Errorf("%w: %q needs a version (name:version) or digest (name@sha256:...); there is no implicit latest", errUtils.ErrArtifactRefInvalid, RedactURL(s)) + } + ref = Ref{Name: s[:index], Version: s[index+1:]} + } + if err := ref.Validate(); err != nil { + // The reference is validated in pieces, so a credential split at its "@" is only visible in + // the whole. + return Ref{}, RedactError(err, s) + } + return ref, nil +} + +// Validate checks the name and that exactly one of version or digest is set and well formed. +func (r Ref) Validate() error { + defer perf.Track(nil, "artifact.Ref.Validate")() + + if err := ValidateName(r.Name); err != nil { + return err + } + switch { + case r.Version != "" && r.Digest != "": + return fmt.Errorf("%w: %q sets both a version and a digest", errUtils.ErrArtifactRefInvalid, RedactURL(r.Name)) + case r.Version == "" && r.Digest == "": + return fmt.Errorf("%w: %q needs a version or a digest", errUtils.ErrArtifactRefInvalid, RedactURL(r.Name)) + case r.Version != "": + return ValidateVersion(r.Version) + default: + return ValidateDigest(r.Digest) + } +} + +// String returns the textual form of the reference. +func (r Ref) String() string { + defer perf.Track(nil, "artifact.Ref.String")() + + if r.Digest != "" { + return r.Name + "@" + r.Digest.String() + } + return r.Name + ":" + r.Version +} + +// IsDigest reports whether the reference selects an immutable digest. +// +//nolint:lintroller // Trivial getter with no I/O. +func (r Ref) IsDigest() bool { return r.Digest != "" } + +// ValidateName checks an artifact name: relative, slash-separated, with no empty, "." or ".." +// segments and each segment matching the OCI path-component grammar. Names are never rewritten. +func ValidateName(name string) error { + defer perf.Track(nil, "artifact.ValidateName")() + + if name == "" { + return fmt.Errorf("%w: name is empty", errUtils.ErrArtifactRefInvalid) + } + if len(name) > MaxNameLength { + return fmt.Errorf("%w: name exceeds %d characters", errUtils.ErrArtifactRefInvalid, MaxNameLength) + } + if strings.HasPrefix(name, "/") || strings.Contains(name, `\`) { + return fmt.Errorf("%w: %w: %q must be relative and slash-separated", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name)) + } + for _, segment := range strings.Split(name, "/") { + switch segment { + case "": + return fmt.Errorf("%w: %w: %q has an empty segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name)) + case ".", "..": + return fmt.Errorf("%w: %w: %q has a %q segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name), segment) + } + if !nameSegmentPattern.MatchString(segment) { + return fmt.Errorf("%w: segment %q of %q must be lowercase alphanumerics separated by '.', '_', '__', or '-'", errUtils.ErrArtifactRefInvalid, RedactURL(segment), RedactURL(name)) + } + } + return nil +} + +// ValidateVersion checks a version against the OCI tag grammar. Versions are never normalized. +func ValidateVersion(version string) error { + defer perf.Track(nil, "artifact.ValidateVersion")() + + if version == "" { + return fmt.Errorf("%w: version is empty", errUtils.ErrArtifactRefInvalid) + } + if !versionPattern.MatchString(version) { + return fmt.Errorf("%w: version %q must match the OCI tag syntax [A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", errUtils.ErrArtifactRefInvalid, RedactURL(version)) + } + return nil +} + +// ValidateDigest checks that d is a well-formed sha256 digest. V1 supports sha256 only. +func ValidateDigest(d digest.Digest) error { + defer perf.Track(nil, "artifact.ValidateDigest")() + + if err := d.Validate(); err != nil { + return fmt.Errorf("%w: digest %q: %w", errUtils.ErrArtifactRefInvalid, RedactURL(string(d)), err) + } + if d.Algorithm() != digest.SHA256 { + return fmt.Errorf("%w: digest %q must use sha256", errUtils.ErrArtifactRefInvalid, RedactURL(string(d))) + } + return nil +} + +// ValidateKind checks an artifact kind such as "files", "component/terraform", or "catalog". +func ValidateKind(kind string) error { + defer perf.Track(nil, "artifact.ValidateKind")() + + if kind == "" || len(kind) > MaxKindLength { + return fmt.Errorf("%w: kind must be 1-%d characters", errUtils.ErrArtifactRecordInvalid, MaxKindLength) + } + for _, segment := range strings.Split(kind, "/") { + if !kindSegmentPattern.MatchString(segment) { + return fmt.Errorf("%w: kind %q must be slash-separated lowercase segments", errUtils.ErrArtifactRecordInvalid, RedactURL(kind)) + } + } + return nil +} diff --git a/pkg/artifact/ref_test.go b/pkg/artifact/ref_test.go new file mode 100644 index 00000000000..2d2b2588151 --- /dev/null +++ b/pkg/artifact/ref_test.go @@ -0,0 +1,117 @@ +package artifact + +import ( + "errors" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +const testSHA = "sha256:0d6e4079e36703ebd37c00722f5891d28b0e2811dc114b129215123adcce3605" + +func TestParseRef(t *testing.T) { + tests := []struct { + name string + input string + want Ref + }{ + {"nested name and version", "components/terraform/vpc:1.2.0", Ref{Name: "components/terraform/vpc", Version: "1.2.0"}}, + {"single segment", "vpc:v1", Ref{Name: "vpc", Version: "v1"}}, + {"CI build identifier version", "app:build-2026.10.30_7", Ref{Name: "app", Version: "build-2026.10.30_7"}}, + {"digest reference", "components/vpc@" + testSHA, Ref{Name: "components/vpc", Digest: testSHA}}, + {"separators in segments", "a.b/c_d/e__f/g--h:1", Ref{Name: "a.b/c_d/e__f/g--h", Version: "1"}}, + {"128 character version", "a:" + strings.Repeat("v", 128), Ref{Name: "a", Version: strings.Repeat("v", 128)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRef(tt.input) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.input, got.String(), "String round-trips the textual form") + assert.Equal(t, tt.want.Digest != "", got.IsDigest()) + }) + } +} + +func TestParseRefRejects(t *testing.T) { + tests := []struct { + name string + input string + unsafe bool + }{ + {name: "empty", input: ""}, + {name: "no version or digest", input: "components/vpc"}, + {name: "empty version", input: "vpc:"}, + {name: "empty name", input: ":1.0"}, + {name: "absolute path", input: "/abs/path:1", unsafe: true}, + {name: "traversal segment", input: "a/../b:1", unsafe: true}, + {name: "leading traversal", input: "../a:1", unsafe: true}, + {name: "dot segment", input: "a/./b:1", unsafe: true}, + {name: "empty segment", input: "a//b:1", unsafe: true}, + {name: "trailing slash", input: "a/b/:1", unsafe: true}, + {name: "backslash", input: `a\b:1`, unsafe: true}, + {name: "uppercase is not normalized", input: "Components/Vpc:1"}, + {name: "space", input: "a b:1"}, + {name: "reserved namespace shaped name", input: ".atmos-artifacts/x:1", unsafe: false}, + {name: "version with plus", input: "a:1.0+build"}, + {name: "version starting with dash", input: "a:-1"}, + {name: "version starting with dot", input: "a:.1"}, + {name: "version too long", input: "a:" + strings.Repeat("v", 129)}, + {name: "version and digest", input: "a:1@" + testSHA}, + {name: "unsupported digest algorithm", input: "a@sha512:" + strings.Repeat("a", 128)}, + {name: "short digest", input: "a@sha256:abc"}, + {name: "uppercase digest hex", input: "a@sha256:" + strings.ToUpper(strings.TrimPrefix(testSHA, "sha256:"))}, + {name: "digest without name", input: "@" + testSHA}, + {name: "name too long", input: strings.Repeat("a/", 128) + "a:1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseRef(tt.input) + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Equal(t, tt.unsafe, isUnsafe(err), "unsafe-path classification of %v", err) + }) + } +} + +func isUnsafe(err error) bool { + return errors.Is(err, errUtils.ErrArtifactUnsafePath) +} + +func TestRefValidate(t *testing.T) { + tests := []struct { + name string + ref Ref + ok bool + }{ + {"version", Ref{Name: "a", Version: "1"}, true}, + {"digest", Ref{Name: "a", Digest: testSHA}, true}, + {"neither", Ref{Name: "a"}, false}, + {"both", Ref{Name: "a", Version: "1", Digest: digest.Digest(testSHA)}, false}, + {"bad name", Ref{Name: "A", Version: "1"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.ref.Validate() + if tt.ok { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) + } +} + +func TestValidateKind(t *testing.T) { + for _, kind := range []string{"files", "component/terraform", "toolchain.package", "ci/planfile", "catalog"} { + assert.NoError(t, ValidateKind(kind), kind) + } + for _, kind := range []string{"", "Files", "a//b", "/a", "a/", "a b", strings.Repeat("k", MaxKindLength+1)} { + assert.ErrorIs(t, ValidateKind(kind), errUtils.ErrArtifactRecordInvalid, kind) + } +} diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go new file mode 100644 index 00000000000..f613e489279 --- /dev/null +++ b/pkg/artifact/registry.go @@ -0,0 +1,258 @@ +package artifact + +import ( + "context" + "fmt" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// IdentityResolver lazily resolves credentials for a named Atmos auth identity. Backends call it +// only when an operation needs credentials; constructing a repository, validating configuration, +// describing configuration, and dry runs never call it. The concrete credential type is owned by +// the backend that asked for it. +type IdentityResolver func(ctx context.Context, identity string) (any, error) + +// RepositoryLookup returns another configured repository by alias, used for delegated locking. +type RepositoryLookup func(alias string) (Repository, error) + +// RepositoryOptions are the construction options a factory receives. +type RepositoryOptions struct { + // ResolveIdentity lazily resolves the repository's identity. It may be nil. + ResolveIdentity IdentityResolver + + // LookupRepository finds another configured repository (for example a lock delegate). It may be nil. + LookupRepository RepositoryLookup +} + +// RepositoryOption configures construction of a repository. +type RepositoryOption func(*RepositoryOptions) + +// WithIdentityResolver supplies the lazy identity resolver. +func WithIdentityResolver(resolver IdentityResolver) RepositoryOption { + defer perf.Track(nil, "artifact.WithIdentityResolver")() + + return func(o *RepositoryOptions) { o.ResolveIdentity = resolver } +} + +// WithRepositoryLookup supplies the lookup used to find delegated repositories. +func WithRepositoryLookup(lookup RepositoryLookup) RepositoryOption { + defer perf.Track(nil, "artifact.WithRepositoryLookup")() + + return func(o *RepositoryOptions) { o.LookupRepository = lookup } +} + +// ApplyRepositoryOptions folds construction options into a RepositoryOptions value for factories. +func ApplyRepositoryOptions(opts ...RepositoryOption) RepositoryOptions { + defer perf.Track(nil, "artifact.ApplyRepositoryOptions")() + + var applied RepositoryOptions + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + return applied +} + +// RepositoryFactory constructs a repository. It validates nothing the registry has not already +// validated, must not touch the network, resolve credentials, or create directories, and returns +// a repository whose Capabilities() is available immediately. +type RepositoryFactory func(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) + +type registeredKind struct { + descriptor KindDescriptor + factory RepositoryFactory +} + +// RepositoryRegistry maps repository kinds to descriptors and factories. There is no central +// backend switch: backend packages register their own kind from a composition root. It is safe +// for concurrent use. +type RepositoryRegistry struct { + mu sync.RWMutex + kinds map[string]registeredKind +} + +// NewRepositoryRegistry returns an empty registry. +func NewRepositoryRegistry() *RepositoryRegistry { + defer perf.Track(nil, "artifact.NewRepositoryRegistry")() + + return &RepositoryRegistry{kinds: make(map[string]registeredKind)} +} + +// Register adds a kind. It rejects an empty kind, a nil factory, and a kind that is already registered. +func (r *RepositoryRegistry) Register(desc *KindDescriptor, factory RepositoryFactory) error { + defer perf.Track(nil, "artifact.RepositoryRegistry.Register")() + + if desc == nil || desc.Kind == "" { + return fmt.Errorf("%w: repository kind must not be empty", errUtils.ErrArtifactConfigInvalid) + } + if err := ValidateKind(desc.Kind); err != nil { + return fmt.Errorf("%w: repository kind %q: %w", errUtils.ErrArtifactConfigInvalid, desc.Kind, err) + } + if factory == nil { + return fmt.Errorf("%w: factory for repository kind %q must not be nil", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + if desc.DefaultPackaging != "" && !desc.Capabilities.HasPackaging(desc.DefaultPackaging) { + return fmt.Errorf("%w: %w: repository kind %q default packaging %q is not in its supported packagings %v", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, desc.DefaultPackaging, desc.Capabilities.Packagings) + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.kinds[desc.Kind]; exists { + return fmt.Errorf("%w: repository kind %q is already registered", errUtils.ErrArtifactConflict, desc.Kind) + } + r.kinds[desc.Kind] = registeredKind{descriptor: cloneKindDescriptor(desc), factory: factory} + return nil +} + +func cloneKindDescriptor(source *KindDescriptor) KindDescriptor { + desc := *source + desc.PathOptions = append([]string(nil), desc.PathOptions...) + desc.Capabilities.Packagings = append([]Packaging(nil), desc.Capabilities.Packagings...) + desc.Capabilities.LockScopes = append([]LockScope(nil), desc.Capabilities.LockScopes...) + return desc +} + +// Lookup returns the descriptor of a kind: the registered descriptor when a backend registered +// the kind, otherwise the built-in V1 descriptor. The boolean is false for an unknown kind. +func (r *RepositoryRegistry) Lookup(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.RepositoryRegistry.Lookup")() + + r.mu.RLock() + entry, ok := r.kinds[kind] + r.mu.RUnlock() + if ok { + desc := cloneKindDescriptor(&entry.descriptor) + return &desc, true + } + return BuiltinKind(kind) +} + +// IsRegistered reports whether a backend registered a factory for the kind. +func (r *RepositoryRegistry) IsRegistered(kind string) bool { + defer perf.Track(nil, "artifact.RepositoryRegistry.IsRegistered")() + + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.kinds[kind] + return ok +} + +// Kinds returns the sorted names of every known kind: the registered ones plus the built-in V1 set. +func (r *RepositoryRegistry) Kinds() []string { + defer perf.Track(nil, "artifact.RepositoryRegistry.Kinds")() + + seen := make(map[string]bool) + for _, kind := range BuiltinKinds() { + seen[kind] = true + } + r.mu.RLock() + for kind := range r.kinds { + seen[kind] = true + } + r.mu.RUnlock() + names := make([]string, 0, len(seen)) + for kind := range seen { + names = append(names, kind) + } + sort.Strings(names) + return names +} + +// New validates the repository configuration against its kind and constructs the repository +// through the registered factory. An unknown kind is a configuration error; a known V1 kind +// whose backend is not registered in this build is an unsupported-capability error. +func (r *RepositoryRegistry) New(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + defer perf.Track(nil, "artifact.RepositoryRegistry.New")() + + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, RedactURL(alias)) + } + desc, known := r.Lookup(cfg.Kind) + if !known { + return nil, fmt.Errorf("%w: repository %q has unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, RedactURL(alias), RedactURL(cfg.Kind), r.Kinds()) + } + r.mu.RLock() + entry, registered := r.kinds[cfg.Kind] + r.mu.RUnlock() + if !registered { + return nil, NewUnsupportedCapability(alias, "construct", "kind/"+cfg.Kind, "no backend is registered for this repository kind in this build") + } + if err := ValidateRepositoryConfig(desc, cfg); err != nil { + return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), err) + } + repo, err := entry.factory(ctx, alias, cfg, opts...) + if err != nil { + return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), RedactError(err, alias)) + } + return repo, nil +} + +// Reset removes every registered kind. It exists for tests that register fake backends. +func (r *RepositoryRegistry) Reset() { + defer perf.Track(nil, "artifact.RepositoryRegistry.Reset")() + + r.mu.Lock() + defer r.mu.Unlock() + r.kinds = make(map[string]registeredKind) +} + +// defaultRepositoryRegistry is the process-wide registry that backend packages register into. +var defaultRepositoryRegistry = NewRepositoryRegistry() + +// DefaultRepositoryRegistry returns the process-wide registry. +func DefaultRepositoryRegistry() *RepositoryRegistry { + defer perf.Track(nil, "artifact.DefaultRepositoryRegistry")() + + return defaultRepositoryRegistry +} + +// RegisterRepositoryKind registers a backend kind in the process-wide registry. Backend packages +// call it from a composition root. Duplicate registration is an error. +func RegisterRepositoryKind(desc *KindDescriptor, factory RepositoryFactory) error { + defer perf.Track(nil, "artifact.RegisterRepositoryKind")() + + return defaultRepositoryRegistry.Register(desc, factory) +} + +// MustRegisterRepositoryKind is RegisterRepositoryKind for init functions: it panics on error. +func MustRegisterRepositoryKind(desc *KindDescriptor, factory RepositoryFactory) { + defer perf.Track(nil, "artifact.MustRegisterRepositoryKind")() + + if err := RegisterRepositoryKind(desc, factory); err != nil { + panic(err) + } +} + +// LookupRepositoryKind looks up a kind in the process-wide registry. +func LookupRepositoryKind(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.LookupRepositoryKind")() + + return defaultRepositoryRegistry.Lookup(kind) +} + +// RepositoryKinds lists the known kinds of the process-wide registry. +func RepositoryKinds() []string { + defer perf.Track(nil, "artifact.RepositoryKinds")() + + return defaultRepositoryRegistry.Kinds() +} + +// NewRepository constructs a repository from the process-wide registry. +func NewRepository(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + defer perf.Track(nil, "artifact.NewRepository")() + + return defaultRepositoryRegistry.New(ctx, alias, cfg, opts...) +} + +// ResetRepositoryKindsForTesting clears the process-wide registry. Tests that register fake +// backends call it from t.Cleanup so registrations do not leak between tests. +func ResetRepositoryKindsForTesting() { + defer perf.Track(nil, "artifact.ResetRepositoryKindsForTesting")() + + defaultRepositoryRegistry.Reset() +} diff --git a/pkg/artifact/registry_test.go b/pkg/artifact/registry_test.go new file mode 100644 index 00000000000..8f00bf4e07e --- /dev/null +++ b/pkg/artifact/registry_test.go @@ -0,0 +1,288 @@ +package artifact + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// fakeKind registers a kind whose factory records its inputs and returns a mock repository. +type fakeKind struct { + ctx context.Context + cfg *schema.ArtifactRepositoryConfig + calls int + alias string + options RepositoryOptions + repo *MockRepository +} + +func (f *fakeKind) factory(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + f.ctx, f.cfg = ctx, cfg + f.calls++ + f.alias = alias + f.options = ApplyRepositoryOptions(opts...) + return f.repo, nil +} + +func localDirConfig(path string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: RepositoryKindLocalDir, Options: map[string]any{"path": path}} +} + +func TestRepositoryRegistry_RegisterRejectsInvalidAndDuplicateKinds(t *testing.T) { + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + desc, ok := BuiltinKind(RepositoryKindLocalDir) + require.True(t, ok) + + tests := []struct { + name string + desc *KindDescriptor + factory RepositoryFactory + wantErr error + }{ + {name: "empty kind", desc: &KindDescriptor{}, factory: fake.factory, wantErr: errUtils.ErrArtifactConfigInvalid}, + {name: "malformed kind", desc: &KindDescriptor{Kind: "Not A Kind"}, factory: fake.factory, wantErr: errUtils.ErrArtifactConfigInvalid}, + {name: "nil factory", desc: desc, factory: nil, wantErr: errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewRepositoryRegistry().Register(tt.desc, tt.factory) + require.ErrorIs(t, err, tt.wantErr) + }) + } + + registry := NewRepositoryRegistry() + require.NoError(t, registry.Register(desc, fake.factory)) + err := registry.Register(desc, fake.factory) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a duplicate kind must be rejected") + assert.Contains(t, err.Error(), RepositoryKindLocalDir) +} + +func TestRepositoryRegistry_LookupFallsBackToBuiltinAndPrefersRegistered(t *testing.T) { + registry := NewRepositoryRegistry() + + builtin, ok := registry.Lookup(RepositoryKindS3) + require.True(t, ok, "V1 kinds validate before any backend registers") + assert.False(t, registry.IsRegistered(RepositoryKindS3)) + assert.Equal(t, PackagingTgz, builtin.DefaultPackaging) + + _, ok = registry.Lookup("ftp/server") + assert.False(t, ok) + + refined := *builtin + refined.DefaultPackaging = PackagingZip + require.NoError(t, registry.Register(&refined, (&fakeKind{}).factory)) + got, ok := registry.Lookup(RepositoryKindS3) + require.True(t, ok) + assert.Equal(t, PackagingZip, got.DefaultPackaging, "a registered descriptor takes precedence") + assert.True(t, registry.IsRegistered(RepositoryKindS3)) +} + +func TestRepositoryRegistry_RegisterRejectsDefaultPackagingOutsideTheMatrix(t *testing.T) { + tests := []struct { + name string + desc KindDescriptor + want bool + }{ + {name: "default outside the matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingTgz, Capabilities: Capabilities{Packagings: []Packaging{PackagingZip}}}}, + {name: "default with an empty matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingTgz}}, + {name: "default inside the matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingZip, Capabilities: Capabilities{Packagings: []Packaging{PackagingZip}}}, want: true}, + {name: "no default", desc: KindDescriptor{Kind: "gcp/gcs"}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry := NewRepositoryRegistry() + err := registry.Register(&tt.desc, (&fakeKind{}).factory) + if tt.want { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.False(t, registry.IsRegistered("gcp/gcs"), "a rejected descriptor is not registered") + }) + } +} + +func TestRepositoryRegistry_KindsListsBuiltinAndRegisteredSorted(t *testing.T) { + registry := NewRepositoryRegistry() + require.NoError(t, registry.Register(&KindDescriptor{Kind: "gcp/gcs"}, (&fakeKind{}).factory)) + assert.Equal(t, []string{"aws/s3", "gcp/gcs", "local/archive", "local/dir", "oci/layout", "oci/registry"}, registry.Kinds()) +} + +func TestRepositoryRegistry_NewConstructsThroughFactoryWithoutNetworkOrCredentials(t *testing.T) { + ctrl := gomock.NewController(t) + repo := NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(Capabilities{Read: true, Write: true, Packagings: []Packaging{PackagingNone}}).AnyTimes() + fake := &fakeKind{repo: repo} + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, fake.factory)) + + resolved := 0 + resolver := func(context.Context, string) (any, error) { resolved++; return nil, nil } + lookup := func(string) (Repository, error) { return nil, nil } + type contextKey struct{} + ctx := context.WithValue(context.Background(), contextKey{}, "test") + cfg := localDirConfig(t.TempDir()) + got, err := registry.New(ctx, "local", cfg, WithIdentityResolver(resolver), WithRepositoryLookup(lookup)) + require.NoError(t, err) + assert.Same(t, repo, got) + assert.Same(t, ctx, fake.ctx) + assert.Same(t, cfg, fake.cfg) + assert.Equal(t, 1, fake.calls) + assert.Equal(t, "local", fake.alias) + assert.NotNil(t, fake.options.ResolveIdentity) + assert.NotNil(t, fake.options.LookupRepository) + assert.Zero(t, resolved, "constructing a repository must not resolve credentials") + assert.True(t, got.Capabilities().Write, "capabilities are available immediately") +} + +func TestRepositoryRegistry_NewRejectsUnknownUnregisteredAndInvalid(t *testing.T) { + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, fake.factory)) + + _, err := registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: "ftp/server"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "unknown kind") + + _, err = registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifacts"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a known V1 kind without a registered backend is unsupported, not unknown") + + _, err = registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: RepositoryKindLocalDir}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "missing required options fail before the factory runs") + + _, err = registry.New(context.Background(), "x", nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Zero(t, fake.calls, "invalid configuration never reaches the factory") +} + +func TestRepositoryRegistry_NewWrapsFactoryErrors(t *testing.T) { + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + boom := errors.New("boom") + require.NoError(t, registry.Register(desc, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...RepositoryOption) (Repository, error) { + return nil, boom + })) + _, err := registry.New(context.Background(), "local", localDirConfig(t.TempDir())) + require.ErrorIs(t, err, boom) + assert.Contains(t, err.Error(), `repository "local"`) +} + +func TestRepositoryRegistry_ResetClearsRegistrations(t *testing.T) { + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, (&fakeKind{}).factory)) + registry.Reset() + assert.False(t, registry.IsRegistered(RepositoryKindLocalDir)) + require.NoError(t, registry.Register(desc, (&fakeKind{}).factory), "a reset registry accepts the kind again") +} + +func TestRepositoryRegistry_ConcurrentRegistrationIsSafe(t *testing.T) { + registry := NewRepositoryRegistry() + var wg sync.WaitGroup + errs := make([]error, 8) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + // Every goroutine races for the same kind; exactly one must win. + errs[i] = registry.Register(&KindDescriptor{Kind: "race/kind"}, (&fakeKind{}).factory) + _ = registry.Kinds() + _, _ = registry.Lookup("race/kind") + }() + } + wg.Wait() + winners := 0 + for _, err := range errs { + if err == nil { + winners++ + continue + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + assert.Equal(t, 1, winners) +} + +func TestDefaultRegistry_RegisterNewAndReset(t *testing.T) { + t.Cleanup(ResetRepositoryKindsForTesting) + ResetRepositoryKindsForTesting() + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + desc, _ := BuiltinKind(RepositoryKindLocalDir) + + require.NoError(t, RegisterRepositoryKind(desc, fake.factory)) + require.ErrorIs(t, RegisterRepositoryKind(desc, fake.factory), errUtils.ErrArtifactConflict) + assert.Panics(t, func() { MustRegisterRepositoryKind(desc, fake.factory) }) + + got, err := NewRepository(context.Background(), "local", localDirConfig(t.TempDir())) + require.NoError(t, err) + assert.Same(t, fake.repo, got) + _, ok := LookupRepositoryKind(RepositoryKindLocalDir) + assert.True(t, ok) + assert.Contains(t, RepositoryKinds(), RepositoryKindLocalDir) + + ResetRepositoryKindsForTesting() + _, err = NewRepository(context.Background(), "local", localDirConfig(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestApplyRepositoryOptions_IgnoresNilOptions(t *testing.T) { + applied := ApplyRepositoryOptions(nil, WithRepositoryLookup(func(alias string) (Repository, error) { + return nil, fmt.Errorf("lookup %s", alias) + })) + require.NotNil(t, applied.LookupRepository) + _, err := applied.LookupRepository("x") + require.EqualError(t, err, "lookup x") +} + +func TestRepositoryRegistryDescriptorIsolation(t *testing.T) { + r := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, r.Register(desc, (&fakeKind{}).factory)) + desc.PathOptions[0] = "changed" + desc.Capabilities.Packagings[0] = PackagingZip + desc.Capabilities.LockScopes[0] = "changed" + got, ok := r.Lookup(desc.Kind) + require.True(t, ok) + assert.Equal(t, OptionPath, got.PathOptions[0]) + assert.Equal(t, PackagingNone, got.Capabilities.Packagings[0]) + assert.NotEqual(t, LockScope("changed"), got.Capabilities.LockScopes[0]) + got.PathOptions[0] = "changed" + got.Capabilities.Packagings[0] = PackagingZip + got.Capabilities.LockScopes[0] = "changed" + again, _ := r.Lookup(desc.Kind) + assert.Equal(t, OptionPath, again.PathOptions[0]) + assert.Equal(t, PackagingNone, again.Capabilities.Packagings[0]) + assert.NotEqual(t, LockScope("changed"), again.Capabilities.LockScopes[0]) +} + +// A factory error that quotes the raw alias must not carry its credential to the caller, and the +// scrubbed error keeps the cause in its chain. +func TestRepositoryRegistryNew_ScrubsFactoryErrorsOfTheAlias(t *testing.T) { + const alias = "https://usr7:S3cr3tA@example.com/repo" + cause := errors.New("dial " + alias + " failed") + r := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, r.Register(desc, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...RepositoryOption) (Repository, error) { + return nil, cause + })) + + _, err := r.New(context.Background(), alias, localDirConfig(t.TempDir())) + require.Error(t, err) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + assert.ErrorIs(t, err, cause, "the cause stays in the chain") +} diff --git a/pkg/artifact/repository.go b/pkg/artifact/repository.go new file mode 100644 index 00000000000..9123c784a77 --- /dev/null +++ b/pkg/artifact/repository.go @@ -0,0 +1,54 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_repository.go github.com/cloudposse/atmos/pkg/artifact Repository,GraphSource + +// Repository is the storage contract implemented by every backend (local directory, local archive, +// S3, OCI layout, OCI registry). Repositories own storage mechanics and capabilities; the service +// owns name and version selection, producer preparation, metadata, digest pinning, conflict policy, +// retries, and result aggregation. Signatures, verification, pinning, and locks are repository +// contracts for every backend, never OCI-only service branches. +// +// An unavailable capability returns a *CapabilityError, never a successful no-op. Backend errors +// are wrapped in the common sentinels (errors/errors.go) while retaining their causes. +type Repository interface { + LockProvider + + // Capabilities describes the repository without network access. + Capabilities() Capabilities + + // Resolve resolves a reference once and returns a pinned handle used for all subsequent reads. + Resolve(ctx context.Context, ref Ref) (PinnedReference, error) + + // Inspect returns the metadata view of a pinned artifact without downloading the payload. + Inspect(ctx context.Context, pinned PinnedReference) (*Inspection, error) + + // OpenDocument opens one document without downloading the payload. A missing optional + // document returns an error wrapping ErrArtifactDocumentMissing. + OpenDocument(ctx context.Context, pinned PinnedReference, role DocumentRole) (*DocumentStream, error) + + // Read returns verified, closeable content streams or an OCI graph source for a pinned artifact. + Read(ctx context.Context, pinned PinnedReference) (*Content, error) + + // Publish stores a prepared artifact. The expected prior reference and force policy have already + // been checked by the service under the writer guard. + Publish(ctx context.Context, prepared *PreparedArtifact, options PublishOptions) (*PublishResult, error) + + // List returns one page of managed references. Pagination is consumed by the service. + List(ctx context.Context, query ListQuery) (*ListPage, error) + + // Delete removes a pinned artifact within an explicit scope after checking the expected current reference. + Delete(ctx context.Context, pinned PinnedReference, options DeleteOptions) (*DeleteResult, error) + + // Pin resolves a reference into an immutable PinnedReference suitable for consumer lockfiles. + // It acquires no write lock and is not a substitute for signature verification. + Pin(ctx context.Context, ref Ref) (PinnedReference, error) + + // Sign asks signer to sign the subject and stores the returned evidence, returning its descriptors. + Sign(ctx context.Context, subject SignatureSubject, signer Signer, options SignOptions) ([]SignatureEvidence, error) + + // Verify authenticates stored evidence against policy. It never trusts metadata claims, catalog + // entries, labels, or cached "verified" booleans. + Verify(ctx context.Context, subject SignatureSubject, policy VerificationPolicy) (*VerificationResult, error) +} diff --git a/pkg/artifact/repository_types.go b/pkg/artifact/repository_types.go new file mode 100644 index 00000000000..3fa92a29840 --- /dev/null +++ b/pkg/artifact/repository_types.go @@ -0,0 +1,228 @@ +package artifact + +import ( + "context" + "fmt" + "io" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Label is an exact key/value selector or assignment. +type Label struct { + // Key is the label key. + Key string `json:"key"` + + // Value is the label value. + Value string `json:"value"` +} + +// Inspection is the metadata view of one artifact, available without downloading the payload. +type Inspection struct { + // Pinned is the immutable reference that was inspected. + Pinned PinnedReference `json:"pinned"` + + // Record is the portable Record; nil for a foreign OCI graph that carries none. + Record *Record `json:"record,omitempty"` + + // Representation describes how the artifact is stored. + Representation Representation `json:"representation"` + + // Evidence lists signature evidence found for the artifact. It is untrusted until verified. + Evidence []SignatureEvidence `json:"evidence,omitempty"` +} + +// DocumentStream is an open, digest-verifying reader over one package document. +type DocumentStream struct { + // Document is the descriptor from the Record. + Document Document + + // Reader yields the document bytes and fails with an integrity error at EOF on mismatch. + // The caller closes it. Callers bound reads to [MaxDocumentSize]. + Reader io.ReadCloser +} + +// ContentStream is one verified, closeable payload stream. +type ContentStream struct { + // Descriptor identifies the blob. + Descriptor ocispec.Descriptor + + // Role is payload or document. + Role BlobRole + + // Path is the inventory path for per-file payloads; empty for archives. + Path string + + // Reader yields the blob bytes and verifies the descriptor digest and size. + Reader io.ReadCloser +} + +// GraphSource exposes an existing OCI graph for transfer without importing a registry SDK. +type GraphSource interface { + // Root returns the original manifest or index descriptor. + Root() ocispec.Descriptor + + // Fetch returns the raw bytes of a graph node by descriptor. + Fetch(ctx context.Context, descriptor ocispec.Descriptor) (io.ReadCloser, error) + + // Successors lists the descriptors a node references (config, layers, children). + Successors(ctx context.Context, descriptor ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// Content is the result of reading an artifact: either verified payload streams or an OCI graph +// source. The caller must Close it. +type Content struct { + // Record is the portable Record; nil for a foreign OCI graph that carries none. + Record *Record + + // Representation describes how the content is stored. + Representation Representation + + // Streams are the verified payload and document streams. Empty when Graph is set. + Streams []ContentStream + + // Graph is the OCI graph source when a native graph is carried. + Graph GraphSource +} + +// Close closes every stream, returning the first error. +func (c *Content) Close() error { + defer perf.Track(nil, "artifact.Content.Close")() + + var first error + for index := range c.Streams { + reader := c.Streams[index].Reader + if reader == nil { + continue + } + if err := reader.Close(); err != nil && first == nil { + first = fmt.Errorf("%w: closing stream %d: %w", errUtils.ErrArtifactStreamFailed, index, err) + } + } + return first +} + +// PublishOptions control one repository publication. +type PublishOptions struct { + // ExpectedPrior is the current reference the service observed under the writer guard; nil + // means the reference must not exist. Repositories use it for conditional updates. + ExpectedPrior *PinnedReference + + // Force permits replacing a conflicting reference. It has already been checked by the service. + Force bool + + // Lock is the writer lock already held by the service; nil if the repository locks internally. + Lock LockHandle +} + +// PublishResult is the outcome of a repository publication. +type PublishResult struct { + // Outcome is created, unchanged, or replaced. + Outcome Outcome + + // Pinned is the immutable reference of the published artifact. + Pinned PinnedReference + + // Representation describes the stored form. + Representation Representation + + // Location is the primary resolved location. + Location string + + // Aliases lists additional locations or tags resolving to the same content. + Aliases []string + + // LockScope is the actual lock scope that coordinated the write. + LockScope LockScope +} + +// ListQuery selects one page of managed references. The service consumes pagination. +type ListQuery struct { + // Name restricts the listing to versions of one exact name; empty enumerates the collection. + Name string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Cursor is the opaque cursor from the previous page; empty starts at the beginning. + Cursor string + + // PageSize bounds the page; zero lets the repository choose. + PageSize int + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// ListEntry summarizes one artifact in a listing. +type ListEntry struct { + // Pinned is the immutable reference. + Pinned PinnedReference `json:"pinned"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the version label. + Version string `json:"version"` + + // Kind is the artifact kind. + Kind string `json:"kind"` + + // Description is the metadata description. + Description string `json:"description,omitempty"` + + // Labels are the metadata labels. + Labels map[string]string `json:"labels,omitempty"` + + // Documents are the document descriptors, never the Markdown itself. + Documents []Document `json:"documents,omitempty"` + + // Repository is the configured repository alias the entry came from. + Repository string `json:"repository,omitempty"` +} + +// ListPage is one page of entries plus a cursor. +type ListPage struct { + // Entries are the page's artifacts, sorted lexically by name then version. + Entries []ListEntry + + // NextCursor is empty when the page is the last. + NextCursor string +} + +// DeleteOptions control one repository deletion. +type DeleteOptions struct { + // ExpectedCurrent is the reference the service observed; nil skips the staleness check. + ExpectedCurrent *PinnedReference + + // Scope is the explicit deletion scope. A tag-safe delete must not remove sibling references. + Scope DeleteSemantics + + // Lock is the writer lock already held by the service. + Lock LockHandle +} + +// DeleteResult reports what a deletion removed and affected. +type DeleteResult struct { + // Deleted lists removed locations. + Deleted []string + + // Affected lists other references that resolved to the removed content. + Affected []Ref + + // Digest is the removed resolved digest. + Digest digest.Digest +} + +// SignOptions control evidence storage. +type SignOptions struct { + // OperationID is the logical operation identifier. + OperationID string + + // Lock is the writer lock already held by the service. + Lock LockHandle +} diff --git a/pkg/artifact/representation.go b/pkg/artifact/representation.go new file mode 100644 index 00000000000..f43c7309ffc --- /dev/null +++ b/pkg/artifact/representation.go @@ -0,0 +1,31 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Representation describes how one Record is physically stored by a repository. It is mutable +// storage metadata and is never part of the Record bytes. +type Representation struct { + // Packaging is the payload packaging mode. + Packaging Packaging `json:"packaging"` + + // Payload lists the payload blob descriptors (archive, per-file objects, or image layers). + Payload []ocispec.Descriptor `json:"payload,omitempty"` + + // Documents lists the README and CHANGELOG blob descriptors stored alongside the payload. + Documents []ocispec.Descriptor `json:"documents,omitempty"` + + // Locator is the backend-specific location (path, object key, or registry reference). + Locator string `json:"locator"` + + // RecordDigest is the digest of the canonical Record bytes this representation belongs to. + RecordDigest digest.Digest `json:"record_digest"` + + // RepresentationDigest is the checksum of an archive or other transferred payload representation. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` + + // OriginalRoot is the original OCI manifest or index descriptor when a native graph is carried. + OriginalRoot *ocispec.Descriptor `json:"original_root,omitempty"` +} diff --git a/pkg/artifact/requests.go b/pkg/artifact/requests.go new file mode 100644 index 00000000000..a2b19a4f13c --- /dev/null +++ b/pkg/artifact/requests.go @@ -0,0 +1,151 @@ +package artifact + +// PublishRequest publishes a local source to one repository. +type PublishRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Source is the local source file or directory. + Source string + + // Name is the explicit artifact name. Precedence: explicit, producer metadata, source basename. + Name string + + // Version is the explicit version. Precedence: explicit, producer metadata; otherwise fail. + Version string + + // Kind is the artifact kind; empty means [KindFiles]. + Kind string + + // Metadata is the explicit user metadata (description and labels). + Metadata Metadata + + // Readme overrides the README source path, relative to the source directory. + Readme string + + // Changelog overrides the CHANGELOG source path, relative to the source directory. + Changelog string + + // Force permits replacing an existing different artifact where the repository allows it. + Force bool + + // DryRun validates and plans without writing. + DryRun bool +} + +// PullRequest materializes an artifact into a local directory. +type PullRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref + + // Destination is the output directory. + Destination string + + // Force permits overwriting existing destination content. + Force bool +} + +// CopyRequest copies an artifact between two configured repositories. +type CopyRequest struct { + // Ref selects the artifact in the source repository. + Ref Ref + + // From is the source repository alias. + From string + + // To is the destination repository alias. + To string + + // Force permits replacing a conflicting destination. It never bypasses verification. + Force bool +} + +// InspectRequest inspects an artifact without downloading its payload. +type InspectRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// ListRequest lists artifacts. List and search intentionally default to all configured repositories. +type ListRequest struct { + // Repositories restricts the listing; empty means all configured repositories. + Repositories []string + + // Name restricts the listing to versions of one exact name. + Name string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// SearchRequest searches artifacts by text. +type SearchRequest struct { + // Repositories restricts the search; empty means all configured repositories. + Repositories []string + + // Query is the search text matched against names, kinds, descriptions, and labels. + Query string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// DeleteRequest deletes an artifact from one repository. +type DeleteRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// ShowDocumentRequest reads one package document. +type ShowDocumentRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref + + // Role is readme or changelog. + Role DocumentRole +} + +// SignRequest adds signature evidence to an immutable subject. +type SignRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref must be a digest reference; sign accepts digest references only. + Ref Ref +} + +// VerifyRequest verifies an artifact's signature evidence against the effective policy. +type VerifyRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// PinRequest creates an immutable reference for consumer lockfiles. +type PinRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} diff --git a/pkg/artifact/results.go b/pkg/artifact/results.go new file mode 100644 index 00000000000..ee7e34a3762 --- /dev/null +++ b/pkg/artifact/results.go @@ -0,0 +1,78 @@ +package artifact + +// InspectResult is the result of Service.Inspect. +type InspectResult struct { + // Operation is the typed operation result stored under StepResult.Metadata["artifact"]. + Operation OperationResult `json:"operation"` + + // Inspection is the metadata view. + Inspection Inspection `json:"inspection"` + + // Verification is set when policy required or constrained verification; nil means unchecked. + Verification *VerificationResult `json:"verification,omitempty"` +} + +// RepositoryListing reports the outcome of listing one repository, so a failing or incapable +// repository is explained rather than silently omitted. +type RepositoryListing struct { + // Repository is the configured repository alias. + Repository string `json:"repository"` + + // Count is the number of entries returned from the repository. + Count int `json:"count"` + + // Diagnostics explain capability or refresh problems. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} + +// ListResult is the result of Service.List and Service.Search. Its JSON name for the entries is +// "artifacts". +type ListResult struct { + // Artifacts are the entries across the selected repositories, sorted lexically. + Artifacts []ListEntry `json:"artifacts"` + + // Repositories reports per-repository outcomes. + Repositories []RepositoryListing `json:"repositories,omitempty"` +} + +// DocumentResult is the result of Service.ShowDocument. Content is bounded by [MaxDocumentSize]. +type DocumentResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Document is the descriptor from the Record. + Document Document `json:"document"` + + // Content is the verified document bytes. + Content []byte `json:"content"` +} + +// SignResult is the result of Service.Sign. +type SignResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Subject is the signed subject. + Subject SignatureSubject `json:"subject"` + + // Evidence lists the stored evidence descriptors. + Evidence []SignatureEvidence `json:"evidence"` +} + +// VerifyResult is the result of Service.Verify. +type VerifyResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Verification is the authenticated verification result. + Verification VerificationResult `json:"verification"` +} + +// PinResult is the result of Service.Pin. +type PinResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Pinned is the immutable reference. + Pinned PinnedReference `json:"pinned"` +} diff --git a/pkg/artifact/service.go b/pkg/artifact/service.go new file mode 100644 index 00000000000..8cfd2fd4f03 --- /dev/null +++ b/pkg/artifact/service.go @@ -0,0 +1,46 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_service.go github.com/cloudposse/atmos/pkg/artifact Service + +// Service is the repository-independent artifact API consumed by commands, steps, hooks, and CI +// adapters. This package defines only the interface and its request and result types; the +// implementation (preparation, packaging, conflict policy, retries) lives in the service package. +// +// Every method takes a context first and a typed request struct, and returns a typed result plus +// an error wrapping the sentinels in errors/errors.go. Display format never changes semantics. +type Service interface { + // Publish prepares the source and stores it in one repository. + Publish(ctx context.Context, request PublishRequest) (*OperationResult, error) + + // Pull materializes an artifact into a local directory after verifying every consumed descriptor. + Pull(ctx context.Context, request PullRequest) (*OperationResult, error) + + // Copy transfers an artifact between repositories, evaluating both source and destination policy. + Copy(ctx context.Context, request CopyRequest) (*OperationResult, error) + + // Inspect returns the metadata view without downloading the payload. + Inspect(ctx context.Context, request InspectRequest) (*InspectResult, error) + + // List enumerates artifacts across the selected repositories. + List(ctx context.Context, request ListRequest) (*ListResult, error) + + // Search filters enumerated artifacts by text. + Search(ctx context.Context, request SearchRequest) (*ListResult, error) + + // Delete removes an artifact within the repository's declared delete capability. + Delete(ctx context.Context, request DeleteRequest) (*OperationResult, error) + + // ShowDocument reads one README or CHANGELOG without downloading the payload. + ShowDocument(ctx context.Context, request ShowDocumentRequest) (*DocumentResult, error) + + // Sign adds signature evidence to a digest reference. + Sign(ctx context.Context, request SignRequest) (*SignResult, error) + + // Verify requires a trusted valid signature for the artifact. + Verify(ctx context.Context, request VerifyRequest) (*VerifyResult, error) + + // Pin returns an immutable reference for consumer lockfiles. + Pin(ctx context.Context, request PinRequest) (*PinResult, error) +} diff --git a/pkg/artifact/signature.go b/pkg/artifact/signature.go new file mode 100644 index 00000000000..0eb446efd1a --- /dev/null +++ b/pkg/artifact/signature.go @@ -0,0 +1,145 @@ +package artifact + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// SubjectKind identifies what a signature is bound to. +type SubjectKind string + +// Signature subject kinds. +const ( + // SubjectRecord is the canonical portable Record of an Atmos file package. Evidence survives + // representation changes because the Record binds name, version, metadata, inventory, + // content_digest, and document digests. + SubjectRecord SubjectKind = "record" + + // SubjectOCIRoot is the original OCI manifest or index digest of a foreign image or an + // existing plugin. It is never rewritten on copy and never replaced by an archive checksum. + SubjectOCIRoot SubjectKind = "oci-root" +) + +// EvidenceFormat identifies a signature evidence encoding. +type EvidenceFormat string + +// Evidence formats supported in V1. Other formats require dedicated tested providers; unsupported +// required evidence fails closed. +const ( + // EvidenceSigstoreBundle is a Sigstore bundle verified against trusted certificate identities. + EvidenceSigstoreBundle EvidenceFormat = "sigstore-bundle" + + // EvidencePublicKeyBundle is a bundle verified against an explicitly configured public key. + // It never satisfies an OIDC issuer/identity requirement. + EvidencePublicKeyBundle EvidenceFormat = "public-key-bundle" +) + +// SignatureSubject is the immutable thing a signature covers. +type SignatureSubject struct { + // Kind is record or oci-root. + Kind SubjectKind `json:"kind"` + + // Digest is the immutable subject digest (record digest or OCI root digest). + Digest digest.Digest `json:"digest"` + + // MediaType is the subject media type. + MediaType string `json:"media_type"` + + // Size is the subject size in bytes. + Size int64 `json:"size"` +} + +// Validate checks the subject is well formed. +func (s SignatureSubject) Validate() error { + defer perf.Track(nil, "artifact.SignatureSubject.Validate")() + + if s.Kind != SubjectRecord && s.Kind != SubjectOCIRoot { + return fmt.Errorf("%w: unsupported signature subject kind %q", errUtils.ErrArtifactRefInvalid, s.Kind) + } + if err := ValidateDigest(s.Digest); err != nil { + return err + } + if s.Size < 0 { + return fmt.Errorf("%w: signature subject has a negative size", errUtils.ErrArtifactRefInvalid) + } + return nil +} + +// SignatureEvidence locates stored evidence for a subject. It is untrusted until verified. +type SignatureEvidence struct { + // Subject is the subject the evidence claims to cover. + Subject SignatureSubject `json:"subject"` + + // Format is the evidence encoding. + Format EvidenceFormat `json:"format"` + + // Descriptor identifies the evidence blob (media type, digest, size). + Descriptor ocispec.Descriptor `json:"descriptor"` + + // Locator is the backend-specific storage location, for example + // ".atmos-artifacts/v1/signatures///.json". + Locator string `json:"locator,omitempty"` +} + +// EvidenceBlob is evidence together with its bytes, bounded by [MaxEvidenceSize]. +type EvidenceBlob struct { + // Evidence locates and describes the blob. + Evidence SignatureEvidence + + // Data holds the evidence bytes. + Data []byte +} + +// VerificationStatus is the outcome of a verification. +type VerificationStatus string + +// Verification statuses. +const ( + // VerificationVerified means a valid permitted signature satisfied every required trust set. + VerificationVerified VerificationStatus = "verified" + + // VerificationUnchecked means no trust configuration was supplied, so nothing was claimed. + VerificationUnchecked VerificationStatus = "unchecked" + + // VerificationFailed means required verification did not succeed. + VerificationFailed VerificationStatus = "failed" +) + +// SignerClaim is an authenticated signer identity extracted by a verifier. Metadata claims, +// labels, catalog entries, and cached booleans never produce claims. +type SignerClaim struct { + // Identity is the certificate identity (SAN) for keyless signing; empty for key signing. + Identity string `json:"identity,omitempty"` + + // Issuer is the OIDC issuer for keyless signing; empty for key signing. + Issuer string `json:"issuer,omitempty"` + + // KeyID is the fingerprint of the trusted public key for key-based signing. + KeyID string `json:"key_id,omitempty"` +} + +// VerificationResult is the authenticated result of verifying a subject against a policy. +type VerificationResult struct { + // Subject is the verified subject. + Subject SignatureSubject `json:"subject"` + + // PolicyFingerprint is [VerificationPolicy.Fingerprint] of the policy that was applied. + PolicyFingerprint string `json:"policy_fingerprint,omitempty"` + + // Claims are the verified signer claims. + Claims []SignerClaim `json:"claims,omitempty"` + + // EvidenceDigests lists the digests of the evidence blobs that were considered. + EvidenceDigests []digest.Digest `json:"evidence_digests,omitempty"` + + // Status is verified, unchecked, or failed. + Status VerificationStatus `json:"status"` + + // Diagnostics describes the failure reason and any bounded malformed or unsupported evidence. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} diff --git a/pkg/artifact/signer.go b/pkg/artifact/signer.go new file mode 100644 index 00000000000..22698e99417 --- /dev/null +++ b/pkg/artifact/signer.go @@ -0,0 +1,28 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_signer.go github.com/cloudposse/atmos/pkg/artifact Signer + +// EvidencePayload is freshly produced signature evidence, bounded by [MaxEvidenceSize]. +type EvidencePayload struct { + // Format is the evidence encoding. + Format EvidenceFormat + + // MediaType is the evidence media type. + MediaType string + + // Data holds the evidence bytes. + Data []byte +} + +// Signer produces signature evidence for a subject. V1 reuses the managed Cosign authoring path +// behind this interface; registry authentication stays separate from signing identity, and no +// custom cryptography is implemented. +type Signer interface { + // Formats lists the evidence formats the signer can produce. + Formats() []EvidenceFormat + + // Sign signs the immutable subject and returns evidence for the repository to store. + Sign(ctx context.Context, subject SignatureSubject) (*EvidencePayload, error) +} diff --git a/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f b/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f new file mode 100644 index 00000000000..b77f8cb9be1 --- /dev/null +++ b/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f @@ -0,0 +1,4 @@ +go test fuzz v1 +string("0") +string("0") +string("%0:%") diff --git a/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c b/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c new file mode 100644 index 00000000000..00b2ddd313c --- /dev/null +++ b/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c @@ -0,0 +1,4 @@ +go test fuzz v1 +string("0") +string("\t") +string("#000") diff --git a/pkg/artifact/timestamp.go b/pkg/artifact/timestamp.go new file mode 100644 index 00000000000..6ecb7560f6c --- /dev/null +++ b/pkg/artifact/timestamp.go @@ -0,0 +1,73 @@ +package artifact + +import ( + "encoding/json" + "fmt" + "regexp" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// timestampLayout is the canonical UTC RFC3339 layout with second precision. +const timestampLayout = "2006-01-02T15:04:05Z" + +// Only the canonical text is admitted. The standard time parser alone would also accept fractional +// seconds even though the layout has none. +var timestampPattern = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$`) + +// Timestamp is an instant serialized as a UTC RFC3339 string with second precision. +// Sub-second precision is discarded on serialization so that an in-memory value and the value +// decoded from its canonical bytes always produce identical digests. +type Timestamp time.Time + +// NewTimestamp converts t to a Timestamp normalized to UTC and truncated to whole seconds. +func NewTimestamp(t time.Time) Timestamp { + defer perf.Track(nil, "artifact.NewTimestamp")() + + return Timestamp(t.UTC().Truncate(time.Second)) +} + +// Time returns the instant as a time.Time. +// +//nolint:lintroller // Trivial conversion with no I/O. +func (t Timestamp) Time() time.Time { return time.Time(t) } + +// IsZero reports whether the timestamp is the zero instant. +// +//nolint:lintroller // Trivial getter with no I/O. +func (t Timestamp) IsZero() bool { return time.Time(t).IsZero() } + +// String returns the canonical representation. +// +//nolint:lintroller // Trivial formatter with no I/O. +func (t Timestamp) String() string { + return time.Time(t).UTC().Truncate(time.Second).Format(timestampLayout) +} + +// MarshalJSON implements json.Marshaler using the canonical layout. +func (t Timestamp) MarshalJSON() ([]byte, error) { + defer perf.Track(nil, "artifact.Timestamp.MarshalJSON")() + + return json.Marshal(t.String()) +} + +// UnmarshalJSON implements json.Unmarshaler. Only the canonical layout is accepted. +func (t *Timestamp) UnmarshalJSON(data []byte) error { + defer perf.Track(nil, "artifact.Timestamp.UnmarshalJSON")() + + var raw string + if err := json.Unmarshal(data, &raw); err != nil { + return fmt.Errorf("%w: timestamp must be a string: %w", errUtils.ErrArtifactRecordInvalid, err) + } + if !timestampPattern.MatchString(raw) { + return fmt.Errorf("%w: timestamp %q must use the form 2006-01-02T15:04:05Z", errUtils.ErrArtifactRecordInvalid, raw) + } + parsed, err := time.Parse(timestampLayout, raw) + if err != nil { + return fmt.Errorf("%w: timestamp %q is not a UTC RFC3339 time: %w", errUtils.ErrArtifactRecordInvalid, raw, err) + } + *t = Timestamp(parsed) + return nil +} diff --git a/pkg/artifact/vectors_test.go b/pkg/artifact/vectors_test.go new file mode 100644 index 00000000000..1384846bf5e --- /dev/null +++ b/pkg/artifact/vectors_test.go @@ -0,0 +1,288 @@ +package artifact + +import ( + "bytes" + "errors" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Frozen canonical byte vectors. These literals are the contract: they were cross-checked with an +// independent JSON canonicalizer and must never be regenerated to hide an incompatible change. +// A change to any value below changes the identity of every previously published artifact. +const ( + // Vector A: files package with every entry type, documents, labels, provenance, escapes, Unicode. + vectorACanonical = `{"content_digest":"sha256:9bd2edcda28887b6bc03cba0a811460d0abfb4f35633a7cf1eb7908f43ec4fea","created_at":"2026-10-30T12:34:56Z","documents":[{"digest":"sha256:6665bc2f57c4a7045d347313a1566d91561ff2a3c9d17874a462ae73b7eb8593","media_type":"text/markdown","role":"changelog","size":9,"source_path":".atmos/artifact/CHANGELOG.md"},{"digest":"sha256:711a6108ba2ce6ca93dd47d6817f2361db10d8ab6eec89460b2dfc2c325efabe","media_type":"text/markdown","role":"readme","size":5,"source_path":".atmos/artifact/README.md"}],"inventory":[{"mode":0,"path":"link.tf","size":0,"target":"main.tf","type":"symlink"},{"digest":"sha256:0d6e4079e36703ebd37c00722f5891d28b0e2811dc114b129215123adcce3605","mode":420,"path":"main.tf","size":8,"type":"file"},{"mode":493,"path":"scripts","size":0,"type":"directory"},{"digest":"sha256:acba25512100f80b56fc3ccd14c65be55d94800cda77585c5f41a887e398f9be","mode":493,"path":"scripts/run.sh","size":12,"type":"file"},{"digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","mode":420,"path":"ünï.txt","size":0,"type":"file"}],"kind":"files","metadata":{"description":"VPC & \"friends\"\u0009é","labels":{"env":"prod","team":"platform"},"provenance":{"ci":{"provider":"github","run_id":"42"}}},"name":"components/terraform/vpc","schema_version":1,"version":"1.2.0"}` + + vectorAContent digest.Digest = "sha256:9bd2edcda28887b6bc03cba0a811460d0abfb4f35633a7cf1eb7908f43ec4fea" + vectorAIntent digest.Digest = "sha256:c542e50200a6bfd14a5663e89c70e6e5aa97b951e682fb20dac8a71ed1777bee" + vectorARecord digest.Digest = "sha256:48797bdf65aa85538095cfa311c57614180796c8200285c10434c32af896e1c3" + + // Vector B: empty package carrying typed kind data (a legacy-planfile-shaped kind). + vectorBCanonical = `{"content_digest":"sha256:6a5cabe4fc108c5947a6ff9fdb256559d9053edb389ee39cdbbcd97fdf0133cb","created_at":"2026-01-02T03:04:05Z","documents":[],"inventory":[],"kind":"ci/planfile","kind_data":{"component":"vpc","extra":{"a":"","b":0},"key":"k","pr_number":7,"stack":"plat-ue2-dev"},"metadata":{},"name":"ci/planfiles/abc","schema_version":1,"version":"deadbeef"}` + + vectorBContent digest.Digest = "sha256:6a5cabe4fc108c5947a6ff9fdb256559d9053edb389ee39cdbbcd97fdf0133cb" + vectorBIntent digest.Digest = "sha256:789b6c268c0f17f8b9744f05c2f309a4ba90a8342be21d4848fb569873fe2702" + vectorBRecord digest.Digest = "sha256:56c1a7c24bd4a1c74fc23f6aa0f351edff24e891bd7fa8fba4846593f51c88de" +) + +type vectorKindData struct { + Stack string `json:"stack"` + Key string `json:"key"` + PRNumber int `json:"pr_number"` + Component string `json:"component"` + Extra struct { + B int `json:"b"` + A string `json:"a"` + } `json:"extra"` +} + +// vectorRecordA builds Vector A with inventory in scrambled order and labels inserted in the given order. +func vectorRecordA(t *testing.T, labelOrder []string) Record { + t.Helper() + all := map[string]string{"team": "platform", "env": "prod"} + labels := make(map[string]string, len(all)) + for _, key := range labelOrder { + labels[key] = all[key] + } + record := Record{ + Name: "components/terraform/vpc", + Version: "1.2.0", + Kind: "files", + // Sub-second precision must be discarded by the canonical form. + CreatedAt: NewTimestamp(time.Date(2026, 10, 30, 12, 34, 56, 789000000, time.UTC)), + Metadata: Metadata{ + Description: "VPC & \"friends\"\té", + Labels: labels, + Provenance: &Provenance{CI: &CIProvenance{Provider: "github", RunID: "42"}}, + }, + Inventory: []Entry{ + {Path: "scripts/run.sh", Type: EntryFile, Size: 12, Mode: 0o755, Digest: digest.FromString("run")}, + {Path: "main.tf", Type: EntryFile, Size: 8, Mode: 0o644, Digest: digest.FromString("main")}, + {Path: "scripts", Type: EntryDirectory, Mode: 0o755}, + {Path: "link.tf", Type: EntrySymlink, Target: "main.tf"}, + {Path: "ünï.txt", Type: EntryFile, Mode: 0o644, Digest: digest.FromString("")}, + }, + Documents: []Document{ + {Role: DocumentReadme, SourcePath: ".atmos/artifact/README.md", MediaType: DocumentMediaType, Size: 5, Digest: digest.FromString("readme")}, + {Role: DocumentChangelog, SourcePath: ".atmos/artifact/CHANGELOG.md", MediaType: DocumentMediaType, Size: 9, Digest: digest.FromString("changelog")}, + }, + } + require.NoError(t, record.Seal()) + return record +} + +func vectorRecordB(t *testing.T) Record { + t.Helper() + data := vectorKindData{Stack: "plat-ue2-dev", Key: "k", PRNumber: 7, Component: "vpc"} + kind, err := NewKindData(data) + require.NoError(t, err) + record := Record{ + Name: "ci/planfiles/abc", + Version: "deadbeef", + Kind: "ci/planfile", + CreatedAt: NewTimestamp(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)), + KindData: kind, + } + require.NoError(t, record.Seal()) + return record +} + +func TestFrozenVectors(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T) Record + canonical string + content digest.Digest + intent digest.Digest + record digest.Digest + }{ + { + name: "A files package, labels env-first", + build: func(t *testing.T) Record { return vectorRecordA(t, []string{"env", "team"}) }, + canonical: vectorACanonical, content: vectorAContent, intent: vectorAIntent, record: vectorARecord, + }, + { + name: "A files package, labels team-first", + build: func(t *testing.T) Record { return vectorRecordA(t, []string{"team", "env"}) }, + canonical: vectorACanonical, content: vectorAContent, intent: vectorAIntent, record: vectorARecord, + }, + { + name: "B typed kind data", + build: vectorRecordB, + canonical: vectorBCanonical, content: vectorBContent, intent: vectorBIntent, record: vectorBRecord, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + record := tt.build(t) + canonical, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, tt.canonical, string(canonical)) + assert.NotEqual(t, byte('\n'), canonical[len(canonical)-1], "no trailing newline") + assert.Equal(t, tt.content, record.ContentDigest) + intent, err := record.IntentDigest() + require.NoError(t, err) + assert.Equal(t, tt.intent, intent) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, tt.record, recordDigest) + assert.Equal(t, digest.FromBytes([]byte(tt.canonical)), recordDigest, "record digest is the SHA-256 of the canonical bytes") + }) + } +} + +func TestFrozenVectorsDecodeRoundTrip(t *testing.T) { + for name, canonical := range map[string]string{"A": vectorACanonical, "B": vectorBCanonical} { + t.Run(name, func(t *testing.T) { + record, err := DecodeRecord([]byte(canonical)) + require.NoError(t, err) + again, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, canonical, string(again)) + }) + } + record, err := DecodeRecord([]byte(vectorBCanonical)) + require.NoError(t, err) + var kind vectorKindData + require.NoError(t, record.KindData.Decode(&kind)) + assert.Equal(t, 7, kind.PRNumber) + assert.Equal(t, "plat-ue2-dev", kind.Stack) +} + +func TestIntentAndRecordDigestSensitivity(t *testing.T) { + base := vectorRecordA(t, []string{"env", "team"}) + baseIntent, err := base.IntentDigest() + require.NoError(t, err) + baseRecord, err := base.RecordDigest() + require.NoError(t, err) + + t.Run("created_at and provenance are excluded from intent but not from the record digest", func(t *testing.T) { + changed := vectorRecordA(t, []string{"env", "team"}) + changed.CreatedAt = NewTimestamp(time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)) + changed.Metadata.Provenance = &Provenance{CI: &CIProvenance{Provider: "github", RunID: "999"}} + intent, err := changed.IntentDigest() + require.NoError(t, err) + assert.Equal(t, baseIntent, intent) + recordDigest, err := changed.RecordDigest() + require.NoError(t, err) + assert.NotEqual(t, baseRecord, recordDigest) + }) + + tests := []struct { + name string + mutate func(r *Record) + }{ + {"label value", func(r *Record) { r.Metadata.Labels["env"] = "dev" }}, + {"description", func(r *Record) { r.Metadata.Description = "other" }}, + {"kind", func(r *Record) { r.Kind = "component/terraform" }}, + {"version", func(r *Record) { r.Version = "1.2.1" }}, + {"name", func(r *Record) { r.Name = "components/terraform/vpc2" }}, + {"kind data", func(r *Record) { + data, err := NewKindData(vectorKindData{Stack: "x"}) + require.NoError(t, err) + r.KindData = data + }}, + {"file mode changes content and intent", func(r *Record) { + r.Inventory[1].Mode = 0o600 + require.NoError(t, r.Seal()) + }}, + } + for _, tt := range tests { + t.Run("intent changes with "+tt.name, func(t *testing.T) { + changed := vectorRecordA(t, []string{"env", "team"}) + tt.mutate(&changed) + if tt.name != "file mode changes content and intent" { + // Fields other than the inventory do not require resealing the content digest. + require.NoError(t, changed.Validate()) + } + intent, err := changed.IntentDigest() + require.NoError(t, err) + assert.NotEqual(t, baseIntent, intent) + }) + } +} + +func TestCanonicalDoesNotMutateRecordAndIsIsolated(t *testing.T) { + record := vectorRecordA(t, []string{"env", "team"}) + firstPath := record.Inventory[0].Path + first, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstPath, record.Inventory[0].Path, "input inventory order is untouched by canonicalization") + assert.Equal(t, "prod", record.Metadata.Labels["env"]) + + // result -> source: scribbling on the returned bytes must not change the record or later results. + scribbled := bytes.Clone(first) + for i := range first { + first[i] = 'x' + } + second, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(second)) + assert.NotEqual(t, string(scribbled), string(first)) + + // source -> result: mutating the record afterwards must not affect bytes already returned. + record.Metadata.Labels["env"] = "changed" + record.Inventory[0].Path = "zzz" + require.NoError(t, record.Seal()) + third, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(second), "earlier result is unaffected by later source mutation") + assert.NotEqual(t, string(second), string(third)) +} + +func TestKindDataOrderIndependence(t *testing.T) { + first, err := NewKindData(struct { + B int `json:"b"` + A int `json:"a"` + }{B: 2, A: 1}) + require.NoError(t, err) + second, err := NewKindData(struct { + A int `json:"a"` + B int `json:"b"` + }{A: 1, B: 2}) + require.NoError(t, err) + assert.Equal(t, `{"a":1,"b":2}`, string(first)) + assert.Equal(t, first, second) + + _, err = NewKindData(struct { + Ratio float64 `json:"ratio"` + }{Ratio: 0.5}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, "floats are not allowed in the canonical path") +} + +func TestDecodeRecordRejectsNonCanonicalBytes(t *testing.T) { + tests := []struct { + name string + raw string + want error + }{ + {"trailing newline", vectorBCanonical + "\n", errUtils.ErrArtifactRecordInvalid}, + {"extra whitespace", `{ ` + vectorBCanonical[1:], errUtils.ErrArtifactRecordInvalid}, + {"unknown field", vectorBCanonical[:len(vectorBCanonical)-1] + `,"extra":1}`, errUtils.ErrArtifactRecordInvalid}, + {"wrong schema version", mutateVectorB(`"schema_version":1`, `"schema_version":2`), errUtils.ErrArtifactRecordInvalid}, + {"tampered content digest", mutateVectorB("6a5cabe4", "00000000"), errUtils.ErrArtifactRecordInvalid}, + {"duplicate key", mutateVectorB(`"kind":"ci/planfile"`, `"kind":"ci/planfile","kind":"files"`), errUtils.ErrArtifactRecordInvalid}, + {"sub-second timestamp", mutateVectorB("03:04:05Z", "03:04:05.5Z"), errUtils.ErrArtifactRecordInvalid}, + {"empty input", ``, errUtils.ErrArtifactRecordInvalid}, + {"oversize", string(make([]byte, MaxRecordSize+1)), errUtils.ErrArtifactTooLarge}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DecodeRecord([]byte(tt.raw)) + require.Error(t, err) + assert.True(t, errors.Is(err, tt.want), "got %v", err) + }) + } +} + +// mutateVectorB returns Vector B with the first occurrence of old replaced. +func mutateVectorB(old, replacement string) string { + return string(bytes.Replace([]byte(vectorBCanonical), []byte(old), []byte(replacement), 1)) +} diff --git a/pkg/artifact/verifier.go b/pkg/artifact/verifier.go new file mode 100644 index 00000000000..1fdb44a6e6c --- /dev/null +++ b/pkg/artifact/verifier.go @@ -0,0 +1,19 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_verifier.go github.com/cloudposse/atmos/pkg/artifact Verifier + +// Verifier authenticates stored evidence against a policy. Repository adapters fetch evidence and +// invoke a Verifier; they may register additional native formats with the same result contract. +type Verifier interface { + // Formats lists the evidence formats the verifier understands. Required evidence in any other + // format fails closed. + Formats() []EvidenceFormat + + // Verify checks subject binding, cryptographic validity, signer or key trust, and the + // applicable log, certificate, and signing-time proofs. At least one valid permitted signature + // must satisfy each required trust set; unrelated invalid attachments cannot override valid + // matching evidence. A public-key signature never satisfies an identity/issuer requirement. + Verify(ctx context.Context, subject SignatureSubject, evidence []EvidenceBlob, policy VerificationPolicy) (*VerificationResult, error) +} diff --git a/pkg/config/artifacts.go b/pkg/config/artifacts.go new file mode 100644 index 00000000000..af81492b6bd --- /dev/null +++ b/pkg/config/artifacts.go @@ -0,0 +1,330 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/spf13/viper" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/merge" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// artifactsSectionKey is the top-level atmos.yaml key of the artifact configuration. +const ( + artifactsSectionKey = "artifacts" + verificationSectionKey = "verification" +) + +// artifactOrigins records, per setting, the configuration file that supplied the final value. It is +// tracked independently of the merged map because deep merging fuses the fields of several files +// into one repository: kind from one file and options.path from another must each keep their own +// declaring file. The last source that declares a setting wins, exactly like the value merge. +type artifactOrigins struct { + repoPath map[string]schema.ArtifactConfigOrigin + repoKeys map[string]schema.ArtifactConfigOrigin + globalKeys schema.ArtifactConfigOrigin +} + +func newArtifactOrigins() *artifactOrigins { + return &artifactOrigins{ + repoPath: make(map[string]schema.ArtifactConfigOrigin), + repoKeys: make(map[string]schema.ArtifactConfigOrigin), + } +} + +// evaluatedArtifacts retains the first evaluation and the file declaring each path-valued setting. +type evaluatedArtifacts struct { + section map[string]any + origins map[string]artifactFileOrigin +} + +type artifactFileOrigin struct { + file string + remote bool +} + +// evaluateArtifactsForLoad resolves tags once before Viper processes the section. +func evaluateArtifactsForLoad(node *yaml.Node, sourceFile string) (*evaluatedArtifacts, error) { + result := &evaluatedArtifacts{origins: make(map[string]artifactFileOrigin)} + decoder := configYamlDecoder{includes: make(map[string]bool)} + decoder.origin = result.recordOrigin + value, err := decoder.decode(node, sourceFile) + if err != nil { + return nil, err + } + if value == nil { + return result, nil + } + section, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("%w: artifacts in %s must be a mapping, got %T", errUtils.ErrArtifactConfigInvalid, sourceFile, value) + } + result.section = section + return result, nil +} + +func (result *evaluatedArtifacts) recordOrigin(path []string, file string, remote bool) { + origin := artifactFileOrigin{file: file, remote: remote} + if len(path) == 2 && strings.EqualFold(path[0], verificationSectionKey) && strings.EqualFold(path[1], "trusted_keys") { + result.origins["global"] = origin + return + } + if len(path) != 4 || !strings.EqualFold(path[0], "repositories") { + return + } + switch strings.ToLower(path[2]) { + case "options": + if strings.EqualFold(path[3], "path") { + result.origins["path\x00"+path[1]] = origin + } + case verificationSectionKey: + if strings.EqualFold(path[3], "trusted_keys") { + result.origins["keys\x00"+path[1]] = origin + } + } +} + +// resolveArtifactsConfig rebuilds atmosConfig.Artifacts from the retained configuration sources. +// +// Viper cannot be used for this section: it lowercases keys (repository aliases, label keys, and +// S3 tag keys are case-sensitive), splits keys on dots, and, once files are merged, no longer knows +// which file supplied a value. So the section is parsed from each retained source in merge order, +// merged as raw data, and decoded strictly once. While doing so the declaring file of each +// path-valued setting is recorded and relative local paths are anchored to that file's directory +// before the origin is lost. A relative path declared only by a remote import is left unresolved +// with its origin marked remote; artifactconfig validation then requires a local or absolute +// override. The key_pattern templates are plain strings here and are never rendered. +func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfiguration) error { + defer perf.Track(atmosConfig, "config.resolveArtifactsConfig")() + + merged := map[string]any{} + origins := newArtifactOrigins() + declared := false + for _, source := range collectConfigSourcesForCasePreservation(v, v.ConfigFileUsed()) { + section, state, err := artifactSectionForReplay(source) + if err != nil { + return err + } + if state == artifactSectionUnset { + merged = map[string]any{} + origins = newArtifactOrigins() + declared = true + continue + } + if state == artifactSectionAbsent { + continue + } + declared = true + origins.record(section, source) + origins.recordEvaluated(source) + merged, err = merge.MergeWithOptions(nil, []map[string]any{merged, section}, false, false) + if err != nil { + return fmt.Errorf("%w: merge artifacts from %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + } + if !declared { + return nil + } + cfg, err := artifactconfig.Decode(merged) + if err != nil { + return err + } + origins.apply(&cfg) + atmosConfig.Artifacts = cfg + return nil +} + +type artifactSectionState uint8 + +const ( + artifactSectionAbsent artifactSectionState = iota + artifactSectionPresent + artifactSectionUnset +) + +// artifactSectionForReplay returns the evaluated section or an explicit removal. +func artifactSectionForReplay(source mergedConfigSource) (map[string]any, artifactSectionState, error) { + var document yaml.Node + if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { + return nil, artifactSectionAbsent, fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + if node := policyMappingValue(&document, artifactsSectionKey); node != nil && node.Tag == "!unset" { + return nil, artifactSectionUnset, nil + } + if source.artifacts != nil { + if source.artifacts.section == nil { + return nil, artifactSectionAbsent, nil + } + return source.artifacts.section, artifactSectionPresent, nil + } + section, present, err := artifactsSectionFromSource(source) + if err != nil { + return nil, artifactSectionAbsent, err + } + if !present { + return nil, artifactSectionAbsent, nil + } + return section, artifactSectionPresent, nil +} + +// recordEvaluated applies the origins of values declared inside included files. +func (o *artifactOrigins) recordEvaluated(source mergedConfigSource) { + if source.artifacts == nil { + return + } + for key, file := range source.artifacts.origins { + origin := artifactconfig.NewOrigin(file.file, source.ephemeral || file.remote) + kind, alias, found := strings.Cut(key, "\x00") + switch { + case key == "global": + o.globalKeys = origin + case found && kind == "path": + o.repoPath[alias] = origin + case found && kind == "keys": + o.repoKeys[alias] = origin + } + } +} + +// artifactsSectionFromSource parses the `artifacts` section of one retained source with its +// original key spelling. The boolean is false when the source declares no such section. +func artifactsSectionFromSource(source mergedConfigSource) (map[string]any, bool, error) { + var document yaml.Node + if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { + return nil, false, fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + node := policyMappingValue(&document, artifactsSectionKey) + if node == nil { + return nil, false, nil + } + value, err := decodeNodeWithYamlFunctionsForFile(node, source.path) + if err != nil { + return nil, false, fmt.Errorf("%w: artifacts in %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + if value == nil { + return nil, false, nil + } + section, ok := value.(map[string]any) + if !ok { + return nil, false, fmt.Errorf("%w: artifacts in %s must be a mapping, got %T", errUtils.ErrArtifactConfigInvalid, source.path, value) + } + return section, true, nil +} + +// record notes which source declares each path-valued setting present in section. +func (o *artifactOrigins) record(section map[string]any, source mergedConfigSource) { + origin := artifactconfig.NewOrigin(source.path, source.ephemeral) + if keysDeclared(mapOf(mapValue(section, verificationSectionKey))) { + o.globalKeys = origin + } + repos, _ := mapValue(section, "repositories").(map[string]any) + for alias, raw := range repos { + repo, ok := raw.(map[string]any) + if !ok { + continue + } + if _, isString := mapValue(mapOf(mapValue(repo, "options")), artifact.OptionPath).(string); isString { + o.repoPath[alias] = origin + } + if keysDeclared(mapOf(mapValue(repo, verificationSectionKey))) { + o.repoKeys[alias] = origin + } + } +} + +// apply stores the origins on the decoded configuration and anchors relative local paths. +func (o *artifactOrigins) apply(cfg *schema.ArtifactConfig) { + if cfg.Verification != nil { + cfg.Verification.KeysOrigin = o.globalKeys + cfg.Verification.TrustedKeys = anchorKeys(cfg.Verification.TrustedKeys, o.globalKeys) + } + for alias := range cfg.Repositories { + repo := cfg.Repositories[alias] + repo.PathOrigin = o.repoPath[alias] + repo.Options = anchorPathOptions(&repo) + if repo.Verification != nil { + repo.Verification.KeysOrigin = o.repoKeys[alias] + repo.Verification.TrustedKeys = anchorKeys(repo.Verification.TrustedKeys, o.repoKeys[alias]) + } + cfg.Repositories[alias] = repo + } +} + +// anchorPathOptions returns the repository options with relative local destination paths made +// absolute against the declaring file. Options are copied, never mutated in place. A path declared +// by a remote import, or with no known declaring file, stays as written so validation can reject it. +func anchorPathOptions(repo *schema.ArtifactRepositoryConfig) map[string]any { + desc, ok := artifact.LookupRepositoryKind(repo.Kind) + if !ok || len(desc.PathOptions) == 0 || repo.PathOrigin.Remote || repo.PathOrigin.Dir == "" { + return repo.Options + } + out := make(map[string]any, len(repo.Options)) + for key, value := range repo.Options { + out[key] = value + } + for _, option := range desc.PathOptions { + anchorOption(out, option, repo.PathOrigin) + } + return out +} + +// anchorOption rewrites one relative path option of options in place to its anchored form. +func anchorOption(options map[string]any, option string, origin schema.ArtifactConfigOrigin) { + path, isString := options[option].(string) + if !isString || path == "" || filepath.IsAbs(path) { + return + } + if anchored, err := artifactconfig.AnchorPath(origin, path); err == nil { + options[option] = anchored + } +} + +// anchorKeys makes relative trusted-key paths absolute against the file that declared them. +func anchorKeys(keys []string, origin schema.ArtifactConfigOrigin) []string { + if len(keys) == 0 || origin.Remote || origin.Dir == "" { + return keys + } + out := make([]string, len(keys)) + for i, key := range keys { + out[i] = key + if key == "" || filepath.IsAbs(key) { + continue + } + if anchored, err := artifactconfig.AnchorPath(origin, key); err == nil { + out[i] = anchored + } + } + return out +} + +// keysDeclared reports whether a verification mapping declares a trusted_keys list. +func keysDeclared(verification map[string]any) bool { + _, ok := mapValue(verification, "trusted_keys").([]any) + return ok +} + +// mapValue looks up a key case-insensitively, matching how the loader decodes struct fields. +func mapValue(m map[string]any, key string) any { + if value, ok := m[key]; ok { + return value + } + for candidate, value := range m { + if strings.EqualFold(candidate, key) { + return value + } + } + return nil +} + +// mapOf returns value as a mapping, or nil when it is not one. +func mapOf(value any) map[string]any { + m, _ := value.(map[string]any) + return m +} diff --git a/pkg/config/artifacts_decode_test.go b/pkg/config/artifacts_decode_test.go new file mode 100644 index 00000000000..e5ec5a97143 --- /dev/null +++ b/pkg/config/artifacts_decode_test.go @@ -0,0 +1,267 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" +) + +func TestLoadConfig_ArtifactsTaggedAliasIsEvaluated(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + t.Setenv("ARTIFACT_STORE", store) + dir := artifactsTestDir(t, `artifacts: + repositories: + first: + kind: local/dir + options: + path: &store !env ARTIFACT_STORE + second: + kind: local/dir + options: + path: *store +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, store, cfg.Artifacts.Repositories["first"].Options["path"]) + assert.Equal(t, store, cfg.Artifacts.Repositories["second"].Options["path"], "the alias sees the evaluated tag, not the literal variable name") + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsAnchoredExecEvaluatedOnceAcrossAliases(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + root := artifactsTestDir(t, fmt.Sprintf(`artifacts: + repositories: + first: + kind: local/dir + options: + path: &store !exec >- + %s -test.run=^TestMergeConfigFileExecHelper$ -- %s + second: + kind: local/dir + options: + path: *store +`, shellQuote(os.Args[0]), shellQuote(countPath))) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count), "an alias reuses the anchor's result instead of running the command again") + want := filepath.Join(root, "resolved-description") + assert.Equal(t, want, cfg.Artifacts.Repositories["first"].Options["path"]) + assert.Equal(t, want, cfg.Artifacts.Repositories["second"].Options["path"]) +} + +func TestLoadConfig_ArtifactsAliasesAndMergeKeysKeepDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./trust/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "artifacts.yaml"), `artifacts: + repositories: + base: &base + kind: local/dir + options: + path: store + verification: + trusted_keys: &keys [release.pub] + aliased: *base + inherited: + <<: *base + overridden: + <<: *base + options: + path: other + keyed: + kind: local/dir + options: + path: keyed + verification: + trusted_keys: *keys +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + trust := filepath.Join(dir, "trust") + repos := cfg.Artifacts.Repositories + for _, alias := range []string{"base", "aliased", "inherited"} { + assert.Equal(t, "local/dir", repos[alias].Kind, alias) + assert.Equal(t, filepath.Join(trust, "store"), repos[alias].Options["path"], alias) + require.NotNil(t, repos[alias].Verification, alias) + assert.Equal(t, []string{filepath.Join(trust, "release.pub")}, repos[alias].Verification.TrustedKeys, alias) + } + assert.Equal(t, "local/dir", repos["overridden"].Kind, "kind is inherited through the merge key") + assert.Equal(t, filepath.Join(trust, "other"), repos["overridden"].Options["path"], "the repository's own options win over the merged ones") + assert.Equal(t, []string{filepath.Join(trust, "release.pub")}, repos["keyed"].Verification.TrustedKeys) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsTrustedKeysIncludeAnchorsToIncludedFile(t *testing.T) { + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, fmt.Sprintf(`artifacts: + verification: + trusted_keys: !include ./trust/keys.yaml%s + repositories: + local: + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./trust/repo-keys.yaml%s +`, suffix, suffix)) + writeArtifactsFile(t, filepath.Join(dir, "trust", "keys.yaml"), "- release.pub\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "repo-keys.yaml"), "- repo.pub\n") + // A same-named file beside the including config must not be the one trusted. + writeArtifactsFile(t, filepath.Join(dir, "release.pub"), "wrong\n") + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{filepath.Join(dir, "trust", "release.pub")}, cfg.Artifacts.Verification.TrustedKeys) + assert.Equal(t, filepath.Join(dir, "trust"), cfg.Artifacts.Verification.KeysOrigin.Dir) + assert.False(t, cfg.Artifacts.Verification.KeysOrigin.Remote) + + repo := cfg.Artifacts.Repositories["local"] + require.NotNil(t, repo.Verification) + assert.Equal(t, []string{filepath.Join(dir, "trust", "repo.pub")}, repo.Verification.TrustedKeys) + assert.Equal(t, filepath.Join(dir, "trust"), repo.Verification.KeysOrigin.Dir) + assert.Equal(t, filepath.Join(dir, "store"), repo.Options["path"], "sibling settings keep the including file") + }) + } +} + +func TestLoadConfig_ArtifactsRemoteTrustedKeysIncludeIsNotAnchored(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, "- release.pub\n") + })) + defer server.Close() + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n verification:\n trusted_keys: !include "+server.URL+"/keys.yaml"+suffix+"\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{"release.pub"}, cfg.Artifacts.Verification.TrustedKeys, "a remote relative key is never anchored to a local directory") + assert.True(t, cfg.Artifacts.Verification.KeysOrigin.Remote) + + err = artifactconfig.Validate(&cfg.Artifacts) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "remote import") + }) + } +} + +func decodeYAMLForTest(t *testing.T, source string) (any, error) { + t.Helper() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &node)) + return decodeNodeWithYamlFunctions(&node) +} + +func TestDecodeNodeWithYamlFunctions_MergeKeys(t *testing.T) { + tests := []struct { + name string + source string + want map[string]any + wantErr bool + }{ + { + name: "alias of a mapping", + source: "base: &base {a: 1, b: 2}\nchild:\n <<: *base\n b: 3\n c: 4\n", + want: map[string]any{ + "base": map[string]any{"a": 1, "b": 2}, + "child": map[string]any{"a": 1, "b": 3, "c": 4}, + }, + }, + { + name: "own keys win regardless of order", + source: "base: &base {a: 1}\nchild:\n a: 9\n <<: *base\n", + want: map[string]any{"base": map[string]any{"a": 1}, "child": map[string]any{"a": 9}}, + }, + { + name: "inline mapping", + source: "child:\n <<: {a: 1}\n b: 2\n", + want: map[string]any{"child": map[string]any{"a": 1, "b": 2}}, + }, + { + name: "sequence of aliases: earlier sources win", + source: "x: &x {a: 1, b: 1}\ny: &y {b: 2, c: 2}\nchild:\n <<: [*x, *y]\n", + want: map[string]any{ + "x": map[string]any{"a": 1, "b": 1}, + "y": map[string]any{"b": 2, "c": 2}, + "child": map[string]any{"a": 1, "b": 1, "c": 2}, + }, + }, + {name: "scalar merge value", source: "child:\n <<: 5\n", wantErr: true}, + {name: "sequence with a scalar", source: "x: &x {a: 1}\nchild:\n <<: [*x, 5]\n", wantErr: true}, + { + name: "quoted merge key is an ordinary key", + source: "child:\n \"<<\": 1\n", + want: map[string]any{"child": map[string]any{"<<": 1}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeYAMLForTest(t, tt.source) + if tt.wantErr { + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestDecodeNodeWithYamlFunctions_AliasPolicy(t *testing.T) { + t.Run("an alias is a copy: mutating it leaves the anchor intact", func(t *testing.T) { + got, err := decodeYAMLForTest(t, "base: &base {list: [1, 2]}\nref: *base\n") + require.NoError(t, err) + decoded, ok := got.(map[string]any) + require.True(t, ok) + ref, ok := decoded["ref"].(map[string]any) + require.True(t, ok) + ref["list"].([]any)[0] = 99 + ref["extra"] = true + assert.Equal(t, map[string]any{"list": []any{1, 2}}, decoded["base"], "result to source isolation") + + base, ok := decoded["base"].(map[string]any) + require.True(t, ok) + base["list"].([]any)[1] = 77 + assert.Equal(t, []any{99, 2}, ref["list"], "source to result isolation") + }) + + t.Run("an alias to itself is a cycle, not a hang", func(t *testing.T) { + _, err := decodeYAMLForTest(t, "loop: &loop [*loop]\n") + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.Contains(t, err.Error(), "cycle") + }) + + t.Run("an anchor outside the decoded subtree is evaluated on demand", func(t *testing.T) { + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("other: &shared {a: 1}\nsection:\n ref: *shared\n"), &document)) + root := document.Content[0] + section := root.Content[3] + got, err := decodeNodeWithYamlFunctions(section) + require.NoError(t, err) + assert.Equal(t, map[string]any{"ref": map[string]any{"a": 1}}, got) + }) +} + +func TestDecodeNodeWithYamlFunctions_IncludeErrorsDoNotEchoCredentials(t *testing.T) { + dir := t.TempDir() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("keys: !include https://user:hunter2@127.0.0.1:1/keys.yaml\n"), &node)) + _, err := decodeNodeWithYamlFunctionsForFile(&node, filepath.Join(dir, "atmos.yaml")) + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.NotContains(t, err.Error(), "hunter2", "include URLs are redacted before they reach an error") +} diff --git a/pkg/config/artifacts_test.go b/pkg/config/artifacts_test.go new file mode 100644 index 00000000000..02ad6106a5e --- /dev/null +++ b/pkg/config/artifacts_test.go @@ -0,0 +1,535 @@ +package config + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/schema" +) + +// prdRepositories is the YAML from the artifacts PRD "Public configuration" section, verbatim. +const prdRepositories = `artifacts: + repositories: + shared: + kind: oci/registry + identity: registry-publisher + options: + registry: ghcr.io + prefix: acme/atmos + defaults: + metadata: + labels: + team: platform + + backups: + kind: aws/s3 + identity: backup-writer + packaging: tgz + options: + bucket: acme-artifacts + prefix: atmos + region: us-east-2 + tags: + purpose: backup + + local: + kind: local/dir + options: + path: .atmos/artifacts + + archives: + kind: local/archive + packaging: tgz + permissions: "0600" + options: + path: .atmos/backups + key_pattern: '{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz' + + offline: + kind: oci/layout + options: + path: .atmos/oci +` + +// artifactsTestDir creates a config root whose atmos.yaml is main, isolated from the host. +func artifactsTestDir(t *testing.T, main string) string { + t.Helper() + setupTestAdapters() + dir := t.TempDir() + t.Setenv("TEST_GIT_ROOT", dir) + t.Setenv("ATMOS_XDG_CACHE_HOME", filepath.Join(dir, "cache")) + writeArtifactsFile(t, filepath.Join(dir, "atmos.yaml"), "base_path: ./\n"+main) + return dir +} + +func writeArtifactsFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func loadArtifactsDir(t *testing.T, dir string) (schema.AtmosConfiguration, error) { + t.Helper() + return LoadConfig(&schema.ConfigAndStacksInfo{AtmosConfigDirsFromArg: []string{dir}}) +} + +func TestLoadConfig_ArtifactsPRDExampleLoadsUnchanged(t *testing.T) { + dir := artifactsTestDir(t, prdRepositories) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repos := cfg.Artifacts.Repositories + require.Len(t, repos, 5) + + shared := repos["shared"] + assert.Equal(t, artifact.RepositoryKindOCIRegistry, shared.Kind) + assert.Equal(t, "registry-publisher", shared.Identity) + assert.Equal(t, map[string]any{"registry": "ghcr.io", "prefix": "acme/atmos"}, shared.Options) + require.NotNil(t, shared.Defaults) + require.NotNil(t, shared.Defaults.Metadata) + assert.Equal(t, map[string]string{"team": "platform"}, shared.Defaults.Metadata.Labels) + + backups := repos["backups"] + assert.Equal(t, "tgz", backups.Packaging) + assert.Equal(t, map[string]any{"purpose": "backup"}, backups.Options["tags"]) + + // Relative local roots are anchored to the declaring file. + assert.Equal(t, filepath.Join(dir, ".atmos", "artifacts"), repos["local"].Options["path"]) + assert.Equal(t, artifact.RepositoryKindOCILayout, repos["offline"].Kind) + assert.Equal(t, filepath.Join(dir, ".atmos", "oci"), repos["offline"].Options["path"]) + + archives := repos["archives"] + assert.Equal(t, "0600", archives.Permissions) + assert.Equal(t, filepath.Join(dir, ".atmos", "backups"), archives.Options["path"]) + // The key_pattern template survives config loading verbatim; it is never rendered here. + assert.Equal(t, `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, archives.Options["key_pattern"]) + + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsCatalogAndVerificationExamplesLoad(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + verification: + require_signature: true + trusted_identities: + - identity: https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main + issuer: https://token.actions.githubusercontent.com + trusted_keys: [] + repositories: + shared: + kind: oci/registry + identity: registry-reader + options: + registry: ghcr.io + prefix: acme/atmos + catalog: oci://ghcr.io/acme/atmos/catalog:stable + locking: + required_scope: repository + repository: coordination + coordination: + kind: aws/s3 + identity: lock-writer + options: + bucket: acme-artifact-coordination +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) + + require.NotNil(t, cfg.Artifacts.Verification) + require.NotNil(t, cfg.Artifacts.Verification.RequireSignature) + assert.True(t, *cfg.Artifacts.Verification.RequireSignature) + assert.Equal(t, "oci://ghcr.io/acme/atmos/catalog:stable", cfg.Artifacts.Repositories["shared"].Catalog) + assert.Equal(t, "coordination", cfg.Artifacts.Repositories["shared"].Locking.Repository) +} + +func TestLoadConfig_ArtifactsPreserveKeyCase(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + Prod.US: + kind: aws/s3 + options: + bucket: acme-artifacts + tags: + Purpose: Backup + defaults: + metadata: + labels: + Team.Name: Platform +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo, ok := cfg.Artifacts.Repositories["Prod.US"] + require.True(t, ok, "repository alias keeps its case and dot, got %v", cfg.Artifacts.Repositories) + assert.Equal(t, map[string]any{"Purpose": "Backup"}, repo.Options["tags"]) + assert.Equal(t, map[string]string{"Team.Name": "Platform"}, repo.Defaults.Metadata.Labels) +} + +func TestLoadConfig_ArtifactsImportedPathsResolveAgainstDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./first/artifacts.yaml\n - ./second/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "first", "artifacts.yaml"), `artifacts: + repositories: + one: + kind: local/dir + options: + path: store-first + two: + kind: local/archive + identity: declared-in-first +`) + writeArtifactsFile(t, filepath.Join(dir, "second", "artifacts.yaml"), `artifacts: + repositories: + one: + identity: declared-in-second + two: + options: + path: store-second +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + one := cfg.Artifacts.Repositories["one"] + two := cfg.Artifacts.Repositories["two"] + // Merged fields come from different files, and each path uses the file that supplied that path. + assert.Equal(t, artifact.RepositoryKindLocalDir, one.Kind) + assert.Equal(t, "declared-in-second", one.Identity) + assert.Equal(t, filepath.Join(dir, "first", "store-first"), one.Options["path"]) + assert.Equal(t, filepath.Join(dir, "first"), one.PathOrigin.Dir) + + assert.Equal(t, artifact.RepositoryKindLocalArchive, two.Kind) + assert.Equal(t, "declared-in-first", two.Identity) + assert.Equal(t, filepath.Join(dir, "second", "store-second"), two.Options["path"]) + assert.Equal(t, filepath.Join(dir, "second"), two.PathOrigin.Dir) + assert.False(t, two.PathOrigin.Remote) + + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsLaterFileOverridesPathAndOrigin(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./base/artifacts.yaml\n"+`artifacts: + repositories: + local: + options: + path: override +`) + writeArtifactsFile(t, filepath.Join(dir, "base", "artifacts.yaml"), `artifacts: + repositories: + local: + kind: local/dir + options: + path: from-base +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "override"), cfg.Artifacts.Repositories["local"].Options["path"]) + assert.Equal(t, dir, cfg.Artifacts.Repositories["local"].PathOrigin.Dir) +} + +func TestLoadConfig_ArtifactsAnchorTrustedKeys(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./trust/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "artifacts.yaml"), `artifacts: + verification: + trusted_keys: + - keys/cosign.pub +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, []string{filepath.Join(dir, "trust", "keys", "cosign.pub")}, cfg.Artifacts.Verification.TrustedKeys) +} + +func TestLoadConfig_ArtifactsRejectsUnquotedNumericPermissions(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + archives: + kind: local/archive + permissions: 0600 + options: + path: out +`) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "quoted octal string") +} + +func TestLoadConfig_ArtifactsRejectsUnknownField(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + local: + kind: local/dir + pakaging: none +`) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "pakaging") +} + +func TestLoadConfig_ArtifactsAbsentLeavesZeroValue(t *testing.T) { + dir := artifactsTestDir(t, "") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Empty(t, cfg.Artifacts.Repositories) + assert.Nil(t, cfg.Artifacts.Verification) +} + +// remoteTestAdapter simulates a remote import: it copies a canned document into the import temp +// directory, exactly as the go-getter adapter downloads into an ephemeral location. +type remoteTestAdapter struct { + documents map[string]string +} + +func (a *remoteTestAdapter) Schemes() []string { return []string{"remote-test://"} } + +func (a *remoteTestAdapter) Resolve(_ context.Context, importPath, _, tempDir string, _, _ int, _ *schema.AtmosConfiguration) ([]ResolvedPaths, error) { + name := strings.TrimPrefix(importPath, "remote-test://") + target := filepath.Join(tempDir, name+".yaml") + if err := os.WriteFile(target, []byte(a.documents[name]), 0o600); err != nil { + return nil, err + } + return []ResolvedPaths{{FilePath: target, ImportPaths: importPath, ImportType: REMOTE}}, nil +} + +func registerRemoteArtifacts(documents map[string]string) { + RegisterImportAdapter(&remoteTestAdapter{documents: documents}) +} + +const remoteRelativeDocument = `artifacts: + repositories: + remote: + kind: local/dir + options: + path: relative-from-remote +` + +func TestLoadConfig_ArtifactsRemoteRelativePathIsNeverAnchoredToDownloadDir(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - remote-test://shared\n") + registerRemoteArtifacts(map[string]string{"shared": remoteRelativeDocument}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, "relative-from-remote", repo.Options["path"], "the ephemeral download directory must not become a write root") + assert.True(t, repo.PathOrigin.Remote) + + err = artifactconfig.Validate(&cfg.Artifacts) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), "remote import") +} + +func TestLoadConfig_ArtifactsRemoteDeclaredPathAcceptsLocalOverride(t *testing.T) { + dir := artifactsTestDir(t, `import: + - remote-test://shared +artifacts: + repositories: + remote: + options: + path: local-override +`) + registerRemoteArtifacts(map[string]string{"shared": remoteRelativeDocument}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, filepath.Join(dir, "local-override"), repo.Options["path"]) + assert.False(t, repo.PathOrigin.Remote) + assert.Equal(t, artifact.RepositoryKindLocalDir, repo.Kind, "the remote file still supplies the other fields") + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsRemoteAbsolutePathIsAccepted(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - remote-test://shared\n") + absolute := filepath.Join(t.TempDir(), "store") + registerRemoteArtifacts(map[string]string{"shared": strings.Replace(remoteRelativeDocument, "relative-from-remote", filepath.ToSlash(absolute), 1)}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + // A path declared by a remote import stays as written, which is slash form here. + assert.Equal(t, filepath.ToSlash(absolute), cfg.Artifacts.Repositories["remote"].Options["path"]) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsLocalImportIsNotEphemeral(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./local/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "local", "artifacts.yaml"), remoteRelativeDocument) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.False(t, cfg.Artifacts.Repositories["remote"].PathOrigin.Remote) + assert.Equal(t, filepath.Join(dir, "local", "relative-from-remote"), cfg.Artifacts.Repositories["remote"].Options["path"]) +} + +func TestLoadConfig_ArtifactsAtmosDFragmentAnchorsToItsOwnDirectory(t *testing.T) { + dir := artifactsTestDir(t, "") + writeArtifactsFile(t, filepath.Join(dir, ".atmos.d", "artifacts.yaml"), `artifacts: + repositories: + fragment: + kind: local/dir + options: + path: store +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, ".atmos.d", "store"), cfg.Artifacts.Repositories["fragment"].Options["path"]) +} + +// TestLoadConfig_ArtifactsDescribeRoundTrip covers `atmos describe config`: the loaded section +// serializes under `artifacts` and reads back as the same values, without leaking load-time origins. +func TestLoadConfig_ArtifactsDescribeRoundTrip(t *testing.T) { + dir := artifactsTestDir(t, prdRepositories) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + jsonBytes, err := json.Marshal(cfg) + require.NoError(t, err) + var describedJSON struct { + Artifacts schema.ArtifactConfig `json:"artifacts"` + } + require.NoError(t, json.Unmarshal(jsonBytes, &describedJSON)) + + yamlBytes, err := yaml.Marshal(cfg) + require.NoError(t, err) + var describedYAML struct { + Artifacts schema.ArtifactConfig `yaml:"artifacts"` + } + require.NoError(t, yaml.Unmarshal(yamlBytes, &describedYAML)) + + expected := cfg.Artifacts + for alias, repo := range expected.Repositories { + repo.PathOrigin = schema.ArtifactConfigOrigin{} + expected.Repositories[alias] = repo + } + assert.Equal(t, expected, describedJSON.Artifacts) + assert.Equal(t, expected, describedYAML.Artifacts) + assert.NotContains(t, string(jsonBytes), "PathOrigin") +} + +func TestLoadConfig_ArtifactsSectionUnsetClearsPriorSources(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./base.yaml\nartifacts: !unset\n") + writeArtifactsFile(t, filepath.Join(dir, "base.yaml"), "artifacts:\n repositories:\n old:\n kind: local/dir\n options:\n path: old\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Empty(t, cfg.Artifacts.Repositories) +} + +func TestPathInsideAnyRoot(t *testing.T) { + root := t.TempDir() + assert.True(t, pathInsideAnyRoot(filepath.Join(root, "nested", "file.yaml"), []string{root})) + assert.False(t, pathInsideAnyRoot(root+"-other/file.yaml", []string{root})) +} + +func TestLoadConfig_ArtifactsIncludesRetainDeclaringFiles(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include ./included/artifacts.yaml | eval\n") + writeArtifactsFile(t, filepath.Join(dir, "included", "artifacts.yaml"), `verification: + trusted_keys: [global.pub] +repositories: + local: !include ./nested/repository.yaml | eval +`) + writeArtifactsFile(t, filepath.Join(dir, "included", "nested", "repository.yaml"), `kind: local/dir +options: + path: store +verification: + trusted_keys: [repo.pub] +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{filepath.Join(dir, "included", "global.pub")}, cfg.Artifacts.Verification.TrustedKeys) + repo := cfg.Artifacts.Repositories["local"] + assert.Equal(t, filepath.Join(dir, "included", "nested", "store"), repo.Options["path"]) + require.NotNil(t, repo.Verification) + assert.Equal(t, []string{filepath.Join(dir, "included", "nested", "repo.pub")}, repo.Verification.TrustedKeys) +} + +func TestLoadConfig_ArtifactsExecEvaluatedOnce(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + config := fmt.Sprintf("artifacts:\n repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", shellQuote(os.Args[0]), shellQuote(countPath)) + root := artifactsTestDir(t, config) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(root, "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestProfileImportArtifactsExecEvaluatedOnce(t *testing.T) { + setupTestAdapters() + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + writeArtifactsFile(t, filepath.Join(dir, "import.yaml"), "logs:\n level: Info\n") + profile := filepath.Join(dir, "profile.yaml") + writeArtifactsFile(t, profile, fmt.Sprintf("base_path: %s\nimport: [./import.yaml]\nartifacts:\n repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", dir, shellQuote(os.Args[0]), shellQuote(countPath))) + v := viper.New() + v.SetConfigType("yaml") + resetMergedConfigFiles(v) + defer mergedFilesReg.finish(v) + require.NoError(t, mergeConfigFileWithImports(profile, v)) + var cfg schema.AtmosConfiguration + require.NoError(t, resolveArtifactsConfig(v, &cfg)) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(dir, "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestLoadConfig_ArtifactsPlainIncludeRetainsDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include ./included/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "included", "artifacts.yaml"), "repositories:\n local:\n kind: local/dir\n options:\n path: store\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "included", "store"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestLoadConfig_ArtifactsRemoteIncludeDoesNotAnchorPaths(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, "repositories:\n remote:\n kind: local/dir\n options:\n path: remote-store\n") + })) + defer server.Close() + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include "+server.URL+"/artifacts.yaml"+suffix+"\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, "remote-store", repo.Options["path"]) + assert.True(t, repo.PathOrigin.Remote) + }) + } +} + +func TestLoadConfig_ArtifactsIncludedExecEvaluatedOnce(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + root := artifactsTestDir(t, "artifacts: !include ./sub/artifacts.yaml | eval\n") + contents := fmt.Sprintf("repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", shellQuote(os.Args[0]), shellQuote(countPath)) + writeArtifactsFile(t, filepath.Join(root, "sub", "artifacts.yaml"), contents) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(root, "sub", "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} diff --git a/pkg/config/global_viper.go b/pkg/config/global_viper.go index d5974afe39f..1c6e01e534e 100644 --- a/pkg/config/global_viper.go +++ b/pkg/config/global_viper.go @@ -89,24 +89,30 @@ func GlobalViper() *SafeViper { // mergedFilesTracker tracks the config files merged during a single LoadConfig // call, guarded by its own mutex. type mergedFilesTracker struct { - mu sync.Mutex - sources []mergedConfigSource + mu sync.Mutex + sources []mergedConfigSource + pendingArtifacts map[string]*evaluatedArtifacts } // mergedConfigSource retains the YAML even after a downloaded import is removed. // Content is immutable so snapshots cannot mutate another reader's source data. type mergedConfigSource struct { - path string - content string + path string + content string + artifacts *evaluatedArtifacts + // ephemeral marks a source downloaded from a remote import. Its local path is a temporary + // location, so relative destinations declared in it must never anchor to that directory. + ephemeral bool } -// track appends a source in effective merge order, including repeated paths. -func (t *mergedFilesTracker) track(source mergedConfigSource) { +// markEphemeral flags every tracked merge of path as a remote (downloaded) source. +func (t *mergedFilesTracker) markEphemeral(path string) { t.mu.Lock() defer t.mu.Unlock() - if source.path != "" { - // Keep repeated merges: their position determines precedence. - t.sources = append(t.sources, source) + for i := range t.sources { + if t.sources[i].path == path { + t.sources[i].ephemeral = true + } } } @@ -189,7 +195,39 @@ func (r *mergedFilesRegistry) track(v *viper.Viper, path string, content []byte) tracker := r.trackers[v] r.mu.Unlock() if tracker != nil { - tracker.track(mergedConfigSource{path: path, content: string(content)}) + tracker.mu.Lock() + artifacts := tracker.pendingArtifacts[path] + delete(tracker.pendingArtifacts, path) + if path != "" { + tracker.sources = append(tracker.sources, mergedConfigSource{path: path, content: string(content), artifacts: artifacts}) + } + tracker.mu.Unlock() + } +} + +// rememberArtifacts retains a source's evaluated, case-preserving artifact section until it is tracked. +func (r *mergedFilesRegistry) rememberArtifacts(v *viper.Viper, path string, value *evaluatedArtifacts) { + r.mu.Lock() + tracker := r.trackers[v] + r.mu.Unlock() + if tracker == nil { + return + } + tracker.mu.Lock() + defer tracker.mu.Unlock() + if tracker.pendingArtifacts == nil { + tracker.pendingArtifacts = make(map[string]*evaluatedArtifacts) + } + tracker.pendingArtifacts[path] = value +} + +// markEphemeral flags a downloaded import source for the load v belongs to. +func (r *mergedFilesRegistry) markEphemeral(v *viper.Viper, path string) { + r.mu.Lock() + tracker := r.trackers[v] + r.mu.Unlock() + if tracker != nil { + tracker.markEphemeral(path) } } diff --git a/pkg/config/import_fuzz_test.go b/pkg/config/import_fuzz_test.go new file mode 100644 index 00000000000..eefb2e762af --- /dev/null +++ b/pkg/config/import_fuzz_test.go @@ -0,0 +1,31 @@ +package config + +import ( + "testing" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" +) + +// TestSanitizeImport_GeneratedInputs runs the generated inputs, built from every delimiter of the +// shared table, through SanitizeImport. +func TestSanitizeImport_GeneratedInputs(t *testing.T) { + for _, separator := range redacttest.Seeds() { + for _, in := range redacttest.Generate(redacttest.SecretOf("A", separator, "B")) { + redacttest.RequireNoLeak(t, "SanitizeImport", in, SanitizeImport(in.Raw)) + } + } +} + +// FuzzSanitizeImport embeds a random secret in many address templates and checks that +// SanitizeImport never leaks the user name or any fragment of the secret. +func FuzzSanitizeImport(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + } + f.Fuzz(func(t *testing.T, first, separator, second string) { + for _, in := range redacttest.Generate(redacttest.SecretOf(first, separator, second)) { + redacttest.RequireNoLeak(t, "SanitizeImport", in, SanitizeImport(in.Raw)) + } + }) +} diff --git a/pkg/config/import_test.go b/pkg/config/import_test.go index fdc18d4f290..c85aee2ae58 100644 --- a/pkg/config/import_test.go +++ b/pkg/config/import_test.go @@ -5,12 +5,15 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/redacttest" log "github.com/cloudposse/atmos/pkg/logger" ) @@ -217,6 +220,21 @@ func TestSanitizeImport(t *testing.T) { input: "gcs::https://storage.googleapis.com/bucket/file?access_token=secret", expected: "gcs::https://storage.googleapis.com/bucket/file", }, + { + name: "forced getter with a double colon password", + input: "git::https://user:pa::ss@example.com/repo.git", + expected: "git::https://example.com/repo.git", + }, + { + name: "password with an at sign", + input: "https://user:pa@ss@second@example.com/file.yaml", + expected: "https://example.com/file.yaml", + }, + { + name: "unparsable URL with credentials is replaced", + input: "https://user:hunter2@exa mple.com/%zz", + expected: artifact.RedactedValue, + }, } for _, tt := range tests { @@ -227,6 +245,22 @@ func TestSanitizeImport(t *testing.T) { } } +// TestSanitizeImport_SharedCredentialCases feeds the table shared by every credential redaction +// helper to SanitizeImport and asserts no user or password of a URL-shaped import survives. +func TestSanitizeImport_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + if !strings.Contains(tc.Raw, "://") { + continue + } + t.Run(tc.Name, func(t *testing.T) { + got := SanitizeImport(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "sanitized form of %q", tc.Raw) + } + }) + } +} + // TestProcessImports_OutsideBaseDirectory verifies that imports outside the base directory // work correctly and only log at trace level (not warning level). func TestProcessImports_OutsideBaseDirectory(t *testing.T) { diff --git a/pkg/config/imports.go b/pkg/config/imports.go index a356e7ce683..c3b806877b7 100644 --- a/pkg/config/imports.go +++ b/pkg/config/imports.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/viper" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/ci" "github.com/cloudposse/atmos/pkg/condition" "github.com/cloudposse/atmos/pkg/filesystem" @@ -28,25 +29,26 @@ import ( // Exported so other config-loading packages (e.g. pkg/config/adapters) can sanitize import // paths before logging them. func SanitizeImport(s string) string { - // Handle go-getter style URLs with :: separator (e.g., git::https://...). - parts := strings.SplitN(s, "::", 2) - var prefix string - urlPart := s - - if len(parts) == 2 { - prefix = parts[0] + "::" - urlPart = parts[1] + // Handle go-getter style URLs with :: separator (e.g., git::https://...). The prefix is only + // recognized when a scheme follows it, so a "::" inside a password never splits the credential. + prefix, urlPart := artifact.SplitGetterPrefix(s) + + // An import that holds an "@" is a credential-bearing URL unless it is provably a local path or + // an SCP-style git address. It is redacted by the helper every include error shares: the + // userinfo ends at the last "@" of the authority, and a value that cannot be proven free of a + // credential is replaced as a whole. + if strings.Contains(urlPart, "@") && !isLocalPath(urlPart) && !artifact.IsSCPAddress(urlPart) { + urlPart = artifact.RedactURL(urlPart) + if urlPart == artifact.RedactedValue { + return urlPart + } } u, err := url.Parse(urlPart) if err != nil { - // Unparsable; return as-is (might be SCP-style git). - return s - } - - // Clear credentials regardless of scheme. - if u.User != nil { - u.User = nil + // Unparsable; keep the text (it might be SCP-style git) but never a query, which may hold tokens. + before, _, _ := strings.Cut(prefix+urlPart, "?") + return before } // Clear query params (may contain tokens/credentials). @@ -57,6 +59,14 @@ func SanitizeImport(s string) string { return prefix + u.String() } +// isLocalPath reports whether an import is a file system path, which is never a URL. +func isLocalPath(s string) bool { + if strings.Contains(s, "://") || strings.HasPrefix(s, "//") { + return false + } + return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") || strings.HasPrefix(s, "~/") +} + type importTypes int const ( @@ -124,7 +134,7 @@ func processConfigImportsWithFSAndBasePathSource(source *schema.AtmosConfigurati log.Debug("processConfigImports resolved paths", "count", len(resolvedPaths)) - return mergeResolvedImports(resolvedPaths, dst, fs) + return mergeResolvedImports(resolvedPaths, dst, fs, tempDir) } // mergeResolvedImports merges each resolved import file into dst and returns the @@ -132,7 +142,7 @@ func processConfigImportsWithFSAndBasePathSource(source *schema.AtmosConfigurati // An optional import (SkipIfMissing) that fails to merge or re-read is skipped so a single // bad import does not abort the whole configuration load; a required import that fails to // merge returns errUtils.ErrRequiredImportFailed (fail closed). -func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs filesystem.FileSystem) (string, error) { +func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs filesystem.FileSystem, temporaryRoots ...string) (string, error) { basePathSourceDir := "" for _, resolvedPath := range resolvedPaths { // Trace: log what we're about to merge (sanitized). @@ -149,6 +159,10 @@ func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs fi continue } log.Trace("successfully merged config from import", "import", SanitizeImport(resolvedPath.ImportPaths), keyFilePath, resolvedPath.FilePath) + if resolvedPath.ImportType != LOCAL || pathInsideAnyRoot(resolvedPath.FilePath, temporaryRoots) { + // Downloaded imports live in a temporary directory that is removed after loading. + mergedFilesReg.markEphemeral(dst, resolvedPath.FilePath) + } content, readErr := fs.ReadFile(resolvedPath.FilePath) if readErr != nil { log.Warn("error re-reading resolved import for base_path detection", "import", SanitizeImport(resolvedPath.ImportPaths), keyFilePath, resolvedPath.FilePath, "error", readErr) @@ -162,6 +176,17 @@ func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs fi return basePathSourceDir, nil } +// pathInsideAnyRoot identifies downloaded files by their actual location, not import ancestry. +func pathInsideAnyRoot(path string, roots []string) bool { + for _, root := range roots { + rel, err := filepath.Rel(root, path) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) { + return true + } + } + return false +} + // ProcessImportsFromAdapter is the public entry point for adapters to process nested imports. // Adapters should call this when they discover nested import statements in resolved configs. // The atmosConfig parameter is threaded through so nested remote imports honor auth settings diff --git a/pkg/config/load.go b/pkg/config/load.go index 1beff1cc317..5afdb2e3e0b 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -485,6 +485,11 @@ func finalizeLoadedConfig(v *viper.Viper, atmosConfig *schema.AtmosConfiguration if err := resolveScaffoldConfig(v, atmosConfig); err != nil { return err } + // Rebuild artifacts from the retained sources: Viper lowercases keys and forgets which + // file declared options.path once imports are merged. + if err := resolveArtifactsConfig(v, atmosConfig); err != nil { + return err + } // Post-process to preserve case-sensitive map keys. // Viper lowercases all YAML map keys, but we need to preserve original case @@ -1431,8 +1436,10 @@ func resolveDotenvMergeIncludeValue(configFile string, node *yaml.Node) (bool, e func loadDotenvIncludeAsYAMLNode(configFile, includeValue string) (*yaml.Node, error) { includeFile, ok := parseDotenvIncludeFile(includeValue) - if !ok { - return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, includeValue) + // A dotenv include is a local file: a URL-shaped target would be joined onto the config + // directory and echoed, with any credential it carries, in the read error. + if !ok || strings.Contains(includeFile, "://") { + return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, redactIncludeValue(includeValue)) } if !filepath.IsAbs(includeFile) { includeFile = filepath.Join(filepath.Dir(configFile), includeFile) @@ -1455,7 +1462,7 @@ func loadDotenvIncludeAsYAMLNode(configFile, includeValue string) (*yaml.Node, e return nil, err } if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode { - return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, includeValue) + return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, redactIncludeValue(includeValue)) } return doc.Content[0], nil } @@ -2068,6 +2075,10 @@ func mergeConfigFile( // top-level `commands` key, which extractCommandsWithYamlFunctionsForFile decodes with its own // include scope. Git hook steps use the same step-aware decoder before generic processing. func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourceFile string) error { + return preprocessConfigYAML(content, v, sourceFile, false) +} + +func preprocessConfigYAML(content []byte, v *viper.Viper, sourceFile string, skipArtifacts bool) error { var root goyaml.Node if err := goyaml.Unmarshal(content, &root); err != nil { return err @@ -2087,6 +2098,10 @@ func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourc } } } + evaluated, err := prepareArtifactsNode(&root, v, sourceFile, skipArtifacts) + if err != nil { + return err + } // Process the parsed tree directly: aliases can refer to anchors under commands, // so serializing the filtered tree would leave dangling aliases on the next parse. if err := preprocessGitHookSteps(&root, v, sourceFile); err != nil { @@ -2096,7 +2111,38 @@ func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourc if err := prepareDevcontainerKeys(&root, v); err != nil { return err } - return processConfigNodeWithAuthSource(&root, v, sourceFile) + if err := processConfigNodeWithAuthSource(&root, v, sourceFile); err != nil { + return err + } + if evaluated != nil { + mergedFilesReg.rememberArtifacts(v, sourceFile, evaluated) + } + return nil +} + +// prepareArtifactsNode evaluates the artifact section once and removes its function tags. +func prepareArtifactsNode(root *goyaml.Node, v *viper.Viper, sourceFile string, skip bool) (*evaluatedArtifacts, error) { + node := findYAMLMappingPath(root, artifactsSectionKey) + if node == nil { + return nil, nil + } + if skip { + // The profile section was already evaluated when it was merged. + *node = goyaml.Node{Kind: goyaml.ScalarNode, Tag: "!!null", Value: "null"} + return nil, nil + } + if node.Tag == "!unset" { + return nil, nil + } + evaluated, err := evaluateArtifactsForLoad(node, sourceFile) + if err != nil { + return nil, err + } + if err := node.Encode(evaluated.section); err != nil { + return nil, err + } + v.Set(artifactsSectionKey, evaluated.section) + return evaluated, nil } // processConfigNodeWithAuthSource retains identities introduced by a whole-auth include. @@ -2198,7 +2244,7 @@ func mergeConfigFileWithImports(path string, v *viper.Viper) error { if len(imports) == 0 { return nil } - if err = preprocessAtmosYamlFuncExceptCommands(content, tempViper, path); err != nil { + if err = preprocessConfigYAML(content, tempViper, path, true); err != nil { return err } overlayProfileSettings(v, tempViper.AllSettings(), "") @@ -2233,7 +2279,7 @@ func applyProfileImports(v, tempViper *viper.Viper, path string, content []byte, func overlayProfileSettings(v *viper.Viper, settings map[string]any, prefix string) { for key, value := range settings { - if key == "import" || key == commandsKey { + if key == "import" || key == commandsKey || (prefix == "" && key == artifactsSectionKey) { continue } path := key diff --git a/pkg/config/process_yaml.go b/pkg/config/process_yaml.go index 647d6d35f36..f89bb82183f 100644 --- a/pkg/config/process_yaml.go +++ b/pkg/config/process_yaml.go @@ -18,7 +18,8 @@ const ( errorFormat = "%w: %v %v error %w" failedToProcess = "failed to process" emptyValueWarning = "execute returned empty value" - includeDataKey = "include_data" + // The includeDataKey mapping key preserves the type of an evaluated include value. + includeDataKey = "include_data" ) var ErrExecuteYamlFunctions = errors.New("failed to execute yaml function") diff --git a/pkg/config/process_yaml_alias_limit_test.go b/pkg/config/process_yaml_alias_limit_test.go new file mode 100644 index 00000000000..4bd57091512 --- /dev/null +++ b/pkg/config/process_yaml_alias_limit_test.go @@ -0,0 +1,94 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// billionLaughs builds the classic exponential alias graph: every level is a sequence of +// `fanout` aliases of the previous one, so level n expands to fanout^n leaves. +func billionLaughs(levels, fanout int) string { + var b strings.Builder + b.WriteString("l0: &l0 [" + strings.TrimSuffix(strings.Repeat("x, ", fanout), ", ") + "]\n") + for level := 1; level <= levels; level++ { + aliases := strings.TrimSuffix(strings.Repeat(fmt.Sprintf("*l%d, ", level-1), fanout), ", ") + fmt.Fprintf(&b, "l%d: &l%d [%s]\n", level, level, aliases) + } + return b.String() +} + +func TestDecodeNodeWithYamlFunctions_AliasExpansionIsBounded(t *testing.T) { + t.Run("an exponential alias graph is rejected before it is expanded", func(t *testing.T) { + start := time.Now() + _, err := decodeYAMLForTest(t, billionLaughs(10, 10)) + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.Contains(t, err.Error(), "l", "the error names the alias that crossed the limit") + assert.Less(t, time.Since(start), 10*time.Second, "the limit stops the expansion instead of allocating it") + }) + + t.Run("a large but bounded document still loads", func(t *testing.T) { + var b strings.Builder + b.WriteString("base: &base\n") + for i := 0; i < 1000; i++ { + fmt.Fprintf(&b, " - {name: item%d, enabled: true, tags: [a, b]}\n", i) + } + for i := 0; i < 50; i++ { + fmt.Fprintf(&b, "copy%d: *base\n", i) + } + got, err := decodeYAMLForTest(t, b.String()) + require.NoError(t, err) + decoded, ok := got.(map[string]any) + require.True(t, ok) + require.Len(t, decoded, 51) + last, ok := decoded["copy49"].([]any) + require.True(t, ok) + require.Len(t, last, 1000) + assert.Equal(t, map[string]any{"name": "item0", "enabled": true, "tags": []any{"a", "b"}}, last[0]) + assert.Equal(t, map[string]any{"name": "item999", "enabled": true, "tags": []any{"a", "b"}}, last[999]) + }) + + t.Run("the budget is checked against the configured limit exactly", func(t *testing.T) { + // The anchor is a sequence of three scalars: four nodes per alias. + source := "a: &a [1, 2, 3]\nb: *a\nc: *a\nd: *a\n" + tests := []struct { + name string + limit int64 + wantErr bool + }{ + {name: "one below the third alias", limit: 11, wantErr: true}, + {name: "exactly the three aliases", limit: 12, wantErr: false}, + {name: "well above", limit: 1000, wantErr: false}, + {name: "below the first alias", limit: 3, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + document := mustParseYAML(t, source) + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: tt.limit} + got, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + assert.Nil(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, map[string]any{"a": []any{1, 2, 3}, "b": []any{1, 2, 3}, "c": []any{1, 2, 3}, "d": []any{1, 2, 3}}, got) + }) + } + }) + + t.Run("merge keys are charged like aliases", func(t *testing.T) { + document := mustParseYAML(t, "base: &base {a: 1, b: 2}\nx:\n <<: *base\ny:\n <<: *base\n") + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: 5} + _, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + }) +} diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go new file mode 100644 index 00000000000..a8d03d3d6af --- /dev/null +++ b/pkg/config/process_yaml_anchor.go @@ -0,0 +1,299 @@ +package config + +import ( + "fmt" + + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// maxAliasExpansion bounds the values that every alias in one decoded document may copy, counted as +// mappings, sequences, and scalars. An alias copies the whole value of its anchor, so a compact +// document of nested aliases can otherwise expand exponentially (the "billion laughs" shape). +// A million values is far above any real configuration, which aliases a few hundred values at most, +// and keeps the worst-case allocation of a hostile document to tens of megabytes. +const maxAliasExpansion = 1_000_000 + +// mergeKeyTag is the resolved tag of the YAML merge key "<<". +const mergeKeyTag = "!!merge" + +// includeOrigin is the file an include (or an alias of an anchored include) produced its value from. +type includeOrigin struct { + file string + remote bool +} + +// originRecord is one origin notification, with its path relative to the node that produced it. +type originRecord struct { + rel []string + file string + remote bool +} + +// anchorEntry is the one evaluation of an anchored node. Aliases and merge keys that refer to the +// anchor reuse it instead of evaluating the node's function tags again: an `!exec`, `!env`, or +// `!random` behind an anchor runs once, at the anchor, and every alias sees that result. +type anchorEntry struct { + value any + origins []originRecord + include *includeOrigin + // size is the number of values an alias of this anchor copies. + size int64 + // remote is the first remote include evaluated while the anchor was evaluated. + remote *includeOrigin +} + +// decodeAnchored evaluates an anchored node once and caches the result for its aliases. +func (d *configYamlDecoder) decodeAnchored(node *yaml.Node, sourceFile string) (any, error) { + entry, err := d.evaluateAnchor(node, sourceFile) + if err != nil { + return nil, err + } + d.nodeOrigin = entry.include + return entry.value, nil +} + +// decodeAlias returns a copy of the anchor's evaluated value. The anchor is evaluated on first use +// when its definition was not visited yet, for example when it sits outside the decoded subtree. +// The origins the anchor recorded are replayed below the alias's own path, so a path-valued setting +// reached through an alias keeps the file that declared it. +func (d *configYamlDecoder) decodeAlias(node *yaml.Node, sourceFile string) (any, error) { + if node.Alias == nil { + return nil, fmt.Errorf("%w: alias %q has no anchor", ErrExecuteYamlFunctions, node.Value) + } + entry, err := d.evaluateAnchor(node.Alias, sourceFile) + if err != nil { + return nil, err + } + if err := d.chargeAliasExpansion(node.Alias, entry); err != nil { + return nil, err + } + d.replayOrigins(entry.origins) + d.nodeOrigin = entry.include + d.noteRemote(entry.remote) + return cloneDecodedValue(entry.value), nil +} + +// chargeAliasExpansion adds the size of an alias's copy to the decoder's budget before the copy is +// made, so an alias that would cross the limit never allocates. +func (d *configYamlDecoder) chargeAliasExpansion(anchor *yaml.Node, entry *anchorEntry) error { + limit := d.aliasLimit + if limit == 0 { + limit = maxAliasExpansion + } + if entry.size > limit-d.aliasExpanded { + return fmt.Errorf("%w: %w: alias %q copies %d values after %d were already copied, over the limit of %d", + ErrExecuteYamlFunctions, errUtils.ErrYamlAliasExpansionLimit, anchor.Anchor, entry.size, d.aliasExpanded, limit) + } + d.aliasExpanded += entry.size + return nil +} + +// evaluateAnchor evaluates an anchored node at most once per decoder. An alias that refers back to +// its own anchor is a cycle and is rejected. +func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) (*anchorEntry, error) { + if entry, ok := d.anchors[node]; ok { + return entry, nil + } + if d.evaluating[node] { + return nil, fmt.Errorf("%w: YAML alias cycle at anchor %q", ErrExecuteYamlFunctions, node.Anchor) + } + if d.anchors == nil { + d.anchors = make(map[*yaml.Node]*anchorEntry) + d.evaluating = make(map[*yaml.Node]bool) + } + d.evaluating[node] = true + defer delete(d.evaluating, node) + + var records []originRecord + previous := d.origin + if previous != nil { + base := len(d.path) + d.origin = func(path []string, file string, remote bool) { + if len(path) > base { + records = append(records, originRecord{rel: append([]string(nil), path[base:]...), file: file, remote: remote}) + } + previous(path, file, remote) + } + defer func() { d.origin = previous }() + } + d.nodeOrigin = nil + outerTaint := d.remoteTaint + d.remoteTaint = nil + value, err := d.decodeNode(node, sourceFile) + entryTaint := d.remoteTaint + d.remoteTaint = outerTaint + d.noteRemote(entryTaint) + if err != nil { + return nil, err + } + entry := &anchorEntry{value: value, origins: records, remote: entryTaint, size: countDecodedValues(value)} + if node.Kind == yaml.ScalarNode && isIncludeTag(node.Tag) { + entry.include = d.nodeOrigin + } + d.nodeOrigin = nil + d.anchors[node] = entry + return entry, nil +} + +// noteRemote records the first remote include seen since the taint was last cleared. +func (d *configYamlDecoder) noteRemote(origin *includeOrigin) { + if origin != nil && origin.remote && d.remoteTaint == nil { + d.remoteTaint = origin + } +} + +// replayOrigins reports the origins an anchor recorded, relative to the current path. +func (d *configYamlDecoder) replayOrigins(records []originRecord) { + if d.origin == nil { + return + } + for _, record := range records { + path := append(append([]string(nil), d.path...), record.rel...) + d.origin(path, record.file, record.remote) + } +} + +// isMergeKey reports whether a mapping key is the YAML merge key "<<". +func isMergeKey(key *yaml.Node) bool { + return key.Kind == yaml.ScalarNode && key.Tag == mergeKeyTag +} + +// mergeSource is one decoded merge source and the origins its values were decoded with. +type mergeSource struct { + value map[string]interface{} + origins []originRecord +} + +// expandMergeKeys copies the entries named by every "<<" key of the mapping into result. Entries +// the mapping declares itself win over merged ones, and earlier merge sources win over later ones, +// as the YAML merge-key rules require. The origins of a merge source are held back while it is +// decoded and reported only for the keys that are actually copied, each at the mapping's own path, +// so a discarded default never overwrites the origin of an explicit key or of an earlier source. +func (d *configYamlDecoder) expandMergeKeys(node *yaml.Node, result map[string]interface{}, sourceFile string) error { + for i := 0; i < len(node.Content); i += 2 { + if !isMergeKey(node.Content[i]) { + continue + } + sources, err := d.mergeSources(node.Content[i+1], sourceFile) + if err != nil { + return err + } + for _, source := range sources { + d.copyMergeSource(source, result) + } + } + return nil +} + +// copyMergeSource copies the entries of one merge source that result does not hold yet and reports +// the held-back origins of each copied key. The origins are grouped by key once, so the work is +// linear in the size of the source: replaying a key by scanning every record would be quadratic for +// an anchored mapping that holds thousands of entries. +func (d *configYamlDecoder) copyMergeSource(source mergeSource, result map[string]interface{}) { + var originsByKey map[string][]originRecord + if d.origin != nil { + originsByKey = groupOriginsByKey(source.origins) + } + for key, value := range source.value { + if _, exists := result[key]; exists { + continue + } + result[key] = value + d.replayOrigins(originsByKey[key]) + } +} + +// groupOriginsByKey groups origin records by the first component of their relative path, which is +// the key of the mapping entry that produced them. The records of one key keep their order, so the +// precedence of the replay is the one of the original scan. +func groupOriginsByKey(records []originRecord) map[string][]originRecord { + grouped := make(map[string][]originRecord) + for _, record := range records { + if len(record.rel) == 0 { + continue + } + grouped[record.rel[0]] = append(grouped[record.rel[0]], record) + } + return grouped +} + +// mergeSources decodes the value of a merge key: a mapping, an alias of one, or a sequence of them. +// The origins each source reports are collected on the source instead of being forwarded. +func (d *configYamlDecoder) mergeSources(value *yaml.Node, sourceFile string) ([]mergeSource, error) { + nodes := []*yaml.Node{value} + if value.Kind == yaml.SequenceNode { + nodes = value.Content + } + sources := make([]mergeSource, 0, len(nodes)) + for _, child := range nodes { + decoded, records, err := d.decodeCollectingOrigins(child, sourceFile) + if err != nil { + return nil, err + } + source, ok := decoded.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("%w: merge key %q requires a mapping or a sequence of mappings, got %T", ErrExecuteYamlFunctions, "<<", decoded) + } + sources = append(sources, mergeSource{value: source, origins: records}) + } + return sources, nil +} + +// decodeCollectingOrigins decodes a node and returns the origins it reported, relative to the +// current path, without forwarding them to the decoder's origin callback. +func (d *configYamlDecoder) decodeCollectingOrigins(node *yaml.Node, sourceFile string) (any, []originRecord, error) { + previous := d.origin + if previous == nil { + value, err := d.decode(node, sourceFile) + return value, nil, err + } + base := len(d.path) + var records []originRecord + d.origin = func(path []string, file string, remote bool) { + if len(path) > base { + records = append(records, originRecord{rel: append([]string(nil), path[base:]...), file: file, remote: remote}) + } + } + defer func() { d.origin = previous }() + value, err := d.decode(node, sourceFile) + return value, records, err +} + +// countDecodedValues counts the mappings, sequences, and scalars of a decoded value. +func countDecodedValues(value any) int64 { + count := int64(1) + switch typed := value.(type) { + case map[string]interface{}: + for _, child := range typed { + count += countDecodedValues(child) + } + case []interface{}: + for _, child := range typed { + count += countDecodedValues(child) + } + } + return count +} + +// cloneDecodedValue deep-copies the mappings and sequences of a decoded value, so an alias never +// shares mutable structure with its anchor. +func cloneDecodedValue(value any) any { + switch typed := value.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(typed)) + for key, child := range typed { + out[key] = cloneDecodedValue(child) + } + return out + case []interface{}: + out := make([]interface{}, len(typed)) + for index, child := range typed { + out[index] = cloneDecodedValue(child) + } + return out + default: + return value + } +} diff --git a/pkg/config/process_yaml_include_eval.go b/pkg/config/process_yaml_include_eval.go index 271f80cf2fd..272534e8694 100644 --- a/pkg/config/process_yaml_include_eval.go +++ b/pkg/config/process_yaml_include_eval.go @@ -12,7 +12,28 @@ const maxConfigIncludeDepth = 100 // configYamlDecoder keeps recursive include evaluation within one config read. // The shared include resolver supplies the correct file context for each subtree. type configYamlDecoder struct { - includes map[string]bool + includes map[string]bool + path []string + origin func([]string, string, bool) + remoteSource string + + // anchors holds each anchored node's single evaluation; evaluating guards against alias cycles. + anchors map[*yaml.Node]*anchorEntry + evaluating map[*yaml.Node]bool + + // aliasLimit overrides maxAliasExpansion when set; aliasExpanded is the number of values the + // aliases decoded so far have copied. + aliasLimit int64 + aliasExpanded int64 + + // nodeOrigin is the file the include or alias just decoded took its value from. The mapping + // entry that owns the value reads it so the setting's origin is the included file. + nodeOrigin *includeOrigin + + // remoteTaint is the first remote include evaluated since it was last cleared. An include whose + // result is not a mapping (a scalar or a list) has no per-key origin records of its own, so a + // remote include nested anywhere inside it marks the whole result as remote-supplied. + remoteTaint *includeOrigin } // evaluateInclude resolves an included subtree using its file context and diff --git a/pkg/config/process_yaml_include_origin.go b/pkg/config/process_yaml_include_origin.go new file mode 100644 index 00000000000..2277e1da0a9 --- /dev/null +++ b/pkg/config/process_yaml_include_origin.go @@ -0,0 +1,68 @@ +package config + +import ( + legacyyaml "gopkg.in/yaml.v3" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +// includeResult is what the origin of one evaluated include is derived from. +type includeResult struct { + // localPath is the local file the include resolved to, empty for a remote include. + localPath string + // target is the include argument as written. + target string + decoded any + // walked reports that the included content was evaluated with per-key origins already recorded. + walked bool + // walkedRemote is the first remote include evaluated inside this one. + walkedRemote *includeOrigin +} + +// runInclude loads an include into resolved. A raw include is read verbatim. Otherwise the included +// YAML is walked, and when origins are tracked a remote include is walked as remote content. +func (d *configYamlDecoder) runInclude(atmosConfig *schema.AtmosConfiguration, resolved *legacyyaml.Node, tag, value, sourceFile string) (bool, error) { + switch { + case tag == u.AtmosYamlFuncIncludeRaw: + return false, u.ProcessIncludeRawTag(atmosConfig, resolved, value, sourceFile) + case d.origin == nil: + return u.ProcessIncludeTagWalked(atmosConfig, resolved, value, sourceFile, d.evaluateInclude) + default: + return u.ProcessIncludeTagWalkedFrom(atmosConfig, resolved, value, sourceFile, u.IncludeWalk{ + Local: d.evaluateInclude, + Remote: func(child *legacyyaml.Node, file, source string) error { + previous := d.remoteSource + d.remoteSource = source + defer func() { d.remoteSource = previous }() + return d.evaluateInclude(child, file) + }, + Source: d.remoteSource, + }) + } +} + +// recordIncludeOrigin records where an include's value came from, so the owning key reads the +// included file rather than the file that merely references it. +func (d *configYamlDecoder) recordIncludeOrigin(result includeResult) { + file, remote := result.localPath, false + if file == "" { + file, remote = result.target, true + } + if d.remoteSource != "" { + file, remote = d.remoteSource, true + } + if !result.walked { + d.recordIncludedOrigins(result.decoded, file, remote) + } + origin := &includeOrigin{file: file, remote: remote} + // A local include never launders remote content: when the result is not a mapping (whose + // keys carry their own origins), the remote include nested inside it keeps its origin. + if _, isMapping := result.decoded.(map[string]any); !origin.remote && result.walkedRemote != nil && !isMapping { + origin = result.walkedRemote + } + d.noteRemote(origin) + // Set last, so the outermost include of a nested chain is the one the owning key reads, + // except that a remote origin is never replaced by a local one. + d.nodeOrigin = origin +} diff --git a/pkg/config/process_yaml_origin_test.go b/pkg/config/process_yaml_origin_test.go new file mode 100644 index 00000000000..fd9bbd23c46 --- /dev/null +++ b/pkg/config/process_yaml_origin_test.go @@ -0,0 +1,213 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// TestLoadConfig_ArtifactsMergeKeysNeverReplayDiscardedOrigins covers the precedence of origin +// records across merge keys: a key the mapping declares itself keeps its own origin, and a key +// copied from a merge source keeps the origin of the earliest source that supplies it. +func TestLoadConfig_ArtifactsMergeKeysNeverReplayDiscardedOrigins(t *testing.T) { + const bases = ` base: &base + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./defaults/keys.yaml + other: &other + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./other/keys.yaml + empty: &empty + kind: local/dir + options: + path: store +` + tests := []struct { + name string + // repository is the body of the repository mapping in atmos.yaml. + repository string + // wantDir is the directory, relative to the config, that declares the effective keys. + wantDir string + wantKeys string + }{ + { + name: "explicit keys declared before the merge key", + repository: ` verification: + trusted_keys: [release.pub] + <<: *base +`, + wantDir: ".", + wantKeys: "release.pub", + }, + { + name: "explicit keys declared after the merge key", + repository: ` <<: *base + verification: + trusted_keys: [release.pub] +`, + wantDir: ".", + wantKeys: "release.pub", + }, + { + name: "keys supplied only by the merge source keep the source file", + repository: " <<: *base\n", + wantDir: "defaults", + wantKeys: "default.pub", + }, + { + name: "the earlier of two merge sources supplies the origin", + repository: " <<: [*base, *other]\n", + wantDir: "defaults", + wantKeys: "default.pub", + }, + { + name: "the later merge source supplies keys the earlier one lacks", + repository: " <<: [*empty, *other]\n", + wantDir: "other", + wantKeys: "other.pub", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n repositories:\n"+bases+" r:\n"+tt.repository) + writeArtifactsFile(t, filepath.Join(dir, "defaults", "keys.yaml"), "- default.pub\n") + writeArtifactsFile(t, filepath.Join(dir, "other", "keys.yaml"), "- other.pub\n") + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo := cfg.Artifacts.Repositories["r"] + require.NotNil(t, repo.Verification) + wantDir := filepath.Join(dir, tt.wantDir) + assert.Equal(t, []string{filepath.Join(wantDir, tt.wantKeys)}, repo.Verification.TrustedKeys) + assert.Equal(t, wantDir, repo.Verification.KeysOrigin.Dir) + assert.False(t, repo.Verification.KeysOrigin.Remote) + assert.Equal(t, filepath.Join(dir, "store"), repo.Options["path"]) + }) + } +} + +// TestDecodeNodeWithYamlFunctions_MergeKeyOriginRecords checks the origin records directly: only +// the keys a merge actually copies are reported, and an explicit key is reported exactly once. +func TestDecodeNodeWithYamlFunctions_MergeKeyOriginRecords(t *testing.T) { + tests := []struct { + name string + source string + want []string + }{ + { + name: "explicit key before the merge key", + source: "child:\n a: 1\n <<: {a: 2, b: 3}\n", + want: []string{"child.a", "child.b"}, + }, + { + name: "explicit key after the merge key", + source: "base: &base {a: 2, b: 3}\nchild:\n <<: *base\n a: 1\n", + want: []string{"child.a", "child.b"}, + }, + { + name: "nested mapping of a discarded key reports nothing", + source: "base: &base {n: {x: 1}, m: 2}\nchild:\n <<: *base\n n: 5\n", + want: []string{"child.n", "child.m"}, + }, + { + name: "nested mapping of a copied key is reported", + source: "base: &base {n: {x: 1}}\nchild:\n <<: *base\n", + want: []string{"child.n", "child.n.x"}, + }, + { + name: "earlier merge sources win", + source: "x: &x {a: {p: 1}}\ny: &y {a: {q: 1}, b: 1}\nchild:\n <<: [*x, *y]\n", + want: []string{"child.a", "child.a.p", "child.b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var seen []string + document := mustParseYAML(t, tt.source) + decoder := configYamlDecoder{includes: map[string]bool{}} + decoder.origin = func(path []string, _ string, _ bool) { + if joined := joinPath(path); strings.HasPrefix(joined, "child.") { + seen = append(seen, joined) + } + } + _, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + require.NoError(t, err) + assert.ElementsMatch(t, tt.want, seen, "each key's origin is reported exactly once, by the source that supplied it") + }) + } +} + +func mustParseYAML(t *testing.T, source string) yaml.Node { + t.Helper() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &node)) + return node +} + +func joinPath(path []string) string { + out := "" + for i, part := range path { + if i > 0 { + out += "." + } + out += part + } + return out +} + +// TestDecodeNodeWithYamlFunctions_MergeKeyOriginsAreLinear merges one large anchored mapping into +// many mappings with origin tracking on. Replaying the origins of each copied key must not rescan +// the whole origin list of the source, which is quadratic in the size of the anchor: with the +// numbers below a rescan makes billions of comparisons and runs for tens of seconds. +func TestDecodeNodeWithYamlFunctions_MergeKeyOriginsAreLinear(t *testing.T) { + const ( + keys = 20000 + merges = 40 + ) + var source strings.Builder + source.WriteString("base: &base\n") + for i := 0; i < keys; i++ { + fmt.Fprintf(&source, " key%d: %d\n", i, i) + } + for i := 0; i < merges; i++ { + fmt.Fprintf(&source, "child%d:\n <<: *base\n", i) + } + + document := mustParseYAML(t, source.String()) + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: 100_000_000} + reported := map[string]int{} + decoder.origin = func(path []string, _ string, _ bool) { + if len(path) == 2 && strings.HasPrefix(path[0], "child") { + reported[joinPath(path)]++ + } + } + + start := time.Now() + got, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + elapsed := time.Since(start) + require.NoError(t, err) + + decoded, ok := got.(map[string]any) + require.True(t, ok) + last, ok := decoded["child39"].(map[string]any) + require.True(t, ok) + require.Len(t, last, keys) + assert.Equal(t, 0, last["key0"]) + assert.Equal(t, keys-1, last[fmt.Sprintf("key%d", keys-1)]) + + require.Len(t, reported, keys*merges, "every copied key of every mapping reports its origin") + assert.Equal(t, 1, reported["child0.key0"], "each key is reported exactly once") + assert.Equal(t, 1, reported["child39.key19999"], "each key is reported exactly once") + assert.Less(t, elapsed, 15*time.Second, "origin replay for a merge must be linear in the anchor size") +} diff --git a/pkg/config/process_yaml_provenance_test.go b/pkg/config/process_yaml_provenance_test.go new file mode 100644 index 00000000000..bf3e9502c0a --- /dev/null +++ b/pkg/config/process_yaml_provenance_test.go @@ -0,0 +1,83 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" +) + +// TestLoadConfig_ArtifactsNestedIncludeKeepsRemoteProvenance covers wrapped includes: the +// innermost remote origin survives an outer local wrapper, and a remote wrapper stays remote. +func TestLoadConfig_ArtifactsNestedIncludeKeepsRemoteProvenance(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/keys.yaml": + _, _ = fmt.Fprint(w, "- release.pub\n") + case "/key.yaml": + _, _ = fmt.Fprint(w, "release.pub\n") + case "/wrap-local.yaml": + _, _ = fmt.Fprint(w, "!include ./local-keys.yaml | eval\n") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + tests := []struct { + name string + wrapper string + wantRemote bool + // section replaces the default `trusted_keys: !include ./trust/wrap.yaml` line when set. + section string + // plain reports that the wrapper holds no tags, so it also loads without `| eval`. + plain bool + }{ + {name: "local wrapper around a remote include", wrapper: "!include " + server.URL + "/keys.yaml | eval\n", wantRemote: true}, + {name: "local wrapper around a sequence holding a remote include", wrapper: "- !include " + server.URL + "/key.yaml | eval\n", wantRemote: true}, + {name: "remote wrapper around a local include", wrapper: "!include " + server.URL + "/wrap-local.yaml | eval\n", wantRemote: true}, + { + name: "local mapping wrapper around a remote key list", + wrapper: "trusted_keys: !include " + server.URL + "/keys.yaml | eval\n", + section: "verification: !include ./trust/wrap.yaml | eval\n", + wantRemote: true, + }, + {name: "local wrapper around local content stays local", wrapper: "- release.pub\n", wantRemote: false, plain: true}, + } + for _, tt := range tests { + suffixes := []string{" | eval"} + if tt.plain { + suffixes = append(suffixes, "") + } + for _, suffix := range suffixes { + t.Run(fmt.Sprintf("%s/eval=%t", tt.name, suffix != ""), func(t *testing.T) { + section := " verification:\n trusted_keys: !include ./trust/wrap.yaml" + suffix + "\n" + if tt.section != "" { + section = " " + tt.section + } + dir := artifactsTestDir(t, "artifacts:\n"+section) + writeArtifactsFile(t, filepath.Join(dir, "trust", "wrap.yaml"), tt.wrapper) + writeArtifactsFile(t, filepath.Join(dir, "trust", "local-keys.yaml"), "- release.pub\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, tt.wantRemote, cfg.Artifacts.Verification.KeysOrigin.Remote) + if !tt.wantRemote { + assert.Equal(t, []string{filepath.Join(dir, "trust", "release.pub")}, cfg.Artifacts.Verification.TrustedKeys) + return + } + assert.Equal(t, []string{"release.pub"}, cfg.Artifacts.Verification.TrustedKeys, "a relative key from remote content is never anchored locally") + err = artifactconfig.Validate(&cfg.Artifacts) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "remote import") + }) + } + } +} diff --git a/pkg/config/process_yaml_redact_test.go b/pkg/config/process_yaml_redact_test.go new file mode 100644 index 00000000000..672ec95ae40 --- /dev/null +++ b/pkg/config/process_yaml_redact_test.go @@ -0,0 +1,60 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const includeRedactSecret = "hunter2" + +// TestConfigIncludeErrorsDoNotEchoCredentials covers every config-level include error path that +// formats the include argument: none may repeat a credential carried by a URL-shaped include. +func TestConfigIncludeErrorsDoNotEchoCredentials(t *testing.T) { + const target = "https://user:" + includeRedactSecret + "@127.0.0.1:1/" + tests := []struct { + name string + source string + }{ + {name: "dotenv include", source: "keys: !include " + target + "settings.env\n"}, + {name: "dotenv include with query", source: "keys: !include " + target + "settings.env .a\n"}, + {name: "yaml include with query", source: "keys: !include " + target + "keys.yaml .a\n"}, + {name: "yaml include with eval", source: "keys: !include " + target + "keys.yaml | eval\n"}, + {name: "raw include", source: "keys: !include.raw " + target + "keys.txt\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n verification:\n "+tt.source) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + + _, err = decodeYAMLForTestFile(t, tt.source, filepath.Join(dir, "atmos.yaml")) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + }) + } +} + +func decodeYAMLForTestFile(t *testing.T, source, file string) (any, error) { + t.Helper() + node := mustParseYAML(t, source) + return decodeNodeWithYamlFunctionsForFile(&node, file) +} + +func TestLoadDotenvIncludeErrorsDoNotEchoCredentials(t *testing.T) { + dir := t.TempDir() + for _, value := range []string{ + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.env", + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.env .a", + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.yaml", + } { + t.Run(value, func(t *testing.T) { + _, err := loadDotenvIncludeAsYAMLNode(filepath.Join(dir, "atmos.yaml"), value) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + }) + } +} diff --git a/pkg/config/schema/overrides.go b/pkg/config/schema/overrides.go index 3d20a36ebcd..318ed68bbef 100644 --- a/pkg/config/schema/overrides.go +++ b/pkg/config/schema/overrides.go @@ -130,6 +130,19 @@ func applyOverrides(r *jsonschema.Reflector, root *jsonschema.Schema) { overrideSchemasSection(r, root) applyPolymorphicOverrides(root) allowYamlFunctions(root) + // Artifact trusted keys are decoded strictly and do not use the generic + // string-to-slice coercion that other configuration fields support. + if def := root.Definitions["ArtifactVerificationConfig"]; def != nil { + if keys, ok := def.Properties.Get("trusted_keys"); ok { + filtered := keys.AnyOf[:0] + for _, branch := range keys.AnyOf { + if branch.Type != typeString { + filtered = append(filtered, branch) + } + } + keys.AnyOf = filtered + } + } } // applyPolymorphicOverrides relaxes shapes that custom unmarshalers accept diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 02c4e281f8a..32bb108ca62 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -35,7 +35,17 @@ func (d *configYamlDecoder) decode(node *yaml.Node, sourceFile string) (any, err if node == nil { return nil, nil } + if node.Kind == yaml.AliasNode { + return d.decodeAlias(node, sourceFile) + } + if node.Anchor != "" { + return d.decodeAnchored(node, sourceFile) + } + return d.decodeNode(node, sourceFile) +} +// decodeNode evaluates one node by kind, without consulting the anchor cache. +func (d *configYamlDecoder) decodeNode(node *yaml.Node, sourceFile string) (any, error) { switch node.Kind { case yaml.DocumentNode: if len(node.Content) == 0 { @@ -59,12 +69,25 @@ func (d *configYamlDecoder) decodeMappingNodeWithYamlFunctions(node *yaml.Node, for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] + if isMergeKey(keyNode) { + continue + } + d.path = append(d.path, keyNode.Value) + d.nodeOrigin = nil value, err := d.decode(valueNode, sourceFile) + if err == nil && d.origin != nil { + d.recordValueOrigin(valueNode, sourceFile) + } + d.nodeOrigin = nil + d.path = d.path[:len(d.path)-1] if err != nil { return nil, err } result[keyNode.Value] = value } + if err := d.expandMergeKeys(node, result, sourceFile); err != nil { + return nil, err + } recordScriptSource(node, result, sourceFile) if err := recordCommandFields(node, result); err != nil { return nil, err @@ -72,6 +95,20 @@ func (d *configYamlDecoder) decodeMappingNodeWithYamlFunctions(node *yaml.Node, return result, nil } +// recordValueOrigin reports the file that declared the value just decoded at the current path. A +// value produced by an include, or by an alias of an anchored include, belongs to the included file +// (and is remote when that file is), not to the file that merely references it. +func (d *configYamlDecoder) recordValueOrigin(valueNode *yaml.Node, sourceFile string) { + file, remote := sourceFile, false + if d.remoteSource != "" { + file, remote = d.remoteSource, true + } + if included := d.nodeOrigin; included != nil && (valueNode.Kind == yaml.AliasNode || isIncludeTag(valueNode.Tag)) { + file, remote = included.file, included.remote + } + d.origin(d.path, file, remote) +} + // decodeSequenceNodeWithYamlFunctions evaluates sequence elements in their original order. func (d *configYamlDecoder) decodeSequenceNodeWithYamlFunctions(node *yaml.Node, sourceFile string) (any, error) { result := make([]interface{}, 0, len(node.Content)) @@ -115,6 +152,12 @@ func (d *configYamlDecoder) processScalarNodeValueForFile(node *yaml.Node, sourc return d.processIncludeNodeValueForFile(node, sourceFile) } +// redactIncludeValue removes credentials from an include argument before it is echoed in an error. +// Only URL-shaped values are touched: a local path or an SCP-style git address keeps its text. +func redactIncludeValue(value string) string { + return u.RedactIncludeArgument(value) +} + func isIncludeTag(tag string) bool { return tag == u.AtmosYamlFuncInclude || tag == u.AtmosYamlFuncIncludeRaw } @@ -128,7 +171,8 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour // Resolve the path against the config file and the project base path, never the working // directory, so the result is the same wherever Atmos runs from. scope := commandIncludeScope(sourceFile) - value := scope.Resolve(node.Value).Value + include := scope.Resolve(node.Value) + value := include.Value resolved := legacyyaml.Node{ Kind: legacyyaml.ScalarNode, Tag: node.Tag, @@ -136,7 +180,7 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour } key := sourceFile + "\x00" + node.Value if d.includes[key] || len(d.includes) >= maxConfigIncludeDepth { - return nil, fmt.Errorf("%w: recursive include: %s", ErrExecuteYamlFunctions, node.Value) + return nil, fmt.Errorf("%w: recursive include: %s", ErrExecuteYamlFunctions, redactIncludeValue(node.Value)) } d.includes[key] = true defer delete(d.includes, key) @@ -144,22 +188,38 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour BasePath: scope.BasePath, BasePathAbsolute: scope.BasePath, } - var err error - if node.Tag == u.AtmosYamlFuncIncludeRaw { - err = u.ProcessIncludeRawTag(atmosConfig, &resolved, value, sourceFile) - } else { - _, err = u.ProcessIncludeTagWalked(atmosConfig, &resolved, value, sourceFile, d.evaluateInclude) - } + outerTaint := d.remoteTaint + d.remoteTaint = nil + walked, err := d.runInclude(atmosConfig, &resolved, node.Tag, value, sourceFile) + walkedRemote := d.remoteTaint + d.remoteTaint = outerTaint if err != nil { - return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } var decoded any if err := resolved.Decode(&decoded); err != nil { return nil, err } + if d.origin != nil { + d.recordIncludeOrigin(includeResult{localPath: include.LocalPath, target: node.Value, decoded: decoded, walked: walked, walkedRemote: walkedRemote}) + } return decoded, nil } +// recordIncludedOrigins records plain, non-evaluated mappings without running their tags. +func (d *configYamlDecoder) recordIncludedOrigins(value any, file string, remote bool) { + mapping, ok := value.(map[string]any) + if !ok { + return + } + for key, child := range mapping { + d.path = append(d.path, key) + d.recordIncludedOrigins(child, file, remote) + d.origin(d.path, file, remote) + d.path = d.path[:len(d.path)-1] + } +} + // includeBasePathForSourceFile returns the directory a config file's bare include paths anchor // to when it declares no base path: the config file's own directory, or the parent of the // nearest enclosing `atmos.d` or `.atmos.d` directory for files loaded from one (at any depth). diff --git a/pkg/config/yaml_functions.go b/pkg/config/yaml_functions.go index d081438c6be..a3db7536c8b 100644 --- a/pkg/config/yaml_functions.go +++ b/pkg/config/yaml_functions.go @@ -39,11 +39,11 @@ func processExecTag(strFunc, nodeValue string) (any, error) { } // processIncludeTag processes the !include tag. -func processIncludeTag(nodeTag, nodeValue, strFunc string) (any, error) { +func processIncludeTag(nodeTag, nodeValue string) (any, error) { includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", "include_data", nodeTag, nodeValue)) if err != nil { - log.Debug(failedToProcess, functionKey, strFunc, "error", err) - return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, nodeValue, err) + log.Debug(failedToProcess, functionKey, nodeTag+" "+redactIncludeValue(nodeValue), "error", err) + return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(nodeValue), err) } if includeValue != nil { if data, ok := includeValue[includeDataKey]; ok { @@ -149,7 +149,7 @@ func processScalarNodeValue(node *yaml.Node) (any, error) { // !literal keeps the value exactly as written. return node.Value, nil case u.AtmosYamlFuncInclude, u.AtmosYamlFuncIncludeRaw: - return processIncludeTag(node.Tag, node.Value, strFunc) + return processIncludeTag(node.Tag, node.Value) default: return processGitScalarNodeValue(node, strFunc) } @@ -323,11 +323,11 @@ func handleExec(node *yaml.Node, v *viper.Viper, currentPath string) error { // handleInclude Process the !include tag and set the value in Viper , returns an error if the processing fails, warns if the value is empty. func handleInclude(node *yaml.Node, v *viper.Viper, currentPath string) error { - strFunc := fmt.Sprintf(tagValueFormat, node.Tag, node.Value) + strFunc := fmt.Sprintf(tagValueFormat, node.Tag, redactIncludeValue(node.Value)) includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", includeDataKey, node.Tag, node.Value)) if err != nil { log.Debug(failedToProcess, functionKey, strFunc, "error", err) - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } if includeValue == nil { log.Debug(emptyValueWarning, functionKey, strFunc) @@ -344,7 +344,7 @@ func handleInclude(node *yaml.Node, v *viper.Viper, currentPath string) error { func storeIncludeValue(node *yaml.Node, v *viper.Viper, currentPath string, data any) error { if err := node.Encode(data); err != nil { - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } encodeDevcontainerKeys(node, currentPath) if err := node.Decode(&data); err != nil { @@ -352,7 +352,7 @@ func storeIncludeValue(node *yaml.Node, v *viper.Viper, currentPath string, data } v.Set(currentPath, data) if err := node.Encode(data); err != nil { - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } if err := preserveEmptyGitPushRetry(node, v, currentPath); err != nil { return err diff --git a/pkg/datafetcher/schema/atmos/config/1.0.json b/pkg/datafetcher/schema/atmos/config/1.0.json index 9fbe3b80bc9..a7dbe774c6e 100644 --- a/pkg/datafetcher/schema/atmos/config/1.0.json +++ b/pkg/datafetcher/schema/atmos/config/1.0.json @@ -1581,6 +1581,380 @@ "type": "object", "description": "Ansible defines configuration for Ansible components." }, + "ArtifactConfig": { + "properties": { + "repositories": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRepositoryConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ] + }, + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Repositories maps a repository alias to its definition. The alias is case-sensitive and is\nwhat `--repository` and the artifact step `repository` field select." + }, + "verification": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactVerificationConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Verification is the global signature verification policy. Repository policies can add\nrequirements but never weaken it." + } + }, + "type": "object", + "description": "ArtifactConfig is the `artifacts` section of atmos.yaml. It holds the single list of artifact\nrepositories (destinations that are also discovery sources) and the optional global signature\nverification policy." + }, + "ArtifactLockingConfig": { + "properties": { + "required_scope": { + "anyOf": [ + { + "type": "string", + "enum": [ + "", + "host", + "repository" + ] + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "RequiredScope is the minimum lock guarantee mutations need: host (the default) or repository." + }, + "repository": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Repository delegates locking to another configured repository that advertises repository scope.\nThe delegate keeps its own identity and storage root. Cycles are rejected." + } + }, + "type": "object", + "description": "ArtifactLockingConfig configures mutation coordination for a repository." + }, + "ArtifactMetadataDefaults": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Description is the default artifact description." + }, + "labels": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Labels are default exact key/value labels for newly published artifact versions." + } + }, + "type": "object", + "description": "ArtifactMetadataDefaults is the default description and labels of newly published artifacts.\nLabels belong to each published artifact version, not to the repository itself." + }, + "ArtifactRepositoryConfig": { + "properties": { + "kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Kind selects the backend: local/dir, local/archive, aws/s3, oci/registry, or oci/layout." + }, + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity names an Atmos auth identity used for this repository. Credentials never belong in\noptions. The identity is resolved lazily, only when an operation needs credentials." + }, + "options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Options are backend-specific settings, for example `path` for local kinds, `bucket`, `prefix`,\n`region`, `tags`, and `key_pattern` for aws/s3, or `registry` and `prefix` for oci/registry.\nThe `key_pattern` template is never expanded while the configuration is loaded." + }, + "packaging": { + "anyOf": [ + { + "type": "string", + "enum": [ + "", + "none", + "tar", + "tgz", + "zip" + ] + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Packaging selects how file payloads are stored: none, tar, tgz, or zip. The supported values\nand the default depend on the repository kind." + }, + "permissions": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-7][0-7][0-7][0-7]?$" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Permissions is the quoted octal mode of the resulting local archive file, for example \"0600\".\nIt is accepted only for local/archive repositories and must be a quoted string." + }, + "catalog": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Catalog is the optional OCI catalog URI of this repository, for example\noci://ghcr.io/acme/atmos/catalog:stable. It is valid only for oci/registry repositories and\nmust name the repository's own registry host." + }, + "defaults": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRepositoryDefaults" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Defaults supplies defaults for newly published artifacts." + }, + "verification": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactVerificationConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Verification is this repository's signature verification policy. It is combined with the\nglobal policy: both must be satisfied." + }, + "locking": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactLockingConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Locking configures mutation coordination for this repository." + } + }, + "type": "object", + "description": "ArtifactRepositoryConfig defines one artifact repository: a collection root that holds many named\nartifacts of possibly different kinds." + }, + "ArtifactRepositoryDefaults": { + "properties": { + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactMetadataDefaults" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Metadata supplies default artifact metadata. Explicit publication metadata overrides matching keys." + } + }, + "type": "object", + "description": "ArtifactRepositoryDefaults holds defaults applied to artifacts newly published to a repository." + }, + "ArtifactTrustedIdentity": { + "properties": { + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity is the expected certificate identity, for example a workflow URL or an email address." + }, + "issuer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Issuer is the expected OIDC issuer URL, for example https://token.actions.githubusercontent.com." + } + }, + "type": "object", + "description": "ArtifactTrustedIdentity is a trusted signer certificate identity and the OIDC issuer that vouches for it." + }, + "ArtifactVerificationConfig": { + "properties": { + "require_signature": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "RequireSignature makes a missing or invalid signature a failure. Unset means false for new\ngeneral artifacts. A repository cannot set it to false when the global policy requires it." + }, + "trusted_identities": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactTrustedIdentity" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ] + }, + "type": "array" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "TrustedIdentities are certificate identity and OIDC issuer pairs. Any one of them satisfies\nthis policy level's trust set." + }, + "trusted_keys": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "TrustedKeys are operator-managed public-key file paths. Relative paths resolve against the\nconfiguration file that declared them." + } + }, + "type": "object", + "description": "ArtifactVerificationConfig is a signature verification policy, allowed globally under\n`artifacts.verification` and per repository." + }, "Artifacts": { "properties": { "paths": { @@ -18845,6 +19219,20 @@ } ], "description": "LSP settings." + }, + "artifacts": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Artifacts configures artifact repositories and signature verification policy." } }, "type": "object", diff --git a/pkg/schema/artifacts.go b/pkg/schema/artifacts.go new file mode 100644 index 00000000000..06667ed2e84 --- /dev/null +++ b/pkg/schema/artifacts.go @@ -0,0 +1,137 @@ +package schema + +// Developer note: these types describe configuration only. Backend-specific `options` stay a +// generic map so this package never depends on a backend; typed decoding and validation live in +// pkg/artifact and pkg/artifact/config. This package must not import pkg/artifact (pkg/perf +// imports pkg/schema and pkg/artifact imports pkg/perf). + +// ArtifactConfig is the `artifacts` section of atmos.yaml. It holds the single list of artifact +// repositories (destinations that are also discovery sources) and the optional global signature +// verification policy. +type ArtifactConfig struct { + // Repositories maps a repository alias to its definition. The alias is case-sensitive and is + // what `--repository` and the artifact step `repository` field select. + Repositories map[string]ArtifactRepositoryConfig `yaml:"repositories,omitempty" json:"repositories,omitempty" mapstructure:"repositories"` + + // Verification is the global signature verification policy. Repository policies can add + // requirements but never weaken it. + Verification *ArtifactVerificationConfig `yaml:"verification,omitempty" json:"verification,omitempty" mapstructure:"verification"` +} + +// ArtifactRepositoryConfig defines one artifact repository: a collection root that holds many named +// artifacts of possibly different kinds. +type ArtifactRepositoryConfig struct { + // Kind selects the backend: local/dir, local/archive, aws/s3, oci/registry, or oci/layout. + Kind string `yaml:"kind,omitempty" json:"kind,omitempty" mapstructure:"kind"` + + // Identity names an Atmos auth identity used for this repository. Credentials never belong in + // options. The identity is resolved lazily, only when an operation needs credentials. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` + + // Options are backend-specific settings, for example `path` for local kinds, `bucket`, `prefix`, + // `region`, `tags`, and `key_pattern` for aws/s3, or `registry` and `prefix` for oci/registry. + // The `key_pattern` template is never expanded while the configuration is loaded. + Options map[string]any `yaml:"options,omitempty" json:"options,omitempty" mapstructure:"options"` + + // Packaging selects how file payloads are stored: none, tar, tgz, or zip. The supported values + // and the default depend on the repository kind. + Packaging string `yaml:"packaging,omitempty" json:"packaging,omitempty" mapstructure:"packaging" jsonschema:"enum=,enum=none,enum=tar,enum=tgz,enum=zip"` + + // Permissions is the quoted octal mode of the resulting local archive file, for example "0600". + // It is accepted only for local/archive repositories and must be a quoted string. + Permissions string `yaml:"permissions,omitempty" json:"permissions,omitempty" mapstructure:"permissions" jsonschema:"pattern=^[0-7][0-7][0-7][0-7]?$"` + + // Catalog is the optional OCI catalog URI of this repository, for example + // oci://ghcr.io/acme/atmos/catalog:stable. It is valid only for oci/registry repositories and + // must name the repository's own registry host. + Catalog string `yaml:"catalog,omitempty" json:"catalog,omitempty" mapstructure:"catalog"` + + // Defaults supplies defaults for newly published artifacts. + Defaults *ArtifactRepositoryDefaults `yaml:"defaults,omitempty" json:"defaults,omitempty" mapstructure:"defaults"` + + // Verification is this repository's signature verification policy. It is combined with the + // global policy: both must be satisfied. + Verification *ArtifactVerificationConfig `yaml:"verification,omitempty" json:"verification,omitempty" mapstructure:"verification"` + + // Locking configures mutation coordination for this repository. + Locking *ArtifactLockingConfig `yaml:"locking,omitempty" json:"locking,omitempty" mapstructure:"locking"` + + // PathOrigin records the configuration file that declared options.path. It is populated by the + // configuration loader, is never read from or written to YAML, and is tracked independently of + // the other (possibly merged) repository fields. + PathOrigin ArtifactConfigOrigin `yaml:"-" json:"-" mapstructure:"-"` +} + +// ArtifactRepositoryDefaults holds defaults applied to artifacts newly published to a repository. +type ArtifactRepositoryDefaults struct { + // Metadata supplies default artifact metadata. Explicit publication metadata overrides matching keys. + Metadata *ArtifactMetadataDefaults `yaml:"metadata,omitempty" json:"metadata,omitempty" mapstructure:"metadata"` +} + +// ArtifactMetadataDefaults is the default description and labels of newly published artifacts. +// Labels belong to each published artifact version, not to the repository itself. +type ArtifactMetadataDefaults struct { + // Description is the default artifact description. + Description string `yaml:"description,omitempty" json:"description,omitempty" mapstructure:"description"` + + // Labels are default exact key/value labels for newly published artifact versions. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty" mapstructure:"labels"` +} + +// ArtifactVerificationConfig is a signature verification policy, allowed globally under +// `artifacts.verification` and per repository. +type ArtifactVerificationConfig struct { + // RequireSignature makes a missing or invalid signature a failure. Unset means false for new + // general artifacts. A repository cannot set it to false when the global policy requires it. + RequireSignature *bool `yaml:"require_signature,omitempty" json:"require_signature,omitempty" mapstructure:"require_signature"` + + // TrustedIdentities are certificate identity and OIDC issuer pairs. Any one of them satisfies + // this policy level's trust set. + TrustedIdentities []ArtifactTrustedIdentity `yaml:"trusted_identities,omitempty" json:"trusted_identities,omitempty" mapstructure:"trusted_identities"` + + // TrustedKeys are operator-managed public-key file paths. Relative paths resolve against the + // configuration file that declared them. + TrustedKeys []string `yaml:"trusted_keys,omitempty" json:"trusted_keys,omitempty" mapstructure:"trusted_keys"` + + // KeysOrigin records the configuration file that declared trusted_keys. It is populated by the + // configuration loader and is never read from or written to YAML. + KeysOrigin ArtifactConfigOrigin `yaml:"-" json:"-" mapstructure:"-"` +} + +// ArtifactTrustedIdentity is a trusted signer certificate identity and the OIDC issuer that vouches for it. +type ArtifactTrustedIdentity struct { + // Identity is the expected certificate identity, for example a workflow URL or an email address. + Identity string `yaml:"identity" json:"identity" mapstructure:"identity"` + + // Issuer is the expected OIDC issuer URL, for example https://token.actions.githubusercontent.com. + Issuer string `yaml:"issuer" json:"issuer" mapstructure:"issuer"` +} + +// ArtifactLockingConfig configures mutation coordination for a repository. +type ArtifactLockingConfig struct { + // RequiredScope is the minimum lock guarantee mutations need: host (the default) or repository. + RequiredScope string `yaml:"required_scope,omitempty" json:"required_scope,omitempty" mapstructure:"required_scope" jsonschema:"enum=,enum=host,enum=repository"` + + // Repository delegates locking to another configured repository that advertises repository scope. + // The delegate keeps its own identity and storage root. Cycles are rejected. + Repository string `yaml:"repository,omitempty" json:"repository,omitempty" mapstructure:"repository"` +} + +// ArtifactConfigOrigin records which configuration file declared a path-valued setting, so that +// relative paths resolve against the declaring file even after imports have been merged. +type ArtifactConfigOrigin struct { + // File is the absolute path of the declaring configuration file. Empty when unknown. + File string `yaml:"-" json:"-" mapstructure:"-"` + + // Dir is the absolute directory of the declaring file. Empty when unknown. + Dir string `yaml:"-" json:"-" mapstructure:"-"` + + // Remote reports that the value was declared by a remote (downloaded) import, whose local + // path is ephemeral and must never anchor a relative destination. + Remote bool `yaml:"-" json:"-" mapstructure:"-"` +} + +// IsZero reports whether no declaring file is known. +func (o ArtifactConfigOrigin) IsZero() bool { + return o.File == "" && o.Dir == "" && !o.Remote +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index aa820039a15..2f88b3f1f0e 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -222,6 +222,8 @@ type AtmosConfiguration struct { MCP MCPSettings `yaml:"mcp,omitempty" json:"mcp,omitempty" mapstructure:"mcp"` // LSP settings. LSP LSPSettings `yaml:"lsp,omitempty" json:"lsp,omitempty" mapstructure:"lsp"` + // Artifacts configures artifact repositories and signature verification policy. + Artifacts ArtifactConfig `yaml:"artifacts,omitempty" json:"artifacts,omitempty" mapstructure:"artifacts"` } // CastConfig configures Atmos cast recording behavior. diff --git a/pkg/utils/yaml_include_by_extension.go b/pkg/utils/yaml_include_by_extension.go index 194f0112013..91f8d5251b6 100644 --- a/pkg/utils/yaml_include_by_extension.go +++ b/pkg/utils/yaml_include_by_extension.go @@ -164,7 +164,7 @@ func processIncludeTagInternal( data, err := os.ReadFile(localFile) if err != nil { return false, fmt.Errorf("%w: %s, stack manifest: %s, error: %w", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } if err := spliceYAMLText(atmosConfig, node, string(data), includeSplice{query: includeQuery, eval: evalFunctions, val: val, file: file}); err != nil { return false, err @@ -198,7 +198,7 @@ func processIncludeTagInternal( text = string(v) default: return false, fmt.Errorf("%w: %s, stack manifest: %s, error: unexpected remote content type %T", - ErrIncludeYamlFunctionFailedStackManifest, val, file, raw) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), raw) } // Nested paths inside remote content keep the outer manifest as // their context; there is no local directory to resolve against. @@ -296,7 +296,7 @@ func spliceYAMLText(atmosConfig *schema.AtmosConfiguration, node *yaml.Node, yam evaluated, err := evaluateIncludeQuery(atmosConfig, yamlText, query) if err != nil { return fmt.Errorf("%w: %s, stack manifest: %s, error: %w", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } yamlText = evaluated } @@ -518,10 +518,17 @@ func processLocalFile(localFile string, forceRaw bool) (any, error) { return filetype.ParseFileByExtension(os.ReadFile, localFile) } -// processRemoteFile downloads and parses a remote file. +// processRemoteFile downloads and parses a remote file. A credential in the include target is +// removed from the text of any error, because the download layers quote the reference they fail on. func processRemoteFile(atmosConfig *schema.AtmosConfiguration, includeFile string, forceRaw bool) (any, error) { defer perf.Track(atmosConfig, "utils.processRemoteFile")() + res, err := fetchRemoteFile(atmosConfig, includeFile, forceRaw) + return res, redactIncludeError(err, includeFile) +} + +// fetchRemoteFile downloads and parses a remote file. +func fetchRemoteFile(atmosConfig *schema.AtmosConfiguration, includeFile string, forceRaw bool) (any, error) { // oci:// sources need one file picked out of the pulled image, which go-getter's own // "//subdir" handling cannot do (it only copies directories), so the pick happens here // after the shared OCI getter has pulled the image. @@ -571,7 +578,7 @@ func processOCIIncludeFile(atmosConfig *schema.AtmosConfiguration, includeFile s imageRef, subdir := getter.SourceDirSubdir(includeFile) if subdir == "" { - return nil, fmt.Errorf("%w: '%s'", ErrIncludeYamlFunctionOCIMissingPath, includeFile) + return nil, fmt.Errorf("%w: '%s'", ErrIncludeYamlFunctionOCIMissingPath, RedactIncludeArgument(includeFile)) } tempDir, err := os.MkdirTemp("", "atmos-include-oci-*") @@ -587,16 +594,18 @@ func processOCIIncludeFile(atmosConfig *schema.AtmosConfiguration, includeFile s // survive. It applies ociIncludeTimeout itself, so no extra deadline is set here. dl := downloader.NewGoGetterDownloader(atmosConfig).(downloader.ContextFileDownloader) if _, err := dl.FetchWithMetadataContext(context.Background(), imageRef, tempDir, downloader.ClientModeDir, ociIncludeTimeout); err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, includeFile, err) + return nil, redactIncludeError(fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(includeFile), err), includeFile) } localFile, err := filesystem.SafeJoin(tempDir, subdir) if err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, includeFile, err) + return nil, redactIncludeError(fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), err), includeFile) } if !FileExists(localFile) { - return nil, fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, imageRef, subdir) + // The subdirectory is cut from the include target at its first "//", so a password that + // holds one leaves a fragment of itself in it: the whole message is scrubbed against the target. + return nil, redactIncludeError(fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(imageRef), subdir), includeFile) } return processLocalFile(localFile, forceRaw) @@ -641,15 +650,15 @@ func unmarshalYamlContent(y string, val string, file string) (*yaml.Node, error) var includedNode yaml.Node err := yaml.Unmarshal([]byte(y), &includedNode) if err != nil { - return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: %v", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: %w", + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } // yaml.Unmarshal creates a DocumentNode, we need to use its content if includedNode.Kind == yaml.DocumentNode { if len(includedNode.Content) == 0 { return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: empty document", - ErrIncludeYamlFunctionFailedStackManifest, val, file) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file)) } return includedNode.Content[0], nil } diff --git a/pkg/utils/yaml_include_not_found.go b/pkg/utils/yaml_include_not_found.go index bc1ea7e710b..7598b560955 100644 --- a/pkg/utils/yaml_include_not_found.go +++ b/pkg/utils/yaml_include_not_found.go @@ -25,13 +25,13 @@ func includeNotFoundError(tag, args, includeFile, manifestFile string, atmosConf switch { case filepath.IsAbs(includeFile): - return fmt.Errorf("%w: could not find local file '%s'", ErrIncludeYamlFunctionInvalidFile, includeFile) + return fmt.Errorf("%w: could not find local file '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile)) case isManifestRelativePath(includeFile): resolved := ResolveRelativePath(includeFile, manifestFile) builder := errUtils.Build(fmt.Errorf( "%w: could not find local file '%s' at '%s' (paths starting with './' or '../' are resolved relative to the including file '%s' only)", - ErrIncludeYamlFunctionInvalidFile, includeFile, resolved, manifestFile, + ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), resolved, manifestFile, )).WithHint("Paths starting with `./` or `../` are relative to the file that contains the `" + tag + "`. Paths without that prefix are relative to `base_path`.") if suggestion := baseRelativeIncludeSuggestion(includeFile, basePath); suggestion != "" { @@ -47,7 +47,7 @@ func includeNotFoundError(tag, args, includeFile, manifestFile string, atmosConf } return errUtils.Build(fmt.Errorf( "%w: could not find local file '%s' (tried relative to base path '%s' and the working directory '%s')", - ErrIncludeYamlFunctionInvalidFile, includeFile, basePath, workingDir, + ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), basePath, workingDir, )).WithHint("Paths without a `./` or `../` prefix are relative to `base_path`. Start the path with `./` or `../` " + "to resolve it relative to the file that contains the `" + tag + "`.").Err() } diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go new file mode 100644 index 00000000000..b51aef39fad --- /dev/null +++ b/pkg/utils/yaml_include_redact.go @@ -0,0 +1,50 @@ +package utils + +import ( + "regexp" + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// schemeLessUserinfoPattern matches the start of an address that has no scheme but opens with +// userinfo, "user:secret@host/path". The user and the password hold no "/" or white space, and the +// password is closed by the first "@". An SCP-style git address ("git@github.com:org/repo.git") +// has no ":" before its "@" and does not match. +var schemeLessUserinfoPattern = regexp.MustCompile(`^[^\s/@]+:[^\s/@]*@`) + +// windowsDrivePattern matches the drive of a Windows path, "C:\" or "C:/", which is not a user name. +var windowsDrivePattern = regexp.MustCompile(`^[A-Za-z]:[\\/]`) + +// RedactIncludeArgument removes credentials from an include argument or include target before it is +// echoed in an error or a log line. Only URL-shaped values are touched: a local path or an +// SCP-style git address keeps its text, and a URL that cannot be proven free of userinfo is +// replaced entirely (see artifact.RedactURL). A value without a scheme that opens with userinfo +// ("user:secret@host/path", "//user:secret@host/path") is URL-shaped too, and is redacted the same way. +func RedactIncludeArgument(value string) string { + defer perf.Track(nil, "utils.RedactIncludeArgument")() + + if !isURLShaped(value) { + return value + } + return artifact.RedactURL(value) +} + +// isURLShaped reports whether value may be an address that carries userinfo. +func isURLShaped(value string) bool { + if strings.Contains(value, "://") || strings.HasPrefix(value, "//") { + return true + } + return !windowsDrivePattern.MatchString(value) && schemeLessUserinfoPattern.MatchString(value) +} + +// redactIncludeError removes the credential of an include target from the text of an error raised +// while fetching it. Fetch errors from the download and OCI layers quote the reference they were +// given (for example "could not parse reference: user:secret@host/image"), so the include +// argument cannot be redacted once for all of them: the credential is cut out of the final text by +// the helper that redacts URLs, so both agree on where a userinfo ends, and any other URL that +// carries userinfo is withheld as well (see artifact.RedactError). +func redactIncludeError(err error, target string) error { + return artifact.RedactError(err, target) +} diff --git a/pkg/utils/yaml_include_redact_fuzz_test.go b/pkg/utils/yaml_include_redact_fuzz_test.go new file mode 100644 index 00000000000..800f0204557 --- /dev/null +++ b/pkg/utils/yaml_include_redact_fuzz_test.go @@ -0,0 +1,69 @@ +package utils + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" +) + +var errIncludeFetch = errors.New("include fetch failed") + +// checkIncludeEntryPoints runs one generated input through every include redaction entry point. +func checkIncludeEntryPoints(t testing.TB, in redacttest.Input) { + t.Helper() + redacttest.RequireNoLeak(t, "RedactIncludeArgument", in, RedactIncludeArgument(in.Raw)) + for _, message := range in.Messages() { + err := redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), in.Raw) + redacttest.RequireNoLeak(t, "redactIncludeError", in, err.Error()) + if !errors.Is(err, errIncludeFetch) { + t.Fatalf("redactIncludeError dropped the cause of %q", message) + } + } +} + +// TestRedactInclude_SharedCredentialCases runs the shared table of hostile inputs through the +// include entry points. A scheme-less input is not URL-shaped, so only its error text is checked. +func TestRedactInclude_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + got := RedactIncludeArgument(tc.Raw) + if strings.Contains(tc.Raw, "://") { + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "RedactIncludeArgument of %q", tc.Raw) + } + } + for _, message := range []string{"fetch " + tc.Raw + " failed", fmt.Sprintf("fetch %q failed", tc.Raw)} { + err := redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, err.Error(), secret, "redactIncludeError of %q", message) + } + } + }) + } +} + +// FuzzRedactInclude embeds a random secret in many address templates and checks that no include +// entry point leaks the user name or any fragment of the secret. +func FuzzRedactInclude(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + } + f.Fuzz(func(t *testing.T, first, separator, second string) { + for _, in := range redacttest.Generate(redacttest.SecretOf(first, separator, second)) { + // An argument that is not URL-shaped is a local path and keeps its text by design. + if !strings.Contains(in.Raw, "://") { + for _, message := range in.Messages() { + redacttest.RequireNoLeak(t, "redactIncludeError", in, redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), in.Raw).Error()) + } + continue + } + checkIncludeEntryPoints(t, in) + } + }) +} diff --git a/pkg/utils/yaml_include_redact_test.go b/pkg/utils/yaml_include_redact_test.go new file mode 100644 index 00000000000..fd9d41d4d8c --- /dev/null +++ b/pkg/utils/yaml_include_redact_test.go @@ -0,0 +1,166 @@ +package utils + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" + "github.com/cloudposse/atmos/pkg/schema" +) + +const redactTestSecret = "hunter2" + +func TestRedactIncludeArgument(t *testing.T) { + tests := []struct { + name string + value string + want string + }{ + {name: "local path", value: "./stacks/base.yaml", want: "./stacks/base.yaml"}, + {name: "local path with query", value: "./base.yaml .a.b | eval", want: "./base.yaml .a.b | eval"}, + {name: "scp style git address", value: "git@github.com:org/repo.git", want: "git@github.com:org/repo.git"}, + {name: "url without credentials", value: "https://example.com/base.yaml .a", want: "https://example.com/base.yaml .a"}, + {name: "url with userinfo", value: "https://user:" + redactTestSecret + "@example.com/base.yaml", want: "https://example.com/base.yaml"}, + {name: "url with userinfo and fragment", value: "https://user:" + redactTestSecret + "@example.com/base.yaml#frag", want: "https://example.com/base.yaml#frag"}, + {name: "forced getter with userinfo", value: "git::https://user:" + redactTestSecret + "@example.com/repo.git", want: "git::https://example.com/repo.git"}, + {name: "scheme-less userinfo", value: "user:" + redactTestSecret + "@example.com/base.yaml", want: ""}, + {name: "scheme-less userinfo with a query", value: "user:" + redactTestSecret + "@example.com/base.yaml .a.b | eval", want: ""}, + {name: "scheme-less token as user", value: "forced::user:" + redactTestSecret + "@example.com/repo.git", want: ""}, + {name: "scheme-relative userinfo", value: "//user:" + redactTestSecret + "@example.com/base.yaml", want: "//example.com/base.yaml"}, + {name: "local path holding a colon", value: "./stacks/a:b.yaml", want: "./stacks/a:b.yaml"}, + {name: "windows path holding an at sign", value: `C:\stacks\user@host\base.yaml`, want: `C:\stacks\user@host\base.yaml`}, + {name: "local path holding an at sign", value: "./stacks/user@host/base.yaml", want: "./stacks/user@host/base.yaml"}, + {name: "unparsable url with userinfo", value: "https://user:" + redactTestSecret + "@exa mple.com/%zz", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := RedactIncludeArgument(tt.value) + assert.Equal(t, tt.want, got) + assert.NotContains(t, got, redactTestSecret) + }) + } +} + +// TestProcessInclude_ErrorsDoNotEchoCredentials runs every error path that formats an include +// argument or target with a credential-bearing URL and checks the credential never reaches the error. +func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/invalid.yaml": + _, _ = fmt.Fprint(w, "value: [\n") + case "/empty.yaml": + _, _ = fmt.Fprint(w, "# only a comment\n---\n") + default: + _, _ = fmt.Fprint(w, "a:\n b: 1\n") + } + })) + defer server.Close() + withUserinfo := strings.Replace(server.URL, "http://", "http://user:"+redactTestSecret+"@", 1) + closed := "http://user:" + redactTestSecret + "@127.0.0.1:1" + + tests := []struct { + name string + include string + // rawOK reports that !include.raw does not parse the body, so it cannot fail on it. + rawOK bool + // also is a second secret that must not reach the error. + also string + }{ + {name: "query evaluation failure", include: withUserinfo + "/config.yaml .a[ | eval"}, + {name: "query evaluation failure without eval", include: withUserinfo + "/config.yaml .a["}, + {name: "invalid remote yaml", include: withUserinfo + "/invalid.yaml | eval", rawOK: true}, + {name: "invalid remote yaml without eval", include: withUserinfo + "/invalid.yaml", rawOK: true}, + {name: "fetch failure", include: closed + "/config.yaml"}, + {name: "fetch failure with eval", include: closed + "/config.yaml | eval"}, + {name: "raw fetch failure", include: closed + "/config.txt"}, + {name: "oci include without a file path", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag"}, + {name: "oci pull failure with a double slash in the password", include: "oci://user:" + redactTestSecret + "//hunter3@127.0.0.1:1/org/image:tag//file.yaml", also: "hunter3"}, + {name: "scheme-less userinfo that is not found", include: "user:" + redactTestSecret + "@127.0.0.1:1/config.yaml"}, + {name: "oci pull failure", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag//file.yaml"}, + } + for _, tag := range []string{"!include", "!include.raw"} { + for _, tt := range tests { + t.Run(tag+"/"+tt.name, func(t *testing.T) { + dir := t.TempDir() + config := &schema.AtmosConfiguration{BasePath: dir} + _, err := UnmarshalYAMLFromFile[map[string]any](config, "included: "+tag+" "+tt.include+"\n", filepath.Join(dir, "stack.yaml")) + if tag == "!include.raw" && tt.rawOK { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.NotContains(t, err.Error(), redactTestSecret, "the include credential must not reach the error text") + if tt.also != "" { + assert.NotContains(t, err.Error(), tt.also, "no part of the include credential may reach the error text") + } + }) + } + } +} + +func TestRedactIncludeError(t *testing.T) { + cause := errors.New("pull failed") + tests := []struct { + name string + target string + message string + want string + }{ + {name: "no credential", target: "https://example.com/a.yaml", message: "boom at example.com", want: "boom at example.com"}, + {name: "userinfo is cut out", target: "https://user:hunter2@example.com/a", message: "could not parse reference: user:hunter2@example.com/a", want: "could not parse reference: example.com/a"}, + {name: "password quoted on its own", target: "https://user:hunter2@example.com/a", message: "auth failed for hunter2", want: "auth failed for "}, + {name: "encoded password is also removed decoded", target: "https://user:p%40ss@example.com/a", message: "bad password p@ss", want: "bad password "}, + {name: "forced getter prefix", target: "git::https://user:hunter2@example.com/r.git", message: "clone https://user:hunter2@example.com/r.git failed", want: "clone https://example.com/r.git failed"}, + {name: "user name alone is not a password", target: "ssh://git@example.com/r.git", message: "git failed", want: "git failed"}, + {name: "double colon is part of the password", target: "https://user:pa::ss@example.com/a", message: "bad https://user:pa::ss@example.com/a", want: "bad https://example.com/a"}, + {name: "double colon password quoted on its own", target: "https://user:pa::ss@example.com/a", message: "bad pa::ss", want: "bad "}, + {name: "last at sign ends the userinfo", target: "https://user:pa@ss@second@example.com/a", message: "bad https://user:pa@ss@second@example.com/a", want: "bad https://example.com/a"}, + {name: "password with an at sign quoted on its own", target: "https://user:pa@ss@second@example.com/a", message: "bad pa@ss@second", want: "bad "}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := redactIncludeError(fmt.Errorf("%s: %w", tt.message, cause), tt.target) + require.Error(t, got) + assert.Equal(t, tt.want+": pull failed", got.Error()) + assert.ErrorIs(t, got, cause, "the cause stays in the chain") + }) + } + assert.NoError(t, redactIncludeError(nil, "https://user:hunter2@example.com")) +} + +// TestRedactIncludeHelpers_SharedCredentialCases feeds the table shared with artifact.RedactURL to +// both include redaction helpers and asserts the one security property: no user or password of the +// input survives, whether the input is echoed as an argument or quoted by a fetch error. +func TestRedactIncludeHelpers_SharedCredentialCases(t *testing.T) { + cause := errors.New("pull failed") + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Raw, "://") { + got := RedactIncludeArgument(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "argument %q", tc.Raw) + } + } + messages := []string{"fetch " + tc.Raw + " failed"} + for _, quoted := range tc.Quoted { + messages = append(messages, "auth failed for "+quoted) + } + for _, message := range messages { + got := redactIncludeError(fmt.Errorf("%s: %w", message, cause), tc.Raw) + require.Error(t, got) + for _, secret := range tc.Secrets { + assert.NotContains(t, got.Error(), secret, "error %q", message) + } + assert.ErrorIs(t, got, cause) + } + }) + } +} diff --git a/scripts/checks/checks.star b/scripts/checks/checks.star index aa2a3a1cd44..47a63c0e3a0 100644 --- a/scripts/checks/checks.star +++ b/scripts/checks/checks.star @@ -126,7 +126,7 @@ def module_compatibility(config, files): # Run the existing Atmos validation checks against all files or the affected scope. def editorconfig(config, files): - args = ["validate", "--exclude", "tests/fixtures/**", "--exclude", "**/*.go", "--exclude", "specs/**", "--exclude", "README.md", "--format", "rich"] + args = ["validate", "--exclude", "tests/fixtures/**", "--exclude", "pkg/artifact/compat/testdata/**", "--exclude", "**/*.go", "--exclude", "specs/**", "--exclude", "README.md", "--format", "rich"] if config["scope"] != "all": args += ["--affected", "--base", config["base"]] atmos.run(args) diff --git a/tests/fixtures/scenarios/artifacts-describe/atmos.yaml b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml new file mode 100644 index 00000000000..36b0cb18133 --- /dev/null +++ b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml @@ -0,0 +1,11 @@ +base_path: . +artifacts: + repositories: + shared: + kind: aws/s3 + options: + bucket: example-artifacts + verification: + require_signature: true + trusted_keys: + - keys/release.pub diff --git a/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden b/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden index 94d5d061aab..3c78ed4fe89 100644 --- a/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden +++ b/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden @@ -404,5 +404,6 @@ "mcp": { "routing": {} }, - "lsp": {} + "lsp": {}, + "artifacts": {} } diff --git a/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden b/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden index 8516cad17b1..6f83d1cad71 100644 --- a/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden +++ b/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden @@ -413,5 +413,6 @@ "mcp": { "routing": {} }, - "lsp": {} + "lsp": {}, + "artifacts": {} } diff --git a/tests/test-cases/artifacts-describe.yaml b/tests/test-cases/artifacts-describe.yaml new file mode 100644 index 00000000000..fe8d69d7a9c --- /dev/null +++ b/tests/test-cases/artifacts-describe.yaml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=schema.json +tests: + - name: describe config with artifacts + enabled: true + snapshot: false + workdir: "fixtures/scenarios/artifacts-describe" + command: "atmos" + env: + ATMOS_PAGER: false + args: + - describe + - config + expect: + valid: + - json + stdout: + - '"shared"' + - '"bucket": "example-artifacts"' + - '"require_signature": true' + # The key is declared by the fixture's atmos.yaml, so it anchors to that directory and no other. + - '"trusted_keys":\s*\[\s*"[^"]*(?:/|\\\\)tests(?:/|\\\\)fixtures(?:/|\\\\)scenarios(?:/|\\\\)artifacts-describe(?:/|\\\\)keys(?:/|\\\\)release\.pub"' + exit_code: 0