From 4b3ed0db036f0d51ff2e2d729ac73c49b7d52a69 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 30 Oct 2026 22:27:15 -0500 Subject: [PATCH 01/32] feat(artifacts): add domain contracts, canonical records, and compatibility fixtures (A00) Co-Authored-By: Claude Sonnet 5.5 --- .atmos-git-hooks.yaml | 10 +- errors/errors.go | 37 +++ pkg/artifact/boundary_test.go | 47 +++ pkg/artifact/canonical.go | 188 ++++++++++++ pkg/artifact/canonicaljson.go | 258 ++++++++++++++++ pkg/artifact/canonicaljson_test.go | 101 ++++++ pkg/artifact/capabilities.go | 95 ++++++ pkg/artifact/compat/compat_test.go | 223 ++++++++++++++ pkg/artifact/compat/doc.go | 8 + pkg/artifact/compat/fixtures_test.go | 152 +++++++++ pkg/artifact/compat/helpers_test.go | 8 + pkg/artifact/compat/testdata/.gitattributes | 2 + ...789abcdef01234567.tfplan.tar.metadata.json | 17 + ...a5b026bd033ac034a0c79d60b0233b30b25cd3c485 | Bin 0 -> 3072 bytes ...d37e4055f47d3cddc851002a5ecee4d8c503156d79 | 1 + ...8363f6241d40127d05b7bc71facdebf0a8855b2380 | 1 + ...76dc7c17213a7c356b9778e19cfe0c52585b2d823f | 1 + ...db2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 | 1 + ...f506478b33d99b8a1dc65de9fc6683e4f26d51af7e | Bin 0 -> 3072 bytes ...476b7123acbfd0a7f37dfa94a397045e22b5c819ee | 1 + .../compat/testdata/plugin-release/index.json | 11 + .../compat/testdata/plugin-release/oci-layout | 3 + pkg/artifact/constants.go | 77 +++++ pkg/artifact/contracts_test.go | 151 +++++++++ pkg/artifact/copier.go | 29 ++ pkg/artifact/digest.go | 81 +++++ pkg/artifact/doc.go | 19 ++ pkg/artifact/entry.go | 205 +++++++++++++ pkg/artifact/errors.go | 107 +++++++ pkg/artifact/errors_test.go | 69 +++++ pkg/artifact/kinddata.go | 71 +++++ pkg/artifact/kinddata_test.go | 38 +++ pkg/artifact/lock.go | 82 +++++ pkg/artifact/lockprovider.go | 14 + pkg/artifact/mock_copier.go | 70 +++++ pkg/artifact/mock_lockprovider.go | 180 +++++++++++ pkg/artifact/mock_producer.go | 70 +++++ pkg/artifact/mock_repository.go | 290 ++++++++++++++++++ pkg/artifact/mock_service.go | 206 +++++++++++++ pkg/artifact/mock_signer.go | 70 +++++ pkg/artifact/mock_verifier.go | 70 +++++ pkg/artifact/operation.go | 124 ++++++++ pkg/artifact/packaging.go | 22 ++ pkg/artifact/pinned.go | 32 ++ pkg/artifact/policy.go | 98 ++++++ pkg/artifact/policy_test.go | 70 +++++ pkg/artifact/prepared.go | 88 ++++++ pkg/artifact/producer.go | 53 ++++ pkg/artifact/record.go | 159 ++++++++++ pkg/artifact/record_test.go | 193 ++++++++++++ pkg/artifact/ref.go | 165 ++++++++++ pkg/artifact/ref_test.go | 117 +++++++ pkg/artifact/repository.go | 54 ++++ pkg/artifact/repository_types.go | 226 ++++++++++++++ pkg/artifact/representation.go | 31 ++ pkg/artifact/requests.go | 151 +++++++++ pkg/artifact/results.go | 78 +++++ pkg/artifact/service.go | 46 +++ pkg/artifact/signature.go | 145 +++++++++ pkg/artifact/signer.go | 28 ++ pkg/artifact/timestamp.go | 73 +++++ pkg/artifact/vectors_test.go | 288 +++++++++++++++++ pkg/artifact/verifier.go | 19 ++ scripts/checks/checks.star | 2 +- 64 files changed, 5320 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/boundary_test.go create mode 100644 pkg/artifact/canonical.go create mode 100644 pkg/artifact/canonicaljson.go create mode 100644 pkg/artifact/canonicaljson_test.go create mode 100644 pkg/artifact/capabilities.go create mode 100644 pkg/artifact/compat/compat_test.go create mode 100644 pkg/artifact/compat/doc.go create mode 100644 pkg/artifact/compat/fixtures_test.go create mode 100644 pkg/artifact/compat/helpers_test.go create mode 100644 pkg/artifact/compat/testdata/.gitattributes create mode 100644 pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f create mode 100755 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e create mode 100644 pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee create mode 100755 pkg/artifact/compat/testdata/plugin-release/index.json create mode 100755 pkg/artifact/compat/testdata/plugin-release/oci-layout create mode 100644 pkg/artifact/constants.go create mode 100644 pkg/artifact/contracts_test.go create mode 100644 pkg/artifact/copier.go create mode 100644 pkg/artifact/digest.go create mode 100644 pkg/artifact/doc.go create mode 100644 pkg/artifact/entry.go create mode 100644 pkg/artifact/errors.go create mode 100644 pkg/artifact/errors_test.go create mode 100644 pkg/artifact/kinddata.go create mode 100644 pkg/artifact/kinddata_test.go create mode 100644 pkg/artifact/lock.go create mode 100644 pkg/artifact/lockprovider.go create mode 100644 pkg/artifact/mock_copier.go create mode 100644 pkg/artifact/mock_lockprovider.go create mode 100644 pkg/artifact/mock_producer.go create mode 100644 pkg/artifact/mock_repository.go create mode 100644 pkg/artifact/mock_service.go create mode 100644 pkg/artifact/mock_signer.go create mode 100644 pkg/artifact/mock_verifier.go create mode 100644 pkg/artifact/operation.go create mode 100644 pkg/artifact/packaging.go create mode 100644 pkg/artifact/pinned.go create mode 100644 pkg/artifact/policy.go create mode 100644 pkg/artifact/policy_test.go create mode 100644 pkg/artifact/prepared.go create mode 100644 pkg/artifact/producer.go create mode 100644 pkg/artifact/record.go create mode 100644 pkg/artifact/record_test.go create mode 100644 pkg/artifact/ref.go create mode 100644 pkg/artifact/ref_test.go create mode 100644 pkg/artifact/repository.go create mode 100644 pkg/artifact/repository_types.go create mode 100644 pkg/artifact/representation.go create mode 100644 pkg/artifact/requests.go create mode 100644 pkg/artifact/results.go create mode 100644 pkg/artifact/service.go create mode 100644 pkg/artifact/signature.go create mode 100644 pkg/artifact/signer.go create mode 100644 pkg/artifact/timestamp.go create mode 100644 pkg/artifact/vectors_test.go create mode 100644 pkg/artifact/verifier.go diff --git a/.atmos-git-hooks.yaml b/.atmos-git-hooks.yaml index dc8c4dd23f2..8b707979d06 100644 --- a/.atmos-git-hooks.yaml +++ b/.atmos-git-hooks.yaml @@ -21,7 +21,7 @@ checks: - id: go-fumpt handler: gofumpt files: '\.go$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/)' fix: true - id: go-mod-tidy @@ -37,7 +37,7 @@ checks: - id: golangci-lint handler: custom_lint files: '\.go$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/)' - id: gomodcheck handler: module_compatibility @@ -57,18 +57,18 @@ checks: - id: trailing-whitespace handler: trailing_whitespace - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|.*\.svg|website/src/components/Screengrabs/)' fix: true - id: end-of-file-fixer handler: end_of_file - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|.*\.svg|website/src/components/Screengrabs/)' fix: true - id: check-yaml handler: yaml_syntax files: '\.(yaml|yml)$' - exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/)' + exclude: '^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|pkg/artifact/compat/testdata/|tests/fixtures/)' - id: check-added-large-files handler: large_files diff --git a/errors/errors.go b/errors/errors.go index cc179229494..309b3ec3ea6 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2080,6 +2080,43 @@ var ( ErrPolicyPlan = errors.New("saved plan policy enforcement failed") ) +// Generic artifact domain sentinels. They describe repository-independent outcomes of the +// pkg/artifact contracts; backend adapters wrap them and keep their SDK causes for diagnostics. +// Missing artifacts use ErrArtifactNotFound and integrity mismatches use ErrArtifactIntegrityFailed. +var ( + // ErrArtifactConfigInvalid is returned when artifact repository or verification configuration is invalid. + ErrArtifactConfigInvalid = errors.New("invalid artifact configuration") + + // ErrArtifactRefInvalid is returned when an artifact name, version, digest, or reference is malformed. + ErrArtifactRefInvalid = errors.New("invalid artifact reference") + + // ErrArtifactConflict is returned when a publication, copy, or delete conflicts with existing state. + ErrArtifactConflict = errors.New("artifact conflict") + + // ErrArtifactPublicationUnavailable is returned when a reference exists but cannot currently be + // published to or read, for example while another operation is pending or after an interrupted replacement. + ErrArtifactPublicationUnavailable = errors.New("artifact publication is unavailable") + + // ErrArtifactUnsupportedCapability is returned when a repository does not provide a requested + // guarantee. It is never a successful no-op. + ErrArtifactUnsupportedCapability = errors.New("artifact repository capability is not supported") + + // ErrArtifactDocumentMissing is returned when a requested package document (README or CHANGELOG) does not exist. + ErrArtifactDocumentMissing = errors.New("artifact document not found") + + // ErrArtifactPackagingInvalid is returned when source content cannot be packaged as an artifact. + ErrArtifactPackagingInvalid = errors.New("invalid artifact packaging") + + // ErrArtifactUnsafePath is returned for absolute, escaping, empty-segment, or reserved paths. + ErrArtifactUnsafePath = errors.New("unsafe artifact path") + + // ErrArtifactRecordInvalid is returned when an artifact record fails validation or is not canonical. + ErrArtifactRecordInvalid = errors.New("invalid artifact record") + + // ErrArtifactTooLarge is returned when a record, document, or evidence blob exceeds its bounded-read limit. + ErrArtifactTooLarge = errors.New("artifact data exceeds the size limit") +) + // ExitCodeError is a typed error that preserves subcommand exit codes. // This allows the root command to exit with the same code as the subcommand. // When Code is 0, it indicates successful completion that should exit cleanly without printing errors. diff --git a/pkg/artifact/boundary_test.go b/pkg/artifact/boundary_test.go new file mode 100644 index 00000000000..63bc5e95542 --- /dev/null +++ b/pkg/artifact/boundary_test.go @@ -0,0 +1,47 @@ +package artifact + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPackageDependencyBoundary keeps the domain package free of CLI, runner, hook, plugin, and CI +// dependencies, as required by the package boundary contract. +func TestPackageDependencyBoundary(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/ci", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/runner", + "github.com/cloudposse/atmos/pkg/hooks", + "github.com/cloudposse/atmos/internal/exec", + "github.com/google/go-containerregistry", + "oras.land/oras-go", + "github.com/aws/aws-sdk-go-v2", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports forbidden package %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/artifact/canonical.go b/pkg/artifact/canonical.go new file mode 100644 index 00000000000..11bbf0cbba8 --- /dev/null +++ b/pkg/artifact/canonical.go @@ -0,0 +1,188 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Validate checks the Record against the V1 schema, including that ContentDigest matches the +// inventory and documents. It does not check size limits of the serialized form; Canonical does. +func (r *Record) Validate() error { + defer perf.Track(nil, "artifact.Record.Validate")() + + if r.SchemaVersion != SchemaVersion { + return fmt.Errorf("%w: unsupported schema_version %d (want %d)", errUtils.ErrArtifactRecordInvalid, r.SchemaVersion, SchemaVersion) + } + if err := ValidateName(r.Name); err != nil { + return err + } + if err := ValidateVersion(r.Version); err != nil { + return err + } + if err := ValidateKind(r.Kind); err != nil { + return err + } + if r.CreatedAt.IsZero() { + return fmt.Errorf("%w: created_at is required", errUtils.ErrArtifactRecordInvalid) + } + if err := r.Metadata.validate(); err != nil { + return err + } + expected, err := ComputeContentDigest(r.Inventory, r.Documents) + if err != nil { + return err + } + if r.ContentDigest != expected { + return fmt.Errorf("%w: content_digest %q does not match the inventory and documents (%s)", errUtils.ErrArtifactRecordInvalid, r.ContentDigest, expected) + } + return nil +} + +func (m Metadata) validate() error { + for key := range m.Labels { + if key == "" { + return fmt.Errorf("%w: label keys must not be empty", errUtils.ErrArtifactRecordInvalid) + } + } + return nil +} + +// Seal sets SchemaVersion (when unset) and ContentDigest from the inventory and documents, then +// validates the Record. Call it after assembling a Record from scratch. +func (r *Record) Seal() error { + defer perf.Track(nil, "artifact.Record.Seal")() + + if r.SchemaVersion == 0 { + r.SchemaVersion = SchemaVersion + } + computed, err := ComputeContentDigest(r.Inventory, r.Documents) + if err != nil { + return err + } + r.ContentDigest = computed + return r.Validate() +} + +// normalized returns a deep-enough copy with sorted arrays, non-nil arrays, normalized time, and +// empty provenance dropped, ready for serialization. The receiver is never mutated. +func (r *Record) normalized() Record { + out := *r + out.CreatedAt = NewTimestamp(r.CreatedAt.Time()) + out.Inventory = sortedInventory(r.Inventory) + out.Documents = sortedDocuments(r.Documents) + out.Metadata.Provenance = r.Metadata.Provenance.normalized() + if len(r.Metadata.Labels) == 0 { + out.Metadata.Labels = nil + } + return out +} + +func (p *Provenance) normalized() *Provenance { + if p == nil { + return nil + } + out := Provenance{} + if p.CI != nil && *p.CI != (CIProvenance{}) { + ci := *p.CI + out.CI = &ci + } + if p.Source != nil && *p.Source != (SourceProvenance{}) { + source := *p.Source + out.Source = &source + } + if out.CI == nil && out.Source == nil { + return nil + } + return &out +} + +// Canonical returns the canonical Record bytes: sorted keys and arrays, UTF-8, integer sizes and +// modes, UTC RFC3339 seconds, no insignificant whitespace, and no trailing newline. The same bytes +// are stored as the OCI config blob and in local and S3 record files. The Record is validated +// first, and records larger than [MaxRecordSize] are rejected. +func (r *Record) Canonical() ([]byte, error) { + defer perf.Track(nil, "artifact.Record.Canonical")() + + if err := r.Validate(); err != nil { + return nil, err + } + canonical, err := canonicalJSON(r.normalized()) + if err != nil { + return nil, err + } + if int64(len(canonical)) > MaxRecordSize { + return nil, fmt.Errorf("%w: record is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(canonical), MaxRecordSize) + } + return canonical, nil +} + +// canonicalJSON marshals v with encoding/json and re-encodes the result canonically. +func canonicalJSON(v any) ([]byte, error) { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return CanonicalizeJSON(buf.Bytes()) +} + +// RecordDigest returns the SHA-256 of the full canonical Record bytes, including created_at. +func (r *Record) RecordDigest() (digest.Digest, error) { + defer perf.Track(nil, "artifact.Record.RecordDigest")() + + canonical, err := r.Canonical() + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} + +// DecodeRecord strictly parses canonical Record bytes. It rejects oversize input, unknown fields, +// invalid records, and any bytes that are not exactly the canonical serialization of the decoded +// Record, so the record digest of the stored bytes is always the digest of the decoded value. +func DecodeRecord(raw []byte) (*Record, error) { + defer perf.Track(nil, "artifact.DecodeRecord")() + + if int64(len(raw)) > MaxRecordSize { + return nil, fmt.Errorf("%w: record is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(raw), MaxRecordSize) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var record Record + if err := decoder.Decode(&record); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactRecordInvalid, err) + } + canonical, err := record.Canonical() + if err != nil { + return nil, err + } + if !bytes.Equal(canonical, raw) { + return nil, fmt.Errorf("%w: bytes are not the canonical serialization", errUtils.ErrArtifactRecordInvalid) + } + return &record, nil +} + +// ReadRecord reads and strictly decodes a Record from r, buffering at most [MaxRecordSize] bytes. +// It returns the canonical bytes alongside the Record so callers can compute the record digest +// of exactly what was stored. +func ReadRecord(r io.Reader) (*Record, []byte, error) { + defer perf.Track(nil, "artifact.ReadRecord")() + + raw, err := ReadBounded(r, MaxRecordSize) + if err != nil { + return nil, nil, err + } + record, err := DecodeRecord(raw) + if err != nil { + return nil, nil, err + } + return record, raw, nil +} diff --git a/pkg/artifact/canonicaljson.go b/pkg/artifact/canonicaljson.go new file mode 100644 index 00000000000..8c277a4c637 --- /dev/null +++ b/pkg/artifact/canonicaljson.go @@ -0,0 +1,258 @@ +package artifact + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + // The nesting limit bounds depth so hostile kind data cannot exhaust the stack. + maxJSONDepth = 64 + + // The numeric base of canonical integers. + decimalBase = 10 + + // The bit width of canonical integers. + int64Bits = 64 + + // The first character that JSON strings need not escape. + firstPrintable = 0x20 +) + +// jsonKind enumerates the node types of the canonicalizer's ordered tree. +type jsonKind int + +const ( + jsonObject jsonKind = iota + jsonArray + jsonString + jsonInteger + jsonBool + jsonNull +) + +// jsonNode is one parsed JSON value. Objects keep their members as an ordered slice rather than a +// map, so no map[string]any or float64 is ever involved in canonicalization. +type jsonNode struct { + kind jsonKind + text string // String value, canonical integer text, or "true"/"false". + members []jsonMember + items []*jsonNode +} + +type jsonMember struct { + key string + value *jsonNode +} + +// CanonicalizeJSON re-encodes JSON text into the canonical form used by Records. +// +// The canonical form has object members sorted by key (bytewise on UTF-8), no insignificant +// whitespace, integers in plain decimal, and strings with only the escapes required by JSON (the +// quote, the backslash, and control characters as lowercase \u00xx); every other character is +// written as literal UTF-8, including <, >, &, U+2028 and U+2029. The result is independent of the +// Go version's encoding/json escaping behavior. The input must be valid UTF-8, contain no +// duplicate object keys, no floating-point numbers, no integers outside the int64 range, no +// U+FFFD replacement characters (which indicate invalid input), and no trailing data. +func CanonicalizeJSON(raw []byte) ([]byte, error) { + defer perf.Track(nil, "artifact.CanonicalizeJSON")() + + if !utf8.Valid(raw) { + return nil, fmt.Errorf("%w: JSON is not valid UTF-8", errUtils.ErrArtifactRecordInvalid) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + root, err := parseJSONNode(decoder, 0) + if err != nil { + return nil, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%w: unexpected data after the JSON value", errUtils.ErrArtifactRecordInvalid) + } + var out bytes.Buffer + root.write(&out) + return out.Bytes(), nil +} + +// parseJSONNode reads one complete JSON value from the decoder's token stream. +func parseJSONNode(decoder *json.Decoder, depth int) (*jsonNode, error) { + if depth > maxJSONDepth { + return nil, fmt.Errorf("%w: JSON nesting exceeds %d levels", errUtils.ErrArtifactRecordInvalid, maxJSONDepth) + } + token, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("%w: malformed JSON: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return parseJSONToken(decoder, token, depth) +} + +// parseJSONToken converts one token (and, for containers, the tokens that follow) into a node. +func parseJSONToken(decoder *json.Decoder, token json.Token, depth int) (*jsonNode, error) { + switch value := token.(type) { + case json.Delim: + if value == '{' { + return parseJSONObject(decoder, depth) + } + return parseJSONArray(decoder, depth) + case string: + return parseJSONStringNode(value) + case json.Number: + return parseJSONInteger(value) + case bool: + return &jsonNode{kind: jsonBool, text: strconv.FormatBool(value)}, nil + case nil: + return &jsonNode{kind: jsonNull}, nil + default: + return nil, fmt.Errorf("%w: unsupported JSON token %v", errUtils.ErrArtifactRecordInvalid, token) + } +} + +func parseJSONStringNode(value string) (*jsonNode, error) { + if err := checkJSONString(value); err != nil { + return nil, err + } + return &jsonNode{kind: jsonString, text: value}, nil +} + +func parseJSONObject(decoder *json.Decoder, depth int) (*jsonNode, error) { + node := &jsonNode{kind: jsonObject} + seen := make(map[string]struct{}) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("%w: malformed JSON object: %w", errUtils.ErrArtifactRecordInvalid, err) + } + key, ok := keyToken.(string) + if !ok { + return nil, fmt.Errorf("%w: JSON object key is not a string", errUtils.ErrArtifactRecordInvalid) + } + if err := checkJSONString(key); err != nil { + return nil, err + } + if _, duplicate := seen[key]; duplicate { + return nil, fmt.Errorf("%w: duplicate JSON object key %q", errUtils.ErrArtifactRecordInvalid, key) + } + seen[key] = struct{}{} + value, err := parseJSONNode(decoder, depth+1) + if err != nil { + return nil, err + } + node.members = append(node.members, jsonMember{key: key, value: value}) + } + if _, err := decoder.Token(); err != nil { // Consume '}'. + return nil, fmt.Errorf("%w: malformed JSON object: %w", errUtils.ErrArtifactRecordInvalid, err) + } + sort.Slice(node.members, func(i, j int) bool { return node.members[i].key < node.members[j].key }) + return node, nil +} + +func parseJSONArray(decoder *json.Decoder, depth int) (*jsonNode, error) { + node := &jsonNode{kind: jsonArray} + for decoder.More() { + item, err := parseJSONNode(decoder, depth+1) + if err != nil { + return nil, err + } + node.items = append(node.items, item) + } + if _, err := decoder.Token(); err != nil { // Consume ']'. + return nil, fmt.Errorf("%w: malformed JSON array: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return node, nil +} + +// parseJSONInteger accepts only plain decimal integers that fit in an int64. +func parseJSONInteger(number json.Number) (*jsonNode, error) { + value, err := strconv.ParseInt(number.String(), decimalBase, int64Bits) + if err != nil { + return nil, fmt.Errorf("%w: number %s is not an int64 integer; floats are not allowed", errUtils.ErrArtifactRecordInvalid, number) + } + return &jsonNode{kind: jsonInteger, text: strconv.FormatInt(value, decimalBase)}, nil +} + +// checkJSONString rejects U+FFFD, which is how encoding/json surfaces invalid UTF-8 and lone +// surrogate escapes. Accepting it would silently corrupt data. +func checkJSONString(s string) error { + if strings.ContainsRune(s, utf8.RuneError) { + return fmt.Errorf("%w: string contains an invalid UTF-8 sequence or U+FFFD", errUtils.ErrArtifactRecordInvalid) + } + return nil +} + +func (n *jsonNode) write(out *bytes.Buffer) { + switch n.kind { + case jsonObject: + out.WriteByte('{') + for i, member := range n.members { + if i > 0 { + out.WriteByte(',') + } + writeJSONString(out, member.key) + out.WriteByte(':') + member.value.write(out) + } + out.WriteByte('}') + case jsonArray: + out.WriteByte('[') + for i, item := range n.items { + if i > 0 { + out.WriteByte(',') + } + item.write(out) + } + out.WriteByte(']') + case jsonString: + writeJSONString(out, n.text) + case jsonInteger, jsonBool: + out.WriteString(n.text) + case jsonNull: + out.WriteString("null") + } +} + +const hexDigits = "0123456789abcdef" + +// writeJSONString writes s with the minimal, Go-version-independent escape set. +func writeJSONString(out *bytes.Buffer, s string) { + out.WriteByte('"') + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '"' || c == '\\': + out.WriteByte('\\') + out.WriteByte(c) + case c < firstPrintable: + out.WriteString(`\u00`) + out.WriteByte(hexDigits[c>>4]) + out.WriteByte(hexDigits[c&0x0f]) + default: + out.WriteByte(c) + } + } + out.WriteByte('"') +} + +// ReadBounded reads at most limit bytes from r and fails with ErrArtifactTooLarge when the stream +// is longer, so callers never buffer unbounded input. +func ReadBounded(r io.Reader, limit int64) ([]byte, error) { + defer perf.Track(nil, "artifact.ReadBounded")() + + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("%w: more than %d bytes", errUtils.ErrArtifactTooLarge, limit) + } + return data, nil +} diff --git a/pkg/artifact/canonicaljson_test.go b/pkg/artifact/canonicaljson_test.go new file mode 100644 index 00000000000..bc4f7dd4b64 --- /dev/null +++ b/pkg/artifact/canonicaljson_test.go @@ -0,0 +1,101 @@ +package artifact + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestCanonicalizeJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"sorts keys recursively", `{"b":{"z":1,"a":2},"a":[3,{"y":true,"x":null}]}`, `{"a":[3,{"x":null,"y":true}],"b":{"a":2,"z":1}}`}, + {"strips whitespace", " {\n \"a\" : [ 1 , 2 ] }\n", `{"a":[1,2]}`}, + {"minimal escapes", `{"s":"<>&
\u007f"}`, "{\"s\":\"<>&
\u007f\"}"}, + {"control characters use lowercase u00xx", `{"s":"\n\t\u001f"}`, `{"s":"\u000a\u0009\u001f"}`}, + {"quote and backslash", `{"s":"\"\\"}`, `{"s":"\"\\"}`}, + {"unicode literal", `{"s":"é é"}`, `{"s":"é é"}`}, + {"integers normalized", `[0,-0,-12,9223372036854775807]`, `[0,0,-12,9223372036854775807]`}, + {"scalar", `true`, `true`}, + {"empty containers", `{"a":{},"b":[]}`, `{"a":{},"b":[]}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CanonicalizeJSON([]byte(tt.input)) + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + again, err := CanonicalizeJSON(got) + require.NoError(t, err) + assert.Equal(t, got, again, "canonicalization is idempotent") + }) + } +} + +func TestCanonicalizeJSONRejects(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"float", `{"a":1.5}`}, + {"exponent", `{"a":1e3}`}, + {"decimal zero", `{"a":1.0}`}, + {"integer overflow", `{"a":9223372036854775808}`}, + {"duplicate key", `{"a":1,"a":2}`}, + {"nested duplicate key", `{"x":{"a":1,"a":1}}`}, + {"invalid utf8", "{\"a\":\"\xff\"}"}, + {"lone surrogate escape", `{"a":"\ud800"}`}, + {"replacement character", "{\"a\":\"�\"}"}, + {"trailing data", `{"a":1} {"b":2}`}, + {"trailing garbage", `{"a":1} x`}, + {"truncated", `{"a":`}, + {"empty", ``}, + {"deep nesting", strings.Repeat("[", maxJSONDepth+2) + strings.Repeat("]", maxJSONDepth+2)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := CanonicalizeJSON([]byte(tt.input)) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) + } +} + +func TestReadBounded(t *testing.T) { + data, err := ReadBounded(strings.NewReader("12345"), 5) + require.NoError(t, err) + assert.Equal(t, "12345", string(data)) + + _, err = ReadBounded(strings.NewReader("123456"), 5) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + boom := errors.New("boom") + _, err = ReadBounded(io.MultiReader(strings.NewReader("1"), errReader{boom}), 5) + require.ErrorIs(t, err, boom) +} + +type errReader struct{ err error } + +func (e errReader) Read([]byte) (int, error) { return 0, e.err } + +func TestReadRecord(t *testing.T) { + record, raw, err := ReadRecord(strings.NewReader(vectorACanonical)) + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(raw)) + assert.Equal(t, "components/terraform/vpc", record.Name) + + _, _, err = ReadRecord(bytes.NewReader(make([]byte, MaxRecordSize+1))) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + _, _, err = ReadRecord(strings.NewReader("{}")) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) +} diff --git a/pkg/artifact/capabilities.go b/pkg/artifact/capabilities.go new file mode 100644 index 00000000000..ddd00f13639 --- /dev/null +++ b/pkg/artifact/capabilities.go @@ -0,0 +1,95 @@ +package artifact + +import ( + "slices" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// DeleteSemantics describes what deleting a reference removes. +type DeleteSemantics string + +// Delete semantics a repository can advertise. +const ( + // DeleteNone means the repository cannot delete. + DeleteNone DeleteSemantics = "none" + + // DeleteReference removes only the named reference (tag-safe). + DeleteReference DeleteSemantics = "reference" + + // DeleteManifest removes a manifest by digest, affecting every reference to it. + DeleteManifest DeleteSemantics = "manifest" + + // DeleteObject removes the stored record and payload objects. + DeleteObject DeleteSemantics = "object" +) + +// Capabilities describes what a repository can do. It is computed without network access. +type Capabilities struct { + // Read reports that Resolve, Inspect, OpenDocument, and Read are supported. + Read bool `json:"read"` + + // Write reports that Publish is supported. + Write bool `json:"write"` + + // EnumerateNamed reports that listing versions of a known name is supported. + EnumerateNamed bool `json:"enumerate_named"` + + // EnumerateCollection reports that listing every name is supported (catalog or native). + EnumerateCollection bool `json:"enumerate_collection"` + + // Delete describes delete semantics. + Delete DeleteSemantics `json:"delete"` + + // ConditionalUpdate reports conditional (compare-and-swap) reference publication. + ConditionalUpdate bool `json:"conditional_update"` + + // Packagings lists supported file payload packaging modes. + Packagings []Packaging `json:"packagings,omitempty"` + + // PreservesModes reports that ordinary permission bits survive a round trip. + PreservesModes bool `json:"preserves_modes"` + + // SubjectKinds lists the immutable signature subjects the repository can bind evidence to. + SubjectKinds []SubjectKind `json:"subject_kinds,omitempty"` + + // EvidenceFormats lists the signature evidence formats the repository can store and verify. + EvidenceFormats []EvidenceFormat `json:"evidence_formats,omitempty"` + + // LockScopes lists the mutation lock scopes the repository can provide. + LockScopes []LockScope `json:"lock_scopes,omitempty"` +} + +// HasPackaging reports whether the repository supports the packaging mode. +func (c *Capabilities) HasPackaging(p Packaging) bool { + defer perf.Track(nil, "artifact.Capabilities.HasPackaging")() + + return slices.Contains(c.Packagings, p) +} + +// HasEvidenceFormat reports whether the repository supports the evidence format. +func (c *Capabilities) HasEvidenceFormat(f EvidenceFormat) bool { + defer perf.Track(nil, "artifact.Capabilities.HasEvidenceFormat")() + + return slices.Contains(c.EvidenceFormats, f) +} + +// HasSubjectKind reports whether the repository supports the signature subject kind. +func (c *Capabilities) HasSubjectKind(k SubjectKind) bool { + defer perf.Track(nil, "artifact.Capabilities.HasSubjectKind")() + + return slices.Contains(c.SubjectKinds, k) +} + +// HasLockScope reports whether the repository can provide at least the requested lock scope. +// A repository-scope lock satisfies a host-scope request. +func (c *Capabilities) HasLockScope(s LockScope) bool { + defer perf.Track(nil, "artifact.Capabilities.HasLockScope")() + + for _, have := range c.LockScopes { + if have.Satisfies(s) { + return true + } + } + return false +} diff --git a/pkg/artifact/compat/compat_test.go b/pkg/artifact/compat/compat_test.go new file mode 100644 index 00000000000..2683546ece6 --- /dev/null +++ b/pkg/artifact/compat/compat_test.go @@ -0,0 +1,223 @@ +package compat + +import ( + "archive/tar" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/ci/artifact" + "github.com/cloudposse/atmos/pkg/ci/artifact/local" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" + "github.com/cloudposse/atmos/pkg/plugin/oci/ocitest" +) + +// Frozen compatibility constants. They were recorded from the code that existed before the +// artifact work and must not be edited to make a test pass: a failure here means a legacy CI +// planfile archive or a plugin release manifest changed identity. +const ( + // The legacy tar SHA-256 of the bundled planfile tar and the sha256 field of its sidecar. + legacyTarSHA256 = "cfb77a4fd8064a9e743ec84d6ed64c6101a5684fc1b7d1fc18a2f6789051118d" + + // The plugin release index digest. + pluginIndexDigest = "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + + // The per-platform manifest digests of the plugin release. + pluginManifestDigestLinuxAMD64 = "sha256:e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee" + pluginManifestDigestDarwinARM64 = "sha256:6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380" + + // The digest of plugin.yaml recorded in the release annotations. + pluginDefinitionDigest = "sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090" +) + +// pluginBlobDigests lists every blob in the plugin release layout (index, two manifests, two +// configs, two bundle layers), sorted. +var pluginBlobDigests = []string{ + "192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485", + "2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79", + "6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380", + "9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f", + "af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59", + "b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e", + "e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee", +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func legacyArchivePath(root string) string { + return filepath.Join(root, filepath.FromSlash(legacyKey)) +} + +func TestLegacyCIArchiveDigestIsStable(t *testing.T) { + committed, err := os.ReadFile(legacyArchivePath(legacyFixtureDir())) + require.NoError(t, err) + require.Positive(t, len(committed), "fixture must not be empty") + assert.Equal(t, legacyTarSHA256, sha256Hex(committed), "committed archive bytes changed") + + t.Run("current code reproduces the committed fixture byte for byte", func(t *testing.T) { + rebuilt := t.TempDir() + buildLegacyStore(t, rebuilt) + got := treeBytes(t, rebuilt) + want := treeBytes(t, legacyFixtureDir()) + require.Len(t, want, 2, "fixture holds the archive and its metadata sidecar") + require.Equal(t, len(want), len(got)) + for name, content := range want { + assert.Equal(t, content, got[name], "%s differs from what the current CreateTarArchive/BundledStore/local backend produce", name) + } + }) + + t.Run("sidecar records the archive checksum", func(t *testing.T) { + raw, err := os.ReadFile(legacyArchivePath(legacyFixtureDir()) + ".metadata.json") + require.NoError(t, err) + assert.Contains(t, string(raw), `"sha256": "`+legacyTarSHA256+`"`) + }) + + t.Run("legacy encoding is mode 0644 with no timestamps in file order", func(t *testing.T) { + raw, err := os.ReadFile(legacyArchivePath(legacyFixtureDir())) + require.NoError(t, err) + reader := tar.NewReader(strings.NewReader(string(raw))) + var names []string + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, header.Name) + assert.Equal(t, int64(0o644), header.Mode, header.Name) + assert.True(t, header.ModTime.IsZero() || header.ModTime.Unix() == 0, "%s has a modification time", header.Name) + } + assert.Equal(t, []string{"plan.tfplan", ".terraform.lock.hcl"}, names) + }) +} + +func TestLegacyCIFixtureStillReadsThroughCurrentStore(t *testing.T) { + copyRoot := t.TempDir() + for name, content := range treeBytes(t, legacyFixtureDir()) { + target := filepath.Join(copyRoot, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.WriteFile(target, content, 0o644)) + } + backend, err := local.NewStore(artifact.StoreOptions{Type: "local", Options: map[string]any{"path": copyRoot}}) + require.NoError(t, err) + store := artifact.NewBundledStore(backend) + + files, metadata, err := store.Download(t.Context(), legacyKey) + require.NoError(t, err) + require.Len(t, files, 2) + bodies := map[string]string{} + for _, file := range files { + body, readErr := io.ReadAll(file.Data) + require.NoError(t, readErr) + require.NoError(t, file.Data.Close()) + bodies[file.Name] = string(body) + } + assert.Equal(t, map[string]string{"plan.tfplan": legacyPlanBody, ".terraform.lock.hcl": legacyLockBody}, bodies) + assert.Equal(t, legacyMetadata().Stack, metadata.Stack) + assert.Equal(t, legacyMetadata().SHA, metadata.SHA) + assert.Equal(t, 42, metadata.PRNumber) + assert.Equal(t, legacyTarSHA256, metadata.SHA256) + assert.Equal(t, map[string]string{"env": "dev", "zone": "a"}, metadata.Custom) + require.NoError(t, metadata.Validate(), "the legacy Metadata.Validate contract is unchanged") + + t.Run("a changed archive fails the integrity check", func(t *testing.T) { + archive := legacyArchivePath(copyRoot) + content, err := os.ReadFile(archive) + require.NoError(t, err) + content[0] ^= 0xff + require.NoError(t, os.WriteFile(archive, content, 0o644)) + _, _, err = store.Download(t.Context(), legacyKey) + require.ErrorIs(t, err, errUtils.ErrArtifactIntegrityFailed) + }) +} + +func TestPluginReleaseFixtureDigestsAreStable(t *testing.T) { + root := pluginFixtureDir() + + t.Run("layout holds exactly the recorded content-addressed blobs", func(t *testing.T) { + tree := treeBytes(t, root) + var blobs []string + for name, content := range tree { + digestHex, found := strings.CutPrefix(name, "blobs/sha256/") + if !found { + continue + } + blobs = append(blobs, digestHex) + assert.Equal(t, digestHex, sha256Hex(content), "blob %s does not hash to its name", digestHex) + } + slices.Sort(blobs) + assert.Equal(t, pluginBlobDigests, blobs) + require.Contains(t, tree, "index.json") + require.Contains(t, tree, "oci-layout") + }) + + t.Run("release inspects to the recorded manifest digests", func(t *testing.T) { + index, err := atmosoci.LoadIndex(root, pluginIndexDigest) + require.NoError(t, err) + release, err := pluginoci.Inspect(index, pluginIndexDigest) + require.NoError(t, err) + assert.Equal(t, pluginIndexDigest, release.Digest) + assert.Equal(t, "example/fixture", release.Name) + assert.Equal(t, "1.2.3", release.Version) + require.Len(t, release.Platforms, 2) + assert.Equal(t, pluginManifestDigestLinuxAMD64, release.Platforms["linux/amd64"].Digest.String()) + assert.Equal(t, pluginManifestDigestDarwinARM64, release.Platforms["darwin/arm64"].Digest.String()) + assert.Equal(t, pluginDefinitionDigest, release.Platforms["linux/amd64"].Annotations["dev.atmos.plugin.manifest.digest"]) + }) + + t.Run("platform payloads extract to the original executables", func(t *testing.T) { + index, err := atmosoci.LoadIndex(root, pluginIndexDigest) + require.NoError(t, err) + release, err := pluginoci.Inspect(index, pluginIndexDigest) + require.NoError(t, err) + for platform, want := range map[string][]byte{ + "linux/amd64": ocitest.Binary("linux", "amd64"), + "darwin/arm64": ocitest.Binary("darwin", "arm64"), + } { + destination := t.TempDir() + require.NoError(t, release.Extract(platform, destination), platform) + got, err := os.ReadFile(filepath.Join(destination, "bin", "plugin")) + require.NoError(t, err) + assert.Equal(t, want, got, platform) + } + }) +} + +func TestPluginReleaseRebuildMatchesFixture(t *testing.T) { + rebuilt := filepath.Join(t.TempDir(), "layout") + packPlugin(t, "1.2.3", rebuilt) + + got := treeBytes(t, rebuilt) + want := treeBytes(t, pluginFixtureDir()) + require.Positive(t, len(want)) + require.Equal(t, len(want), len(got), "layout file set changed") + for name, content := range want { + assert.Equal(t, content, got[name], "%s differs from what the current plugin packer produces", name) + } +} + +// TestPluginDigestChangeIsDetected proves the constants above would fail if the packer's output +// changed: a one-field change in the packed release moves the index digest. +func TestPluginDigestChangeIsDetected(t *testing.T) { + changed := filepath.Join(t.TempDir(), "layout") + packPlugin(t, "1.2.4", changed) + + index, err := atmosoci.LoadIndex(changed, "") + require.NoError(t, err) + digest, err := index.Digest() + require.NoError(t, err) + assert.NotEqual(t, pluginIndexDigest, digest.String()) +} diff --git a/pkg/artifact/compat/doc.go b/pkg/artifact/compat/doc.go new file mode 100644 index 00000000000..d6fe0290b1a --- /dev/null +++ b/pkg/artifact/compat/doc.go @@ -0,0 +1,8 @@ +// Package compat holds test-only compatibility fixtures that freeze the byte-level behavior of the +// existing CI planfile archive format and the existing plugin OCI release format before the +// artifact implementation replaces any of their code paths. +// +// It lives apart from pkg/artifact so the core domain package never imports pkg/ci or pkg/plugin. +// Fixture bytes under testdata are committed and must never be regenerated to hide an +// incompatible change; see compat_test.go for the generation rules. +package compat diff --git a/pkg/artifact/compat/fixtures_test.go b/pkg/artifact/compat/fixtures_test.go new file mode 100644 index 00000000000..e731a19f6e8 --- /dev/null +++ b/pkg/artifact/compat/fixtures_test.go @@ -0,0 +1,152 @@ +package compat + +import ( + "flag" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ci/artifact" + "github.com/cloudposse/atmos/pkg/ci/artifact/local" + "github.com/cloudposse/atmos/pkg/plugin/extension" + pluginoci "github.com/cloudposse/atmos/pkg/plugin/oci" + "github.com/cloudposse/atmos/pkg/plugin/oci/ocitest" +) + +// generateFixtures writes missing fixtures under testdata. It never overwrites an existing +// fixture: to refresh one on purpose, a human deletes it and reviews the resulting digest change, +// because a changed legacy archive or plugin manifest digest is an incompatibility, not a refresh. +var generateFixtures = flag.Bool("generate-fixtures", false, "write missing fixtures to testdata (never overwrites)") + +const ( + // The legacy planfile key follows the default key pattern {{ .Stack }}/{{ .Component }}/{{ .SHA }}.tfplan.tar. + legacyKey = "plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar" + + legacyPlanBody = "legacy-planfile-bytes\n" + legacyLockBody = "# legacy lock file\nprovider \"registry.terraform.io/hashicorp/aws\" {\n version = \"5.0.0\"\n}\n" +) + +// legacyFixtureDir is the committed pre-change CI planfile store (local/dir backend layout). +func legacyFixtureDir() string { return filepath.Join("testdata", "legacy-ci") } + +// pluginFixtureDir is the committed plugin release OCI layout. +func pluginFixtureDir() string { return filepath.Join("testdata", "plugin-release") } + +// legacyMetadata is fully fixed so the archive and sidecar are byte-deterministic. +func legacyMetadata() *artifact.Metadata { + return &artifact.Metadata{ + Stack: "plat-ue2-dev", + Component: "vpc", + SHA: "0123456789abcdef0123456789abcdef01234567", + BaseSHA: "89abcdef0123456789abcdef0123456789abcdef", + Branch: "feature/fixture", + PRNumber: 42, + RunID: "run-1", + Repository: "acme/infra", + CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + AtmosVersion: "0.0.0-fixture", + Custom: map[string]string{"zone": "a", "env": "dev"}, + } +} + +// buildLegacyStore writes the legacy planfile through the CURRENT BundledStore and local backend +// into root and returns the relative slash paths of every file produced. +func buildLegacyStore(t *testing.T, root string) { + t.Helper() + backend, err := local.NewStore(artifact.StoreOptions{Type: "local", Options: map[string]any{"path": root}}) + require.NoError(t, err) + store := artifact.NewBundledStore(backend) + files := []artifact.FileEntry{ + {Name: "plan.tfplan", Data: bytesReader(legacyPlanBody), Size: -1}, + {Name: ".terraform.lock.hcl", Data: bytesReader(legacyLockBody), Size: -1}, + } + require.NoError(t, store.Upload(t.Context(), legacyKey, files, legacyMetadata())) +} + +// pluginPackageDir assembles the plugin source tree from fixed inputs. Executables are valid +// but never-executed native headers, so the build needs no toolchain. +func pluginPackageDir(t *testing.T, version string) (string, *extension.Registry) { + t.Helper() + directory := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(directory, "bin"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "bin", "linux"), ocitest.Binary("linux", "amd64"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "bin", "mac"), ocitest.Binary("darwin", "arm64"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "plugin.yaml"), []byte(pluginManifest(version)), 0o644)) + descriptors := extension.New() + require.NoError(t, descriptors.Register(extension.Descriptor{ + Capability: "fixture", Path: []string{"fixtures"}, Version: 1, + Validate: func(extension.Binding) error { return nil }, + })) + return directory, descriptors +} + +func pluginManifest(version string) string { + return `apiVersion: atmos/v1 +kind: AtmosPluginConfig +metadata: + name: example/fixture + version: ` + version + ` +spec: + runtime: + type: native + protocol: 1 + executable: bin/plugin + compatibility: + protocol: ">=1 <2" + platforms: + linux/amd64: + executable: bin/linux + darwin/arm64: + executable: bin/mac + capabilities: + fixtures: + example/fixture: {} +` +} + +// packPlugin builds the plugin OCI layout at output (which must not exist) with the CURRENT packer. +func packPlugin(t *testing.T, version, output string) { + t.Helper() + directory, descriptors := pluginPackageDir(t, version) + require.NoError(t, pluginoci.Pack(directory, output, descriptors)) +} + +// treeBytes returns every regular file below root keyed by slash-separated relative path. +func treeBytes(t *testing.T, root string) map[string][]byte { + t.Helper() + tree := make(map[string][]byte) + require.NoError(t, filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + tree[filepath.ToSlash(relative)] = content + return nil + })) + return tree +} + +// TestGenerateFixtures is skipped unless -generate-fixtures is passed. It only fills in missing +// fixtures and fails if one already exists, so a regeneration can never mask a change. +func TestGenerateFixtures(t *testing.T) { + if !*generateFixtures { + t.Skipf("pass -generate-fixtures to write missing fixtures") + } + for _, target := range []string{legacyFixtureDir(), pluginFixtureDir()} { + _, err := os.Lstat(target) + require.ErrorIs(t, err, fs.ErrNotExist, "%s already exists; fixtures are frozen. Delete it deliberately and review the digest change", target) + } + buildLegacyStore(t, legacyFixtureDir()) + packPlugin(t, "1.2.3", pluginFixtureDir()) +} diff --git a/pkg/artifact/compat/helpers_test.go b/pkg/artifact/compat/helpers_test.go new file mode 100644 index 00000000000..317f713022a --- /dev/null +++ b/pkg/artifact/compat/helpers_test.go @@ -0,0 +1,8 @@ +package compat + +import ( + "bytes" + "io" +) + +func bytesReader(s string) io.Reader { return bytes.NewReader([]byte(s)) } diff --git a/pkg/artifact/compat/testdata/.gitattributes b/pkg/artifact/compat/testdata/.gitattributes new file mode 100644 index 00000000000..425ff5530e0 --- /dev/null +++ b/pkg/artifact/compat/testdata/.gitattributes @@ -0,0 +1,2 @@ +# Frozen compatibility fixtures are byte-exact; never normalize line endings or treat them as text. +* -text diff --git a/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json new file mode 100644 index 00000000000..a10a8071294 --- /dev/null +++ b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar.metadata.json @@ -0,0 +1,17 @@ +{ + "stack": "plat-ue2-dev", + "component": "vpc", + "sha": "0123456789abcdef0123456789abcdef01234567", + "base_sha": "89abcdef0123456789abcdef0123456789abcdef", + "branch": "feature/fixture", + "pr_number": 42, + "run_id": "run-1", + "repository": "acme/infra", + "created_at": "2026-01-02T03:04:05Z", + "sha256": "cfb77a4fd8064a9e743ec84d6ed64c6101a5684fc1b7d1fc18a2f6789051118d", + "atmos_version": "0.0.0-fixture", + "custom": { + "env": "dev", + "zone": "a" + } +} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485 new file mode 100644 index 0000000000000000000000000000000000000000..cc666b04eeb6a9b321ec53aca50ccb7e22296f1a GIT binary patch literal 3072 zcmeHIOHRWu5DhFN4q#HLc( zO=e@y_|3fWvqsLjij1gNTm^$!jJBA~0>7diuW_92W8FEWbddxk9;~B-!gb}qVoL|1 zleX>E4lZ|6k@F{get-Sk1i_VOzCJ(w$arA|y4jy>K#IXS@uM+M)8o0{`SEO??Dq%_ zVoH;7{s()GUSTl&h1rU0l@V}KSJ9e=uUwTGdGzjc`!_skwdAW%G6y9%$U;ICNR|=S zfHZi$OvtH*07>> zkPC$6`yUFA3M~z$&L>5Bd3-agz@ayPw2oDtxb+RCcK2(IcT0$I+}E3TIoQtqxnyv-I@I$oogw(|>eE1x5u%1umt)FUV7$jQ{`u literal 0 HcmV?d00001 diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 new file mode 100644 index 00000000000..cb7ec038172 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79 @@ -0,0 +1 @@ +{"architecture":"arm64","created":"0001-01-01T00:00:00Z","history":[{"created":"0001-01-01T00:00:00Z"}],"os":"darwin","rootfs":{"type":"layers","diff_ids":["sha256:192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485"]},"config":{"Labels":{"dev.atmos.plugin.manifest":"apiVersion: atmos/v1\nkind: AtmosPluginConfig\nmetadata:\n name: example/fixture\n version: 1.2.3\nspec:\n runtime:\n type: native\n protocol: 1\n executable: bin/plugin\n compatibility:\n protocol: \"\u003e=1 \u003c2\"\n platforms:\n linux/amd64:\n executable: bin/linux\n darwin/arm64:\n executable: bin/mac\n capabilities:\n fixtures:\n example/fixture: {}\n"}}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 new file mode 100644 index 00000000000..2774d4bd44e --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.atmos.plugin.config.v1+json","size":688,"digest":"sha256:2e33831c0fd207c7001a3ad37e4055f47d3cddc851002a5ecee4d8c503156d79"},"layers":[{"mediaType":"application/vnd.atmos.plugin.bundle.v1+tar","size":3072,"digest":"sha256:192cdb60e7ddc392b8048fa5b026bd033ac034a0c79d60b0233b30b25cd3c485"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f new file mode 100644 index 00000000000..812e6f2e133 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f @@ -0,0 +1 @@ +{"architecture":"amd64","created":"0001-01-01T00:00:00Z","history":[{"created":"0001-01-01T00:00:00Z"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e"]},"config":{"Labels":{"dev.atmos.plugin.manifest":"apiVersion: atmos/v1\nkind: AtmosPluginConfig\nmetadata:\n name: example/fixture\n version: 1.2.3\nspec:\n runtime:\n type: native\n protocol: 1\n executable: bin/plugin\n compatibility:\n protocol: \"\u003e=1 \u003c2\"\n platforms:\n linux/amd64:\n executable: bin/linux\n darwin/arm64:\n executable: bin/mac\n capabilities:\n fixtures:\n example/fixture: {}\n"}}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 new file mode 100755 index 00000000000..bd6d4cd17b7 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","size":606,"digest":"sha256:6a988f06a391ca23572eb78363f6241d40127d05b7bc71facdebf0a8855b2380","annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"},"platform":{"architecture":"arm64","os":"darwin"},"artifactType":"application/vnd.atmos.plugin.config.v1+json"},{"mediaType":"application/vnd.oci.image.manifest.v1+json","size":606,"digest":"sha256:e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee","annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"},"platform":{"architecture":"amd64","os":"linux"},"artifactType":"application/vnd.atmos.plugin.config.v1+json"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e new file mode 100644 index 0000000000000000000000000000000000000000..14dd56846e19b46fb4d87d40d5f53c3474199b9f GIT binary patch literal 3072 zcmeHI$!fze5Os3&SBQK-Y{gl?w4tT6ryhFheQYa1R2#CKI+W61uVg1)0=)!GC|DQG z8@*Zd#Gr(VK(S&^g*Vs=?b@tXp4;uZ+(X>@>T`_oCiGA^SVx`2MQ%XbmO8Gh%<5|G zMXt(mgP@=1*O%#J(o~*$TdzfpyJtM~&i6leMw4?L9_!i<#vq24*KU&ub2&q9_C4*?L(!knOx* zqM}sAGti&5CTRXQn?vfl0R>enl$;8kbT}bbrKeAfzP|mA`lC52Fe)%Aa61Kl0cBjB Ap8x;= literal 0 HcmV?d00001 diff --git a/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee new file mode 100644 index 00000000000..623c41d20c8 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/blobs/sha256/e249a05906c791f6881749476b7123acbfd0a7f37dfa94a397045e22b5c819ee @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.atmos.plugin.config.v1+json","size":687,"digest":"sha256:9ecd9e0a855de88e01dc9e76dc7c17213a7c356b9778e19cfe0c52585b2d823f"},"layers":[{"mediaType":"application/vnd.atmos.plugin.bundle.v1+tar","size":3072,"digest":"sha256:b0c7d7c2a4ded93805cb35f506478b33d99b8a1dc65de9fc6683e4f26d51af7e"}],"annotations":{"dev.atmos.plugin.manifest.digest":"sha256:ca012d44c6a9ea139972472614caa656eb40d86e298ddcde501c216eb953b090","dev.atmos.plugin.name":"example/fixture","dev.atmos.plugin.version":"1.2.3"}} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/index.json b/pkg/artifact/compat/testdata/plugin-release/index.json new file mode 100755 index 00000000000..837db573436 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/index.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": 1220, + "digest": "sha256:af34a37555cc97c7f1043fdb2ad3404a062d97d1ce07d96c5a594f6b5ac83b59" + } + ] +} \ No newline at end of file diff --git a/pkg/artifact/compat/testdata/plugin-release/oci-layout b/pkg/artifact/compat/testdata/plugin-release/oci-layout new file mode 100755 index 00000000000..224a8698125 --- /dev/null +++ b/pkg/artifact/compat/testdata/plugin-release/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} \ No newline at end of file diff --git a/pkg/artifact/constants.go b/pkg/artifact/constants.go new file mode 100644 index 00000000000..88f1be98568 --- /dev/null +++ b/pkg/artifact/constants.go @@ -0,0 +1,77 @@ +package artifact + +// Serialization and bounded-read limits shared by every repository adapter. +const ( + // SchemaVersion is the only Record schema version defined by V1. + SchemaVersion = 1 + + // MaxRecordSize is the largest canonical Record (and catalog JSON) read or written, in bytes. + MaxRecordSize int64 = 8 << 20 + + // MaxDocumentSize is the largest README or CHANGELOG document, in bytes. + MaxDocumentSize int64 = 4 << 20 + + // MaxEvidenceSize is the largest single signature evidence blob read into memory, in bytes. + MaxEvidenceSize int64 = 1 << 20 + + // MaxNameLength is the longest artifact name, matching the OCI repository name limit. + MaxNameLength = 255 + + // MaxVersionLength is the longest artifact version, matching the OCI tag limit. + MaxVersionLength = 128 + + // MaxKindLength is the longest artifact kind. + MaxKindLength = 128 + + // DefaultConcurrency is the default transfer and discovery concurrency. + DefaultConcurrency = 4 + + // ReservedNamespace is the managed-record directory below a local or S3 root. Payload + // names and key patterns must never enter it. + ReservedNamespace = ".atmos-artifacts" + + // DocumentMediaType is the default media type of README and CHANGELOG documents. + DocumentMediaType = "text/markdown" +) + +// OCI media types from the artifact PRD. Existing plugin and foreign OCI graphs keep their own types. +const ( + // MediaTypeArtifact is the generic artifact type. + MediaTypeArtifact = "application/vnd.atmos.artifact.v1" + + // MediaTypeScaffold is the scaffold artifact type. + MediaTypeScaffold = "application/vnd.atmos.scaffold.v1" + + // MediaTypeComponent is the component artifact type. + MediaTypeComponent = "application/vnd.atmos.component.v1" + + // MediaTypeToolchainPackage is the toolchain package artifact type. + MediaTypeToolchainPackage = "application/vnd.atmos.toolchain.package.v1" + + // MediaTypeCatalog is the artifact type of a catalog Record. + MediaTypeCatalog = "application/vnd.atmos.artifact.catalog.v1" + + // MediaTypeCatalogPayload is the media type of a catalog's direct JSON payload layer. + MediaTypeCatalogPayload = "application/vnd.atmos.artifact.catalog.v1+json" + + // MediaTypeConfig is the media type of the OCI config blob holding the canonical Record bytes. + MediaTypeConfig = "application/vnd.atmos.artifact.config.v1+json" + + // MediaTypeTar is the media type of an uncompressed tar payload. + MediaTypeTar = "application/vnd.atmos.artifact.content.v1.tar" + + // MediaTypeTarGzip is the media type of a gzip-compressed tar payload. + MediaTypeTarGzip = "application/vnd.atmos.artifact.content.v1.tar+gzip" + + // MediaTypeZip is the media type of a zip payload. + MediaTypeZip = "application/vnd.atmos.artifact.content.v1.zip" +) + +// Well-known artifact kinds. +const ( + // KindFiles is the default kind for generic file packages. + KindFiles = "files" + + // KindCatalog is the kind of an artifact catalog Record. + KindCatalog = "catalog" +) diff --git a/pkg/artifact/contracts_test.go b/pkg/artifact/contracts_test.go new file mode 100644 index 00000000000..6d5006f1fce --- /dev/null +++ b/pkg/artifact/contracts_test.go @@ -0,0 +1,151 @@ +package artifact + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Compile-time proof that every generated mock satisfies its interface. +var ( + _ Repository = (*MockRepository)(nil) + _ LockProvider = (*MockLockProvider)(nil) + _ LockProvider = Repository(nil) + _ LockHandle = (*MockLockHandle)(nil) + _ Producer = (*MockProducer)(nil) + _ Signer = (*MockSigner)(nil) + _ Verifier = (*MockVerifier)(nil) + _ Copier = (*MockCopier)(nil) + _ Service = (*MockService)(nil) + _ GraphSource = (*MockGraphSource)(nil) +) + +func TestLockScopeSatisfies(t *testing.T) { + tests := []struct { + have, need LockScope + want bool + }{ + {LockScopeHost, LockScopeHost, true}, + {LockScopeHost, "", true}, + {LockScopeRepository, LockScopeHost, true}, + {LockScopeRepository, LockScopeRepository, true}, + {LockScopeHost, LockScopeRepository, false}, + {"", LockScopeHost, false}, + {LockScopeRepository, "cluster", false}, + } + for _, tt := range tests { + t.Run(string(tt.have)+"->"+string(tt.need), func(t *testing.T) { + assert.Equal(t, tt.want, tt.have.Satisfies(tt.need)) + }) + } +} + +func TestCapabilitiesLookups(t *testing.T) { + capabilities := Capabilities{ + Packagings: []Packaging{PackagingTgz, PackagingZip}, + EvidenceFormats: []EvidenceFormat{EvidenceSigstoreBundle}, + SubjectKinds: []SubjectKind{SubjectRecord}, + LockScopes: []LockScope{LockScopeHost}, + } + assert.True(t, capabilities.HasPackaging(PackagingTgz)) + assert.False(t, capabilities.HasPackaging(PackagingNone)) + assert.True(t, capabilities.HasEvidenceFormat(EvidenceSigstoreBundle)) + assert.False(t, capabilities.HasEvidenceFormat(EvidencePublicKeyBundle), "a missing format is not silently supported") + assert.True(t, capabilities.HasSubjectKind(SubjectRecord)) + assert.False(t, capabilities.HasSubjectKind(SubjectOCIRoot)) + assert.True(t, capabilities.HasLockScope(LockScopeHost)) + assert.False(t, capabilities.HasLockScope(LockScopeRepository)) + + capabilities.LockScopes = []LockScope{LockScopeRepository} + assert.True(t, capabilities.HasLockScope(LockScopeHost), "repository scope satisfies a host request") +} + +func TestPreparedArtifactCloseRunsCleanupOnce(t *testing.T) { + calls := 0 + boom := errors.New("cleanup failed") + prepared := &PreparedArtifact{Cleanup: func() error { calls++; return boom }} + require.ErrorIs(t, prepared.Close(), boom) + require.ErrorIs(t, prepared.Close(), boom, "later calls return the first result") + assert.Equal(t, 1, calls) + + assert.NoError(t, (&PreparedArtifact{}).Close(), "nil cleanup is fine") +} + +type trackingCloser struct { + io.Reader + closed *int + err error +} + +func (c trackingCloser) Close() error { *c.closed++; return c.err } + +func TestContentCloseClosesEveryStream(t *testing.T) { + closed := 0 + first := errors.New("first") + content := &Content{Streams: []ContentStream{ + {Reader: trackingCloser{Reader: strings.NewReader("a"), closed: &closed, err: first}}, + {Reader: nil}, + {Reader: trackingCloser{Reader: strings.NewReader("b"), closed: &closed, err: errors.New("second")}}, + }} + require.ErrorIs(t, content.Close(), first) + assert.Equal(t, 2, closed, "an early failure does not leak later streams") +} + +func TestSignatureSubjectValidate(t *testing.T) { + good := SignatureSubject{Kind: SubjectRecord, Digest: digest.FromString("r"), MediaType: MediaTypeConfig, Size: 10} + require.NoError(t, good.Validate()) + + bad := good + bad.Kind = "archive" + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) + bad = good + bad.Digest = "sha256:short" + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) + bad = good + bad.Size = -1 + require.ErrorIs(t, bad.Validate(), errUtils.ErrArtifactRefInvalid) +} + +// TestRepositoryContractThroughMocks drives the interfaces the way the service will: ask a +// repository for a guarantee it lacks and observe a typed error, not a no-op. +func TestRepositoryContractThroughMocks(t *testing.T) { + ctrl := gomock.NewController(t) + repository := NewMockRepository(ctrl) + + repository.EXPECT().Capabilities().Return(Capabilities{LockScopes: []LockScope{LockScopeHost}}) + repository.EXPECT(). + Lock(gomock.Any(), LockRequest{Resource: "vpc:1.0", Owner: "host-1", Required: LockScopeRepository}). + Return(nil, NewUnsupportedCapability("shared", "lock", "lock/repository", "configure locking.repository")) + + request := LockRequest{Resource: "vpc:1.0", Owner: "host-1", Required: LockScopeRepository} + var repo Repository = repository + capabilities := repo.Capabilities() + require.False(t, capabilities.HasLockScope(request.Required)) + + handle, err := repo.Lock(context.Background(), request) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.Nil(t, handle) +} + +func TestOptionalCopierIsDetectedByTypeAssertion(t *testing.T) { + ctrl := gomock.NewController(t) + var plain Repository = NewMockRepository(ctrl) + _, ok := plain.(Copier) + assert.False(t, ok, "a repository without native copy falls back to staged copy") + + var native Repository = struct { + *MockRepository + *MockCopier + }{NewMockRepository(ctrl), NewMockCopier(ctrl)} + _, ok = native.(Copier) + assert.True(t, ok) +} diff --git a/pkg/artifact/copier.go b/pkg/artifact/copier.go new file mode 100644 index 00000000000..fc4a537983f --- /dev/null +++ b/pkg/artifact/copier.go @@ -0,0 +1,29 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_copier.go github.com/cloudposse/atmos/pkg/artifact Copier + +// CopyFromOptions control a native transfer. +type CopyFromOptions struct { + // ExpectedPrior is the destination reference observed under the writer guard; nil means none. + ExpectedPrior *PinnedReference + + // Force permits replacing a conflicting destination. It never bypasses verification. + Force bool + + // Lock is the destination lock already held by the service. + Lock LockHandle +} + +// Copier is an optional Repository extension for a compatible native transfer (for example an OCI +// registry-to-registry graph copy that preserves every original descriptor). A repository that does +// not implement it, or whose CanCopyFrom returns false, uses the shared staging fallback. +type Copier interface { + // CanCopyFrom reports whether source is compatible with a native transfer into this repository. + CanCopyFrom(source Repository) bool + + // CopyFrom transfers the pinned artifact from source into this repository, preserving Record + // bytes and content identity. + CopyFrom(ctx context.Context, source Repository, pinned PinnedReference, options CopyFromOptions) (*PublishResult, error) +} diff --git a/pkg/artifact/digest.go b/pkg/artifact/digest.go new file mode 100644 index 00000000000..9d7f1f2f9c3 --- /dev/null +++ b/pkg/artifact/digest.go @@ -0,0 +1,81 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// contentView is the exact document hashed for content_digest. +type contentView struct { + Documents []Document `json:"documents"` + Inventory []Entry `json:"inventory"` + SchemaVersion int `json:"schema_version"` +} + +// userMetadata is the part of Metadata that participates in intent identity. +type userMetadata struct { + Description string `json:"description,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// intentView is the exact document hashed for intent_digest. +type intentView struct { + ContentDigest digest.Digest `json:"content_digest"` + Kind string `json:"kind"` + KindData KindData `json:"kind_data,omitempty"` + Metadata userMetadata `json:"metadata"` + Name string `json:"name"` + SchemaVersion int `json:"schema_version"` + Version string `json:"version"` +} + +// ComputeContentDigest returns the content_digest: the SHA-256 of the canonical JSON object +// {"documents":[...],"inventory":[...],"schema_version":1} built from the sorted inventory and +// document descriptors, including paths, modes, and link targets. It excludes storage locators +// and generated operation data. Inputs are validated first. +func ComputeContentDigest(inventory []Entry, documents []Document) (digest.Digest, error) { + defer perf.Track(nil, "artifact.ComputeContentDigest")() + + if err := validateInventory(inventory); err != nil { + return "", err + } + if err := validateDocuments(documents); err != nil { + return "", err + } + canonical, err := canonicalJSON(contentView{ + Documents: sortedDocuments(documents), + Inventory: sortedInventory(inventory), + SchemaVersion: SchemaVersion, + }) + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} + +// IntentDigest returns the intent_digest: the SHA-256 of the canonical JSON object holding +// name, version, kind, content_digest, the effective user metadata (description and labels), and +// the kind data. It excludes created_at and generated provenance so that a retry of the same +// intent yields the same digest and a no-op can be detected. +func (r *Record) IntentDigest() (digest.Digest, error) { + defer perf.Track(nil, "artifact.Record.IntentDigest")() + + if err := r.Validate(); err != nil { + return "", err + } + normalized := r.normalized() + canonical, err := canonicalJSON(intentView{ + ContentDigest: normalized.ContentDigest, + Kind: normalized.Kind, + KindData: normalized.KindData, + Metadata: userMetadata{Description: normalized.Metadata.Description, Labels: normalized.Metadata.Labels}, + Name: normalized.Name, + SchemaVersion: normalized.SchemaVersion, + Version: normalized.Version, + }) + if err != nil { + return "", err + } + return digest.FromBytes(canonical), nil +} diff --git a/pkg/artifact/doc.go b/pkg/artifact/doc.go new file mode 100644 index 00000000000..4e6e7839b69 --- /dev/null +++ b/pkg/artifact/doc.go @@ -0,0 +1,19 @@ +// Package artifact defines the generic, repository-independent domain contracts for Atmos +// artifacts: references, portable records, canonical serialization and digests, the repository, +// producer, signer, verifier, and lock-provider interfaces, and the typed service requests and +// results. +// +// The package deliberately imports no CLI, runner, hook, plugin, or CI packages. Backends and +// producers register through composition roots and implement the interfaces defined here. +// Backend SDK types (AWS clients, go-containerregistry images, ORAS remote repositories) never +// appear in this API; OCI seams use opencontainers image-spec descriptors and go-digest values. +// +// # Canonical records +// +// A [Record] serializes to one canonical byte sequence (see [Record.Canonical]): typed JSON with +// a fixed field order, sorted inventory and document arrays, sorted map keys, UTF-8 strings, +// integer sizes and modes, UTC RFC3339 timestamps at second precision, no insignificant +// whitespace, no HTML escaping, and no trailing newline. A Record never contains its own hash. +// The content, intent, and record digests are SHA-256 digests over canonical byte sequences +// defined in this package and frozen by test vectors. +package artifact diff --git a/pkg/artifact/entry.go b/pkg/artifact/entry.go new file mode 100644 index 00000000000..dff6d0ff251 --- /dev/null +++ b/pkg/artifact/entry.go @@ -0,0 +1,205 @@ +package artifact + +import ( + "fmt" + "path" + "sort" + "strings" + "unicode/utf8" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// maxPathLength bounds one inventory path, in bytes. +const maxPathLength = 4096 + +// firstPrintableRune is the first rune that is not an ASCII control character. +const firstPrintableRune = 0x20 + +// deleteRune is the ASCII DEL control character. +const deleteRune = 0x7f + +// permissionMask holds the ordinary permission bits an Entry may carry. +const permissionMask uint32 = 0o777 + +// ValidatePayloadPath checks a relative, slash-separated path inside a package. It rejects empty, +// absolute, backslash, control-character, "." / ".." segments, invalid UTF-8, and any path that +// enters the managed [ReservedNamespace]. Paths are never rewritten. +func ValidatePayloadPath(p string) error { + defer perf.Track(nil, "artifact.ValidatePayloadPath")() + + if err := validateRelativePath(p, false); err != nil { + return err + } + if first, _, _ := strings.Cut(p, "/"); first == ReservedNamespace { + return fmt.Errorf("%w: %q enters the reserved %s namespace", errUtils.ErrArtifactUnsafePath, p, ReservedNamespace) + } + return nil +} + +// validateRelativePath checks a relative slash-separated path. Link targets pass allowDots because +// "." and ".." segments are legitimate there as long as the resolved target stays inside the package. +func validateRelativePath(p string, allowDots bool) error { + if err := validatePathShape(p); err != nil { + return err + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || (!allowDots && (segment == "." || segment == "..")) { + return fmt.Errorf("%w: path %q has an empty, '.', or '..' segment", errUtils.ErrArtifactUnsafePath, p) + } + } + return nil +} + +// validatePathShape checks length, encoding, rootedness, separators, and control characters. +func validatePathShape(p string) error { + if p == "" || len(p) > maxPathLength { + return fmt.Errorf("%w: path must be 1-%d bytes", errUtils.ErrArtifactUnsafePath, maxPathLength) + } + if !utf8.ValidString(p) { + return fmt.Errorf("%w: path %q is not valid UTF-8", errUtils.ErrArtifactUnsafePath, p) + } + if strings.HasPrefix(p, "/") || strings.Contains(p, `\`) || hasControlRune(p) { + return fmt.Errorf("%w: path %q must be relative, slash-separated, and free of control characters", errUtils.ErrArtifactUnsafePath, p) + } + return nil +} + +func hasControlRune(p string) bool { + return strings.IndexFunc(p, func(r rune) bool { return r < firstPrintableRune || r == deleteRune }) >= 0 +} + +// Validate checks one inventory entry in isolation. +func (e *Entry) Validate() error { + defer perf.Track(nil, "artifact.Entry.Validate")() + + if err := ValidatePayloadPath(e.Path); err != nil { + return err + } + if e.Mode&^permissionMask != 0 { + return fmt.Errorf("%w: entry %q mode %#o has bits outside 0777; privileged bits are cleared", errUtils.ErrArtifactRecordInvalid, e.Path, e.Mode) + } + switch e.Type { + case EntryFile: + return e.validateFile() + case EntryDirectory: + return e.validateDirectory() + case EntrySymlink: + return e.validateSymlink() + default: + return fmt.Errorf("%w: entry %q has unsupported type %q", errUtils.ErrArtifactPackagingInvalid, e.Path, e.Type) + } +} + +func (e *Entry) validateFile() error { + if e.Size < 0 || e.Target != "" { + return fmt.Errorf("%w: file %q needs a non-negative size and no link target", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if e.Digest == "" { + return fmt.Errorf("%w: file %q needs a digest", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if err := ValidateDigest(e.Digest); err != nil { + return fmt.Errorf("%w: file %q: %w", errUtils.ErrArtifactRecordInvalid, e.Path, err) + } + return nil +} + +func (e *Entry) validateDirectory() error { + if e.Size != 0 || e.Digest != "" || e.Target != "" { + return fmt.Errorf("%w: directory %q must have no size, digest, or target", errUtils.ErrArtifactRecordInvalid, e.Path) + } + return nil +} + +func (e *Entry) validateSymlink() error { + if e.Size != 0 || e.Mode != 0 || e.Digest != "" { + return fmt.Errorf("%w: symlink %q must have no size, mode, or digest", errUtils.ErrArtifactRecordInvalid, e.Path) + } + if err := validateRelativePath(e.Target, true); err != nil { + return fmt.Errorf("%w: symlink %q target: %w", errUtils.ErrArtifactUnsafePath, e.Path, err) + } + resolved := path.Clean(path.Dir(e.Path) + "/" + e.Target) + if resolved == ".." || strings.HasPrefix(resolved, "../") { + return fmt.Errorf("%w: symlink %q target %q escapes the package", errUtils.ErrArtifactUnsafePath, e.Path, e.Target) + } + return nil +} + +// validateInventory checks every entry, rejects duplicate paths, and rejects entries nested under +// a file or symlink entry. +func validateInventory(inventory []Entry) error { + types := make(map[string]EntryType, len(inventory)) + for _, entry := range inventory { + if err := entry.Validate(); err != nil { + return err + } + if _, duplicate := types[entry.Path]; duplicate { + return fmt.Errorf("%w: duplicate inventory path %q", errUtils.ErrArtifactPackagingInvalid, entry.Path) + } + types[entry.Path] = entry.Type + } + for _, entry := range inventory { + for parent := path.Dir(entry.Path); parent != "."; parent = path.Dir(parent) { + if parentType, found := types[parent]; found && parentType != EntryDirectory { + return fmt.Errorf("%w: %q is nested under non-directory %q", errUtils.ErrArtifactPackagingInvalid, entry.Path, parent) + } + } + } + return nil +} + +// Validate checks one document descriptor in isolation. +func (d Document) Validate() error { + defer perf.Track(nil, "artifact.Document.Validate")() + + if d.Role != DocumentReadme && d.Role != DocumentChangelog { + return fmt.Errorf("%w: unsupported document role %q", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if err := validateRelativePath(d.SourcePath, false); err != nil { + return fmt.Errorf("%w: %s source path: %w", errUtils.ErrArtifactRecordInvalid, d.Role, err) + } + if d.MediaType == "" { + return fmt.Errorf("%w: %s needs a media type", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if d.Size < 0 { + return fmt.Errorf("%w: %s has a negative size", errUtils.ErrArtifactRecordInvalid, d.Role) + } + if d.Size > MaxDocumentSize { + return fmt.Errorf("%w: %s is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, d.Role, d.Size, MaxDocumentSize) + } + if err := ValidateDigest(d.Digest); err != nil { + return fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactRecordInvalid, d.Role, err) + } + return nil +} + +func validateDocuments(documents []Document) error { + seen := make(map[DocumentRole]struct{}, len(documents)) + for _, document := range documents { + if err := document.Validate(); err != nil { + return err + } + if _, duplicate := seen[document.Role]; duplicate { + return fmt.Errorf("%w: duplicate %s document", errUtils.ErrArtifactRecordInvalid, document.Role) + } + seen[document.Role] = struct{}{} + } + return nil +} + +// sortedInventory returns a copy of inventory sorted by path (bytewise). +func sortedInventory(inventory []Entry) []Entry { + out := make([]Entry, len(inventory)) + copy(out, inventory) + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// sortedDocuments returns a copy of documents sorted by role. +func sortedDocuments(documents []Document) []Document { + out := make([]Document, len(documents)) + copy(out, documents) + sort.Slice(out, func(i, j int) bool { return out[i].Role < out[j].Role }) + return out +} diff --git a/pkg/artifact/errors.go b/pkg/artifact/errors.go new file mode 100644 index 00000000000..0f5c702f371 --- /dev/null +++ b/pkg/artifact/errors.go @@ -0,0 +1,107 @@ +package artifact + +import ( + "errors" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// CapabilityError reports an operation that a repository cannot honor. It is never a successful +// no-op: callers receive it instead of silently degraded behavior. It wraps a sentinel +// (ErrArtifactUnsupportedCapability by default, or ErrArtifactPublicationUnavailable) and an +// optional cause, so errors.Is works against both. +type CapabilityError struct { + // Repository is the repository alias or kind that lacks the capability. + Repository string + + // Operation is the attempted operation, for example "lock" or "sign". + Operation string + + // Capability names the missing guarantee, for example "lock/repository" or "enumerate/collection". + Capability string + + // Reason explains the limitation and, where useful, the supported alternative. + Reason string + + // Sentinel is the wrapped sentinel; nil means ErrArtifactUnsupportedCapability. + Sentinel error + + // Cause is an optional underlying error kept for diagnostics. + Cause error +} + +// NewUnsupportedCapability returns a CapabilityError wrapping ErrArtifactUnsupportedCapability. +func NewUnsupportedCapability(repository, operation, capability, reason string) *CapabilityError { + defer perf.Track(nil, "artifact.NewUnsupportedCapability")() + + return &CapabilityError{ + Repository: repository, + Operation: operation, + Capability: capability, + Reason: reason, + Sentinel: errUtils.ErrArtifactUnsupportedCapability, + } +} + +// NewPublicationUnavailable returns a CapabilityError wrapping ErrArtifactPublicationUnavailable. +func NewPublicationUnavailable(repository, operation, reason string, cause error) *CapabilityError { + defer perf.Track(nil, "artifact.NewPublicationUnavailable")() + + return &CapabilityError{ + Repository: repository, + Operation: operation, + Reason: reason, + Sentinel: errUtils.ErrArtifactPublicationUnavailable, + Cause: cause, + } +} + +// Error implements error. +func (e *CapabilityError) Error() string { + defer perf.Track(nil, "artifact.CapabilityError.Error")() + + message := e.sentinel().Error() + if e.Repository != "" { + message = fmt.Sprintf("%s: repository %q", message, e.Repository) + } + if e.Operation != "" { + message = fmt.Sprintf("%s: %s", message, e.Operation) + } + if e.Capability != "" { + message = fmt.Sprintf("%s: capability %s", message, e.Capability) + } + if e.Reason != "" { + message = fmt.Sprintf("%s: %s", message, e.Reason) + } + return message +} + +// Unwrap exposes the sentinel and the cause to errors.Is and errors.As. +func (e *CapabilityError) Unwrap() []error { + defer perf.Track(nil, "artifact.CapabilityError.Unwrap")() + + if e.Cause == nil { + return []error{e.sentinel()} + } + return []error{e.sentinel(), e.Cause} +} + +func (e *CapabilityError) sentinel() error { + if e.Sentinel != nil { + return e.Sentinel + } + return errUtils.ErrArtifactUnsupportedCapability +} + +// AsCapabilityError extracts a *CapabilityError from err's chain. +func AsCapabilityError(err error) (*CapabilityError, bool) { + defer perf.Track(nil, "artifact.AsCapabilityError")() + + var target *CapabilityError + if errors.As(err, &target) { + return target, true + } + return nil, false +} diff --git a/pkg/artifact/errors_test.go b/pkg/artifact/errors_test.go new file mode 100644 index 00000000000..2eee7f8ed90 --- /dev/null +++ b/pkg/artifact/errors_test.go @@ -0,0 +1,69 @@ +package artifact + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// TestSentinelsAreDistinct guards against two sentinels aliasing each other, which would make +// errors.Is match the wrong failure class. +func TestSentinelsAreDistinct(t *testing.T) { + sentinels := []error{ + errUtils.ErrArtifactConfigInvalid, + errUtils.ErrArtifactRefInvalid, + errUtils.ErrArtifactConflict, + errUtils.ErrArtifactNotFound, + errUtils.ErrArtifactPublicationUnavailable, + errUtils.ErrArtifactUnsupportedCapability, + errUtils.ErrArtifactIntegrityFailed, + errUtils.ErrArtifactDocumentMissing, + errUtils.ErrArtifactPackagingInvalid, + errUtils.ErrArtifactUnsafePath, + errUtils.ErrArtifactRecordInvalid, + errUtils.ErrArtifactTooLarge, + } + for i, sentinel := range sentinels { + wrapped := fmt.Errorf("%w: context", sentinel) + for j, other := range sentinels { + assert.Equal(t, i == j, errors.Is(wrapped, other), "%v vs %v", sentinel, other) + } + } +} + +func TestCapabilityError(t *testing.T) { + t.Run("unsupported capability wraps the sentinel", func(t *testing.T) { + err := NewUnsupportedCapability("shared", "lock", "lock/repository", "OCI tags cannot provide distributed exclusion") + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + assert.NotErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + assert.Contains(t, err.Error(), "shared") + assert.Contains(t, err.Error(), "lock/repository") + }) + + t.Run("survives further wrapping and is extractable", func(t *testing.T) { + wrapped := fmt.Errorf("%w: publishing vpc:1.0", NewUnsupportedCapability("shared", "lock", "lock/repository", "")) + require.ErrorIs(t, wrapped, errUtils.ErrArtifactUnsupportedCapability) + capability, ok := AsCapabilityError(wrapped) + require.True(t, ok) + assert.Equal(t, "lock/repository", capability.Capability) + _, ok = AsCapabilityError(errors.New("plain")) + assert.False(t, ok) + }) + + t.Run("publication unavailable keeps its cause", func(t *testing.T) { + cause := errors.New("sdk: 503") + err := NewPublicationUnavailable("backups", "publish", "pending operation op-1", cause) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationUnavailable) + require.ErrorIs(t, err, cause, "backend causes stay available for diagnostics") + assert.NotErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + }) + + t.Run("zero value defaults to unsupported", func(t *testing.T) { + require.ErrorIs(t, &CapabilityError{}, errUtils.ErrArtifactUnsupportedCapability) + }) +} diff --git a/pkg/artifact/kinddata.go b/pkg/artifact/kinddata.go new file mode 100644 index 00000000000..b567a043515 --- /dev/null +++ b/pkg/artifact/kinddata.go @@ -0,0 +1,71 @@ +package artifact + +import ( + "encoding/json" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// KindData is optional, kind-specific record data held as canonical JSON bytes. +// +// Design: a Record cannot carry map[string]any or float64 in its canonical path, yet kinds (for +// example legacy CI planfiles) need typed extra data. KindData is therefore a typed raw-JSON +// holder. [NewKindData] marshals a caller-owned typed value and immediately re-encodes it with +// [CanonicalizeJSON], which sorts object keys, writes stable string escapes, and rejects floats, +// duplicate keys, and invalid UTF-8. A KindData value is always canonical, so it embeds into the +// Record bytes unchanged and participates in the intent and record digests byte-for-byte. +// Consumers read it back into their own typed struct with [KindData.Decode]. +type KindData []byte + +// NewKindData canonicalizes v, which should be a typed struct, into KindData. +func NewKindData(v any) (KindData, error) { + defer perf.Track(nil, "artifact.NewKindData")() + + raw, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("%w: kind data: %w", errUtils.ErrArtifactRecordInvalid, err) + } + canonical, err := CanonicalizeJSON(raw) + if err != nil { + return nil, err + } + return KindData(canonical), nil +} + +// Decode unmarshals the kind data into v, a pointer to the kind's typed struct. +func (k KindData) Decode(v any) error { + defer perf.Track(nil, "artifact.KindData.Decode")() + + if len(k) == 0 { + return fmt.Errorf("%w: record has no kind data", errUtils.ErrArtifactRecordInvalid) + } + return json.Unmarshal(k, v) +} + +// MarshalJSON implements json.Marshaler, emitting the canonical bytes. +func (k KindData) MarshalJSON() ([]byte, error) { + defer perf.Track(nil, "artifact.KindData.MarshalJSON")() + + if len(k) == 0 { + return []byte("null"), nil + } + return CanonicalizeJSON(k) +} + +// UnmarshalJSON implements json.Unmarshaler, canonicalizing the input. +func (k *KindData) UnmarshalJSON(data []byte) error { + defer perf.Track(nil, "artifact.KindData.UnmarshalJSON")() + + canonical, err := CanonicalizeJSON(data) + if err != nil { + return err + } + if string(canonical) == "null" { + *k = nil + return nil + } + *k = KindData(canonical) + return nil +} diff --git a/pkg/artifact/kinddata_test.go b/pkg/artifact/kinddata_test.go new file mode 100644 index 00000000000..58850747a6c --- /dev/null +++ b/pkg/artifact/kinddata_test.go @@ -0,0 +1,38 @@ +package artifact + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestKindDataJSONBehavior(t *testing.T) { + t.Run("empty kind data marshals to null and cannot be decoded", func(t *testing.T) { + raw, err := KindData(nil).MarshalJSON() + require.NoError(t, err) + assert.Equal(t, "null", string(raw)) + require.ErrorIs(t, KindData(nil).Decode(&struct{}{}), errUtils.ErrArtifactRecordInvalid) + }) + + t.Run("unmarshal canonicalizes and treats null as absent", func(t *testing.T) { + var data KindData + require.NoError(t, json.Unmarshal([]byte(` {"b": 1, "a": 2} `), &data)) + assert.Equal(t, `{"a":2,"b":1}`, string(data)) + require.NoError(t, data.UnmarshalJSON([]byte("null"))) + assert.Empty(t, data) + }) + + t.Run("unmarshal rejects floats", func(t *testing.T) { + var data KindData + require.ErrorIs(t, json.Unmarshal([]byte(`{"a":1.5}`), &data), errUtils.ErrArtifactRecordInvalid) + }) + + t.Run("marshal failure surfaces as an invalid record", func(t *testing.T) { + _, err := NewKindData(make(chan int)) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) + }) +} diff --git a/pkg/artifact/lock.go b/pkg/artifact/lock.go new file mode 100644 index 00000000000..24d70712f6b --- /dev/null +++ b/pkg/artifact/lock.go @@ -0,0 +1,82 @@ +package artifact + +import ( + "context" + "time" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// LockScope is the strength of mutation coordination a lock provides. +type LockScope string + +// Lock scopes. No S3 Object Lock retention, OCI tag, or local-cache lock is a distributed lease. +const ( + // LockScopeHost serializes writers on one host. It is the default for single-host use. + LockScopeHost LockScope = "host" + + // LockScopeRepository serializes writers across hosts through the repository's own + // coordination (for example S3 conditional owner claims or a configured lock repository). + LockScopeRepository LockScope = "repository" +) + +// Satisfies reports whether a lock of scope s meets a request for scope required. A repository +// scope lock satisfies a host scope request; an empty requirement is always satisfied. +func (s LockScope) Satisfies(required LockScope) bool { + defer perf.Track(nil, "artifact.LockScope.Satisfies")() + + switch required { + case "", LockScopeHost: + return s == LockScopeHost || s == LockScopeRepository + case LockScopeRepository: + return s == LockScopeRepository + default: + return false + } +} + +// LockRequest asks a repository to coordinate a mutation of one resource. Multi-resource work +// acquires locks in canonical repository/resource order to avoid deadlocks. +type LockRequest struct { + // Resource identifies what is being mutated, for example a canonical reference key. + Resource string + + // Owner identifies the writer, for example host and process. + Owner string + + // OperationID is the logical operation identifier recorded in the claim. + OperationID string + + // Required is the minimum guarantee the caller needs. Empty means host scope. + Required LockScope + + // Lease is the requested lease duration for providers that expire claims. V1 claims do not + // expire automatically, so zero means no expiry and providers without fencing must reject + // a non-zero lease with an unsupported-capability error. + Lease time.Duration +} + +// LockHandle is an acquired, ownership-checked lock. Release verifies the ownership token and +// never releases another operation's claim. Providers never steal a lock automatically. +type LockHandle interface { + // Resource returns the locked resource. + Resource() string + + // Owner returns the owner recorded in the claim. + Owner() string + + // Scope returns the actual scope provided, which results report. + Scope() LockScope + + // Token returns the ownership token checked on renew and release. + Token() string + + // ExpiresAt returns the lease expiry and whether the lock expires at all. + ExpiresAt() (time.Time, bool) + + // Renew extends a leased lock; providers without leases return an unsupported-capability error. + Renew(ctx context.Context, lease time.Duration) error + + // Release relinquishes the lock after checking the ownership token. + Release(ctx context.Context) error +} diff --git a/pkg/artifact/lockprovider.go b/pkg/artifact/lockprovider.go new file mode 100644 index 00000000000..636101f676c --- /dev/null +++ b/pkg/artifact/lockprovider.go @@ -0,0 +1,14 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_lockprovider.go github.com/cloudposse/atmos/pkg/artifact LockProvider,LockHandle + +// LockProvider coordinates artifact mutations. Repositories implement it directly or by composing +// a shared provider (filesystem locks, S3 conditional owner claims, a delegated lock repository). +type LockProvider interface { + // Lock acquires an ownership-checked lock satisfying request.Required, or returns a + // *CapabilityError wrapping ErrArtifactUnsupportedCapability when the guarantee is unavailable. + // It never steals another operation's lock and never silently downgrades the scope. + Lock(ctx context.Context, request LockRequest) (LockHandle, error) +} diff --git a/pkg/artifact/mock_copier.go b/pkg/artifact/mock_copier.go new file mode 100644 index 00000000000..9e3519a0648 --- /dev/null +++ b/pkg/artifact/mock_copier.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Copier) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_copier.go github.com/cloudposse/atmos/pkg/artifact Copier +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockCopier is a mock of Copier interface. +type MockCopier struct { + ctrl *gomock.Controller + recorder *MockCopierMockRecorder + isgomock struct{} +} + +// MockCopierMockRecorder is the mock recorder for MockCopier. +type MockCopierMockRecorder struct { + mock *MockCopier +} + +// NewMockCopier creates a new mock instance. +func NewMockCopier(ctrl *gomock.Controller) *MockCopier { + mock := &MockCopier{ctrl: ctrl} + mock.recorder = &MockCopierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCopier) EXPECT() *MockCopierMockRecorder { + return m.recorder +} + +// CanCopyFrom mocks base method. +func (m *MockCopier) CanCopyFrom(source Repository) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CanCopyFrom", source) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CanCopyFrom indicates an expected call of CanCopyFrom. +func (mr *MockCopierMockRecorder) CanCopyFrom(source any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CanCopyFrom", reflect.TypeOf((*MockCopier)(nil).CanCopyFrom), source) +} + +// CopyFrom mocks base method. +func (m *MockCopier) CopyFrom(ctx context.Context, source Repository, pinned PinnedReference, options CopyFromOptions) (*PublishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CopyFrom", ctx, source, pinned, options) + ret0, _ := ret[0].(*PublishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CopyFrom indicates an expected call of CopyFrom. +func (mr *MockCopierMockRecorder) CopyFrom(ctx, source, pinned, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFrom", reflect.TypeOf((*MockCopier)(nil).CopyFrom), ctx, source, pinned, options) +} diff --git a/pkg/artifact/mock_lockprovider.go b/pkg/artifact/mock_lockprovider.go new file mode 100644 index 00000000000..ff9096387c9 --- /dev/null +++ b/pkg/artifact/mock_lockprovider.go @@ -0,0 +1,180 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: LockProvider,LockHandle) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_lockprovider.go github.com/cloudposse/atmos/pkg/artifact LockProvider,LockHandle +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + time "time" + + gomock "go.uber.org/mock/gomock" +) + +// MockLockProvider is a mock of LockProvider interface. +type MockLockProvider struct { + ctrl *gomock.Controller + recorder *MockLockProviderMockRecorder + isgomock struct{} +} + +// MockLockProviderMockRecorder is the mock recorder for MockLockProvider. +type MockLockProviderMockRecorder struct { + mock *MockLockProvider +} + +// NewMockLockProvider creates a new mock instance. +func NewMockLockProvider(ctrl *gomock.Controller) *MockLockProvider { + mock := &MockLockProvider{ctrl: ctrl} + mock.recorder = &MockLockProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLockProvider) EXPECT() *MockLockProviderMockRecorder { + return m.recorder +} + +// Lock mocks base method. +func (m *MockLockProvider) Lock(ctx context.Context, request LockRequest) (LockHandle, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lock", ctx, request) + ret0, _ := ret[0].(LockHandle) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Lock indicates an expected call of Lock. +func (mr *MockLockProviderMockRecorder) Lock(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockLockProvider)(nil).Lock), ctx, request) +} + +// MockLockHandle is a mock of LockHandle interface. +type MockLockHandle struct { + ctrl *gomock.Controller + recorder *MockLockHandleMockRecorder + isgomock struct{} +} + +// MockLockHandleMockRecorder is the mock recorder for MockLockHandle. +type MockLockHandleMockRecorder struct { + mock *MockLockHandle +} + +// NewMockLockHandle creates a new mock instance. +func NewMockLockHandle(ctrl *gomock.Controller) *MockLockHandle { + mock := &MockLockHandle{ctrl: ctrl} + mock.recorder = &MockLockHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLockHandle) EXPECT() *MockLockHandleMockRecorder { + return m.recorder +} + +// ExpiresAt mocks base method. +func (m *MockLockHandle) ExpiresAt() (time.Time, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExpiresAt") + ret0, _ := ret[0].(time.Time) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// ExpiresAt indicates an expected call of ExpiresAt. +func (mr *MockLockHandleMockRecorder) ExpiresAt() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpiresAt", reflect.TypeOf((*MockLockHandle)(nil).ExpiresAt)) +} + +// Owner mocks base method. +func (m *MockLockHandle) Owner() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Owner") + ret0, _ := ret[0].(string) + return ret0 +} + +// Owner indicates an expected call of Owner. +func (mr *MockLockHandleMockRecorder) Owner() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Owner", reflect.TypeOf((*MockLockHandle)(nil).Owner)) +} + +// Release mocks base method. +func (m *MockLockHandle) Release(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Release", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Release indicates an expected call of Release. +func (mr *MockLockHandleMockRecorder) Release(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockLockHandle)(nil).Release), ctx) +} + +// Renew mocks base method. +func (m *MockLockHandle) Renew(ctx context.Context, lease time.Duration) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Renew", ctx, lease) + ret0, _ := ret[0].(error) + return ret0 +} + +// Renew indicates an expected call of Renew. +func (mr *MockLockHandleMockRecorder) Renew(ctx, lease any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Renew", reflect.TypeOf((*MockLockHandle)(nil).Renew), ctx, lease) +} + +// Resource mocks base method. +func (m *MockLockHandle) Resource() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resource") + ret0, _ := ret[0].(string) + return ret0 +} + +// Resource indicates an expected call of Resource. +func (mr *MockLockHandleMockRecorder) Resource() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resource", reflect.TypeOf((*MockLockHandle)(nil).Resource)) +} + +// Scope mocks base method. +func (m *MockLockHandle) Scope() LockScope { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Scope") + ret0, _ := ret[0].(LockScope) + return ret0 +} + +// Scope indicates an expected call of Scope. +func (mr *MockLockHandleMockRecorder) Scope() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Scope", reflect.TypeOf((*MockLockHandle)(nil).Scope)) +} + +// Token mocks base method. +func (m *MockLockHandle) Token() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Token") + ret0, _ := ret[0].(string) + return ret0 +} + +// Token indicates an expected call of Token. +func (mr *MockLockHandleMockRecorder) Token() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockLockHandle)(nil).Token)) +} diff --git a/pkg/artifact/mock_producer.go b/pkg/artifact/mock_producer.go new file mode 100644 index 00000000000..d98247d3332 --- /dev/null +++ b/pkg/artifact/mock_producer.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Producer) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_producer.go github.com/cloudposse/atmos/pkg/artifact Producer +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockProducer is a mock of Producer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder + isgomock struct{} +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// Kind mocks base method. +func (m *MockProducer) Kind() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Kind") + ret0, _ := ret[0].(string) + return ret0 +} + +// Kind indicates an expected call of Kind. +func (mr *MockProducerMockRecorder) Kind() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kind", reflect.TypeOf((*MockProducer)(nil).Kind)) +} + +// Prepare mocks base method. +func (m *MockProducer) Prepare(ctx context.Context, request PrepareRequest) (*PreparedArtifact, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Prepare", ctx, request) + ret0, _ := ret[0].(*PreparedArtifact) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Prepare indicates an expected call of Prepare. +func (mr *MockProducerMockRecorder) Prepare(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prepare", reflect.TypeOf((*MockProducer)(nil).Prepare), ctx, request) +} diff --git a/pkg/artifact/mock_repository.go b/pkg/artifact/mock_repository.go new file mode 100644 index 00000000000..ac160ce6daf --- /dev/null +++ b/pkg/artifact/mock_repository.go @@ -0,0 +1,290 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Repository,GraphSource) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_repository.go github.com/cloudposse/atmos/pkg/artifact Repository,GraphSource +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + io "io" + reflect "reflect" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockRepository is a mock of Repository interface. +type MockRepository struct { + ctrl *gomock.Controller + recorder *MockRepositoryMockRecorder + isgomock struct{} +} + +// MockRepositoryMockRecorder is the mock recorder for MockRepository. +type MockRepositoryMockRecorder struct { + mock *MockRepository +} + +// NewMockRepository creates a new mock instance. +func NewMockRepository(ctrl *gomock.Controller) *MockRepository { + mock := &MockRepository{ctrl: ctrl} + mock.recorder = &MockRepositoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepository) EXPECT() *MockRepositoryMockRecorder { + return m.recorder +} + +// Capabilities mocks base method. +func (m *MockRepository) Capabilities() Capabilities { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Capabilities") + ret0, _ := ret[0].(Capabilities) + return ret0 +} + +// Capabilities indicates an expected call of Capabilities. +func (mr *MockRepositoryMockRecorder) Capabilities() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Capabilities", reflect.TypeOf((*MockRepository)(nil).Capabilities)) +} + +// Delete mocks base method. +func (m *MockRepository) Delete(ctx context.Context, pinned PinnedReference, options DeleteOptions) (*DeleteResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, pinned, options) + ret0, _ := ret[0].(*DeleteResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockRepositoryMockRecorder) Delete(ctx, pinned, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRepository)(nil).Delete), ctx, pinned, options) +} + +// Inspect mocks base method. +func (m *MockRepository) Inspect(ctx context.Context, pinned PinnedReference) (*Inspection, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inspect", ctx, pinned) + ret0, _ := ret[0].(*Inspection) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Inspect indicates an expected call of Inspect. +func (mr *MockRepositoryMockRecorder) Inspect(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inspect", reflect.TypeOf((*MockRepository)(nil).Inspect), ctx, pinned) +} + +// List mocks base method. +func (m *MockRepository) List(ctx context.Context, query ListQuery) (*ListPage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, query) + ret0, _ := ret[0].(*ListPage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockRepositoryMockRecorder) List(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRepository)(nil).List), ctx, query) +} + +// Lock mocks base method. +func (m *MockRepository) Lock(ctx context.Context, request LockRequest) (LockHandle, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lock", ctx, request) + ret0, _ := ret[0].(LockHandle) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Lock indicates an expected call of Lock. +func (mr *MockRepositoryMockRecorder) Lock(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockRepository)(nil).Lock), ctx, request) +} + +// OpenDocument mocks base method. +func (m *MockRepository) OpenDocument(ctx context.Context, pinned PinnedReference, role DocumentRole) (*DocumentStream, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OpenDocument", ctx, pinned, role) + ret0, _ := ret[0].(*DocumentStream) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OpenDocument indicates an expected call of OpenDocument. +func (mr *MockRepositoryMockRecorder) OpenDocument(ctx, pinned, role any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenDocument", reflect.TypeOf((*MockRepository)(nil).OpenDocument), ctx, pinned, role) +} + +// Pin mocks base method. +func (m *MockRepository) Pin(ctx context.Context, ref Ref) (PinnedReference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pin", ctx, ref) + ret0, _ := ret[0].(PinnedReference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pin indicates an expected call of Pin. +func (mr *MockRepositoryMockRecorder) Pin(ctx, ref any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pin", reflect.TypeOf((*MockRepository)(nil).Pin), ctx, ref) +} + +// Publish mocks base method. +func (m *MockRepository) Publish(ctx context.Context, prepared *PreparedArtifact, options PublishOptions) (*PublishResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, prepared, options) + ret0, _ := ret[0].(*PublishResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Publish indicates an expected call of Publish. +func (mr *MockRepositoryMockRecorder) Publish(ctx, prepared, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockRepository)(nil).Publish), ctx, prepared, options) +} + +// Read mocks base method. +func (m *MockRepository) Read(ctx context.Context, pinned PinnedReference) (*Content, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Read", ctx, pinned) + ret0, _ := ret[0].(*Content) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Read indicates an expected call of Read. +func (mr *MockRepositoryMockRecorder) Read(ctx, pinned any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockRepository)(nil).Read), ctx, pinned) +} + +// Resolve mocks base method. +func (m *MockRepository) Resolve(ctx context.Context, ref Ref) (PinnedReference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resolve", ctx, ref) + ret0, _ := ret[0].(PinnedReference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Resolve indicates an expected call of Resolve. +func (mr *MockRepositoryMockRecorder) Resolve(ctx, ref any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resolve", reflect.TypeOf((*MockRepository)(nil).Resolve), ctx, ref) +} + +// Sign mocks base method. +func (m *MockRepository) Sign(ctx context.Context, subject SignatureSubject, signer Signer, options SignOptions) ([]SignatureEvidence, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, subject, signer, options) + ret0, _ := ret[0].([]SignatureEvidence) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockRepositoryMockRecorder) Sign(ctx, subject, signer, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockRepository)(nil).Sign), ctx, subject, signer, options) +} + +// Verify mocks base method. +func (m *MockRepository) Verify(ctx context.Context, subject SignatureSubject, policy VerificationPolicy) (*VerificationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, subject, policy) + ret0, _ := ret[0].(*VerificationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockRepositoryMockRecorder) Verify(ctx, subject, policy any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockRepository)(nil).Verify), ctx, subject, policy) +} + +// MockGraphSource is a mock of GraphSource interface. +type MockGraphSource struct { + ctrl *gomock.Controller + recorder *MockGraphSourceMockRecorder + isgomock struct{} +} + +// MockGraphSourceMockRecorder is the mock recorder for MockGraphSource. +type MockGraphSourceMockRecorder struct { + mock *MockGraphSource +} + +// NewMockGraphSource creates a new mock instance. +func NewMockGraphSource(ctrl *gomock.Controller) *MockGraphSource { + mock := &MockGraphSource{ctrl: ctrl} + mock.recorder = &MockGraphSourceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGraphSource) EXPECT() *MockGraphSourceMockRecorder { + return m.recorder +} + +// Fetch mocks base method. +func (m *MockGraphSource) Fetch(ctx context.Context, descriptor v1.Descriptor) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fetch", ctx, descriptor) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch indicates an expected call of Fetch. +func (mr *MockGraphSourceMockRecorder) Fetch(ctx, descriptor any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fetch", reflect.TypeOf((*MockGraphSource)(nil).Fetch), ctx, descriptor) +} + +// Root mocks base method. +func (m *MockGraphSource) Root() v1.Descriptor { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Root") + ret0, _ := ret[0].(v1.Descriptor) + return ret0 +} + +// Root indicates an expected call of Root. +func (mr *MockGraphSourceMockRecorder) Root() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockGraphSource)(nil).Root)) +} + +// Successors mocks base method. +func (m *MockGraphSource) Successors(ctx context.Context, descriptor v1.Descriptor) ([]v1.Descriptor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Successors", ctx, descriptor) + ret0, _ := ret[0].([]v1.Descriptor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Successors indicates an expected call of Successors. +func (mr *MockGraphSourceMockRecorder) Successors(ctx, descriptor any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Successors", reflect.TypeOf((*MockGraphSource)(nil).Successors), ctx, descriptor) +} diff --git a/pkg/artifact/mock_service.go b/pkg/artifact/mock_service.go new file mode 100644 index 00000000000..cd99d9cda7d --- /dev/null +++ b/pkg/artifact/mock_service.go @@ -0,0 +1,206 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Service) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_service.go github.com/cloudposse/atmos/pkg/artifact Service +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockService is a mock of Service interface. +type MockService struct { + ctrl *gomock.Controller + recorder *MockServiceMockRecorder + isgomock struct{} +} + +// MockServiceMockRecorder is the mock recorder for MockService. +type MockServiceMockRecorder struct { + mock *MockService +} + +// NewMockService creates a new mock instance. +func NewMockService(ctrl *gomock.Controller) *MockService { + mock := &MockService{ctrl: ctrl} + mock.recorder = &MockServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockService) EXPECT() *MockServiceMockRecorder { + return m.recorder +} + +// Copy mocks base method. +func (m *MockService) Copy(ctx context.Context, request CopyRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Copy", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Copy indicates an expected call of Copy. +func (mr *MockServiceMockRecorder) Copy(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockService)(nil).Copy), ctx, request) +} + +// Delete mocks base method. +func (m *MockService) Delete(ctx context.Context, request DeleteRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Delete indicates an expected call of Delete. +func (mr *MockServiceMockRecorder) Delete(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockService)(nil).Delete), ctx, request) +} + +// Inspect mocks base method. +func (m *MockService) Inspect(ctx context.Context, request InspectRequest) (*InspectResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Inspect", ctx, request) + ret0, _ := ret[0].(*InspectResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Inspect indicates an expected call of Inspect. +func (mr *MockServiceMockRecorder) Inspect(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inspect", reflect.TypeOf((*MockService)(nil).Inspect), ctx, request) +} + +// List mocks base method. +func (m *MockService) List(ctx context.Context, request ListRequest) (*ListResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, request) + ret0, _ := ret[0].(*ListResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockServiceMockRecorder) List(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockService)(nil).List), ctx, request) +} + +// Pin mocks base method. +func (m *MockService) Pin(ctx context.Context, request PinRequest) (*PinResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pin", ctx, request) + ret0, _ := ret[0].(*PinResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pin indicates an expected call of Pin. +func (mr *MockServiceMockRecorder) Pin(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pin", reflect.TypeOf((*MockService)(nil).Pin), ctx, request) +} + +// Publish mocks base method. +func (m *MockService) Publish(ctx context.Context, request PublishRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Publish indicates an expected call of Publish. +func (mr *MockServiceMockRecorder) Publish(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockService)(nil).Publish), ctx, request) +} + +// Pull mocks base method. +func (m *MockService) Pull(ctx context.Context, request PullRequest) (*OperationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pull", ctx, request) + ret0, _ := ret[0].(*OperationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Pull indicates an expected call of Pull. +func (mr *MockServiceMockRecorder) Pull(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pull", reflect.TypeOf((*MockService)(nil).Pull), ctx, request) +} + +// Search mocks base method. +func (m *MockService) Search(ctx context.Context, request SearchRequest) (*ListResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, request) + ret0, _ := ret[0].(*ListResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockServiceMockRecorder) Search(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockService)(nil).Search), ctx, request) +} + +// ShowDocument mocks base method. +func (m *MockService) ShowDocument(ctx context.Context, request ShowDocumentRequest) (*DocumentResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ShowDocument", ctx, request) + ret0, _ := ret[0].(*DocumentResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ShowDocument indicates an expected call of ShowDocument. +func (mr *MockServiceMockRecorder) ShowDocument(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShowDocument", reflect.TypeOf((*MockService)(nil).ShowDocument), ctx, request) +} + +// Sign mocks base method. +func (m *MockService) Sign(ctx context.Context, request SignRequest) (*SignResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, request) + ret0, _ := ret[0].(*SignResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockServiceMockRecorder) Sign(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockService)(nil).Sign), ctx, request) +} + +// Verify mocks base method. +func (m *MockService) Verify(ctx context.Context, request VerifyRequest) (*VerifyResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, request) + ret0, _ := ret[0].(*VerifyResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockServiceMockRecorder) Verify(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockService)(nil).Verify), ctx, request) +} diff --git a/pkg/artifact/mock_signer.go b/pkg/artifact/mock_signer.go new file mode 100644 index 00000000000..85ac8a921ba --- /dev/null +++ b/pkg/artifact/mock_signer.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Signer) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_signer.go github.com/cloudposse/atmos/pkg/artifact Signer +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockSigner is a mock of Signer interface. +type MockSigner struct { + ctrl *gomock.Controller + recorder *MockSignerMockRecorder + isgomock struct{} +} + +// MockSignerMockRecorder is the mock recorder for MockSigner. +type MockSignerMockRecorder struct { + mock *MockSigner +} + +// NewMockSigner creates a new mock instance. +func NewMockSigner(ctrl *gomock.Controller) *MockSigner { + mock := &MockSigner{ctrl: ctrl} + mock.recorder = &MockSignerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSigner) EXPECT() *MockSignerMockRecorder { + return m.recorder +} + +// Formats mocks base method. +func (m *MockSigner) Formats() []EvidenceFormat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Formats") + ret0, _ := ret[0].([]EvidenceFormat) + return ret0 +} + +// Formats indicates an expected call of Formats. +func (mr *MockSignerMockRecorder) Formats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Formats", reflect.TypeOf((*MockSigner)(nil).Formats)) +} + +// Sign mocks base method. +func (m *MockSigner) Sign(ctx context.Context, subject SignatureSubject) (*EvidencePayload, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sign", ctx, subject) + ret0, _ := ret[0].(*EvidencePayload) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sign indicates an expected call of Sign. +func (mr *MockSignerMockRecorder) Sign(ctx, subject any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sign", reflect.TypeOf((*MockSigner)(nil).Sign), ctx, subject) +} diff --git a/pkg/artifact/mock_verifier.go b/pkg/artifact/mock_verifier.go new file mode 100644 index 00000000000..cfcc9f1c384 --- /dev/null +++ b/pkg/artifact/mock_verifier.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact (interfaces: Verifier) +// +// Generated by this command: +// +// mockgen -package artifact -destination mock_verifier.go github.com/cloudposse/atmos/pkg/artifact Verifier +// + +// Package artifact is a generated GoMock package. +package artifact + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockVerifier is a mock of Verifier interface. +type MockVerifier struct { + ctrl *gomock.Controller + recorder *MockVerifierMockRecorder + isgomock struct{} +} + +// MockVerifierMockRecorder is the mock recorder for MockVerifier. +type MockVerifierMockRecorder struct { + mock *MockVerifier +} + +// NewMockVerifier creates a new mock instance. +func NewMockVerifier(ctrl *gomock.Controller) *MockVerifier { + mock := &MockVerifier{ctrl: ctrl} + mock.recorder = &MockVerifierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVerifier) EXPECT() *MockVerifierMockRecorder { + return m.recorder +} + +// Formats mocks base method. +func (m *MockVerifier) Formats() []EvidenceFormat { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Formats") + ret0, _ := ret[0].([]EvidenceFormat) + return ret0 +} + +// Formats indicates an expected call of Formats. +func (mr *MockVerifierMockRecorder) Formats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Formats", reflect.TypeOf((*MockVerifier)(nil).Formats)) +} + +// Verify mocks base method. +func (m *MockVerifier) Verify(ctx context.Context, subject SignatureSubject, evidence []EvidenceBlob, policy VerificationPolicy) (*VerificationResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Verify", ctx, subject, evidence, policy) + ret0, _ := ret[0].(*VerificationResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Verify indicates an expected call of Verify. +func (mr *MockVerifierMockRecorder) Verify(ctx, subject, evidence, policy any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Verify", reflect.TypeOf((*MockVerifier)(nil).Verify), ctx, subject, evidence, policy) +} diff --git a/pkg/artifact/operation.go b/pkg/artifact/operation.go new file mode 100644 index 00000000000..165c7c8936b --- /dev/null +++ b/pkg/artifact/operation.go @@ -0,0 +1,124 @@ +package artifact + +import ( + "time" + + "github.com/opencontainers/go-digest" +) + +// Action names an artifact operation. +type Action string + +// Artifact operations. +const ( + ActionPublish Action = "publish" + ActionPull Action = "pull" + ActionCopy Action = "copy" + ActionInspect Action = "inspect" + ActionList Action = "list" + ActionSearch Action = "search" + ActionDelete Action = "delete" + ActionShow Action = "show" + ActionSign Action = "sign" + ActionVerify Action = "verify" + ActionPin Action = "pin" +) + +// Outcome is the result classification of an operation. +type Outcome string + +// Operation outcomes. A no-op returns the existing Record unchanged. +const ( + // OutcomeCreated means new state was written. + OutcomeCreated Outcome = "created" + + // OutcomeUnchanged means the same intent already existed and nothing was written. + OutcomeUnchanged Outcome = "unchanged" + + // OutcomeReplaced means existing state was replaced under force. + OutcomeReplaced Outcome = "replaced" + + // OutcomeDeleted means state was removed. + OutcomeDeleted Outcome = "deleted" + + // OutcomeRead means state was read or materialized without mutation. + OutcomeRead Outcome = "read" + + // OutcomeDryRun means the operation was validated and planned but nothing was written. + OutcomeDryRun Outcome = "dry-run" +) + +// Severity classifies a diagnostic. +type Severity string + +// Diagnostic severities. +const ( + SeverityInfo Severity = "info" + SeverityWarning Severity = "warning" + SeverityError Severity = "error" +) + +// Diagnostic is a structured, non-fatal message attached to a result. +type Diagnostic struct { + // Severity is info, warning, or error. + Severity Severity `json:"severity"` + + // Code is a stable machine-readable identifier. + Code string `json:"code,omitempty"` + + // Message is the human-readable text. It never contains credentials. + Message string `json:"message"` +} + +// OperationResult is the typed result of one operation. Its JSON field names match the named step +// outputs and structured CLI output; optional values are omitted rather than filled with +// misleading empty strings. +type OperationResult struct { + // OperationID is the logical operation identifier. + OperationID string `json:"operation_id"` + + // Action is the operation performed. + Action Action `json:"action"` + + // Outcome classifies the result. + Outcome Outcome `json:"outcome"` + + // Repository is the configured repository alias the operation targeted. + Repository string `json:"repository,omitempty"` + + // Requested is the reference as requested by the caller. + Requested string `json:"requested_ref,omitempty"` + + // Ref is the resolved reference. + Ref string `json:"ref,omitempty"` + + // Location is the primary resolved storage location. + Location string `json:"location,omitempty"` + + // Aliases lists additional locations or tags that resolve to the same content. + Aliases []string `json:"aliases,omitempty"` + + // ResolvedDigest is the OCI manifest/index digest, or the record digest for local and S3. + ResolvedDigest digest.Digest `json:"resolved_digest,omitempty"` + + // RecordDigest is the digest of the canonical Record bytes. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // ContentDigest is the portable content digest from the Record. + ContentDigest digest.Digest `json:"content_digest,omitempty"` + + // RepresentationDigest is the checksum of the transferred payload representation. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` + + // Size is the transferred or described payload size in bytes. + Size int64 `json:"size,omitempty"` + + // Duration is the wall-clock duration of the operation. + Duration time.Duration `json:"duration_ns,omitempty"` + + // LockScope is the actual lock scope that coordinated a mutation. + LockScope LockScope `json:"lock_scope,omitempty"` + + // Diagnostics are non-fatal messages. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} diff --git a/pkg/artifact/packaging.go b/pkg/artifact/packaging.go new file mode 100644 index 00000000000..fd06445137b --- /dev/null +++ b/pkg/artifact/packaging.go @@ -0,0 +1,22 @@ +package artifact + +// Packaging identifies how a file payload is stored in a repository. +type Packaging string + +// Payload packaging modes. +const ( + // PackagingNone stores copied files (local directories, S3 object-per-file). + PackagingNone Packaging = "none" + + // PackagingTar stores one uncompressed tar archive. + PackagingTar Packaging = "tar" + + // PackagingTgz stores one gzip-compressed tar archive. + PackagingTgz Packaging = "tgz" + + // PackagingZip stores one zip archive. + PackagingZip Packaging = "zip" + + // PackagingOCIGraph identifies an existing native OCI graph carried unchanged. + PackagingOCIGraph Packaging = "oci-graph" +) diff --git a/pkg/artifact/pinned.go b/pkg/artifact/pinned.go new file mode 100644 index 00000000000..015a8cdd5bc --- /dev/null +++ b/pkg/artifact/pinned.go @@ -0,0 +1,32 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// PinnedReference is an immutable reference suitable for consumer lockfiles. It is not a write +// lock and not a substitute for signature verification; trust policy is reevaluated on use. +type PinnedReference struct { + // Ref is the reference that was pinned, as requested. + Ref Ref `json:"ref"` + + // Repository is the repository locator (a registry path, bucket and prefix, or directory), + // never credentials. + Repository string `json:"repository"` + + // ResolvedDigest is the OCI manifest/index digest, or the record digest for local and S3. + ResolvedDigest digest.Digest `json:"resolved_digest"` + + // RecordDigest is the digest of the canonical Record bytes, when a Record is bound. + RecordDigest digest.Digest `json:"record_digest,omitempty"` + + // ContentDigest is the portable content digest from the Record, when a Record is bound. + ContentDigest digest.Digest `json:"content_digest,omitempty"` + + // Evidence lists the signature evidence descriptors known at pin time. + Evidence []ocispec.Descriptor `json:"evidence,omitempty"` + + // Platform is the selected platform when a multi-platform index was narrowed. + Platform *ocispec.Platform `json:"platform,omitempty"` +} diff --git a/pkg/artifact/policy.go b/pkg/artifact/policy.go new file mode 100644 index 00000000000..3a129e53228 --- /dev/null +++ b/pkg/artifact/policy.go @@ -0,0 +1,98 @@ +package artifact + +import ( + "fmt" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// TrustedIdentity is a certificate identity and issuer pair that may satisfy a trust set. +type TrustedIdentity struct { + // Identity is the expected certificate identity. + Identity string `json:"identity"` + + // Issuer is the expected OIDC issuer. + Issuer string `json:"issuer"` +} + +// TrustSet is one set of acceptable signers. Within a trust set any one signer satisfies it. +type TrustSet struct { + // Identities are trusted certificate identities. + Identities []TrustedIdentity `json:"identities,omitempty"` + + // Keys are operator-managed public-key file paths. + Keys []string `json:"keys,omitempty"` +} + +// VerificationPolicy is the effective policy passed to Verify. When a global and a repository +// trust restriction both exist, the policy lists both TrustSets and every one must be satisfied; +// they are never unioned into a weaker policy. +type VerificationPolicy struct { + // RequireSignature makes a missing or invalid signature a failure. + RequireSignature bool `json:"require_signature"` + + // TrustSets are the sets that must each be satisfied by at least one valid permitted signature. + TrustSets []TrustSet `json:"trust_sets,omitempty"` + + // Offline forbids network access; only preserved evidence and provisioned trust material are used. + Offline bool `json:"offline"` +} + +// HasTrustConstraints reports whether the policy supplies any trust material. Without it a +// verifier reports unchecked rather than verified. +func (p VerificationPolicy) HasTrustConstraints() bool { + defer perf.Track(nil, "artifact.VerificationPolicy.HasTrustConstraints")() + + for _, set := range p.TrustSets { + if len(set.Identities) > 0 || len(set.Keys) > 0 { + return true + } + } + return false +} + +// Fingerprint returns a stable SHA-256 fingerprint of the policy, independent of the order of +// trust sets, identities, and keys, for recording in verification results. +func (p VerificationPolicy) Fingerprint() (string, error) { + defer perf.Track(nil, "artifact.VerificationPolicy.Fingerprint")() + + normalized := VerificationPolicy{RequireSignature: p.RequireSignature, Offline: p.Offline} + for _, set := range p.TrustSets { + next := TrustSet{ + Identities: append([]TrustedIdentity(nil), set.Identities...), + Keys: append([]string(nil), set.Keys...), + } + sort.Slice(next.Identities, func(i, j int) bool { + if next.Identities[i].Identity != next.Identities[j].Identity { + return next.Identities[i].Identity < next.Identities[j].Identity + } + return next.Identities[i].Issuer < next.Identities[j].Issuer + }) + sort.Strings(next.Keys) + normalized.TrustSets = append(normalized.TrustSets, next) + } + sort.Slice(normalized.TrustSets, func(i, j int) bool { + return trustSetKey(normalized.TrustSets[i]) < trustSetKey(normalized.TrustSets[j]) + }) + canonical, err := canonicalJSON(normalized) + if err != nil { + return "", fmt.Errorf("%w: policy fingerprint: %w", errUtils.ErrArtifactConfigInvalid, err) + } + return digest.FromBytes(canonical).String(), nil +} + +// trustSetKey is a deterministic sort key for a trust set. +func trustSetKey(set TrustSet) string { + key := "" + for _, identity := range set.Identities { + key += identity.Identity + "\x00" + identity.Issuer + "\x00" + } + for _, path := range set.Keys { + key += path + "\x01" + } + return key +} diff --git a/pkg/artifact/policy_test.go b/pkg/artifact/policy_test.go new file mode 100644 index 00000000000..318adc66d15 --- /dev/null +++ b/pkg/artifact/policy_test.go @@ -0,0 +1,70 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVerificationPolicyFingerprint(t *testing.T) { + identityA := TrustedIdentity{Identity: "https://github.com/acme/a/.github/workflows/r.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + identityB := TrustedIdentity{Identity: "https://github.com/acme/b/.github/workflows/r.yml@refs/heads/main", Issuer: "https://token.actions.githubusercontent.com"} + + base := VerificationPolicy{ + RequireSignature: true, + TrustSets: []TrustSet{ + {Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"b.pub", "a.pub"}}, + {Keys: []string{"c.pub"}}, + }, + } + fingerprint, err := base.Fingerprint() + require.NoError(t, err) + assert.Regexp(t, `^sha256:[0-9a-f]{64}$`, fingerprint) + + reordered := VerificationPolicy{ + RequireSignature: true, + TrustSets: []TrustSet{ + {Keys: []string{"c.pub"}}, + {Identities: []TrustedIdentity{identityB, identityA}, Keys: []string{"a.pub", "b.pub"}}, + }, + } + same, err := reordered.Fingerprint() + require.NoError(t, err) + assert.Equal(t, fingerprint, same, "order of trust sets, identities, and keys does not matter") + assert.Equal(t, "b.pub", base.TrustSets[0].Keys[0], "fingerprinting does not reorder the caller's policy") + + tests := []struct { + name string + mutate func(p *VerificationPolicy) + }{ + {"require signature", func(p *VerificationPolicy) { p.RequireSignature = false }}, + {"offline", func(p *VerificationPolicy) { p.Offline = true }}, + {"dropped trust set", func(p *VerificationPolicy) { p.TrustSets = p.TrustSets[:1] }}, + {"different key", func(p *VerificationPolicy) { + p.TrustSets = []TrustSet{{Keys: []string{"x.pub"}}, {Keys: []string{"c.pub"}}} + }}, + {"unioned instead of separate sets", func(p *VerificationPolicy) { + p.TrustSets = []TrustSet{{Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"a.pub", "b.pub", "c.pub"}}} + }}, + } + for _, tt := range tests { + t.Run("differs with "+tt.name, func(t *testing.T) { + policy := VerificationPolicy{RequireSignature: true, TrustSets: []TrustSet{ + {Identities: []TrustedIdentity{identityA, identityB}, Keys: []string{"a.pub", "b.pub"}}, + {Keys: []string{"c.pub"}}, + }} + tt.mutate(&policy) + other, err := policy.Fingerprint() + require.NoError(t, err) + assert.NotEqual(t, fingerprint, other) + }) + } +} + +func TestHasTrustConstraints(t *testing.T) { + assert.False(t, VerificationPolicy{}.HasTrustConstraints()) + assert.False(t, VerificationPolicy{RequireSignature: true, TrustSets: []TrustSet{{}}}.HasTrustConstraints()) + assert.True(t, VerificationPolicy{TrustSets: []TrustSet{{Keys: []string{"a.pub"}}}}.HasTrustConstraints()) + assert.True(t, VerificationPolicy{TrustSets: []TrustSet{{Identities: []TrustedIdentity{{Identity: "i", Issuer: "o"}}}}}.HasTrustConstraints()) +} diff --git a/pkg/artifact/prepared.go b/pkg/artifact/prepared.go new file mode 100644 index 00000000000..f3d880a0172 --- /dev/null +++ b/pkg/artifact/prepared.go @@ -0,0 +1,88 @@ +package artifact + +import ( + "context" + "io" + "sync" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// BlobRole identifies what a staged blob contains. +type BlobRole string + +// Staged blob roles. +const ( + // BlobPayload is payload content (an archive or one file). + BlobPayload BlobRole = "payload" + + // BlobDocument is a README or CHANGELOG document. + BlobDocument BlobRole = "document" +) + +// StagedBlob is one blob ready to be uploaded. Open returns a fresh reader each time, so a retry +// within one invocation reuses the same prepared content. +type StagedBlob struct { + // Descriptor identifies the blob by media type, digest, and size. + Descriptor ocispec.Descriptor + + // Role is payload or document. + Role BlobRole + + // DocumentRole is the document role when Role is [BlobDocument]. + DocumentRole DocumentRole + + // Path is the inventory path for per-file payloads; empty for archives. + Path string + + // Open returns a new reader positioned at the start of the blob. The caller closes it. + Open func(ctx context.Context) (io.ReadCloser, error) +} + +// PreparedArtifact is the output of a [Producer]: a sealed Record plus staged blobs. The preparer +// captured one UTC preparation time and one logical operation ID before the first attempt. +// +// A PreparedArtifact owns temporary resources. Whoever finally consumes it must call Close, which +// runs the cleanup function exactly once. It must not be copied after first use. +type PreparedArtifact struct { + // Record is the sealed Record to publish. + Record Record + + // IntentDigest is the Record's intent digest. + IntentDigest digest.Digest + + // OperationID is the logical operation identifier, stable across retries. + OperationID string + + // PreparedAt is the single UTC preparation time. + PreparedAt time.Time + + // Packaging is the packaging mode of the staged payload. + Packaging Packaging + + // Blobs are the staged payload and document blobs. + Blobs []StagedBlob + + // Cleanup releases temporary resources owned by this operation; may be nil. + Cleanup func() error + + closeOnce sync.Once + closeErr error +} + +// Close releases the temporary resources owned by the PreparedArtifact. It is idempotent: the +// cleanup function runs at most once and later calls return the first result. +func (p *PreparedArtifact) Close() error { + defer perf.Track(nil, "artifact.PreparedArtifact.Close")() + + p.closeOnce.Do(func() { + if p.Cleanup != nil { + p.closeErr = p.Cleanup() + } + }) + return p.closeErr +} diff --git a/pkg/artifact/producer.go b/pkg/artifact/producer.go new file mode 100644 index 00000000000..f55863b9ea5 --- /dev/null +++ b/pkg/artifact/producer.go @@ -0,0 +1,53 @@ +package artifact + +import ( + "context" + "time" +) + +//go:generate mockgen -package artifact -destination mock_producer.go github.com/cloudposse/atmos/pkg/artifact Producer + +// PrepareRequest carries everything a Producer needs. The service captures PreparedAt (UTC) and +// OperationID once, before the first attempt, and reuses them on every retry. +type PrepareRequest struct { + // Source is the local source file or directory. + Source string + + // Name is the explicit artifact name; empty lets the producer supply one. + Name string + + // Version is the explicit version; empty lets the producer supply one or the operation fails. + Version string + + // Kind is the artifact kind; empty means [KindFiles]. + Kind string + + // Metadata is the effective user metadata (description, labels) merged by the service. + Metadata Metadata + + // Readme overrides the README source path, relative to the source directory. + Readme string + + // Changelog overrides the CHANGELOG source path, relative to the source directory. + Changelog string + + // Packaging is the target repository's effective packaging mode. + Packaging Packaging + + // OperationID is the logical operation identifier. + OperationID string + + // PreparedAt is the single UTC preparation time. + PreparedAt time.Time +} + +// Producer turns a source into a sealed Record plus staged blobs. Producers walk sources with +// lstat, never follow links outside the source, and never mutate the user's source tree. The +// caller owns the returned PreparedArtifact and must Close it. +type Producer interface { + // Kind returns the artifact kind this producer supplies, for example "files". + Kind() string + + // Prepare reads the source and returns a stable, sealed PreparedArtifact. + Prepare(ctx context.Context, request PrepareRequest) (*PreparedArtifact, error) +} diff --git a/pkg/artifact/record.go b/pkg/artifact/record.go new file mode 100644 index 00000000000..abca49d080c --- /dev/null +++ b/pkg/artifact/record.go @@ -0,0 +1,159 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" +) + +// Record is the portable, canonical description of one published artifact. It is independent of +// the repository alias, credentials, resolved storage paths, upload duration, and archive or OCI +// envelope digests; those belong to [Representation] and [OperationResult]. +// +// A Record never contains its own hash. Serialize it with [Record.Canonical]; the struct's field +// order below is the canonical field order and is frozen by test vectors. +type Record struct { + // SchemaVersion is always [SchemaVersion] (1). + SchemaVersion int `json:"schema_version"` + + // Name is the artifact name, e.g. "components/terraform/vpc". + Name string `json:"name"` + + // Version is the explicit version label. + Version string `json:"version"` + + // Kind is the artifact kind, e.g. "files" or "component/terraform". + Kind string `json:"kind"` + + // CreatedAt is the creation time, captured once per publication in UTC. + CreatedAt Timestamp `json:"created_at"` + + // Metadata is the user-visible description, labels, and optional provenance. + Metadata Metadata `json:"metadata"` + + // Inventory lists every payload entry, sorted by path in the canonical form. + Inventory []Entry `json:"inventory"` + + // Documents lists the README and CHANGELOG descriptors, sorted by role in the canonical form. + Documents []Document `json:"documents"` + + // ContentDigest is the SHA-256 of the canonical inventory and document descriptors. + ContentDigest digest.Digest `json:"content_digest"` + + // KindData is optional kind-specific data. It is held as canonical JSON; see [KindData]. + KindData KindData `json:"kind_data,omitempty"` +} + +// Metadata is the descriptive metadata of a Record. No stack, component, or SHA is required. +type Metadata struct { + // Description is a short human-readable description. + Description string `json:"description,omitempty"` + + // Labels are opaque string key/value pairs set by the user. + Labels map[string]string `json:"labels,omitempty"` + + // Provenance is generated CI and source context. It is part of the record digest but excluded + // from the intent digest so that retries and no-op detection are not defeated by run IDs. + Provenance *Provenance `json:"provenance,omitempty"` +} + +// Provenance carries optional CI and source context captured at publication time. +type Provenance struct { + // CI describes the CI run that produced the artifact. + CI *CIProvenance `json:"ci,omitempty"` + + // Source describes the source tree the artifact was built from. + Source *SourceProvenance `json:"source,omitempty"` +} + +// CIProvenance identifies the CI run that published an artifact. +type CIProvenance struct { + // Provider is the CI provider name, e.g. "github". + Provider string `json:"provider,omitempty"` + + // RunID is the provider's run identifier. + RunID string `json:"run_id,omitempty"` + + // RunURL links to the run. + RunURL string `json:"run_url,omitempty"` + + // Job is the job or step name. + Job string `json:"job,omitempty"` +} + +// SourceProvenance identifies the source revision an artifact was built from. +type SourceProvenance struct { + // Repository is the source repository identifier or URL (without credentials). + Repository string `json:"repository,omitempty"` + + // Commit is the source commit SHA. + Commit string `json:"commit,omitempty"` + + // Ref is the branch or tag name. + Ref string `json:"ref,omitempty"` +} + +// EntryType is the type of an inventory entry. +type EntryType string + +// Inventory entry types. Devices, sockets, and other special files are never representable. +const ( + // EntryFile is a regular file. + EntryFile EntryType = "file" + + // EntryDirectory is a directory. + EntryDirectory EntryType = "directory" + + // EntrySymlink is a symbolic link whose target stays inside the package. + EntrySymlink EntryType = "symlink" +) + +// Entry is one inventory item. Hard links are represented as independent regular files. +type Entry struct { + // Path is the relative, slash-separated path inside the package. + Path string `json:"path"` + + // Type is file, directory, or symlink. + Type EntryType `json:"type"` + + // Size is the file size in bytes; zero for directories and symlinks. + Size int64 `json:"size"` + + // Mode holds ordinary permission bits (0o777 mask); privileged bits are always cleared. + // Symlinks have mode zero so equivalent trees share one canonical form. + Mode uint32 `json:"mode"` + + // Digest is the SHA-256 of a regular file's bytes; empty for other types. + Digest digest.Digest `json:"digest,omitempty"` + + // Target is a symlink's relative target; empty for other types. + Target string `json:"target,omitempty"` +} + +// DocumentRole is the role of a package document. +type DocumentRole string + +// Package document roles. +const ( + // DocumentReadme is the package README. + DocumentReadme DocumentRole = "readme" + + // DocumentChangelog is the package CHANGELOG. + DocumentChangelog DocumentRole = "changelog" +) + +// Document describes one distribution document. Catalogs carry descriptors, never full Markdown. +type Document struct { + // Role is readme or changelog. At most one document exists per role. + Role DocumentRole `json:"role"` + + // SourcePath is the original source path, relative to the publish source. + SourcePath string `json:"source_path"` + + // MediaType is the document media type, normally [DocumentMediaType]. + MediaType string `json:"media_type"` + + // Size is the document size in bytes (at most [MaxDocumentSize]). + Size int64 `json:"size"` + + // Digest is the SHA-256 of the document bytes. + Digest digest.Digest `json:"digest"` +} diff --git a/pkg/artifact/record_test.go b/pkg/artifact/record_test.go new file mode 100644 index 00000000000..f70b554d09c --- /dev/null +++ b/pkg/artifact/record_test.go @@ -0,0 +1,193 @@ +package artifact + +import ( + "strings" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestEntryValidate(t *testing.T) { + fileDigest := digest.FromString("x") + tests := []struct { + name string + entry Entry + want error + }{ + {"file", Entry{Path: "a/b.txt", Type: EntryFile, Size: 1, Mode: 0o644, Digest: fileDigest}, nil}, + {"executable file", Entry{Path: "run.sh", Type: EntryFile, Mode: 0o755, Digest: fileDigest}, nil}, + {"directory", Entry{Path: "a", Type: EntryDirectory, Mode: 0o755}, nil}, + {"contained symlink", Entry{Path: "a/link", Type: EntrySymlink, Target: "../b.txt"}, nil}, + {"symlink to own directory", Entry{Path: "a/link", Type: EntrySymlink, Target: "."}, nil}, + {"symlink empty segment", Entry{Path: "a/link", Type: EntrySymlink, Target: "b//c"}, errUtils.ErrArtifactUnsafePath}, + {"symlink escaping the package", Entry{Path: "a/link", Type: EntrySymlink, Target: "../../b"}, errUtils.ErrArtifactUnsafePath}, + {"symlink at root escaping", Entry{Path: "link", Type: EntrySymlink, Target: "../x"}, errUtils.ErrArtifactUnsafePath}, + {"absolute symlink target", Entry{Path: "link", Type: EntrySymlink, Target: "/etc/passwd"}, errUtils.ErrArtifactUnsafePath}, + {"empty symlink target", Entry{Path: "link", Type: EntrySymlink}, errUtils.ErrArtifactUnsafePath}, + {"symlink with mode", Entry{Path: "link", Type: EntrySymlink, Target: "a", Mode: 0o777}, errUtils.ErrArtifactRecordInvalid}, + {"setuid bit", Entry{Path: "su", Type: EntryFile, Mode: 0o4755, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"file without digest", Entry{Path: "a", Type: EntryFile}, errUtils.ErrArtifactRecordInvalid}, + {"file with bad digest", Entry{Path: "a", Type: EntryFile, Digest: "sha256:zz"}, errUtils.ErrArtifactRecordInvalid}, + {"file with negative size", Entry{Path: "a", Type: EntryFile, Size: -1, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"directory with digest", Entry{Path: "a", Type: EntryDirectory, Digest: fileDigest}, errUtils.ErrArtifactRecordInvalid}, + {"device type", Entry{Path: "dev", Type: "device"}, errUtils.ErrArtifactPackagingInvalid}, + {"absolute path", Entry{Path: "/a", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"traversal path", Entry{Path: "a/../b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"empty segment", Entry{Path: "a//b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"backslash path", Entry{Path: `a\b`, Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"control character", Entry{Path: "a\x00b", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"invalid UTF-8", Entry{Path: "a\xffb", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"reserved namespace", Entry{Path: ReservedNamespace + "/v1/refs/x.json", Type: EntryDirectory}, errUtils.ErrArtifactUnsafePath}, + {"reserved namespace prefix only as first segment", Entry{Path: "a/" + ReservedNamespace, Type: EntryDirectory}, nil}, + {"unicode name", Entry{Path: "ünï/名前.txt", Type: EntryFile, Digest: fileDigest}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.entry.Validate() + if tt.want == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.want) + }) + } +} + +func TestValidateInventoryStructure(t *testing.T) { + file := func(p string) Entry { return Entry{Path: p, Type: EntryFile, Digest: digest.FromString(p)} } + dir := func(p string) Entry { return Entry{Path: p, Type: EntryDirectory} } + + _, err := ComputeContentDigest([]Entry{file("a"), file("a")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "duplicate path") + + _, err = ComputeContentDigest([]Entry{file("a"), file("a/b")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "entry nested under a file") + + _, err = ComputeContentDigest([]Entry{{Path: "l", Type: EntrySymlink, Target: "x"}, file("l/b")}, nil) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid, "entry nested under a symlink") + + _, err = ComputeContentDigest([]Entry{dir("a"), file("a/b")}, nil) + require.NoError(t, err) + + // The digest is order independent for the same logical inventory. + first, err := ComputeContentDigest([]Entry{file("b"), file("a")}, nil) + require.NoError(t, err) + second, err := ComputeContentDigest([]Entry{file("a"), file("b")}, nil) + require.NoError(t, err) + assert.Equal(t, first, second) +} + +func TestDocumentValidate(t *testing.T) { + good := Document{Role: DocumentReadme, SourcePath: "README.md", MediaType: DocumentMediaType, Size: 10, Digest: digest.FromString("r")} + require.NoError(t, good.Validate()) + + tests := []struct { + name string + mutate func(d *Document) + want error + }{ + {"unknown role", func(d *Document) { d.Role = "license" }, errUtils.ErrArtifactRecordInvalid}, + {"absolute source", func(d *Document) { d.SourcePath = "/README.md" }, errUtils.ErrArtifactRecordInvalid}, + {"traversing source", func(d *Document) { d.SourcePath = "../README.md" }, errUtils.ErrArtifactRecordInvalid}, + {"no media type", func(d *Document) { d.MediaType = "" }, errUtils.ErrArtifactRecordInvalid}, + {"negative size", func(d *Document) { d.Size = -1 }, errUtils.ErrArtifactRecordInvalid}, + {"oversize document", func(d *Document) { d.Size = MaxDocumentSize + 1 }, errUtils.ErrArtifactTooLarge}, + {"bad digest", func(d *Document) { d.Digest = "nope" }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + doc := good + tt.mutate(&doc) + require.ErrorIs(t, doc.Validate(), tt.want) + }) + } + + boundary := good + boundary.Size = MaxDocumentSize + require.NoError(t, boundary.Validate(), "exactly the limit is allowed") + + _, err := ComputeContentDigest(nil, []Document{good, good}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, "duplicate role") +} + +func TestRecordValidate(t *testing.T) { + tests := []struct { + name string + mutate func(r *Record) + want error + }{ + {"valid", func(*Record) {}, nil}, + {"wrong schema version", func(r *Record) { r.SchemaVersion = 2 }, errUtils.ErrArtifactRecordInvalid}, + {"bad name", func(r *Record) { r.Name = "../x" }, errUtils.ErrArtifactUnsafePath}, + {"bad version", func(r *Record) { r.Version = "1+x" }, errUtils.ErrArtifactRefInvalid}, + {"bad kind", func(r *Record) { r.Kind = "Files" }, errUtils.ErrArtifactRecordInvalid}, + {"missing created_at", func(r *Record) { r.CreatedAt = Timestamp{} }, errUtils.ErrArtifactRecordInvalid}, + {"empty label key", func(r *Record) { r.Metadata.Labels = map[string]string{"": "v"} }, errUtils.ErrArtifactRecordInvalid}, + {"stale content digest", func(r *Record) { r.ContentDigest = digest.FromString("other") }, errUtils.ErrArtifactRecordInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + record := vectorRecordB(t) + tt.mutate(&record) + err := record.Validate() + if tt.want == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.want) + _, canonicalErr := record.Canonical() + require.ErrorIs(t, canonicalErr, tt.want, "an invalid record is never serialized") + }) + } +} + +func TestRecordOversizeRejected(t *testing.T) { + record := vectorRecordB(t) + record.Metadata.Description = strings.Repeat("d", int(MaxRecordSize)) + _, err := record.Canonical() + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + _, err = record.RecordDigest() + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestEmptyProvenanceNormalizesAway(t *testing.T) { + bare := vectorRecordB(t) + withEmpty := vectorRecordB(t) + withEmpty.Metadata.Provenance = &Provenance{CI: &CIProvenance{}, Source: &SourceProvenance{}} + withEmpty.Metadata.Labels = map[string]string{} + + first, err := bare.Canonical() + require.NoError(t, err) + second, err := withEmpty.Canonical() + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) +} + +func TestSealSetsSchemaVersionAndContentDigest(t *testing.T) { + record := Record{Name: "a", Version: "1", Kind: KindFiles, CreatedAt: NewTimestamp(time.Unix(1, 0))} + require.NoError(t, record.Seal()) + assert.Equal(t, SchemaVersion, record.SchemaVersion) + assert.NotEmpty(t, record.ContentDigest) +} + +func TestTimestamp(t *testing.T) { + zone := time.FixedZone("x", 5*3600) + stamp := NewTimestamp(time.Date(2026, 10, 30, 17, 0, 0, 999, zone)) + assert.Equal(t, "2026-10-30T12:00:00Z", stamp.String()) + assert.Equal(t, time.UTC, stamp.Time().Location()) + + var parsed Timestamp + require.NoError(t, parsed.UnmarshalJSON([]byte(`"2026-10-30T12:00:00Z"`))) + assert.Equal(t, stamp.Time(), parsed.Time()) + + for _, bad := range []string{`"2026-10-30T12:00:00+00:00"`, `"2026-10-30T12:00:00.5Z"`, `"2026-10-30 12:00:00"`, `12`, `""`} { + var ts Timestamp + require.ErrorIs(t, ts.UnmarshalJSON([]byte(bad)), errUtils.ErrArtifactRecordInvalid, bad) + } + assert.True(t, Timestamp{}.IsZero()) +} diff --git a/pkg/artifact/ref.go b/pkg/artifact/ref.go new file mode 100644 index 00000000000..667acbf5892 --- /dev/null +++ b/pkg/artifact/ref.go @@ -0,0 +1,165 @@ +package artifact + +import ( + "fmt" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +var ( + // Name segments use the OCI distribution path-component grammar. It is lowercase-only so + // that a name valid here is valid on every backend without case folding or other rewriting. + nameSegmentPattern = regexp.MustCompile(`^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$`) + + // Versions use the OCI tag grammar. Versions are opaque labels, not required to be SemVer. + versionPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) + + // Kind segments are lowercase tokens, for example "component" in "component/terraform". + kindSegmentPattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*$`) +) + +// Ref addresses an artifact by name and exactly one of version or digest. The repository alias is +// a separate request field and never part of a Ref. +// +// The textual forms are "name:version" and "name@sha256:". There is no implicit "latest". +type Ref struct { + // Name is the slash-separated relative artifact name, e.g. "components/terraform/vpc". + Name string `json:"name"` + + // Version is the explicit version label. Mutually exclusive with Digest. + Version string `json:"version,omitempty"` + + // Digest is an immutable record or manifest digest. Mutually exclusive with Version. + Digest digest.Digest `json:"digest,omitempty"` +} + +// ParseRef parses "name:version" or "name@sha256:" and validates the result. +func ParseRef(s string) (Ref, error) { + defer perf.Track(nil, "artifact.ParseRef")() + + if s == "" { + return Ref{}, fmt.Errorf("%w: reference is empty", errUtils.ErrArtifactRefInvalid) + } + var ref Ref + if name, dgst, found := strings.Cut(s, "@"); found { + ref = Ref{Name: name, Digest: digest.Digest(dgst)} + } else { + index := strings.LastIndex(s, ":") + if index < 0 { + return Ref{}, fmt.Errorf("%w: %q needs a version (name:version) or digest (name@sha256:...); there is no implicit latest", errUtils.ErrArtifactRefInvalid, s) + } + ref = Ref{Name: s[:index], Version: s[index+1:]} + } + if err := ref.Validate(); err != nil { + return Ref{}, err + } + return ref, nil +} + +// Validate checks the name and that exactly one of version or digest is set and well formed. +func (r Ref) Validate() error { + defer perf.Track(nil, "artifact.Ref.Validate")() + + if err := ValidateName(r.Name); err != nil { + return err + } + switch { + case r.Version != "" && r.Digest != "": + return fmt.Errorf("%w: %q sets both a version and a digest", errUtils.ErrArtifactRefInvalid, r.Name) + case r.Version == "" && r.Digest == "": + return fmt.Errorf("%w: %q needs a version or a digest", errUtils.ErrArtifactRefInvalid, r.Name) + case r.Version != "": + return ValidateVersion(r.Version) + default: + return ValidateDigest(r.Digest) + } +} + +// String returns the textual form of the reference. +func (r Ref) String() string { + defer perf.Track(nil, "artifact.Ref.String")() + + if r.Digest != "" { + return r.Name + "@" + r.Digest.String() + } + return r.Name + ":" + r.Version +} + +// IsDigest reports whether the reference selects an immutable digest. +// +//nolint:lintroller // Trivial getter with no I/O. +func (r Ref) IsDigest() bool { return r.Digest != "" } + +// ValidateName checks an artifact name: relative, slash-separated, with no empty, "." or ".." +// segments and each segment matching the OCI path-component grammar. Names are never rewritten. +func ValidateName(name string) error { + defer perf.Track(nil, "artifact.ValidateName")() + + if name == "" { + return fmt.Errorf("%w: name is empty", errUtils.ErrArtifactRefInvalid) + } + if len(name) > MaxNameLength { + return fmt.Errorf("%w: name exceeds %d characters", errUtils.ErrArtifactRefInvalid, MaxNameLength) + } + if strings.HasPrefix(name, "/") || strings.Contains(name, `\`) { + return fmt.Errorf("%w: %w: %q must be relative and slash-separated", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name) + } + for _, segment := range strings.Split(name, "/") { + switch segment { + case "": + return fmt.Errorf("%w: %w: %q has an empty segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name) + case ".", "..": + return fmt.Errorf("%w: %w: %q has a %q segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name, segment) + } + if !nameSegmentPattern.MatchString(segment) { + return fmt.Errorf("%w: segment %q of %q must be lowercase alphanumerics separated by '.', '_', '__', or '-'", errUtils.ErrArtifactRefInvalid, segment, name) + } + } + return nil +} + +// ValidateVersion checks a version against the OCI tag grammar. Versions are never normalized. +func ValidateVersion(version string) error { + defer perf.Track(nil, "artifact.ValidateVersion")() + + if version == "" { + return fmt.Errorf("%w: version is empty", errUtils.ErrArtifactRefInvalid) + } + if !versionPattern.MatchString(version) { + return fmt.Errorf("%w: version %q must match the OCI tag syntax [A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", errUtils.ErrArtifactRefInvalid, version) + } + return nil +} + +// ValidateDigest checks that d is a well-formed sha256 digest. V1 supports sha256 only. +func ValidateDigest(d digest.Digest) error { + defer perf.Track(nil, "artifact.ValidateDigest")() + + if err := d.Validate(); err != nil { + return fmt.Errorf("%w: digest %q: %w", errUtils.ErrArtifactRefInvalid, d, err) + } + if d.Algorithm() != digest.SHA256 { + return fmt.Errorf("%w: digest %q must use sha256", errUtils.ErrArtifactRefInvalid, d) + } + return nil +} + +// ValidateKind checks an artifact kind such as "files", "component/terraform", or "catalog". +func ValidateKind(kind string) error { + defer perf.Track(nil, "artifact.ValidateKind")() + + if kind == "" || len(kind) > MaxKindLength { + return fmt.Errorf("%w: kind must be 1-%d characters", errUtils.ErrArtifactRecordInvalid, MaxKindLength) + } + for _, segment := range strings.Split(kind, "/") { + if !kindSegmentPattern.MatchString(segment) { + return fmt.Errorf("%w: kind %q must be slash-separated lowercase segments", errUtils.ErrArtifactRecordInvalid, kind) + } + } + return nil +} diff --git a/pkg/artifact/ref_test.go b/pkg/artifact/ref_test.go new file mode 100644 index 00000000000..2d2b2588151 --- /dev/null +++ b/pkg/artifact/ref_test.go @@ -0,0 +1,117 @@ +package artifact + +import ( + "errors" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +const testSHA = "sha256:0d6e4079e36703ebd37c00722f5891d28b0e2811dc114b129215123adcce3605" + +func TestParseRef(t *testing.T) { + tests := []struct { + name string + input string + want Ref + }{ + {"nested name and version", "components/terraform/vpc:1.2.0", Ref{Name: "components/terraform/vpc", Version: "1.2.0"}}, + {"single segment", "vpc:v1", Ref{Name: "vpc", Version: "v1"}}, + {"CI build identifier version", "app:build-2026.10.30_7", Ref{Name: "app", Version: "build-2026.10.30_7"}}, + {"digest reference", "components/vpc@" + testSHA, Ref{Name: "components/vpc", Digest: testSHA}}, + {"separators in segments", "a.b/c_d/e__f/g--h:1", Ref{Name: "a.b/c_d/e__f/g--h", Version: "1"}}, + {"128 character version", "a:" + strings.Repeat("v", 128), Ref{Name: "a", Version: strings.Repeat("v", 128)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRef(tt.input) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.input, got.String(), "String round-trips the textual form") + assert.Equal(t, tt.want.Digest != "", got.IsDigest()) + }) + } +} + +func TestParseRefRejects(t *testing.T) { + tests := []struct { + name string + input string + unsafe bool + }{ + {name: "empty", input: ""}, + {name: "no version or digest", input: "components/vpc"}, + {name: "empty version", input: "vpc:"}, + {name: "empty name", input: ":1.0"}, + {name: "absolute path", input: "/abs/path:1", unsafe: true}, + {name: "traversal segment", input: "a/../b:1", unsafe: true}, + {name: "leading traversal", input: "../a:1", unsafe: true}, + {name: "dot segment", input: "a/./b:1", unsafe: true}, + {name: "empty segment", input: "a//b:1", unsafe: true}, + {name: "trailing slash", input: "a/b/:1", unsafe: true}, + {name: "backslash", input: `a\b:1`, unsafe: true}, + {name: "uppercase is not normalized", input: "Components/Vpc:1"}, + {name: "space", input: "a b:1"}, + {name: "reserved namespace shaped name", input: ".atmos-artifacts/x:1", unsafe: false}, + {name: "version with plus", input: "a:1.0+build"}, + {name: "version starting with dash", input: "a:-1"}, + {name: "version starting with dot", input: "a:.1"}, + {name: "version too long", input: "a:" + strings.Repeat("v", 129)}, + {name: "version and digest", input: "a:1@" + testSHA}, + {name: "unsupported digest algorithm", input: "a@sha512:" + strings.Repeat("a", 128)}, + {name: "short digest", input: "a@sha256:abc"}, + {name: "uppercase digest hex", input: "a@sha256:" + strings.ToUpper(strings.TrimPrefix(testSHA, "sha256:"))}, + {name: "digest without name", input: "@" + testSHA}, + {name: "name too long", input: strings.Repeat("a/", 128) + "a:1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseRef(tt.input) + require.Error(t, err) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Equal(t, tt.unsafe, isUnsafe(err), "unsafe-path classification of %v", err) + }) + } +} + +func isUnsafe(err error) bool { + return errors.Is(err, errUtils.ErrArtifactUnsafePath) +} + +func TestRefValidate(t *testing.T) { + tests := []struct { + name string + ref Ref + ok bool + }{ + {"version", Ref{Name: "a", Version: "1"}, true}, + {"digest", Ref{Name: "a", Digest: testSHA}, true}, + {"neither", Ref{Name: "a"}, false}, + {"both", Ref{Name: "a", Version: "1", Digest: digest.Digest(testSHA)}, false}, + {"bad name", Ref{Name: "A", Version: "1"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.ref.Validate() + if tt.ok { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) + } +} + +func TestValidateKind(t *testing.T) { + for _, kind := range []string{"files", "component/terraform", "toolchain.package", "ci/planfile", "catalog"} { + assert.NoError(t, ValidateKind(kind), kind) + } + for _, kind := range []string{"", "Files", "a//b", "/a", "a/", "a b", strings.Repeat("k", MaxKindLength+1)} { + assert.ErrorIs(t, ValidateKind(kind), errUtils.ErrArtifactRecordInvalid, kind) + } +} diff --git a/pkg/artifact/repository.go b/pkg/artifact/repository.go new file mode 100644 index 00000000000..9123c784a77 --- /dev/null +++ b/pkg/artifact/repository.go @@ -0,0 +1,54 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_repository.go github.com/cloudposse/atmos/pkg/artifact Repository,GraphSource + +// Repository is the storage contract implemented by every backend (local directory, local archive, +// S3, OCI layout, OCI registry). Repositories own storage mechanics and capabilities; the service +// owns name and version selection, producer preparation, metadata, digest pinning, conflict policy, +// retries, and result aggregation. Signatures, verification, pinning, and locks are repository +// contracts for every backend, never OCI-only service branches. +// +// An unavailable capability returns a *CapabilityError, never a successful no-op. Backend errors +// are wrapped in the common sentinels (errors/errors.go) while retaining their causes. +type Repository interface { + LockProvider + + // Capabilities describes the repository without network access. + Capabilities() Capabilities + + // Resolve resolves a reference once and returns a pinned handle used for all subsequent reads. + Resolve(ctx context.Context, ref Ref) (PinnedReference, error) + + // Inspect returns the metadata view of a pinned artifact without downloading the payload. + Inspect(ctx context.Context, pinned PinnedReference) (*Inspection, error) + + // OpenDocument opens one document without downloading the payload. A missing optional + // document returns an error wrapping ErrArtifactDocumentMissing. + OpenDocument(ctx context.Context, pinned PinnedReference, role DocumentRole) (*DocumentStream, error) + + // Read returns verified, closeable content streams or an OCI graph source for a pinned artifact. + Read(ctx context.Context, pinned PinnedReference) (*Content, error) + + // Publish stores a prepared artifact. The expected prior reference and force policy have already + // been checked by the service under the writer guard. + Publish(ctx context.Context, prepared *PreparedArtifact, options PublishOptions) (*PublishResult, error) + + // List returns one page of managed references. Pagination is consumed by the service. + List(ctx context.Context, query ListQuery) (*ListPage, error) + + // Delete removes a pinned artifact within an explicit scope after checking the expected current reference. + Delete(ctx context.Context, pinned PinnedReference, options DeleteOptions) (*DeleteResult, error) + + // Pin resolves a reference into an immutable PinnedReference suitable for consumer lockfiles. + // It acquires no write lock and is not a substitute for signature verification. + Pin(ctx context.Context, ref Ref) (PinnedReference, error) + + // Sign asks signer to sign the subject and stores the returned evidence, returning its descriptors. + Sign(ctx context.Context, subject SignatureSubject, signer Signer, options SignOptions) ([]SignatureEvidence, error) + + // Verify authenticates stored evidence against policy. It never trusts metadata claims, catalog + // entries, labels, or cached "verified" booleans. + Verify(ctx context.Context, subject SignatureSubject, policy VerificationPolicy) (*VerificationResult, error) +} diff --git a/pkg/artifact/repository_types.go b/pkg/artifact/repository_types.go new file mode 100644 index 00000000000..9e92982da29 --- /dev/null +++ b/pkg/artifact/repository_types.go @@ -0,0 +1,226 @@ +package artifact + +import ( + "context" + "io" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Label is an exact key/value selector or assignment. +type Label struct { + // Key is the label key. + Key string `json:"key"` + + // Value is the label value. + Value string `json:"value"` +} + +// Inspection is the metadata view of one artifact, available without downloading the payload. +type Inspection struct { + // Pinned is the immutable reference that was inspected. + Pinned PinnedReference `json:"pinned"` + + // Record is the portable Record; nil for a foreign OCI graph that carries none. + Record *Record `json:"record,omitempty"` + + // Representation describes how the artifact is stored. + Representation Representation `json:"representation"` + + // Evidence lists signature evidence found for the artifact. It is untrusted until verified. + Evidence []SignatureEvidence `json:"evidence,omitempty"` +} + +// DocumentStream is an open, digest-verifying reader over one package document. +type DocumentStream struct { + // Document is the descriptor from the Record. + Document Document + + // Reader yields the document bytes and fails with an integrity error at EOF on mismatch. + // The caller closes it. Callers bound reads to [MaxDocumentSize]. + Reader io.ReadCloser +} + +// ContentStream is one verified, closeable payload stream. +type ContentStream struct { + // Descriptor identifies the blob. + Descriptor ocispec.Descriptor + + // Role is payload or document. + Role BlobRole + + // Path is the inventory path for per-file payloads; empty for archives. + Path string + + // Reader yields the blob bytes and verifies the descriptor digest and size. + Reader io.ReadCloser +} + +// GraphSource exposes an existing OCI graph for transfer without importing a registry SDK. +type GraphSource interface { + // Root returns the original manifest or index descriptor. + Root() ocispec.Descriptor + + // Fetch returns the raw bytes of a graph node by descriptor. + Fetch(ctx context.Context, descriptor ocispec.Descriptor) (io.ReadCloser, error) + + // Successors lists the descriptors a node references (config, layers, children). + Successors(ctx context.Context, descriptor ocispec.Descriptor) ([]ocispec.Descriptor, error) +} + +// Content is the result of reading an artifact: either verified payload streams or an OCI graph +// source. The caller must Close it. +type Content struct { + // Record is the portable Record; nil for a foreign OCI graph that carries none. + Record *Record + + // Representation describes how the content is stored. + Representation Representation + + // Streams are the verified payload and document streams. Empty when Graph is set. + Streams []ContentStream + + // Graph is the OCI graph source when a native graph is carried. + Graph GraphSource +} + +// Close closes every stream, returning the first error. +func (c *Content) Close() error { + defer perf.Track(nil, "artifact.Content.Close")() + + var first error + for index := range c.Streams { + reader := c.Streams[index].Reader + if reader == nil { + continue + } + if err := reader.Close(); err != nil && first == nil { + first = err + } + } + return first +} + +// PublishOptions control one repository publication. +type PublishOptions struct { + // ExpectedPrior is the current reference the service observed under the writer guard; nil + // means the reference must not exist. Repositories use it for conditional updates. + ExpectedPrior *PinnedReference + + // Force permits replacing a conflicting reference. It has already been checked by the service. + Force bool + + // Lock is the writer lock already held by the service; nil if the repository locks internally. + Lock LockHandle +} + +// PublishResult is the outcome of a repository publication. +type PublishResult struct { + // Outcome is created, unchanged, or replaced. + Outcome Outcome + + // Pinned is the immutable reference of the published artifact. + Pinned PinnedReference + + // Representation describes the stored form. + Representation Representation + + // Location is the primary resolved location. + Location string + + // Aliases lists additional locations or tags resolving to the same content. + Aliases []string + + // LockScope is the actual lock scope that coordinated the write. + LockScope LockScope +} + +// ListQuery selects one page of managed references. The service consumes pagination. +type ListQuery struct { + // Name restricts the listing to versions of one exact name; empty enumerates the collection. + Name string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Cursor is the opaque cursor from the previous page; empty starts at the beginning. + Cursor string + + // PageSize bounds the page; zero lets the repository choose. + PageSize int + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// ListEntry summarizes one artifact in a listing. +type ListEntry struct { + // Pinned is the immutable reference. + Pinned PinnedReference `json:"pinned"` + + // Name is the artifact name. + Name string `json:"name"` + + // Version is the version label. + Version string `json:"version"` + + // Kind is the artifact kind. + Kind string `json:"kind"` + + // Description is the metadata description. + Description string `json:"description,omitempty"` + + // Labels are the metadata labels. + Labels map[string]string `json:"labels,omitempty"` + + // Documents are the document descriptors, never the Markdown itself. + Documents []Document `json:"documents,omitempty"` + + // Repository is the configured repository alias the entry came from. + Repository string `json:"repository,omitempty"` +} + +// ListPage is one page of entries plus a cursor. +type ListPage struct { + // Entries are the page's artifacts, sorted lexically by name then version. + Entries []ListEntry + + // NextCursor is empty when the page is the last. + NextCursor string +} + +// DeleteOptions control one repository deletion. +type DeleteOptions struct { + // ExpectedCurrent is the reference the service observed; nil skips the staleness check. + ExpectedCurrent *PinnedReference + + // Scope is the explicit deletion scope. A tag-safe delete must not remove sibling references. + Scope DeleteSemantics + + // Lock is the writer lock already held by the service. + Lock LockHandle +} + +// DeleteResult reports what a deletion removed and affected. +type DeleteResult struct { + // Deleted lists removed locations. + Deleted []string + + // Affected lists other references that resolved to the removed content. + Affected []Ref + + // Digest is the removed resolved digest. + Digest digest.Digest +} + +// SignOptions control evidence storage. +type SignOptions struct { + // OperationID is the logical operation identifier. + OperationID string + + // Lock is the writer lock already held by the service. + Lock LockHandle +} diff --git a/pkg/artifact/representation.go b/pkg/artifact/representation.go new file mode 100644 index 00000000000..f43c7309ffc --- /dev/null +++ b/pkg/artifact/representation.go @@ -0,0 +1,31 @@ +package artifact + +import ( + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Representation describes how one Record is physically stored by a repository. It is mutable +// storage metadata and is never part of the Record bytes. +type Representation struct { + // Packaging is the payload packaging mode. + Packaging Packaging `json:"packaging"` + + // Payload lists the payload blob descriptors (archive, per-file objects, or image layers). + Payload []ocispec.Descriptor `json:"payload,omitempty"` + + // Documents lists the README and CHANGELOG blob descriptors stored alongside the payload. + Documents []ocispec.Descriptor `json:"documents,omitempty"` + + // Locator is the backend-specific location (path, object key, or registry reference). + Locator string `json:"locator"` + + // RecordDigest is the digest of the canonical Record bytes this representation belongs to. + RecordDigest digest.Digest `json:"record_digest"` + + // RepresentationDigest is the checksum of an archive or other transferred payload representation. + RepresentationDigest digest.Digest `json:"representation_digest,omitempty"` + + // OriginalRoot is the original OCI manifest or index descriptor when a native graph is carried. + OriginalRoot *ocispec.Descriptor `json:"original_root,omitempty"` +} diff --git a/pkg/artifact/requests.go b/pkg/artifact/requests.go new file mode 100644 index 00000000000..a2b19a4f13c --- /dev/null +++ b/pkg/artifact/requests.go @@ -0,0 +1,151 @@ +package artifact + +// PublishRequest publishes a local source to one repository. +type PublishRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Source is the local source file or directory. + Source string + + // Name is the explicit artifact name. Precedence: explicit, producer metadata, source basename. + Name string + + // Version is the explicit version. Precedence: explicit, producer metadata; otherwise fail. + Version string + + // Kind is the artifact kind; empty means [KindFiles]. + Kind string + + // Metadata is the explicit user metadata (description and labels). + Metadata Metadata + + // Readme overrides the README source path, relative to the source directory. + Readme string + + // Changelog overrides the CHANGELOG source path, relative to the source directory. + Changelog string + + // Force permits replacing an existing different artifact where the repository allows it. + Force bool + + // DryRun validates and plans without writing. + DryRun bool +} + +// PullRequest materializes an artifact into a local directory. +type PullRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref + + // Destination is the output directory. + Destination string + + // Force permits overwriting existing destination content. + Force bool +} + +// CopyRequest copies an artifact between two configured repositories. +type CopyRequest struct { + // Ref selects the artifact in the source repository. + Ref Ref + + // From is the source repository alias. + From string + + // To is the destination repository alias. + To string + + // Force permits replacing a conflicting destination. It never bypasses verification. + Force bool +} + +// InspectRequest inspects an artifact without downloading its payload. +type InspectRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// ListRequest lists artifacts. List and search intentionally default to all configured repositories. +type ListRequest struct { + // Repositories restricts the listing; empty means all configured repositories. + Repositories []string + + // Name restricts the listing to versions of one exact name. + Name string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// SearchRequest searches artifacts by text. +type SearchRequest struct { + // Repositories restricts the search; empty means all configured repositories. + Repositories []string + + // Query is the search text matched against names, kinds, descriptions, and labels. + Query string + + // Labels keep only entries whose labels match every selector exactly. + Labels []Label + + // Refresh bypasses cached discovery snapshots. + Refresh bool +} + +// DeleteRequest deletes an artifact from one repository. +type DeleteRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// ShowDocumentRequest reads one package document. +type ShowDocumentRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref + + // Role is readme or changelog. + Role DocumentRole +} + +// SignRequest adds signature evidence to an immutable subject. +type SignRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref must be a digest reference; sign accepts digest references only. + Ref Ref +} + +// VerifyRequest verifies an artifact's signature evidence against the effective policy. +type VerifyRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} + +// PinRequest creates an immutable reference for consumer lockfiles. +type PinRequest struct { + // Repository is the required configured repository alias. + Repository string + + // Ref selects the artifact. + Ref Ref +} diff --git a/pkg/artifact/results.go b/pkg/artifact/results.go new file mode 100644 index 00000000000..ee7e34a3762 --- /dev/null +++ b/pkg/artifact/results.go @@ -0,0 +1,78 @@ +package artifact + +// InspectResult is the result of Service.Inspect. +type InspectResult struct { + // Operation is the typed operation result stored under StepResult.Metadata["artifact"]. + Operation OperationResult `json:"operation"` + + // Inspection is the metadata view. + Inspection Inspection `json:"inspection"` + + // Verification is set when policy required or constrained verification; nil means unchecked. + Verification *VerificationResult `json:"verification,omitempty"` +} + +// RepositoryListing reports the outcome of listing one repository, so a failing or incapable +// repository is explained rather than silently omitted. +type RepositoryListing struct { + // Repository is the configured repository alias. + Repository string `json:"repository"` + + // Count is the number of entries returned from the repository. + Count int `json:"count"` + + // Diagnostics explain capability or refresh problems. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} + +// ListResult is the result of Service.List and Service.Search. Its JSON name for the entries is +// "artifacts". +type ListResult struct { + // Artifacts are the entries across the selected repositories, sorted lexically. + Artifacts []ListEntry `json:"artifacts"` + + // Repositories reports per-repository outcomes. + Repositories []RepositoryListing `json:"repositories,omitempty"` +} + +// DocumentResult is the result of Service.ShowDocument. Content is bounded by [MaxDocumentSize]. +type DocumentResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Document is the descriptor from the Record. + Document Document `json:"document"` + + // Content is the verified document bytes. + Content []byte `json:"content"` +} + +// SignResult is the result of Service.Sign. +type SignResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Subject is the signed subject. + Subject SignatureSubject `json:"subject"` + + // Evidence lists the stored evidence descriptors. + Evidence []SignatureEvidence `json:"evidence"` +} + +// VerifyResult is the result of Service.Verify. +type VerifyResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Verification is the authenticated verification result. + Verification VerificationResult `json:"verification"` +} + +// PinResult is the result of Service.Pin. +type PinResult struct { + // Operation is the typed operation result. + Operation OperationResult `json:"operation"` + + // Pinned is the immutable reference. + Pinned PinnedReference `json:"pinned"` +} diff --git a/pkg/artifact/service.go b/pkg/artifact/service.go new file mode 100644 index 00000000000..8cfd2fd4f03 --- /dev/null +++ b/pkg/artifact/service.go @@ -0,0 +1,46 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_service.go github.com/cloudposse/atmos/pkg/artifact Service + +// Service is the repository-independent artifact API consumed by commands, steps, hooks, and CI +// adapters. This package defines only the interface and its request and result types; the +// implementation (preparation, packaging, conflict policy, retries) lives in the service package. +// +// Every method takes a context first and a typed request struct, and returns a typed result plus +// an error wrapping the sentinels in errors/errors.go. Display format never changes semantics. +type Service interface { + // Publish prepares the source and stores it in one repository. + Publish(ctx context.Context, request PublishRequest) (*OperationResult, error) + + // Pull materializes an artifact into a local directory after verifying every consumed descriptor. + Pull(ctx context.Context, request PullRequest) (*OperationResult, error) + + // Copy transfers an artifact between repositories, evaluating both source and destination policy. + Copy(ctx context.Context, request CopyRequest) (*OperationResult, error) + + // Inspect returns the metadata view without downloading the payload. + Inspect(ctx context.Context, request InspectRequest) (*InspectResult, error) + + // List enumerates artifacts across the selected repositories. + List(ctx context.Context, request ListRequest) (*ListResult, error) + + // Search filters enumerated artifacts by text. + Search(ctx context.Context, request SearchRequest) (*ListResult, error) + + // Delete removes an artifact within the repository's declared delete capability. + Delete(ctx context.Context, request DeleteRequest) (*OperationResult, error) + + // ShowDocument reads one README or CHANGELOG without downloading the payload. + ShowDocument(ctx context.Context, request ShowDocumentRequest) (*DocumentResult, error) + + // Sign adds signature evidence to a digest reference. + Sign(ctx context.Context, request SignRequest) (*SignResult, error) + + // Verify requires a trusted valid signature for the artifact. + Verify(ctx context.Context, request VerifyRequest) (*VerifyResult, error) + + // Pin returns an immutable reference for consumer lockfiles. + Pin(ctx context.Context, request PinRequest) (*PinResult, error) +} diff --git a/pkg/artifact/signature.go b/pkg/artifact/signature.go new file mode 100644 index 00000000000..0eb446efd1a --- /dev/null +++ b/pkg/artifact/signature.go @@ -0,0 +1,145 @@ +package artifact + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// SubjectKind identifies what a signature is bound to. +type SubjectKind string + +// Signature subject kinds. +const ( + // SubjectRecord is the canonical portable Record of an Atmos file package. Evidence survives + // representation changes because the Record binds name, version, metadata, inventory, + // content_digest, and document digests. + SubjectRecord SubjectKind = "record" + + // SubjectOCIRoot is the original OCI manifest or index digest of a foreign image or an + // existing plugin. It is never rewritten on copy and never replaced by an archive checksum. + SubjectOCIRoot SubjectKind = "oci-root" +) + +// EvidenceFormat identifies a signature evidence encoding. +type EvidenceFormat string + +// Evidence formats supported in V1. Other formats require dedicated tested providers; unsupported +// required evidence fails closed. +const ( + // EvidenceSigstoreBundle is a Sigstore bundle verified against trusted certificate identities. + EvidenceSigstoreBundle EvidenceFormat = "sigstore-bundle" + + // EvidencePublicKeyBundle is a bundle verified against an explicitly configured public key. + // It never satisfies an OIDC issuer/identity requirement. + EvidencePublicKeyBundle EvidenceFormat = "public-key-bundle" +) + +// SignatureSubject is the immutable thing a signature covers. +type SignatureSubject struct { + // Kind is record or oci-root. + Kind SubjectKind `json:"kind"` + + // Digest is the immutable subject digest (record digest or OCI root digest). + Digest digest.Digest `json:"digest"` + + // MediaType is the subject media type. + MediaType string `json:"media_type"` + + // Size is the subject size in bytes. + Size int64 `json:"size"` +} + +// Validate checks the subject is well formed. +func (s SignatureSubject) Validate() error { + defer perf.Track(nil, "artifact.SignatureSubject.Validate")() + + if s.Kind != SubjectRecord && s.Kind != SubjectOCIRoot { + return fmt.Errorf("%w: unsupported signature subject kind %q", errUtils.ErrArtifactRefInvalid, s.Kind) + } + if err := ValidateDigest(s.Digest); err != nil { + return err + } + if s.Size < 0 { + return fmt.Errorf("%w: signature subject has a negative size", errUtils.ErrArtifactRefInvalid) + } + return nil +} + +// SignatureEvidence locates stored evidence for a subject. It is untrusted until verified. +type SignatureEvidence struct { + // Subject is the subject the evidence claims to cover. + Subject SignatureSubject `json:"subject"` + + // Format is the evidence encoding. + Format EvidenceFormat `json:"format"` + + // Descriptor identifies the evidence blob (media type, digest, size). + Descriptor ocispec.Descriptor `json:"descriptor"` + + // Locator is the backend-specific storage location, for example + // ".atmos-artifacts/v1/signatures///.json". + Locator string `json:"locator,omitempty"` +} + +// EvidenceBlob is evidence together with its bytes, bounded by [MaxEvidenceSize]. +type EvidenceBlob struct { + // Evidence locates and describes the blob. + Evidence SignatureEvidence + + // Data holds the evidence bytes. + Data []byte +} + +// VerificationStatus is the outcome of a verification. +type VerificationStatus string + +// Verification statuses. +const ( + // VerificationVerified means a valid permitted signature satisfied every required trust set. + VerificationVerified VerificationStatus = "verified" + + // VerificationUnchecked means no trust configuration was supplied, so nothing was claimed. + VerificationUnchecked VerificationStatus = "unchecked" + + // VerificationFailed means required verification did not succeed. + VerificationFailed VerificationStatus = "failed" +) + +// SignerClaim is an authenticated signer identity extracted by a verifier. Metadata claims, +// labels, catalog entries, and cached booleans never produce claims. +type SignerClaim struct { + // Identity is the certificate identity (SAN) for keyless signing; empty for key signing. + Identity string `json:"identity,omitempty"` + + // Issuer is the OIDC issuer for keyless signing; empty for key signing. + Issuer string `json:"issuer,omitempty"` + + // KeyID is the fingerprint of the trusted public key for key-based signing. + KeyID string `json:"key_id,omitempty"` +} + +// VerificationResult is the authenticated result of verifying a subject against a policy. +type VerificationResult struct { + // Subject is the verified subject. + Subject SignatureSubject `json:"subject"` + + // PolicyFingerprint is [VerificationPolicy.Fingerprint] of the policy that was applied. + PolicyFingerprint string `json:"policy_fingerprint,omitempty"` + + // Claims are the verified signer claims. + Claims []SignerClaim `json:"claims,omitempty"` + + // EvidenceDigests lists the digests of the evidence blobs that were considered. + EvidenceDigests []digest.Digest `json:"evidence_digests,omitempty"` + + // Status is verified, unchecked, or failed. + Status VerificationStatus `json:"status"` + + // Diagnostics describes the failure reason and any bounded malformed or unsupported evidence. + Diagnostics []Diagnostic `json:"diagnostics,omitempty"` +} diff --git a/pkg/artifact/signer.go b/pkg/artifact/signer.go new file mode 100644 index 00000000000..22698e99417 --- /dev/null +++ b/pkg/artifact/signer.go @@ -0,0 +1,28 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_signer.go github.com/cloudposse/atmos/pkg/artifact Signer + +// EvidencePayload is freshly produced signature evidence, bounded by [MaxEvidenceSize]. +type EvidencePayload struct { + // Format is the evidence encoding. + Format EvidenceFormat + + // MediaType is the evidence media type. + MediaType string + + // Data holds the evidence bytes. + Data []byte +} + +// Signer produces signature evidence for a subject. V1 reuses the managed Cosign authoring path +// behind this interface; registry authentication stays separate from signing identity, and no +// custom cryptography is implemented. +type Signer interface { + // Formats lists the evidence formats the signer can produce. + Formats() []EvidenceFormat + + // Sign signs the immutable subject and returns evidence for the repository to store. + Sign(ctx context.Context, subject SignatureSubject) (*EvidencePayload, error) +} diff --git a/pkg/artifact/timestamp.go b/pkg/artifact/timestamp.go new file mode 100644 index 00000000000..6ecb7560f6c --- /dev/null +++ b/pkg/artifact/timestamp.go @@ -0,0 +1,73 @@ +package artifact + +import ( + "encoding/json" + "fmt" + "regexp" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// timestampLayout is the canonical UTC RFC3339 layout with second precision. +const timestampLayout = "2006-01-02T15:04:05Z" + +// Only the canonical text is admitted. The standard time parser alone would also accept fractional +// seconds even though the layout has none. +var timestampPattern = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$`) + +// Timestamp is an instant serialized as a UTC RFC3339 string with second precision. +// Sub-second precision is discarded on serialization so that an in-memory value and the value +// decoded from its canonical bytes always produce identical digests. +type Timestamp time.Time + +// NewTimestamp converts t to a Timestamp normalized to UTC and truncated to whole seconds. +func NewTimestamp(t time.Time) Timestamp { + defer perf.Track(nil, "artifact.NewTimestamp")() + + return Timestamp(t.UTC().Truncate(time.Second)) +} + +// Time returns the instant as a time.Time. +// +//nolint:lintroller // Trivial conversion with no I/O. +func (t Timestamp) Time() time.Time { return time.Time(t) } + +// IsZero reports whether the timestamp is the zero instant. +// +//nolint:lintroller // Trivial getter with no I/O. +func (t Timestamp) IsZero() bool { return time.Time(t).IsZero() } + +// String returns the canonical representation. +// +//nolint:lintroller // Trivial formatter with no I/O. +func (t Timestamp) String() string { + return time.Time(t).UTC().Truncate(time.Second).Format(timestampLayout) +} + +// MarshalJSON implements json.Marshaler using the canonical layout. +func (t Timestamp) MarshalJSON() ([]byte, error) { + defer perf.Track(nil, "artifact.Timestamp.MarshalJSON")() + + return json.Marshal(t.String()) +} + +// UnmarshalJSON implements json.Unmarshaler. Only the canonical layout is accepted. +func (t *Timestamp) UnmarshalJSON(data []byte) error { + defer perf.Track(nil, "artifact.Timestamp.UnmarshalJSON")() + + var raw string + if err := json.Unmarshal(data, &raw); err != nil { + return fmt.Errorf("%w: timestamp must be a string: %w", errUtils.ErrArtifactRecordInvalid, err) + } + if !timestampPattern.MatchString(raw) { + return fmt.Errorf("%w: timestamp %q must use the form 2006-01-02T15:04:05Z", errUtils.ErrArtifactRecordInvalid, raw) + } + parsed, err := time.Parse(timestampLayout, raw) + if err != nil { + return fmt.Errorf("%w: timestamp %q is not a UTC RFC3339 time: %w", errUtils.ErrArtifactRecordInvalid, raw, err) + } + *t = Timestamp(parsed) + return nil +} diff --git a/pkg/artifact/vectors_test.go b/pkg/artifact/vectors_test.go new file mode 100644 index 00000000000..1384846bf5e --- /dev/null +++ b/pkg/artifact/vectors_test.go @@ -0,0 +1,288 @@ +package artifact + +import ( + "bytes" + "errors" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// Frozen canonical byte vectors. These literals are the contract: they were cross-checked with an +// independent JSON canonicalizer and must never be regenerated to hide an incompatible change. +// A change to any value below changes the identity of every previously published artifact. +const ( + // Vector A: files package with every entry type, documents, labels, provenance, escapes, Unicode. + vectorACanonical = `{"content_digest":"sha256:9bd2edcda28887b6bc03cba0a811460d0abfb4f35633a7cf1eb7908f43ec4fea","created_at":"2026-10-30T12:34:56Z","documents":[{"digest":"sha256:6665bc2f57c4a7045d347313a1566d91561ff2a3c9d17874a462ae73b7eb8593","media_type":"text/markdown","role":"changelog","size":9,"source_path":".atmos/artifact/CHANGELOG.md"},{"digest":"sha256:711a6108ba2ce6ca93dd47d6817f2361db10d8ab6eec89460b2dfc2c325efabe","media_type":"text/markdown","role":"readme","size":5,"source_path":".atmos/artifact/README.md"}],"inventory":[{"mode":0,"path":"link.tf","size":0,"target":"main.tf","type":"symlink"},{"digest":"sha256:0d6e4079e36703ebd37c00722f5891d28b0e2811dc114b129215123adcce3605","mode":420,"path":"main.tf","size":8,"type":"file"},{"mode":493,"path":"scripts","size":0,"type":"directory"},{"digest":"sha256:acba25512100f80b56fc3ccd14c65be55d94800cda77585c5f41a887e398f9be","mode":493,"path":"scripts/run.sh","size":12,"type":"file"},{"digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","mode":420,"path":"ünï.txt","size":0,"type":"file"}],"kind":"files","metadata":{"description":"VPC & \"friends\"\u0009é","labels":{"env":"prod","team":"platform"},"provenance":{"ci":{"provider":"github","run_id":"42"}}},"name":"components/terraform/vpc","schema_version":1,"version":"1.2.0"}` + + vectorAContent digest.Digest = "sha256:9bd2edcda28887b6bc03cba0a811460d0abfb4f35633a7cf1eb7908f43ec4fea" + vectorAIntent digest.Digest = "sha256:c542e50200a6bfd14a5663e89c70e6e5aa97b951e682fb20dac8a71ed1777bee" + vectorARecord digest.Digest = "sha256:48797bdf65aa85538095cfa311c57614180796c8200285c10434c32af896e1c3" + + // Vector B: empty package carrying typed kind data (a legacy-planfile-shaped kind). + vectorBCanonical = `{"content_digest":"sha256:6a5cabe4fc108c5947a6ff9fdb256559d9053edb389ee39cdbbcd97fdf0133cb","created_at":"2026-01-02T03:04:05Z","documents":[],"inventory":[],"kind":"ci/planfile","kind_data":{"component":"vpc","extra":{"a":"","b":0},"key":"k","pr_number":7,"stack":"plat-ue2-dev"},"metadata":{},"name":"ci/planfiles/abc","schema_version":1,"version":"deadbeef"}` + + vectorBContent digest.Digest = "sha256:6a5cabe4fc108c5947a6ff9fdb256559d9053edb389ee39cdbbcd97fdf0133cb" + vectorBIntent digest.Digest = "sha256:789b6c268c0f17f8b9744f05c2f309a4ba90a8342be21d4848fb569873fe2702" + vectorBRecord digest.Digest = "sha256:56c1a7c24bd4a1c74fc23f6aa0f351edff24e891bd7fa8fba4846593f51c88de" +) + +type vectorKindData struct { + Stack string `json:"stack"` + Key string `json:"key"` + PRNumber int `json:"pr_number"` + Component string `json:"component"` + Extra struct { + B int `json:"b"` + A string `json:"a"` + } `json:"extra"` +} + +// vectorRecordA builds Vector A with inventory in scrambled order and labels inserted in the given order. +func vectorRecordA(t *testing.T, labelOrder []string) Record { + t.Helper() + all := map[string]string{"team": "platform", "env": "prod"} + labels := make(map[string]string, len(all)) + for _, key := range labelOrder { + labels[key] = all[key] + } + record := Record{ + Name: "components/terraform/vpc", + Version: "1.2.0", + Kind: "files", + // Sub-second precision must be discarded by the canonical form. + CreatedAt: NewTimestamp(time.Date(2026, 10, 30, 12, 34, 56, 789000000, time.UTC)), + Metadata: Metadata{ + Description: "VPC & \"friends\"\té", + Labels: labels, + Provenance: &Provenance{CI: &CIProvenance{Provider: "github", RunID: "42"}}, + }, + Inventory: []Entry{ + {Path: "scripts/run.sh", Type: EntryFile, Size: 12, Mode: 0o755, Digest: digest.FromString("run")}, + {Path: "main.tf", Type: EntryFile, Size: 8, Mode: 0o644, Digest: digest.FromString("main")}, + {Path: "scripts", Type: EntryDirectory, Mode: 0o755}, + {Path: "link.tf", Type: EntrySymlink, Target: "main.tf"}, + {Path: "ünï.txt", Type: EntryFile, Mode: 0o644, Digest: digest.FromString("")}, + }, + Documents: []Document{ + {Role: DocumentReadme, SourcePath: ".atmos/artifact/README.md", MediaType: DocumentMediaType, Size: 5, Digest: digest.FromString("readme")}, + {Role: DocumentChangelog, SourcePath: ".atmos/artifact/CHANGELOG.md", MediaType: DocumentMediaType, Size: 9, Digest: digest.FromString("changelog")}, + }, + } + require.NoError(t, record.Seal()) + return record +} + +func vectorRecordB(t *testing.T) Record { + t.Helper() + data := vectorKindData{Stack: "plat-ue2-dev", Key: "k", PRNumber: 7, Component: "vpc"} + kind, err := NewKindData(data) + require.NoError(t, err) + record := Record{ + Name: "ci/planfiles/abc", + Version: "deadbeef", + Kind: "ci/planfile", + CreatedAt: NewTimestamp(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)), + KindData: kind, + } + require.NoError(t, record.Seal()) + return record +} + +func TestFrozenVectors(t *testing.T) { + tests := []struct { + name string + build func(t *testing.T) Record + canonical string + content digest.Digest + intent digest.Digest + record digest.Digest + }{ + { + name: "A files package, labels env-first", + build: func(t *testing.T) Record { return vectorRecordA(t, []string{"env", "team"}) }, + canonical: vectorACanonical, content: vectorAContent, intent: vectorAIntent, record: vectorARecord, + }, + { + name: "A files package, labels team-first", + build: func(t *testing.T) Record { return vectorRecordA(t, []string{"team", "env"}) }, + canonical: vectorACanonical, content: vectorAContent, intent: vectorAIntent, record: vectorARecord, + }, + { + name: "B typed kind data", + build: vectorRecordB, + canonical: vectorBCanonical, content: vectorBContent, intent: vectorBIntent, record: vectorBRecord, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + record := tt.build(t) + canonical, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, tt.canonical, string(canonical)) + assert.NotEqual(t, byte('\n'), canonical[len(canonical)-1], "no trailing newline") + assert.Equal(t, tt.content, record.ContentDigest) + intent, err := record.IntentDigest() + require.NoError(t, err) + assert.Equal(t, tt.intent, intent) + recordDigest, err := record.RecordDigest() + require.NoError(t, err) + assert.Equal(t, tt.record, recordDigest) + assert.Equal(t, digest.FromBytes([]byte(tt.canonical)), recordDigest, "record digest is the SHA-256 of the canonical bytes") + }) + } +} + +func TestFrozenVectorsDecodeRoundTrip(t *testing.T) { + for name, canonical := range map[string]string{"A": vectorACanonical, "B": vectorBCanonical} { + t.Run(name, func(t *testing.T) { + record, err := DecodeRecord([]byte(canonical)) + require.NoError(t, err) + again, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, canonical, string(again)) + }) + } + record, err := DecodeRecord([]byte(vectorBCanonical)) + require.NoError(t, err) + var kind vectorKindData + require.NoError(t, record.KindData.Decode(&kind)) + assert.Equal(t, 7, kind.PRNumber) + assert.Equal(t, "plat-ue2-dev", kind.Stack) +} + +func TestIntentAndRecordDigestSensitivity(t *testing.T) { + base := vectorRecordA(t, []string{"env", "team"}) + baseIntent, err := base.IntentDigest() + require.NoError(t, err) + baseRecord, err := base.RecordDigest() + require.NoError(t, err) + + t.Run("created_at and provenance are excluded from intent but not from the record digest", func(t *testing.T) { + changed := vectorRecordA(t, []string{"env", "team"}) + changed.CreatedAt = NewTimestamp(time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)) + changed.Metadata.Provenance = &Provenance{CI: &CIProvenance{Provider: "github", RunID: "999"}} + intent, err := changed.IntentDigest() + require.NoError(t, err) + assert.Equal(t, baseIntent, intent) + recordDigest, err := changed.RecordDigest() + require.NoError(t, err) + assert.NotEqual(t, baseRecord, recordDigest) + }) + + tests := []struct { + name string + mutate func(r *Record) + }{ + {"label value", func(r *Record) { r.Metadata.Labels["env"] = "dev" }}, + {"description", func(r *Record) { r.Metadata.Description = "other" }}, + {"kind", func(r *Record) { r.Kind = "component/terraform" }}, + {"version", func(r *Record) { r.Version = "1.2.1" }}, + {"name", func(r *Record) { r.Name = "components/terraform/vpc2" }}, + {"kind data", func(r *Record) { + data, err := NewKindData(vectorKindData{Stack: "x"}) + require.NoError(t, err) + r.KindData = data + }}, + {"file mode changes content and intent", func(r *Record) { + r.Inventory[1].Mode = 0o600 + require.NoError(t, r.Seal()) + }}, + } + for _, tt := range tests { + t.Run("intent changes with "+tt.name, func(t *testing.T) { + changed := vectorRecordA(t, []string{"env", "team"}) + tt.mutate(&changed) + if tt.name != "file mode changes content and intent" { + // Fields other than the inventory do not require resealing the content digest. + require.NoError(t, changed.Validate()) + } + intent, err := changed.IntentDigest() + require.NoError(t, err) + assert.NotEqual(t, baseIntent, intent) + }) + } +} + +func TestCanonicalDoesNotMutateRecordAndIsIsolated(t *testing.T) { + record := vectorRecordA(t, []string{"env", "team"}) + firstPath := record.Inventory[0].Path + first, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, firstPath, record.Inventory[0].Path, "input inventory order is untouched by canonicalization") + assert.Equal(t, "prod", record.Metadata.Labels["env"]) + + // result -> source: scribbling on the returned bytes must not change the record or later results. + scribbled := bytes.Clone(first) + for i := range first { + first[i] = 'x' + } + second, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(second)) + assert.NotEqual(t, string(scribbled), string(first)) + + // source -> result: mutating the record afterwards must not affect bytes already returned. + record.Metadata.Labels["env"] = "changed" + record.Inventory[0].Path = "zzz" + require.NoError(t, record.Seal()) + third, err := record.Canonical() + require.NoError(t, err) + assert.Equal(t, vectorACanonical, string(second), "earlier result is unaffected by later source mutation") + assert.NotEqual(t, string(second), string(third)) +} + +func TestKindDataOrderIndependence(t *testing.T) { + first, err := NewKindData(struct { + B int `json:"b"` + A int `json:"a"` + }{B: 2, A: 1}) + require.NoError(t, err) + second, err := NewKindData(struct { + A int `json:"a"` + B int `json:"b"` + }{A: 1, B: 2}) + require.NoError(t, err) + assert.Equal(t, `{"a":1,"b":2}`, string(first)) + assert.Equal(t, first, second) + + _, err = NewKindData(struct { + Ratio float64 `json:"ratio"` + }{Ratio: 0.5}) + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, "floats are not allowed in the canonical path") +} + +func TestDecodeRecordRejectsNonCanonicalBytes(t *testing.T) { + tests := []struct { + name string + raw string + want error + }{ + {"trailing newline", vectorBCanonical + "\n", errUtils.ErrArtifactRecordInvalid}, + {"extra whitespace", `{ ` + vectorBCanonical[1:], errUtils.ErrArtifactRecordInvalid}, + {"unknown field", vectorBCanonical[:len(vectorBCanonical)-1] + `,"extra":1}`, errUtils.ErrArtifactRecordInvalid}, + {"wrong schema version", mutateVectorB(`"schema_version":1`, `"schema_version":2`), errUtils.ErrArtifactRecordInvalid}, + {"tampered content digest", mutateVectorB("6a5cabe4", "00000000"), errUtils.ErrArtifactRecordInvalid}, + {"duplicate key", mutateVectorB(`"kind":"ci/planfile"`, `"kind":"ci/planfile","kind":"files"`), errUtils.ErrArtifactRecordInvalid}, + {"sub-second timestamp", mutateVectorB("03:04:05Z", "03:04:05.5Z"), errUtils.ErrArtifactRecordInvalid}, + {"empty input", ``, errUtils.ErrArtifactRecordInvalid}, + {"oversize", string(make([]byte, MaxRecordSize+1)), errUtils.ErrArtifactTooLarge}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DecodeRecord([]byte(tt.raw)) + require.Error(t, err) + assert.True(t, errors.Is(err, tt.want), "got %v", err) + }) + } +} + +// mutateVectorB returns Vector B with the first occurrence of old replaced. +func mutateVectorB(old, replacement string) string { + return string(bytes.Replace([]byte(vectorBCanonical), []byte(old), []byte(replacement), 1)) +} diff --git a/pkg/artifact/verifier.go b/pkg/artifact/verifier.go new file mode 100644 index 00000000000..1fdb44a6e6c --- /dev/null +++ b/pkg/artifact/verifier.go @@ -0,0 +1,19 @@ +package artifact + +import "context" + +//go:generate mockgen -package artifact -destination mock_verifier.go github.com/cloudposse/atmos/pkg/artifact Verifier + +// Verifier authenticates stored evidence against a policy. Repository adapters fetch evidence and +// invoke a Verifier; they may register additional native formats with the same result contract. +type Verifier interface { + // Formats lists the evidence formats the verifier understands. Required evidence in any other + // format fails closed. + Formats() []EvidenceFormat + + // Verify checks subject binding, cryptographic validity, signer or key trust, and the + // applicable log, certificate, and signing-time proofs. At least one valid permitted signature + // must satisfy each required trust set; unrelated invalid attachments cannot override valid + // matching evidence. A public-key signature never satisfies an identity/issuer requirement. + Verify(ctx context.Context, subject SignatureSubject, evidence []EvidenceBlob, policy VerificationPolicy) (*VerificationResult, error) +} diff --git a/scripts/checks/checks.star b/scripts/checks/checks.star index aa2a3a1cd44..47a63c0e3a0 100644 --- a/scripts/checks/checks.star +++ b/scripts/checks/checks.star @@ -126,7 +126,7 @@ def module_compatibility(config, files): # Run the existing Atmos validation checks against all files or the affected scope. def editorconfig(config, files): - args = ["validate", "--exclude", "tests/fixtures/**", "--exclude", "**/*.go", "--exclude", "specs/**", "--exclude", "README.md", "--format", "rich"] + args = ["validate", "--exclude", "tests/fixtures/**", "--exclude", "pkg/artifact/compat/testdata/**", "--exclude", "**/*.go", "--exclude", "specs/**", "--exclude", "README.md", "--format", "rich"] if config["scope"] != "all": args += ["--affected", "--base", config["base"]] atmos.run(args) From bb356a22617798cedb6db6ba29c10517659ef990 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 31 Oct 2026 08:45:34 -0500 Subject: [PATCH 02/32] feat(artifacts): add artifacts configuration, validation, and repository registry (A01) Also escapes the pipe in the contracts Documents table row. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/catalog.go | 80 ++++ pkg/artifact/config/decode.go | 122 ++++++ pkg/artifact/config/decode_test.go | 249 +++++++++++ pkg/artifact/config/doc.go | 25 ++ pkg/artifact/config/locking.go | 105 +++++ pkg/artifact/config/locking_test.go | 169 ++++++++ pkg/artifact/config/origin.go | 47 +++ pkg/artifact/config/validate.go | 198 +++++++++ pkg/artifact/config/validate_test.go | 242 +++++++++++ pkg/artifact/config/verification.go | 115 +++++ pkg/artifact/config/verification_test.go | 151 +++++++ pkg/artifact/kinds.go | 206 +++++++++ pkg/artifact/kinds_test.go | 144 +++++++ pkg/artifact/labels.go | 73 ++++ pkg/artifact/labels_test.go | 61 +++ pkg/artifact/options.go | 410 ++++++++++++++++++ pkg/artifact/options_test.go | 200 +++++++++ pkg/artifact/permissions.go | 47 +++ pkg/artifact/permissions_test.go | 46 ++ pkg/artifact/registry.go | 247 +++++++++++ pkg/artifact/registry_test.go | 215 ++++++++++ pkg/config/artifacts.go | 217 ++++++++++ pkg/config/artifacts_test.go | 418 +++++++++++++++++++ pkg/config/global_viper.go | 24 ++ pkg/config/imports.go | 4 + pkg/config/load.go | 5 + pkg/datafetcher/schema/atmos/config/1.0.json | 391 +++++++++++++++++ pkg/schema/artifacts.go | 137 ++++++ pkg/schema/schema.go | 2 + 29 files changed, 4350 insertions(+) create mode 100644 pkg/artifact/config/catalog.go create mode 100644 pkg/artifact/config/decode.go create mode 100644 pkg/artifact/config/decode_test.go create mode 100644 pkg/artifact/config/doc.go create mode 100644 pkg/artifact/config/locking.go create mode 100644 pkg/artifact/config/locking_test.go create mode 100644 pkg/artifact/config/origin.go create mode 100644 pkg/artifact/config/validate.go create mode 100644 pkg/artifact/config/validate_test.go create mode 100644 pkg/artifact/config/verification.go create mode 100644 pkg/artifact/config/verification_test.go create mode 100644 pkg/artifact/kinds.go create mode 100644 pkg/artifact/kinds_test.go create mode 100644 pkg/artifact/labels.go create mode 100644 pkg/artifact/labels_test.go create mode 100644 pkg/artifact/options.go create mode 100644 pkg/artifact/options_test.go create mode 100644 pkg/artifact/permissions.go create mode 100644 pkg/artifact/permissions_test.go create mode 100644 pkg/artifact/registry.go create mode 100644 pkg/artifact/registry_test.go create mode 100644 pkg/config/artifacts.go create mode 100644 pkg/config/artifacts_test.go create mode 100644 pkg/schema/artifacts.go diff --git a/pkg/artifact/config/catalog.go b/pkg/artifact/config/catalog.go new file mode 100644 index 00000000000..1dc4773b921 --- /dev/null +++ b/pkg/artifact/config/catalog.go @@ -0,0 +1,80 @@ +package config + +import ( + "fmt" + "net/url" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// validateCatalog checks the optional catalog reference. A catalog belongs directly to its +// repository: it must be an oci:// reference with a tag or digest (there is no implicit latest), +// carry no credentials, and name the repository's own registry host, because the repository's +// identity must never authenticate a different host. +func validateCatalog(desc *artifact.KindDescriptor, repo *schema.ArtifactRepositoryConfig) error { + if repo.Catalog == "" { + return nil + } + if !desc.AllowsCatalog { + return fmt.Errorf("%w: kind %q does not support a catalog; direct named operations and managed-record enumeration work without one", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + parsed, err := parseCatalogURI(repo.Catalog) + if err != nil { + return err + } + registry, _ := repo.Options["registry"].(string) + if registry != "" && !strings.EqualFold(registry, parsed.Host) { + return fmt.Errorf("%w: catalog host %q does not match the repository registry %q; a catalog cannot direct the repository identity to another host", errUtils.ErrArtifactConfigInvalid, parsed.Host, registry) + } + return nil +} + +// parseCatalogURI parses and checks the shape of an oci:// catalog reference. +func parseCatalogURI(raw string) (*url.URL, error) { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "oci" || parsed.Host == "" { + return nil, fmt.Errorf("%w: catalog %q must be an oci:///: reference", errUtils.ErrArtifactConfigInvalid, sanitizedCatalog(raw)) + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return nil, fmt.Errorf("%w: catalog %q must not contain credentials, a query, or a fragment", errUtils.ErrArtifactConfigInvalid, sanitizedCatalog(raw)) + } + if err := validateCatalogReference(strings.TrimPrefix(parsed.Path, "/")); err != nil { + return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, raw, err) + } + return parsed, nil +} + +// validateCatalogReference checks ":" or "@sha256:". +func validateCatalogReference(reference string) error { + path, tag, dgst := reference, "", "" + if before, after, found := strings.Cut(reference, "@"); found { + path, dgst = before, after + } else if index := strings.LastIndex(reference, ":"); index >= 0 { + path, tag = reference[:index], reference[index+1:] + } + if tag == "" && dgst == "" { + return fmt.Errorf("%w: a tag or digest is required (there is no implicit latest)", errUtils.ErrArtifactRefInvalid) + } + if err := artifact.ValidateName(path); err != nil { + return err + } + if dgst != "" { + return artifact.ValidateDigest(digest.Digest(dgst)) + } + return artifact.ValidateVersion(tag) +} + +// sanitizedCatalog hides userinfo so a mistaken credential is not echoed back in the error. +func sanitizedCatalog(raw string) string { + parsed, err := url.Parse(raw) + if err != nil || parsed.User == nil { + return raw + } + parsed.User = nil + return parsed.String() +} diff --git a/pkg/artifact/config/decode.go b/pkg/artifact/config/decode.go new file mode 100644 index 00000000000..958823670f3 --- /dev/null +++ b/pkg/artifact/config/decode.go @@ -0,0 +1,122 @@ +package config + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/go-viper/mapstructure/v2" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + keyRepositories = "repositories" + keyPermissions = "permissions" +) + +// Decode strictly decodes a raw `artifacts` section (as parsed from YAML, with original key +// spelling) into the schema type. Unknown fields, mistyped values, and non-string permissions are +// errors, so `permissions: 0600` (which YAML reads as the integer 384) never silently becomes the +// wrong mode. A nil or empty section decodes to the zero value. +func Decode(raw map[string]any) (schema.ArtifactConfig, error) { + defer perf.Track(nil, "artifactconfig.Decode")() + + var cfg schema.ArtifactConfig + if len(raw) == 0 { + return cfg, nil + } + if err := checkPermissionTypes(raw); err != nil { + return cfg, err + } + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &cfg, + TagName: "mapstructure", + ErrorUnused: true, + }) + if err != nil { + return cfg, fmt.Errorf("%w: decoder: %w", errUtils.ErrArtifactConfigInvalid, err) + } + if err := decoder.Decode(raw); err != nil { + return cfg, fmt.Errorf("%w: artifacts: %w", errUtils.ErrArtifactConfigInvalid, err) + } + normalize(&cfg) + return cfg, nil +} + +// normalize turns empty collections into nil so that a value decoded from `trusted_keys: []` +// is identical to the same value after a YAML, JSON, or mapstructure round trip, all of which omit +// empty collections. +func normalize(cfg *schema.ArtifactConfig) { + if len(cfg.Repositories) == 0 { + cfg.Repositories = nil + } + normalizeVerification(cfg.Verification) + for alias := range cfg.Repositories { + repo := cfg.Repositories[alias] + if len(repo.Options) == 0 { + repo.Options = nil + } + if repo.Defaults != nil && repo.Defaults.Metadata != nil && len(repo.Defaults.Metadata.Labels) == 0 { + repo.Defaults.Metadata.Labels = nil + } + normalizeVerification(repo.Verification) + cfg.Repositories[alias] = repo + } +} + +func normalizeVerification(verification *schema.ArtifactVerificationConfig) { + if verification == nil { + return + } + if len(verification.TrustedKeys) == 0 { + verification.TrustedKeys = nil + } + if len(verification.TrustedIdentities) == 0 { + verification.TrustedIdentities = nil + } +} + +// checkPermissionTypes rejects any repository whose permissions value is not a string. +func checkPermissionTypes(raw map[string]any) error { + repos, ok := lookupKey(raw, keyRepositories).(map[string]any) + if !ok { + return nil + } + aliases := make([]string, 0, len(repos)) + for alias := range repos { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + var errs []error + for _, alias := range aliases { + repo, ok := repos[alias].(map[string]any) + if !ok { + continue + } + value := lookupKey(repo, keyPermissions) + if value == nil { + continue + } + if _, isString := value.(string); !isString { + errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T %v (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, alias, value, value)) + } + } + return errors.Join(errs...) +} + +// lookupKey finds a key case-insensitively, matching the loader's case-insensitive key handling. +func lookupKey(m map[string]any, key string) any { + if value, ok := m[key]; ok { + return value + } + for candidate, value := range m { + if strings.EqualFold(candidate, key) { + return value + } + } + return nil +} diff --git a/pkg/artifact/config/decode_test.go b/pkg/artifact/config/decode_test.go new file mode 100644 index 00000000000..dc1f1743d9d --- /dev/null +++ b/pkg/artifact/config/decode_test.go @@ -0,0 +1,249 @@ +package config + +import ( + "encoding/json" + "testing" + + "github.com/go-viper/mapstructure/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// prdDocument is every YAML example from the artifacts PRD that sits under `artifacts:`, merged +// into one document: the public-configuration repositories plus the catalog, verification, and +// locking examples. +const prdDocument = ` +verification: + require_signature: true + trusted_identities: + - identity: https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main + issuer: https://token.actions.githubusercontent.com + trusted_keys: [] +repositories: + shared: + kind: oci/registry + identity: registry-publisher + options: + registry: ghcr.io + prefix: acme/atmos + defaults: + metadata: + description: Shared platform artifacts + labels: + team: platform + catalog: oci://ghcr.io/acme/atmos/catalog:stable + locking: + required_scope: repository + repository: coordination + backups: + kind: aws/s3 + identity: backup-writer + packaging: tgz + options: + bucket: acme-artifacts + prefix: atmos + region: us-east-2 + tags: + purpose: backup + local: + kind: local/dir + options: + path: .atmos/artifacts + archives: + kind: local/archive + packaging: tgz + permissions: "0600" + options: + path: .atmos/backups + key_pattern: '{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz' + offline: + kind: oci/layout + options: + path: .atmos/oci + coordination: + kind: aws/s3 + identity: lock-writer + options: + bucket: acme-artifact-coordination + verification: + require_signature: true + trusted_keys: + - /etc/atmos/cosign.pub +` + +func decodeYAMLDocument(t *testing.T, document string) (schema.ArtifactConfig, map[string]any) { + t.Helper() + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(document), &raw)) + cfg, err := Decode(raw) + require.NoError(t, err) + return cfg, raw +} + +func TestDecode_PRDDocumentLoadsUnchanged(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, prdDocument) + + require.Len(t, cfg.Repositories, 6) + archives := cfg.Repositories["archives"] + assert.Equal(t, "0600", archives.Permissions) + assert.Equal(t, `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, archives.Options["key_pattern"], "key_pattern survives decoding verbatim") + assert.Equal(t, "Shared platform artifacts", cfg.Repositories["shared"].Defaults.Metadata.Description) + assert.Equal(t, []string{"/etc/atmos/cosign.pub"}, cfg.Repositories["coordination"].Verification.TrustedKeys) + assert.Empty(t, cfg.Verification.TrustedKeys) + require.Len(t, cfg.Verification.TrustedIdentities, 1) + assert.Equal(t, "https://token.actions.githubusercontent.com", cfg.Verification.TrustedIdentities[0].Issuer) +} + +// TestDecode_YAMLJSONAndMapstructureAgree decodes the same document through YAML, JSON, and a +// plain mapstructure round trip and requires identical values. +func TestDecode_YAMLJSONAndMapstructureAgree(t *testing.T) { + fromYAML, raw := decodeYAMLDocument(t, prdDocument) + + // YAML round trip: marshal the typed value and decode it again. + yamlBytes, err := yaml.Marshal(fromYAML) + require.NoError(t, err) + var yamlRoundTrip schema.ArtifactConfig + require.NoError(t, yaml.Unmarshal(yamlBytes, &yamlRoundTrip)) + assert.Equal(t, fromYAML, yamlRoundTrip) + + // JSON round trip. + jsonBytes, err := json.Marshal(fromYAML) + require.NoError(t, err) + var jsonRoundTrip schema.ArtifactConfig + require.NoError(t, json.Unmarshal(jsonBytes, &jsonRoundTrip)) + assert.Equal(t, fromYAML, jsonRoundTrip) + + // The raw map re-encoded as JSON, then decoded strictly, matches too. + var rawFromJSON map[string]any + require.NoError(t, json.Unmarshal(mustJSON(t, raw), &rawFromJSON)) + fromJSONMap, err := Decode(rawFromJSON) + require.NoError(t, err) + assert.Equal(t, fromYAML, fromJSONMap) + + // mapstructure struct -> map -> struct round trip with weak typing disabled. + var asMap map[string]any + require.NoError(t, mapstructure.Decode(fromYAML, &asMap)) + var mapstructureRoundTrip schema.ArtifactConfig + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &mapstructureRoundTrip, TagName: "mapstructure"}) + require.NoError(t, err) + require.NoError(t, decoder.Decode(asMap)) + assert.Equal(t, fromYAML, mapstructureRoundTrip) +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + out, err := json.Marshal(value) + require.NoError(t, err) + return out +} + +func TestDecode_OriginsAreNeverSerialized(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, prdDocument) + repo := cfg.Repositories["local"] + repo.PathOrigin = NewOrigin("/etc/atmos/atmos.yaml", true) + cfg.Repositories["local"] = repo + cfg.Verification.KeysOrigin = NewOrigin("/etc/atmos/atmos.yaml", false) + + jsonBytes := mustJSON(t, cfg) + yamlBytes, err := yaml.Marshal(cfg) + require.NoError(t, err) + for _, encoded := range []string{string(jsonBytes), string(yamlBytes)} { + assert.NotContains(t, encoded, "/etc/atmos/atmos.yaml") + assert.NotContains(t, encoded, "origin") + } +} + +func TestDecode_RejectsUnquotedNumericPermissions(t *testing.T) { + tests := []struct { + name string + document string + }{ + {name: "octal literal", document: "repositories:\n a:\n kind: local/archive\n permissions: 0600\n"}, + {name: "decimal literal", document: "repositories:\n a:\n kind: local/archive\n permissions: 384\n"}, + {name: "float", document: "repositories:\n a:\n kind: local/archive\n permissions: 6.0\n"}, + {name: "bool", document: "repositories:\n a:\n kind: local/archive\n permissions: true\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(tt.document), &raw)) + _, err := Decode(raw) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "quoted octal string") + assert.Contains(t, err.Error(), `"a"`) + }) + } + + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte("repositories:\n a:\n kind: local/archive\n permissions: \"0600\"\n"), &raw)) + cfg, err := Decode(raw) + require.NoError(t, err) + assert.Equal(t, "0600", cfg.Repositories["a"].Permissions) +} + +func TestDecode_StrictAboutUnknownAndMistypedFields(t *testing.T) { + tests := []struct { + name string + document string + wantErr string + }{ + {name: "unknown repository field", document: "repositories:\n a:\n kind: local/dir\n pakaging: none\n", wantErr: "pakaging"}, + {name: "unknown top level field", document: "repositorys: {}\n", wantErr: "repositorys"}, + {name: "unknown verification field", document: "verification:\n require_sig: true\n", wantErr: "require_sig"}, + {name: "non-boolean require_signature", document: "verification:\n require_signature: sometimes\n", wantErr: "require_signature"}, + {name: "non-string label value", document: "repositories:\n a:\n kind: local/dir\n defaults:\n metadata:\n labels:\n cost: 1234\n", wantErr: "cost"}, + {name: "repositories not a map", document: "repositories: [a]\n", wantErr: "repositories"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(tt.document), &raw)) + _, err := Decode(raw) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestDecode_EmptySectionIsTheZeroValue(t *testing.T) { + cfg, err := Decode(nil) + require.NoError(t, err) + assert.Equal(t, schema.ArtifactConfig{}, cfg) + cfg, err = Decode(map[string]any{}) + require.NoError(t, err) + assert.Empty(t, cfg.Repositories) +} + +func TestDecode_KeysAreCaseSensitiveForAliasesLabelsAndTags(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, ` +repositories: + Prod.US: + kind: aws/s3 + options: + bucket: acme-artifacts + tags: + Purpose: Backup + defaults: + metadata: + labels: + Team.Name: Platform +`) + repo, ok := cfg.Repositories["Prod.US"] + require.True(t, ok) + assert.Equal(t, map[string]string{"Team.Name": "Platform"}, repo.Defaults.Metadata.Labels) + assert.Equal(t, map[string]any{"Purpose": "Backup"}, repo.Options["tags"]) +} + +func TestDecodedPRDDocumentValidatesWithAnchoredPaths(t *testing.T) { + cfg, _ := decodeYAMLDocument(t, prdDocument) + origin := NewOrigin("/project/atmos.yaml", false) + for alias, repo := range cfg.Repositories { + repo.PathOrigin = origin + cfg.Repositories[alias] = repo + } + require.NoError(t, Validate(&cfg)) +} diff --git a/pkg/artifact/config/doc.go b/pkg/artifact/config/doc.go new file mode 100644 index 00000000000..99400dd6941 --- /dev/null +++ b/pkg/artifact/config/doc.go @@ -0,0 +1,25 @@ +// Package config decodes, validates, and interprets the `artifacts` section of atmos.yaml. +// +// It is separate from pkg/artifact so that the domain package stays free of configuration policy, +// while pkg/config (the loader) and the artifact service share one implementation of: +// +// - strict decoding of the raw section into schema.ArtifactConfig, including rejecting +// unquoted numeric permissions that YAML would silently turn into the wrong octal value; +// - whole-configuration validation (kinds, per-kind options, packaging, permissions, catalogs, +// identities, labels, verification policy, and delegated lock chains); +// - the effective verification policy model, where a global requirement can be strengthened but +// never weakened by a repository and both trust sets must be satisfied; +// - anchoring relative local paths to the configuration file that declared them. +// +// Validation performs no network access, resolves no credentials, and creates no directories. +// An `identity` is only checked for shape (and, when the caller supplies a lookup, existence); +// it is resolved lazily by the operation that needs credentials. +// +// # Repository kinds +// +// Kind names are checked against artifact.RepositoryRegistry, which returns a registered backend's +// descriptor when one exists and otherwise the documented static set of V1 kinds (local/dir, +// local/archive, aws/s3, oci/registry, oci/layout). Validation therefore works before backend +// packages register, and a backend that registers a kind refines the descriptor rather than +// changing the accepted names. +package config diff --git a/pkg/artifact/config/locking.go b/pkg/artifact/config/locking.go new file mode 100644 index 00000000000..91424172833 --- /dev/null +++ b/pkg/artifact/config/locking.go @@ -0,0 +1,105 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// LockPlan is the resolved lock coordination of one repository. +type LockPlan struct { + // Provider is the alias of the repository that provides the lock. It is the repository itself + // unless locking is delegated. + Provider string + + // Required is the minimum scope mutations need. + Required artifact.LockScope + + // Chain lists the aliases visited from the repository to the provider, in order. + Chain []string +} + +// ResolveLockPlan follows `locking.repository` delegation from alias to the repository that +// provides the lock and reports the required scope. It fails on an unknown delegate, on a cycle, +// and on an invalid scope. It does not check the provider's capability; [Validator.Validate] does. +func ResolveLockPlan(cfg *schema.ArtifactConfig, alias string) (LockPlan, error) { + defer perf.Track(nil, "artifactconfig.ResolveLockPlan")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return LockPlan{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + required, err := requiredScope(repo.Locking) + if err != nil { + return LockPlan{}, err + } + plan := LockPlan{Provider: alias, Required: required, Chain: []string{alias}} + visited := map[string]bool{alias: true} + for { + current := cfg.Repositories[plan.Provider] + if current.Locking == nil || current.Locking.Repository == "" { + return plan, nil + } + next := current.Locking.Repository + if _, exists := cfg.Repositories[next]; !exists { + return LockPlan{}, fmt.Errorf("%w: repository %q delegates locking to %q, which is not configured", errUtils.ErrArtifactConfigInvalid, plan.Provider, next) + } + if visited[next] { + return LockPlan{}, fmt.Errorf("%w: lock delegation cycle: %s -> %s", errUtils.ErrArtifactConfigInvalid, strings.Join(plan.Chain, " -> "), next) + } + visited[next] = true + plan.Chain = append(plan.Chain, next) + plan.Provider = next + } +} + +// requiredScope returns the configured scope, defaulting to host. +func requiredScope(locking *schema.ArtifactLockingConfig) (artifact.LockScope, error) { + if locking == nil || locking.RequiredScope == "" { + return artifact.LockScopeHost, nil + } + switch scope := artifact.LockScope(locking.RequiredScope); scope { + case artifact.LockScopeHost, artifact.LockScopeRepository: + return scope, nil + default: + return "", fmt.Errorf("%w: locking.required_scope %q must be %q or %q", errUtils.ErrArtifactConfigInvalid, locking.RequiredScope, artifact.LockScopeHost, artifact.LockScopeRepository) + } +} + +// validateLocking checks the lock configuration: scope value, delegate existence, delegation +// cycles, and that the repository that finally provides the lock advertises the required scope. +func validateLocking(v *Validator, cfg *schema.ArtifactConfig, alias string, repo *schema.ArtifactRepositoryConfig) error { + if repo.Locking == nil { + return nil + } + plan, err := ResolveLockPlan(cfg, alias) + if err != nil { + return err + } + provider := cfg.Repositories[plan.Provider] + desc, ok := v.registry.Lookup(provider.Kind) + if !ok { + // The provider's own unknown kind is reported when that repository is validated. + return nil + } + if desc.Capabilities.HasLockScope(plan.Required) { + return nil + } + return errors.Join( + errUtils.ErrArtifactConfigInvalid, + fmt.Errorf("%w: locking.required_scope %q is not provided by %q (kind %q advertises %v)%s", + errUtils.ErrArtifactUnsupportedCapability, plan.Required, plan.Provider, provider.Kind, desc.Capabilities.LockScopes, delegateHint(plan)), + ) +} + +func delegateHint(plan LockPlan) string { + if len(plan.Chain) > 1 { + return "" + } + return "; set locking.repository to a repository (for example an aws/s3 coordinator) that advertises repository scope" +} diff --git a/pkg/artifact/config/locking_test.go b/pkg/artifact/config/locking_test.go new file mode 100644 index 00000000000..4422fc52c20 --- /dev/null +++ b/pkg/artifact/config/locking_test.go @@ -0,0 +1,169 @@ +package config + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +func ptr[T any](v T) *T { return &v } + +func withLocking(repo *schema.ArtifactRepositoryConfig, scope, delegate string) schema.ArtifactRepositoryConfig { + out := *repo + out.Locking = &schema.ArtifactLockingConfig{RequiredScope: scope, Repository: delegate} + return out +} + +func TestValidate_PRDLockingExample(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": withLocking(ptr(registryRepo()), "repository", "coordination"), + "coordination": s3Repo(), + }} + require.NoError(t, Validate(cfg)) + + plan, err := ResolveLockPlan(cfg, "shared") + require.NoError(t, err) + assert.Equal(t, LockPlan{Provider: "coordination", Required: artifact.LockScopeRepository, Chain: []string{"shared", "coordination"}}, plan) +} + +func TestValidate_LockingScopes(t *testing.T) { + tests := []struct { + name string + repos map[string]schema.ArtifactRepositoryConfig + alias string + wantErr string + wantCap bool + }{ + { + name: "host scope is the default and always available", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "", "")}, + alias: "r", + }, + { + name: "registry cannot provide repository scope itself", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "repository", "")}, + alias: "r", + wantErr: "not provided by", + wantCap: true, + }, + { + name: "local directory cannot provide repository scope", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(localRepo(t.TempDir())), "repository", "")}, + alias: "r", + wantErr: "not provided by", + wantCap: true, + }, + { + name: "s3 provides repository scope itself", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(s3Repo()), "repository", "")}, + alias: "r", + }, + { + name: "delegate lacking repository scope is insufficient", + repos: map[string]schema.ArtifactRepositoryConfig{ + "r": withLocking(ptr(registryRepo()), "repository", "other"), + "other": func() schema.ArtifactRepositoryConfig { + repo := registryRepo() + repo.Options["prefix"] = "other" + return repo + }(), + }, + alias: "r", + wantErr: `not provided by "other"`, + wantCap: true, + }, + { + name: "unknown scope", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(s3Repo()), "global", "")}, + alias: "r", + wantErr: "required_scope", + }, + { + name: "unknown delegate", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "repository", "missing")}, + alias: "r", + wantErr: `delegates locking to "missing", which is not configured`, + }, + { + name: "host scope with a stronger delegate is fine", + repos: map[string]schema.ArtifactRepositoryConfig{"r": withLocking(ptr(registryRepo()), "host", "c"), "c": s3Repo()}, + alias: "r", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{Repositories: tt.repos}, tt.alias) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantCap, errors.Is(err, errUtils.ErrArtifactUnsupportedCapability)) + }) + } +} + +func TestValidate_DelegatedLockCyclesAreRejected(t *testing.T) { + tests := []struct { + name string + repos map[string]schema.ArtifactRepositoryConfig + want string + }{ + { + name: "self delegation", + repos: map[string]schema.ArtifactRepositoryConfig{"a": withLocking(ptr(s3Repo()), "repository", "a")}, + want: "a -> a", + }, + { + name: "two repositories", + repos: map[string]schema.ArtifactRepositoryConfig{ + "a": withLocking(ptr(s3Repo()), "repository", "b"), + "b": withLocking(ptr(s3Repo()), "repository", "a"), + }, + want: "a -> b -> a", + }, + { + name: "three repositories", + repos: map[string]schema.ArtifactRepositoryConfig{ + "a": withLocking(ptr(s3Repo()), "repository", "b"), + "b": withLocking(ptr(s3Repo()), "repository", "c"), + "c": withLocking(ptr(s3Repo()), "repository", "a"), + }, + want: "a -> b -> c -> a", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Validate(&schema.ArtifactConfig{Repositories: tt.repos}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "lock delegation cycle") + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestResolveLockPlan_FollowsChainToTheProvider(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "registry": withLocking(ptr(registryRepo()), "repository", "middle"), + "middle": withLocking(ptr(s3Repo()), "repository", "final"), + "final": s3Repo(), + }} + plan, err := ResolveLockPlan(cfg, "registry") + require.NoError(t, err) + assert.Equal(t, "final", plan.Provider) + assert.Equal(t, []string{"registry", "middle", "final"}, plan.Chain) + + plain, err := ResolveLockPlan(cfg, "final") + require.NoError(t, err) + assert.Equal(t, LockPlan{Provider: "final", Required: artifact.LockScopeHost, Chain: []string{"final"}}, plain) + + _, err = ResolveLockPlan(cfg, "nope") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} diff --git a/pkg/artifact/config/origin.go b/pkg/artifact/config/origin.go new file mode 100644 index 00000000000..5b8baade3b7 --- /dev/null +++ b/pkg/artifact/config/origin.go @@ -0,0 +1,47 @@ +package config + +import ( + "fmt" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// AnchorPath returns the absolute, cleaned form of p. An absolute path is only cleaned. A relative +// path resolves against the directory of the configuration file that declared it, so the result +// does not depend on the working directory or on which file was merged last. +// +// A relative path declared by a remote import cannot be anchored: its local copy lives in an +// ephemeral download directory that must never become a write root. A relative path with no known +// declaring file is likewise rejected. In both cases the caller must provide an absolute path or +// override the value in a locally declared configuration file. +func AnchorPath(origin schema.ArtifactConfigOrigin, p string) (string, error) { + defer perf.Track(nil, "artifactconfig.AnchorPath")() + + if filepath.IsAbs(p) { + return filepath.Clean(p), nil + } + switch { + case origin.Remote: + return "", fmt.Errorf("%w: %w: relative path %q is declared only by a remote import (%s); declare it in a local configuration file or use an absolute path", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, p, origin.File) + case origin.Dir == "": + return "", fmt.Errorf("%w: %w: relative path %q has no declaring configuration file to resolve against; use an absolute path", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, p) + } + return filepath.Join(origin.Dir, p), nil +} + +// NewOrigin builds the origin of a value declared in file. Remote marks a downloaded import +// whose local path is ephemeral. +func NewOrigin(file string, remote bool) schema.ArtifactConfigOrigin { + defer perf.Track(nil, "artifactconfig.NewOrigin")() + + if file == "" { + return schema.ArtifactConfigOrigin{Remote: remote} + } + if abs, err := filepath.Abs(file); err == nil { + file = abs + } + return schema.ArtifactConfigOrigin{File: file, Dir: filepath.Dir(file), Remote: remote} +} diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go new file mode 100644 index 00000000000..b76bb3ae160 --- /dev/null +++ b/pkg/artifact/config/validate.go @@ -0,0 +1,198 @@ +package config + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" + "unicode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + // The longest auth identity reference. + maxIdentityLength = 256 + + // The longest default artifact description. + maxDescriptionLength = 4096 +) + +// aliasPattern is the shape of a repository alias, which appears in `--repository` and step fields. +var aliasPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) + +// Validator validates artifact configuration. The zero value is not usable; use [NewValidator]. +type Validator struct { + registry *artifact.RepositoryRegistry + identityExists func(name string) bool +} + +// Option configures a Validator. +type Option func(*Validator) + +// WithRegistry validates kinds against a specific registry instead of the process-wide one. +func WithRegistry(registry *artifact.RepositoryRegistry) Option { + defer perf.Track(nil, "artifactconfig.WithRegistry")() + + return func(v *Validator) { v.registry = registry } +} + +// WithIdentityLookup also checks that each referenced identity exists. Without it only the shape +// of the identity name is checked: identities are resolved lazily and may be provisioned later. +func WithIdentityLookup(exists func(name string) bool) Option { + defer perf.Track(nil, "artifactconfig.WithIdentityLookup")() + + return func(v *Validator) { v.identityExists = exists } +} + +// NewValidator returns a Validator. +func NewValidator(opts ...Option) *Validator { + defer perf.Track(nil, "artifactconfig.NewValidator")() + + v := &Validator{registry: artifact.DefaultRepositoryRegistry()} + for _, opt := range opts { + if opt != nil { + opt(v) + } + } + return v +} + +// Validate validates a whole artifact configuration with a default Validator. +func Validate(cfg *schema.ArtifactConfig, opts ...Option) error { + defer perf.Track(nil, "artifactconfig.Validate")() + + return NewValidator(opts...).Validate(cfg) +} + +// Validate checks the global verification policy and every repository, returning all problems +// joined in a deterministic order. Every error wraps errUtils.ErrArtifactConfigInvalid. It +// performs no network access, credential resolution, or directory creation. +func (v *Validator) Validate(cfg *schema.ArtifactConfig) error { + defer perf.Track(nil, "artifactconfig.Validator.Validate")() + + if cfg == nil { + return nil + } + var errs []error + if err := validateVerification("artifacts.verification", cfg.Verification); err != nil { + errs = append(errs, err) + } + for _, alias := range sortedAliases(cfg) { + repo := cfg.Repositories[alias] + if err := v.validateRepository(cfg, alias, &repo); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// ValidateRepository validates one repository in the context of the whole configuration, which +// the lock delegation and verification checks need. +func (v *Validator) ValidateRepository(cfg *schema.ArtifactConfig, alias string) error { + defer perf.Track(nil, "artifactconfig.Validator.ValidateRepository")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return fmt.Errorf("%w: repository %q is not configured (configured: %v)", errUtils.ErrArtifactConfigInvalid, alias, sortedAliases(cfg)) + } + return v.validateRepository(cfg, alias, &repo) +} + +func (v *Validator) validateRepository(cfg *schema.ArtifactConfig, alias string, repo *schema.ArtifactRepositoryConfig) error { + prefix := fmt.Sprintf("repository %q", alias) + if !aliasPattern.MatchString(alias) { + return fmt.Errorf("%w: %s: alias must start with a letter or digit and contain only letters, digits, '_', '.' and '-'", errUtils.ErrArtifactConfigInvalid, prefix) + } + desc, err := v.resolveKind(repo.Kind) + if err != nil { + return fmt.Errorf("%s: %w", prefix, err) + } + errs := []error{ + v.validateIdentity(repo.Identity), + artifact.ValidateRepositoryConfig(desc, repo), + validatePathOptions(desc, repo), + validateCatalog(desc, repo), + validateDefaults(repo.Defaults), + validateVerification("verification", repo.Verification), + validateWeakening(cfg.Verification, repo.Verification), + validateLocking(v, cfg, alias, repo), + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("%s: %w", prefix, err) + } + return nil +} + +// resolveKind finds the descriptor for a kind, rejecting empty and unknown kinds. +func (v *Validator) resolveKind(kind string) (*artifact.KindDescriptor, error) { + if kind == "" { + return nil, fmt.Errorf("%w: kind is required (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, v.registry.Kinds()) + } + desc, ok := v.registry.Lookup(kind) + if !ok { + return nil, fmt.Errorf("%w: unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, kind, v.registry.Kinds()) + } + return desc, nil +} + +// validateIdentity checks the shape of an auth identity reference. Resolution is lazy. +func (v *Validator) validateIdentity(identity string) error { + if identity == "" { + return nil + } + switch { + case len(identity) > maxIdentityLength: + return fmt.Errorf("%w: identity exceeds %d characters", errUtils.ErrArtifactConfigInvalid, maxIdentityLength) + case strings.IndexFunc(identity, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) >= 0: + return fmt.Errorf("%w: identity %q must be an auth identity name without whitespace", errUtils.ErrArtifactConfigInvalid, identity) + case strings.Contains(identity, "://"): + return fmt.Errorf("%w: identity %q looks like a credential or URL; reference an Atmos auth identity name", errUtils.ErrArtifactConfigInvalid, identity) + case v.identityExists != nil && !v.identityExists(identity): + return fmt.Errorf("%w: identity %q is not defined under auth.identities", errUtils.ErrArtifactConfigInvalid, identity) + } + return nil +} + +// validatePathOptions rejects local path options that cannot be anchored to a local file. +func validatePathOptions(desc *artifact.KindDescriptor, repo *schema.ArtifactRepositoryConfig) error { + var errs []error + for _, option := range desc.PathOptions { + value, ok := repo.Options[option].(string) + if !ok || value == "" { + continue + } + if _, err := AnchorPath(repo.PathOrigin, value); err != nil { + errs = append(errs, fmt.Errorf("options.%s: %w", option, err)) + } + } + return errors.Join(errs...) +} + +// validateDefaults checks the default artifact metadata. +func validateDefaults(defaults *schema.ArtifactRepositoryDefaults) error { + if defaults == nil || defaults.Metadata == nil { + return nil + } + var errs []error + if len(defaults.Metadata.Description) > maxDescriptionLength { + errs = append(errs, fmt.Errorf("%w: defaults.metadata.description exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, maxDescriptionLength)) + } + if err := artifact.ValidateLabels(defaults.Metadata.Labels); err != nil { + errs = append(errs, fmt.Errorf("defaults.metadata.labels: %w", err)) + } + return errors.Join(errs...) +} + +func sortedAliases(cfg *schema.ArtifactConfig) []string { + aliases := make([]string, 0, len(cfg.Repositories)) + for alias := range cfg.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + return aliases +} diff --git a/pkg/artifact/config/validate_test.go b/pkg/artifact/config/validate_test.go new file mode 100644 index 00000000000..167d254ae36 --- /dev/null +++ b/pkg/artifact/config/validate_test.go @@ -0,0 +1,242 @@ +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinels: renaming a schema field these tests rely on must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: "", Identity: "", Packaging: "", Permissions: "", Catalog: ""} + +var _ = schema.ArtifactLockingConfig{RequiredScope: "", Repository: ""} + +func localRepo(path string) schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": path}} +} + +func s3Repo() schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifact-coordination"}} +} + +func registryRepo() schema.ArtifactRepositoryConfig { + return schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "ghcr.io", "prefix": "acme/atmos"}} +} + +func validateOne(t *testing.T, alias string, repo *schema.ArtifactRepositoryConfig, opts ...Option) error { + t.Helper() + return Validate(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{alias: *repo}}, opts...) +} + +func TestValidate_NilAndEmptyConfigurationAreValid(t *testing.T) { + require.NoError(t, Validate(nil)) + require.NoError(t, Validate(&schema.ArtifactConfig{})) +} + +func TestValidate_RepositoryBasics(t *testing.T) { + root := t.TempDir() + tests := []struct { + name string + alias string + mutate func(*schema.ArtifactRepositoryConfig) + wantErr string + }{ + {name: "valid local", alias: "local"}, + {name: "alias with dots and case", alias: "Prod.US-1"}, + {name: "alias with slash", alias: "a/b", wantErr: "alias must start"}, + {name: "alias leading dash", alias: "-x", wantErr: "alias must start"}, + {name: "missing kind", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Kind = "" }, wantErr: "kind is required"}, + {name: "unknown kind", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Kind = "ftp/server" }, wantErr: `unknown kind "ftp/server"`}, + {name: "missing required option", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options = nil }, wantErr: `"path" is required`}, + {name: "unknown option", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["bucket"] = "b" }, wantErr: "bucket"}, + {name: "credentials in options", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["access_key"] = "AKIA" }, wantErr: "auth identity"}, + {name: "identity with whitespace", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "a b" }, wantErr: "without whitespace"}, + {name: "identity that is a URL", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "https://u:p@host" }, wantErr: "credential or URL"}, + {name: "permissions on a directory", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Permissions = "0600" }, wantErr: "does not accept permissions"}, + {name: "packaging outside the matrix", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Packaging = "tgz" }, wantErr: "does not support packaging"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := localRepo(root) + if tt.mutate != nil { + tt.mutate(&repo) + } + err := validateOne(t, tt.alias, &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestValidate_JoinsErrorsAcrossRepositoriesInAliasOrder(t *testing.T) { + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "zeta": {Kind: "bad/one"}, + "alpha": {Kind: "bad/two"}, + }} + err := Validate(cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + message := err.Error() + assert.Less(t, strings.Index(message, `"alpha"`), strings.Index(message, `"zeta"`)) +} + +func TestValidate_IdentityLookupIsOptional(t *testing.T) { + repo := localRepo(t.TempDir()) + repo.Identity = "registry-publisher" + require.NoError(t, validateOne(t, "x", &repo), "identity is resolved lazily; shape only by default") + + err := validateOne(t, "x", &repo, WithIdentityLookup(func(string) bool { return false })) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "not defined under auth.identities") + + require.NoError(t, validateOne(t, "x", &repo, WithIdentityLookup(func(name string) bool { return name == "registry-publisher" }))) +} + +func TestValidate_UsesInjectedRegistryDescriptor(t *testing.T) { + registry := artifact.NewRepositoryRegistry() + require.NoError(t, registry.Register(&artifact.KindDescriptor{Kind: "gcp/gcs", Capabilities: artifact.Capabilities{Packagings: []artifact.Packaging{artifact.PackagingTgz}}, DefaultPackaging: artifact.PackagingTgz}, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...artifact.RepositoryOption) (artifact.Repository, error) { + return nil, nil + })) + require.NoError(t, validateOne(t, "x", &schema.ArtifactRepositoryConfig{Kind: "gcp/gcs"}, WithRegistry(registry))) + require.Error(t, validateOne(t, "x", &schema.ArtifactRepositoryConfig{Kind: "gcp/gcs"}, WithRegistry(artifact.NewRepositoryRegistry()))) +} + +func TestValidateRepository_UnknownAlias(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{}, "nope") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestValidate_RelativePathsNeedALocalDeclaringFile(t *testing.T) { + dir := t.TempDir() + tests := []struct { + name string + origin schema.ArtifactConfigOrigin + path string + wantErr string + }{ + {name: "relative with local origin", origin: NewOrigin(filepath.Join(dir, "atmos.yaml"), false), path: "store"}, + {name: "absolute needs no origin", path: filepath.Join(dir, "store")}, + {name: "relative from remote import", origin: NewOrigin(filepath.Join(dir, "tmp", "remote.yaml"), true), path: "store", wantErr: "remote import"}, + {name: "relative with unknown origin", path: "store", wantErr: "no declaring configuration file"}, + {name: "absolute from remote import is allowed", origin: NewOrigin(filepath.Join(dir, "tmp", "remote.yaml"), true), path: filepath.Join(dir, "store")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := localRepo(tt.path) + repo.PathOrigin = tt.origin + err := validateOne(t, "x", &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestAnchorPath(t *testing.T) { + dir := t.TempDir() + origin := NewOrigin(filepath.Join(dir, "conf", "atmos.yaml"), false) + assert.Equal(t, filepath.Join(dir, "conf"), origin.Dir) + + got, err := AnchorPath(origin, filepath.Join("..", "shared", "store")) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "shared", "store"), got) + + absolute := filepath.Join(dir, "abs", "..", "abs2") + got, err = AnchorPath(origin, absolute) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "abs2"), got) + + assert.True(t, NewOrigin("", true).Remote) + assert.True(t, NewOrigin("", false).IsZero()) +} + +func TestValidate_DefaultsMetadata(t *testing.T) { + repo := localRepo(t.TempDir()) + repo.Defaults = &schema.ArtifactRepositoryDefaults{Metadata: &schema.ArtifactMetadataDefaults{ + Description: "backups", + Labels: map[string]string{"team": "platform", "bad=key": "x", "also": "bad,value"}, + }} + err := validateOne(t, "x", &repo) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "defaults.metadata.labels") + assert.Contains(t, err.Error(), "bad=key") + assert.Contains(t, err.Error(), `"also"`) + + repo.Defaults.Metadata.Labels = map[string]string{"team": "platform"} + require.NoError(t, validateOne(t, "x", &repo)) + + repo.Defaults.Metadata.Description = strings.Repeat("d", 4097) + require.ErrorIs(t, validateOne(t, "x", &repo), errUtils.ErrArtifactConfigInvalid) +} + +func TestValidate_CatalogReferences(t *testing.T) { + tests := []struct { + name string + mutate func(*schema.ArtifactRepositoryConfig) + wantErr string + }{ + {name: "PRD example", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/atmos/catalog:stable" }}, + {name: "digest reference", mutate: func(r *schema.ArtifactRepositoryConfig) { + r.Catalog = "oci://ghcr.io/acme/atmos/catalog@sha256:" + strings.Repeat("a", 64) + }}, + {name: "host compared case-insensitively", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://GHCR.io/acme/catalog:stable" }}, + {name: "another registry host", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://evil.example/acme/catalog:stable" }, wantErr: "does not match the repository registry"}, + {name: "wrong scheme", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "https://ghcr.io/acme/catalog:stable" }, wantErr: "oci://"}, + {name: "no tag or digest", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog" }, wantErr: "no implicit latest"}, + {name: "credentials", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/acme/catalog:stable" }, wantErr: "credentials"}, + {name: "query", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog:stable?token=x" }, wantErr: "credentials, a query"}, + {name: "bad tag", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog:-bad" }, wantErr: "tag syntax"}, + {name: "bad digest", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog@sha256:xyz" }, wantErr: "digest"}, + {name: "uppercase path", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/Acme/catalog:stable" }, wantErr: "lowercase"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := registryRepo() + tt.mutate(&repo) + err := validateOne(t, "shared", &repo) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "credentials are never echoed") + }) + } + + s3 := s3Repo() + s3.Catalog = "oci://ghcr.io/acme/catalog:stable" + err := validateOne(t, "backups", &s3) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "does not support a catalog") +} + +func TestValidate_PerformsNoFilesystemSideEffects(t *testing.T) { + root := t.TempDir() + cfg := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "local": localRepo(filepath.Join(root, "dest", "local")), + "archives": {Kind: artifact.RepositoryKindLocalArchive, Permissions: "0600", Options: map[string]any{"path": filepath.Join(root, "dest", "archives"), "key_pattern": "{{ .Name }}.tgz"}}, + "layout": {Kind: artifact.RepositoryKindOCILayout, Options: map[string]any{"path": filepath.Join(root, "dest", "layout")}}, + }} + require.NoError(t, Validate(cfg)) + + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries, "validating or describing configuration must not create destination directories") +} diff --git a/pkg/artifact/config/verification.go b/pkg/artifact/config/verification.go new file mode 100644 index 00000000000..c3b21a43536 --- /dev/null +++ b/pkg/artifact/config/verification.go @@ -0,0 +1,115 @@ +package config + +import ( + "errors" + "fmt" + "net/url" + "strings" + "unicode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// validateVerification checks one verification policy level (global or repository). +func validateVerification(scope string, policy *schema.ArtifactVerificationConfig) error { + if policy == nil { + return nil + } + var errs []error + for index, trusted := range policy.TrustedIdentities { + if err := validateTrustedIdentity(trusted); err != nil { + errs = append(errs, fmt.Errorf("trusted_identities[%d]: %w", index, err)) + } + } + for index, key := range policy.TrustedKeys { + if err := validateTrustedKey(policy, key); err != nil { + errs = append(errs, fmt.Errorf("trusted_keys[%d]: %w", index, err)) + } + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("%s: %w", scope, err) + } + return nil +} + +func validateTrustedIdentity(trusted schema.ArtifactTrustedIdentity) error { + var errs []error + if strings.TrimSpace(trusted.Identity) == "" || strings.IndexFunc(trusted.Identity, unicode.IsControl) >= 0 { + errs = append(errs, fmt.Errorf("%w: identity is required and must not contain control characters", errUtils.ErrArtifactConfigInvalid)) + } + issuer, err := url.Parse(trusted.Issuer) + if trusted.Issuer == "" || err != nil || issuer.Scheme != "https" || issuer.Host == "" || issuer.User != nil { + errs = append(errs, fmt.Errorf("%w: issuer %q must be an https URL without credentials", errUtils.ErrArtifactConfigInvalid, trusted.Issuer)) + } + return errors.Join(errs...) +} + +func validateTrustedKey(policy *schema.ArtifactVerificationConfig, key string) error { + if key == "" || strings.ContainsRune(key, 0) { + return fmt.Errorf("%w: key path must be a non-empty file path", errUtils.ErrArtifactConfigInvalid) + } + if _, err := AnchorPath(policy.KeysOrigin, key); err != nil { + return err + } + return nil +} + +// validateWeakening rejects a repository that tries to relax the global policy. +func validateWeakening(global, repo *schema.ArtifactVerificationConfig) error { + if global == nil || repo == nil || global.RequireSignature == nil || repo.RequireSignature == nil { + return nil + } + if *global.RequireSignature && !*repo.RequireSignature { + return fmt.Errorf("%w: verification.require_signature: false cannot weaken the global artifacts.verification.require_signature: true", errUtils.ErrArtifactConfigInvalid) + } + return nil +} + +// EffectivePolicy returns the verification policy that applies to a repository. +// +// A global requirement can be strengthened but never weakened: RequireSignature is true when +// either level requires it. Within one trust set any configured signer satisfies it; when both +// the global and the repository define trust (identities or keys), the policy carries two trust +// sets and both must be satisfied, so a repository can narrow trust but never widen it. The +// global trust set comes first. +func EffectivePolicy(cfg *schema.ArtifactConfig, alias string) (artifact.VerificationPolicy, error) { + defer perf.Track(nil, "artifactconfig.EffectivePolicy")() + + repo, ok := cfg.Repositories[alias] + if !ok { + return artifact.VerificationPolicy{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + } + return CombinePolicy(cfg.Verification, repo.Verification), nil +} + +// CombinePolicy combines a global and a repository policy level. Either may be nil. +func CombinePolicy(global, repo *schema.ArtifactVerificationConfig) artifact.VerificationPolicy { + defer perf.Track(nil, "artifactconfig.CombinePolicy")() + + var policy artifact.VerificationPolicy + for _, level := range []*schema.ArtifactVerificationConfig{global, repo} { + if level == nil { + continue + } + if level.RequireSignature != nil && *level.RequireSignature { + policy.RequireSignature = true + } + if set := trustSetOf(level); len(set.Identities) > 0 || len(set.Keys) > 0 { + policy.TrustSets = append(policy.TrustSets, set) + } + } + return policy +} + +// trustSetOf converts one policy level to a trust set, copying slices so the policy never aliases configuration. +func trustSetOf(level *schema.ArtifactVerificationConfig) artifact.TrustSet { + var set artifact.TrustSet + for _, trusted := range level.TrustedIdentities { + set.Identities = append(set.Identities, artifact.TrustedIdentity{Identity: trusted.Identity, Issuer: trusted.Issuer}) + } + set.Keys = append(set.Keys, level.TrustedKeys...) + return set +} diff --git a/pkg/artifact/config/verification_test.go b/pkg/artifact/config/verification_test.go new file mode 100644 index 00000000000..25422eefc17 --- /dev/null +++ b/pkg/artifact/config/verification_test.go @@ -0,0 +1,151 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + workflowIdentity = "https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main" + githubIssuer = "https://token.actions.githubusercontent.com" +) + +func boolPtr(b bool) *bool { return &b } + +func identityPolicy(requireSignature *bool, identity string) *schema.ArtifactVerificationConfig { + return &schema.ArtifactVerificationConfig{ + RequireSignature: requireSignature, + TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: identity, Issuer: githubIssuer}}, + } +} + +func TestValidate_VerificationShape(t *testing.T) { + keyDir := t.TempDir() + tests := []struct { + name string + policy *schema.ArtifactVerificationConfig + wantErr string + }{ + {name: "PRD example", policy: identityPolicy(boolPtr(true), workflowIdentity)}, + {name: "empty key list is allowed", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{}}}, + {name: "missing identity", policy: identityPolicy(nil, ""), wantErr: "identity is required"}, + {name: "issuer without https", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "http://issuer.example"}}}, wantErr: "https URL"}, + {name: "missing issuer", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me"}}}, wantErr: "https URL"}, + {name: "issuer with credentials", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "https://u:p@issuer.example"}}}, wantErr: "https URL"}, + {name: "absolute key", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{filepath.Join(keyDir, "cosign.pub")}}}, + {name: "empty key path", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{""}}, wantErr: "non-empty file path"}, + {name: "relative key without a declaring file", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}}, wantErr: "no declaring configuration file"}, + {name: "relative key from a remote import", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}, KeysOrigin: NewOrigin(filepath.Join(keyDir, "r.yaml"), true)}, wantErr: "remote import"}, + {name: "relative key from a local file", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}, KeysOrigin: NewOrigin(filepath.Join(keyDir, "atmos.yaml"), false)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Validate(&schema.ArtifactConfig{Verification: tt.policy}) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestValidate_RepositoryCannotWeakenGlobalSignatureRequirement(t *testing.T) { + repo := registryRepo() + repo.Verification = &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(false)} + cfg := &schema.ArtifactConfig{ + Verification: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true)}, + Repositories: map[string]schema.ArtifactRepositoryConfig{"shared": repo}, + } + err := Validate(cfg) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "cannot weaken") + + // Strengthening, or leaving it unset, is fine. + repo.Verification.RequireSignature = boolPtr(true) + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) + repo.Verification.RequireSignature = nil + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) + + // A repository may be stricter than a permissive global policy. + cfg.Verification.RequireSignature = boolPtr(false) + repo.Verification.RequireSignature = boolPtr(true) + cfg.Repositories["shared"] = repo + require.NoError(t, Validate(cfg)) +} + +func TestEffectivePolicy(t *testing.T) { + repoIdentity := "https://github.com/acme/other/.github/workflows/release.yml@refs/heads/main" + tests := []struct { + name string + global, repo *schema.ArtifactVerificationConfig + want artifact.VerificationPolicy + }{ + {name: "nothing configured", want: artifact.VerificationPolicy{}}, + { + name: "global only", + global: identityPolicy(boolPtr(true), workflowIdentity), + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: workflowIdentity, Issuer: githubIssuer}}}, + }}, + }, + { + name: "repository only", + repo: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true), TrustedKeys: []string{"/keys/a.pub"}}, + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{{Keys: []string{"/keys/a.pub"}}}}, + }, + { + name: "both levels keep two trust sets, global first, never unioned", + global: identityPolicy(boolPtr(true), workflowIdentity), + repo: identityPolicy(nil, repoIdentity), + want: artifact.VerificationPolicy{RequireSignature: true, TrustSets: []artifact.TrustSet{ + {Identities: []artifact.TrustedIdentity{{Identity: workflowIdentity, Issuer: githubIssuer}}}, + {Identities: []artifact.TrustedIdentity{{Identity: repoIdentity, Issuer: githubIssuer}}}, + }}, + }, + { + name: "repository false cannot lower a global requirement", + global: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(true)}, + repo: &schema.ArtifactVerificationConfig{RequireSignature: boolPtr(false)}, + want: artifact.VerificationPolicy{RequireSignature: true}, + }, + { + name: "empty trust level adds no trust set", + global: &schema.ArtifactVerificationConfig{TrustedKeys: []string{}}, + want: artifact.VerificationPolicy{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := registryRepo() + repo.Verification = tt.repo + cfg := &schema.ArtifactConfig{Verification: tt.global, Repositories: map[string]schema.ArtifactRepositoryConfig{"shared": repo}} + got, err := EffectivePolicy(cfg, "shared") + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + _, err := EffectivePolicy(&schema.ArtifactConfig{}, "missing") + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestCombinePolicy_DoesNotAliasConfiguration(t *testing.T) { + global := identityPolicy(boolPtr(true), workflowIdentity) + global.TrustedKeys = []string{"/keys/a.pub"} + policy := CombinePolicy(global, nil) + policy.TrustSets[0].Keys[0] = "/mutated" + policy.TrustSets[0].Identities[0].Identity = "mutated" + assert.Equal(t, []string{"/keys/a.pub"}, global.TrustedKeys) + assert.Equal(t, workflowIdentity, global.TrustedIdentities[0].Identity) +} diff --git a/pkg/artifact/kinds.go b/pkg/artifact/kinds.go new file mode 100644 index 00000000000..6525624529d --- /dev/null +++ b/pkg/artifact/kinds.go @@ -0,0 +1,206 @@ +package artifact + +import ( + "errors" + "fmt" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Repository kinds defined by V1. They name repository backends and are independent of artifact +// kinds such as "files" or "component/terraform". +const ( + // RepositoryKindLocalDir stores copied files below a local directory. + RepositoryKindLocalDir = "local/dir" + + // RepositoryKindLocalArchive stores one archive per artifact below a local directory. + RepositoryKindLocalArchive = "local/archive" + + // RepositoryKindS3 stores objects in an S3 bucket. + RepositoryKindS3 = "aws/s3" + + // RepositoryKindOCIRegistry stores artifacts in an OCI registry. + RepositoryKindOCIRegistry = "oci/registry" + + // RepositoryKindOCILayout stores artifacts in an OCI image layout directory. + RepositoryKindOCILayout = "oci/layout" +) + +// KindDescriptor is the static, network-free description of a repository kind. Configuration +// validation, path anchoring, and lock delegation checks use it before any backend exists or +// any credential is resolved. +// +// The V1 kinds below have built-in descriptors so that validation works before backend packages +// register. A backend that registers its kind supplies its own descriptor, which then takes +// precedence (typically a copy of the built-in one with refinements). +type KindDescriptor struct { + // Kind is the repository kind name, for example "aws/s3". + Kind string + + // Capabilities are the capabilities known from configuration alone (supported packagings and + // lock scopes). A constructed repository's own Capabilities refine them. + Capabilities Capabilities + + // DefaultPackaging is the packaging used when the configuration omits it. + DefaultPackaging Packaging + + // PathOptions names the option keys that are local destination paths. Relative values resolve + // against the declaring configuration file when the configuration is loaded. + PathOptions []string + + // AllowsPermissions reports that the kind accepts the `permissions` field. + AllowsPermissions bool + + // AllowsCatalog reports that the kind accepts the `catalog` field. + AllowsCatalog bool + + // NewOptions returns a fresh typed options struct for strict decoding and validation. + NewOptions func() Options +} + +// UsesLocalPath reports whether the named option of the kind is a local destination path. +func (d *KindDescriptor) UsesLocalPath(option string) bool { + defer perf.Track(nil, "artifact.KindDescriptor.UsesLocalPath")() + + for _, name := range d.PathOptions { + if name == option { + return true + } + } + return false +} + +// EffectivePackaging returns the configured packaging, or the kind's default when it is omitted. +func (d *KindDescriptor) EffectivePackaging(configured string) Packaging { + defer perf.Track(nil, "artifact.KindDescriptor.EffectivePackaging")() + + if configured == "" { + return d.DefaultPackaging + } + return Packaging(configured) +} + +// builtinKinds returns the documented static set of V1 repository kinds. +func builtinKinds() map[string]KindDescriptor { + archive := []Packaging{PackagingTar, PackagingTgz, PackagingZip} + return map[string]KindDescriptor{ + RepositoryKindLocalDir: { + Kind: RepositoryKindLocalDir, + Capabilities: Capabilities{Read: true, Write: true, Packagings: []Packaging{PackagingNone}, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingNone, + PathOptions: []string{OptionPath}, + NewOptions: func() Options { return &LocalDirOptions{} }, + }, + RepositoryKindLocalArchive: { + Kind: RepositoryKindLocalArchive, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + PathOptions: []string{OptionPath}, + AllowsPermissions: true, + NewOptions: func() Options { return &LocalArchiveOptions{} }, + }, + RepositoryKindS3: { + Kind: RepositoryKindS3, + Capabilities: Capabilities{Read: true, Write: true, Packagings: append([]Packaging{PackagingNone}, archive...), LockScopes: []LockScope{LockScopeHost, LockScopeRepository}}, + DefaultPackaging: PackagingTgz, + NewOptions: func() Options { return &S3Options{} }, + }, + RepositoryKindOCIRegistry: { + Kind: RepositoryKindOCIRegistry, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + AllowsCatalog: true, + NewOptions: func() Options { return &OCIRegistryOptions{} }, + }, + RepositoryKindOCILayout: { + Kind: RepositoryKindOCILayout, + Capabilities: Capabilities{Read: true, Write: true, Packagings: archive, LockScopes: []LockScope{LockScopeHost}}, + DefaultPackaging: PackagingTgz, + PathOptions: []string{OptionPath}, + NewOptions: func() Options { return &OCILayoutOptions{} }, + }, + } +} + +// BuiltinKind returns the built-in descriptor of a V1 repository kind. +func BuiltinKind(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.BuiltinKind")() + + desc, ok := builtinKinds()[kind] + if !ok { + return nil, false + } + return &desc, true +} + +// BuiltinKinds returns the sorted names of the V1 repository kinds. +func BuiltinKinds() []string { + defer perf.Track(nil, "artifact.BuiltinKinds")() + + kinds := builtinKinds() + names := make([]string, 0, len(kinds)) + for name := range kinds { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// ValidateRepositoryConfig checks one repository configuration against its kind descriptor: +// the packaging matrix, permissions (accepted only where the kind allows them, and only as a +// quoted octal string), and strict typed options with their required values. It performs no +// network access, credential resolution, or directory creation. +func ValidateRepositoryConfig(desc *KindDescriptor, cfg *schema.ArtifactRepositoryConfig) error { + defer perf.Track(nil, "artifact.ValidateRepositoryConfig")() + + if desc == nil || cfg == nil { + return fmt.Errorf("%w: repository configuration is nil", errUtils.ErrArtifactConfigInvalid) + } + var errs []error + packaging, err := validatePackagingFor(desc, cfg.Packaging) + errs = append(errs, err) + errs = append(errs, validatePermissionsFor(desc, cfg.Permissions)) + errs = append(errs, validateOptionsFor(desc, cfg, packaging)) + return errors.Join(errs...) +} + +// validatePackagingFor checks the packaging against the kind's matrix and returns the effective packaging. +func validatePackagingFor(desc *KindDescriptor, configured string) (Packaging, error) { + packaging := desc.EffectivePackaging(configured) + if configured == "" { + return packaging, nil + } + if !desc.Capabilities.HasPackaging(packaging) { + return packaging, fmt.Errorf("%w: %w: kind %q does not support packaging %q (supported: %v)", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, configured, desc.Capabilities.Packagings) + } + return packaging, nil +} + +func validatePermissionsFor(desc *KindDescriptor, permissions string) error { + if permissions == "" { + return nil + } + if !desc.AllowsPermissions { + return fmt.Errorf("%w: kind %q does not accept permissions; they apply only to local archive files", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + _, err := ParsePermissions(permissions) + return err +} + +func validateOptionsFor(desc *KindDescriptor, cfg *schema.ArtifactRepositoryConfig, packaging Packaging) error { + if desc.NewOptions == nil { + return nil + } + options := desc.NewOptions() + if err := DecodeOptions(cfg.Options, options); err != nil { + return err + } + errs := []error{options.Validate()} + if aware, ok := options.(PackagingValidator); ok { + errs = append(errs, aware.ValidateForPackaging(packaging)) + } + return errors.Join(errs...) +} diff --git a/pkg/artifact/kinds_test.go b/pkg/artifact/kinds_test.go new file mode 100644 index 00000000000..f4ee61c65c5 --- /dev/null +++ b/pkg/artifact/kinds_test.go @@ -0,0 +1,144 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestValidateRepositoryConfig_PackagingMatrix(t *testing.T) { + tests := []struct { + kind string + options map[string]any + packaging string + valid bool + }{ + {kind: RepositoryKindLocalDir, packaging: "", valid: true}, + {kind: RepositoryKindLocalDir, packaging: "none", valid: true}, + {kind: RepositoryKindLocalDir, packaging: "tar"}, + {kind: RepositoryKindLocalDir, packaging: "tgz"}, + {kind: RepositoryKindLocalDir, packaging: "zip"}, + {kind: RepositoryKindLocalArchive, packaging: "", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "none"}, + {kind: RepositoryKindLocalArchive, packaging: "tar", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "tgz", valid: true}, + {kind: RepositoryKindLocalArchive, packaging: "zip", valid: true}, + {kind: RepositoryKindS3, packaging: "", valid: true}, + {kind: RepositoryKindS3, packaging: "none", valid: true}, + {kind: RepositoryKindS3, packaging: "tar", valid: true}, + {kind: RepositoryKindS3, packaging: "tgz", valid: true}, + {kind: RepositoryKindS3, packaging: "zip", valid: true}, + {kind: RepositoryKindS3, packaging: "rar"}, + {kind: RepositoryKindOCIRegistry, packaging: "", valid: true}, + {kind: RepositoryKindOCIRegistry, packaging: "none"}, + {kind: RepositoryKindOCIRegistry, packaging: "tar", valid: true}, + {kind: RepositoryKindOCILayout, packaging: "none"}, + {kind: RepositoryKindOCILayout, packaging: "zip", valid: true}, + } + for _, tt := range tests { + t.Run(tt.kind+"/"+tt.packaging, func(t *testing.T) { + desc, ok := BuiltinKind(tt.kind) + require.True(t, ok) + cfg := &schema.ArtifactRepositoryConfig{Kind: tt.kind, Packaging: tt.packaging, Options: validOptionsFor(tt.kind)} + err := ValidateRepositoryConfig(desc, cfg) + if tt.valid { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) + } +} + +func validOptionsFor(kind string) map[string]any { + switch kind { + case RepositoryKindS3: + return map[string]any{"bucket": "acme-artifacts"} + case RepositoryKindOCIRegistry: + return map[string]any{"registry": "ghcr.io"} + default: + return map[string]any{"path": "out"} + } +} + +func TestValidateRepositoryConfig_Permissions(t *testing.T) { + tests := []struct { + name string + kind string + permissions string + wantErr bool + }{ + {name: "archive accepts quoted octal", kind: RepositoryKindLocalArchive, permissions: "0600"}, + {name: "archive accepts three digits", kind: RepositoryKindLocalArchive, permissions: "640"}, + {name: "archive rejects preserve", kind: RepositoryKindLocalArchive, permissions: "preserve", wantErr: true}, + {name: "archive rejects privileged bits", kind: RepositoryKindLocalArchive, permissions: "4755", wantErr: true}, + {name: "archive rejects 0o prefix", kind: RepositoryKindLocalArchive, permissions: "0o600", wantErr: true}, + {name: "archive rejects decimal-looking 384", kind: RepositoryKindLocalArchive, permissions: "384", wantErr: true}, + {name: "directory rejects permissions", kind: RepositoryKindLocalDir, permissions: "0600", wantErr: true}, + {name: "s3 rejects permissions", kind: RepositoryKindS3, permissions: "0600", wantErr: true}, + {name: "registry rejects permissions", kind: RepositoryKindOCIRegistry, permissions: "0600", wantErr: true}, + {name: "layout rejects permissions", kind: RepositoryKindOCILayout, permissions: "0600", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + desc, _ := BuiltinKind(tt.kind) + err := ValidateRepositoryConfig(desc, &schema.ArtifactRepositoryConfig{Kind: tt.kind, Permissions: tt.permissions, Options: validOptionsFor(tt.kind)}) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + return + } + require.NoError(t, err) + }) + } +} + +func TestValidateRepositoryConfig_JoinsEveryProblem(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + err := ValidateRepositoryConfig(desc, &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, Packaging: "none", Permissions: "preserve", Options: map[string]any{"pth": "x"}, + }) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "preserve") + assert.Contains(t, err.Error(), "pth") +} + +func TestValidateRepositoryConfig_KeyPatternAgreesWithEffectivePackaging(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + cfg := &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, Packaging: "zip", + Options: map[string]any{"path": "out", "key_pattern": "{{ .Name }}/{{ .Version }}.tgz"}, + } + require.ErrorIs(t, ValidateRepositoryConfig(desc, cfg), errUtils.ErrArtifactConfigInvalid) + + cfg.Packaging = "" + require.NoError(t, ValidateRepositoryConfig(desc, cfg), "the default packaging for local/archive is tgz") +} + +func TestValidateRepositoryConfig_NilConfig(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.ErrorIs(t, ValidateRepositoryConfig(desc, nil), errUtils.ErrArtifactConfigInvalid) +} + +func TestKindDescriptorHelpers(t *testing.T) { + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + assert.True(t, desc.UsesLocalPath("path")) + assert.False(t, desc.UsesLocalPath("key_pattern")) + assert.Equal(t, PackagingTgz, desc.EffectivePackaging("")) + assert.Equal(t, PackagingZip, desc.EffectivePackaging("zip")) + + _, ok := BuiltinKind("nope") + assert.False(t, ok) + assert.Equal(t, []string{"aws/s3", "local/archive", "local/dir", "oci/layout", "oci/registry"}, BuiltinKinds()) +} + +func TestBuiltinKindsAreIndependentCopies(t *testing.T) { + first, _ := BuiltinKind(RepositoryKindS3) + first.Capabilities.Packagings[0] = PackagingZip + second, _ := BuiltinKind(RepositoryKindS3) + assert.Equal(t, PackagingNone, second.Capabilities.Packagings[0], "mutating a returned descriptor must not change the built-in set") +} diff --git a/pkg/artifact/labels.go b/pkg/artifact/labels.go new file mode 100644 index 00000000000..58dc9ca41c9 --- /dev/null +++ b/pkg/artifact/labels.go @@ -0,0 +1,73 @@ +package artifact + +import ( + "errors" + "fmt" + "sort" + "strings" + "unicode" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Label limits. Labels must be expressible and selectable with `--labels 'key=value,key2=value2'`, +// which splits pairs on ',' and each pair on the first '=' or ':'. +const ( + // MaxLabelKeyLength is the longest label key, in bytes. + MaxLabelKeyLength = 128 + + // MaxLabelValueLength is the longest label value, in bytes. + MaxLabelValueLength = 1024 + + // MaxLabels is the largest number of labels on one artifact version or repository default. + MaxLabels = 64 +) + +// ValidateLabel checks one exact key/value label. The key must be non-empty, free of control +// characters and surrounding whitespace, and free of '=', ':' and ',' so that it can be selected +// with `--labels`; the value must be free of control characters and ','. +func ValidateLabel(key, value string) error { + defer perf.Track(nil, "artifact.ValidateLabel")() + + switch { + case key == "": + return fmt.Errorf("%w: label key must not be empty", errUtils.ErrArtifactConfigInvalid) + case len(key) > MaxLabelKeyLength: + return fmt.Errorf("%w: label key %q exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, key, MaxLabelKeyLength) + case strings.TrimSpace(key) != key: + return fmt.Errorf("%w: label key %q must not have leading or trailing whitespace", errUtils.ErrArtifactConfigInvalid, key) + case strings.ContainsAny(key, "=:,"): + return fmt.Errorf("%w: label key %q must not contain '=', ':' or ',' (they delimit --labels selectors)", errUtils.ErrArtifactConfigInvalid, key) + case strings.IndexFunc(key, unicode.IsControl) >= 0: + return fmt.Errorf("%w: label key %q must not contain control characters", errUtils.ErrArtifactConfigInvalid, key) + case len(value) > MaxLabelValueLength: + return fmt.Errorf("%w: label %q value exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, key, MaxLabelValueLength) + case strings.Contains(value, ","): + return fmt.Errorf("%w: label %q value must not contain ',' (it delimits --labels selectors)", errUtils.ErrArtifactConfigInvalid, key) + case strings.IndexFunc(value, unicode.IsControl) >= 0: + return fmt.Errorf("%w: label %q value must not contain control characters", errUtils.ErrArtifactConfigInvalid, key) + } + return nil +} + +// ValidateLabels checks a label set, returning every malformed label joined in key order. +func ValidateLabels(labels map[string]string) error { + defer perf.Track(nil, "artifact.ValidateLabels")() + + var errs []error + if len(labels) > MaxLabels { + errs = append(errs, fmt.Errorf("%w: %d labels exceed the limit of %d", errUtils.ErrArtifactConfigInvalid, len(labels), MaxLabels)) + } + keys := make([]string, 0, len(labels)) + for key := range labels { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if err := ValidateLabel(key, labels[key]); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/pkg/artifact/labels_test.go b/pkg/artifact/labels_test.go new file mode 100644 index 00000000000..c28bfde40c0 --- /dev/null +++ b/pkg/artifact/labels_test.go @@ -0,0 +1,61 @@ +package artifact + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestValidateLabel(t *testing.T) { + tests := []struct { + name string + key, value string + wantErr string + }{ + {name: "plain", key: "team", value: "platform"}, + {name: "empty value is allowed", key: "team", value: ""}, + {name: "dots and dashes in key", key: "cost-center.id", value: "1234"}, + {name: "empty key", key: "", value: "x", wantErr: "must not be empty"}, + {name: "space-padded key", key: " team", value: "x", wantErr: "whitespace"}, + {name: "equals in key", key: "a=b", value: "x", wantErr: "--labels"}, + {name: "colon in key", key: "a:b", value: "x", wantErr: "--labels"}, + {name: "comma in key", key: "a,b", value: "x", wantErr: "--labels"}, + {name: "control char in key", key: "a\nb", value: "x", wantErr: "control"}, + {name: "oversized key", key: strings.Repeat("k", 129), value: "x", wantErr: "exceeds"}, + {name: "comma in value", key: "team", value: "a,b", wantErr: "','"}, + {name: "control char in value", key: "team", value: "a\tb", wantErr: "control"}, + {name: "oversized value", key: "team", value: strings.Repeat("v", 1025), wantErr: "exceeds"}, + {name: "colon in value is selectable", key: "ref", value: "refs/heads/main:abc"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateLabel(tt.key, tt.value) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestValidateLabels_JoinsInKeyOrderAndBoundsCount(t *testing.T) { + err := ValidateLabels(map[string]string{"b,": "x", "a=": "y", "ok": "fine"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + message := err.Error() + assert.Less(t, strings.Index(message, `"a="`), strings.Index(message, `"b,"`), "errors follow sorted key order") + assert.NotContains(t, message, `"ok"`) + + require.NoError(t, ValidateLabels(nil)) + + many := make(map[string]string) + for i := range MaxLabels + 1 { + many[strings.Repeat("k", 1)+string(rune('a'+i%26))+strings.Repeat("z", i/26)] = "v" + } + require.Error(t, ValidateLabels(many)) +} diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go new file mode 100644 index 00000000000..e75944f9343 --- /dev/null +++ b/pkg/artifact/options.go @@ -0,0 +1,410 @@ +package artifact + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + + "github.com/go-viper/mapstructure/v2" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Options is a typed, validated view of one repository kind's `options` map. The configuration +// schema keeps `options` a generic map so it never depends on a backend; each kind decodes it into +// its own struct with [DecodeOptions] and checks it with Validate. Validate never performs +// network access, credential resolution, or directory creation. +type Options interface { + // Validate checks required options and value shapes. + Validate() error +} + +// PackagingValidator is implemented by Options that depend on the effective packaging, for example +// to check that a key_pattern extension agrees with an archive format. +type PackagingValidator interface { + // ValidateForPackaging checks the options against the effective packaging. + ValidateForPackaging(packaging Packaging) error +} + +// Option keys shared by several kinds. +const ( + // OptionPath is the local destination root of local/dir, local/archive, and oci/layout. + OptionPath = "path" + + // OptionKeyPattern overrides the path below the stable root for local and S3 payloads. + OptionKeyPattern = "key_pattern" +) + +// S3 object tag limits from the S3 documentation. +const ( + maxS3Tags = 10 + maxS3TagKeyLen = 128 + maxS3TagValLen = 256 + reservedTagHead = "aws:" +) + +var ( + // Flags option keys that look like credentials. Credentials never belong + // in repository options; the `identity` field references an Atmos auth identity instead. + credentialOptionPattern = regexp.MustCompile(`(?i)secret|password|passwd|token|credential|access[_-]?key|api[_-]?key`) + + // The S3 general-purpose bucket naming rule (3-63 lowercase letters, digits, dots, hyphens). + s3BucketPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) + + // The shape of a cloud region name such as us-east-2. + regionPattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) + + // A registry host[:port] without a scheme, path, or credentials. + registryHostPattern = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]{1,5})?$`) + + // Matches one Go template action. + templateActionPattern = regexp.MustCompile(`(?s)\{\{.*?\}\}`) + + // Matches Go template string literals so field references are not read from them. + templateStringPattern = regexp.MustCompile("\"(?:[^\"\\\\]|\\\\.)*\"|`[^`]*`") + + // Matches a field reference such as .Name. + templateFieldPattern = regexp.MustCompile(`\.([A-Za-z_][A-Za-z0-9_]*)`) +) + +// Path segments that would escape or empty a path. +const ( + currentDirSegment = "." + parentDirSegment = ".." +) + +// keyPatternFields are the fields a key_pattern may reference. +var keyPatternFields = map[string]bool{"Name": true, "Version": true, "Kind": true, "CreatedAt": true} + +// LocalDirOptions are the options of a local/dir repository. +type LocalDirOptions struct { + // Path is the destination root directory. + Path string `mapstructure:"path"` + + // KeyPattern overrides the payload directory path below the root. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *LocalDirOptions) Validate() error { + defer perf.Track(nil, "artifact.LocalDirOptions.Validate")() + + return errors.Join(validateLocalPath(o.Path), validateKeyPattern(o.KeyPattern, "")) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *LocalDirOptions) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.LocalDirOptions.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// LocalArchiveOptions are the options of a local/archive repository. +type LocalArchiveOptions struct { + // Path is the destination root directory. + Path string `mapstructure:"path"` + + // KeyPattern overrides the archive file path below the root. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *LocalArchiveOptions) Validate() error { + defer perf.Track(nil, "artifact.LocalArchiveOptions.Validate")() + + return errors.Join(validateLocalPath(o.Path), validateKeyPattern(o.KeyPattern, "")) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *LocalArchiveOptions) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.LocalArchiveOptions.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// S3Options are the options of an aws/s3 repository. +type S3Options struct { + // Bucket is the S3 bucket name. + Bucket string `mapstructure:"bucket"` + + // Prefix is the collection root below the bucket. + Prefix string `mapstructure:"prefix"` + + // Region is the bucket region. + Region string `mapstructure:"region"` + + // Tags are S3 object tags applied to every object of a publication. They are not artifact labels. + Tags map[string]string `mapstructure:"tags"` + + // KeyPattern overrides the object key below the prefix. + KeyPattern string `mapstructure:"key_pattern"` +} + +// Validate implements Options. +func (o *S3Options) Validate() error { + defer perf.Track(nil, "artifact.S3Options.Validate")() + + var errs []error + if o.Bucket == "" { + errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "bucket")) + } else if !s3BucketPattern.MatchString(o.Bucket) { + errs = append(errs, fmt.Errorf("%w: bucket %q is not a valid S3 bucket name", errUtils.ErrArtifactConfigInvalid, o.Bucket)) + } + if o.Region != "" && !regionPattern.MatchString(o.Region) { + errs = append(errs, fmt.Errorf("%w: region %q is not a valid region name", errUtils.ErrArtifactConfigInvalid, o.Region)) + } + errs = append(errs, validatePrefix(o.Prefix), validateS3Tags(o.Tags), validateKeyPattern(o.KeyPattern, "")) + return errors.Join(errs...) +} + +// ValidateForPackaging implements PackagingValidator. +func (o *S3Options) ValidateForPackaging(packaging Packaging) error { + defer perf.Track(nil, "artifact.S3Options.ValidateForPackaging")() + + return validateKeyPattern(o.KeyPattern, packaging) +} + +// OCIRegistryOptions are the options of an oci/registry repository. The prefix is an Atmos +// collection-root convention, not an ORAS namespace field: Atmos maps a collection and artifact +// name onto an OCI repository path below it. +type OCIRegistryOptions struct { + // Registry is the registry host, with an optional port, for example ghcr.io. + Registry string `mapstructure:"registry"` + + // Prefix is the collection root path below the registry, for example acme/atmos. + Prefix string `mapstructure:"prefix"` +} + +// Validate implements Options. +func (o *OCIRegistryOptions) Validate() error { + defer perf.Track(nil, "artifact.OCIRegistryOptions.Validate")() + + var errs []error + if o.Registry == "" { + errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) + } else if !registryHostPattern.MatchString(o.Registry) { + errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, o.Registry)) + } + errs = append(errs, validatePrefix(o.Prefix)) + if o.Prefix != "" { + for _, segment := range strings.Split(o.Prefix, "/") { + if err := ValidateName(segment); err != nil { + errs = append(errs, fmt.Errorf("%w: prefix %q is not a valid OCI repository path: %w", errUtils.ErrArtifactConfigInvalid, o.Prefix, err)) + break + } + } + } + return errors.Join(errs...) +} + +// OCILayoutOptions are the options of an oci/layout repository. +type OCILayoutOptions struct { + // Path is the OCI image layout directory. + Path string `mapstructure:"path"` +} + +// Validate implements Options. +func (o *OCILayoutOptions) Validate() error { + defer perf.Track(nil, "artifact.OCILayoutOptions.Validate")() + + return validateLocalPath(o.Path) +} + +// DecodeOptions decodes a generic options map into a typed options struct. Decoding is strict: +// unknown keys and mistyped values are errors, so a typo such as `buckt` never silently becomes a +// missing option. Keys that look like credentials get a pointer to the `identity` field. +func DecodeOptions(raw map[string]any, out any) error { + defer perf.Track(nil, "artifact.DecodeOptions")() + + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: out, + TagName: "mapstructure", + ErrorUnused: true, + }) + if err != nil { + return fmt.Errorf("%w: options decoder: %w", errUtils.ErrArtifactConfigInvalid, err) + } + if err := decoder.Decode(raw); err != nil { + return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, err, credentialHint(raw)) + } + return nil +} + +// credentialHint explains where credentials belong when an option key looks like one. +func credentialHint(raw map[string]any) string { + keys := make([]string, 0, len(raw)) + for key := range raw { + if credentialOptionPattern.MatchString(key) { + keys = append(keys, key) + } + } + if len(keys) == 0 { + return "" + } + sort.Strings(keys) + return fmt.Sprintf(" (option %s looks like a credential; credentials never belong in repository options, reference an auth identity with `identity` instead)", strings.Join(keys, ", ")) +} + +// validateLocalPath checks a local destination root. A relative path is legitimate (it resolves +// against the declaring configuration file), so only emptiness and NUL bytes are rejected here. +func validateLocalPath(path string) error { + switch { + case path == "": + return fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, OptionPath) + case strings.ContainsRune(path, 0): + return fmt.Errorf("%w: %w: path contains a NUL byte", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath) + } + return nil +} + +// ValidatePrefix checks a collection-root prefix such as acme/atmos below a bucket or registry. It +// must be a relative, slash-separated path with no empty, "." or ".." segments, so the prefix +// cannot escape its root, and it must not enter the reserved managed-record namespace. +func ValidatePrefix(prefix string) error { + defer perf.Track(nil, "artifact.ValidatePrefix")() + + return validatePrefix(prefix) +} + +func validatePrefix(prefix string) error { + if prefix == "" { + return nil + } + if strings.HasPrefix(prefix, "/") || strings.Contains(prefix, `\`) || strings.ContainsRune(prefix, 0) { + return fmt.Errorf("%w: %w: prefix %q must be relative and slash-separated", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix) + } + for _, segment := range strings.Split(prefix, "/") { + switch segment { + case "", currentDirSegment, parentDirSegment: + return fmt.Errorf("%w: %w: prefix %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix, currentDirSegment, parentDirSegment) + case ReservedNamespace: + return fmt.Errorf("%w: %w: prefix %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix, ReservedNamespace) + } + } + return nil +} + +// ValidateKeyPattern statically checks a key_pattern template against the effective packaging +// without expanding it. Expansion needs the artifact name, version, kind, and creation time, so +// only the literal structure is checked here: balanced template delimiters, referenced fields +// limited to .Name, .Version, .Kind and .CreatedAt, no literal root escape or empty segment, no +// entry into the reserved namespace, and an archive extension that agrees with the packaging. +// The pattern is never rendered during configuration loading. +func ValidateKeyPattern(pattern string, packaging Packaging) error { + defer perf.Track(nil, "artifact.ValidateKeyPattern")() + + return validateKeyPattern(pattern, packaging) +} + +func validateKeyPattern(pattern string, packaging Packaging) error { + if pattern == "" { + return nil + } + if strings.ContainsRune(pattern, 0) || strings.Contains(pattern, `\`) { + return fmt.Errorf("%w: %w: key_pattern %q must not contain NUL bytes or backslashes", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern) + } + actions := templateActionPattern.FindAllString(pattern, -1) + literal := templateActionPattern.ReplaceAllString(pattern, "\x01") + if strings.Contains(literal, "{{") || strings.Contains(literal, "}}") { + return fmt.Errorf("%w: key_pattern %q has unbalanced template delimiters", errUtils.ErrArtifactConfigInvalid, pattern) + } + if err := validateKeyPatternFields(pattern, actions); err != nil { + return err + } + if err := validateKeyPatternSegments(pattern, literal); err != nil { + return err + } + return validateKeyPatternExtension(pattern, literal, packaging) +} + +func validateKeyPatternFields(pattern string, actions []string) error { + for _, action := range actions { + stripped := templateStringPattern.ReplaceAllString(action, `""`) + for _, match := range templateFieldPattern.FindAllStringSubmatch(stripped, -1) { + if !keyPatternFields[match[1]] { + return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, pattern, match[1]) + } + } + } + return nil +} + +func validateKeyPatternSegments(pattern, literal string) error { + if strings.HasPrefix(literal, "/") { + return fmt.Errorf("%w: %w: key_pattern %q must be relative to the repository root", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern) + } + for _, segment := range strings.Split(literal, "/") { + switch segment { + case "", currentDirSegment, parentDirSegment: + return fmt.Errorf("%w: %w: key_pattern %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern, currentDirSegment, parentDirSegment) + case ReservedNamespace: + return fmt.Errorf("%w: %w: key_pattern %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern, ReservedNamespace) + } + } + return nil +} + +// archiveExtensions maps the literal trailing extension of a key_pattern to the packaging it implies. +var archiveExtensions = []struct { + suffix string + packaging Packaging +}{ + {".tar.gz", PackagingTgz}, + {".tgz", PackagingTgz}, + {".tar", PackagingTar}, + {".zip", PackagingZip}, +} + +func validateKeyPatternExtension(pattern, literal string, packaging Packaging) error { + // Only a literal tail (after the last template action) has a known extension. + tail := literal[strings.LastIndex(literal, "\x01")+1:] + for _, candidate := range archiveExtensions { + if !strings.HasSuffix(tail, candidate.suffix) { + continue + } + if packaging != "" && candidate.packaging != packaging { + return fmt.Errorf("%w: key_pattern %q ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, pattern, candidate.suffix, packaging) + } + return nil + } + return nil +} + +func validateS3Tags(tags map[string]string) error { + if len(tags) > maxS3Tags { + return fmt.Errorf("%w: %d S3 tags exceed the S3 limit of %d", errUtils.ErrArtifactConfigInvalid, len(tags), maxS3Tags) + } + keys := make([]string, 0, len(tags)) + for key := range tags { + keys = append(keys, key) + } + sort.Strings(keys) + var errs []error + for _, key := range keys { + switch { + case key == "" || len(key) > maxS3TagKeyLen: + errs = append(errs, fmt.Errorf("%w: S3 tag key %q must be 1-%d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagKeyLen)) + case strings.HasPrefix(strings.ToLower(key), reservedTagHead): + errs = append(errs, fmt.Errorf("%w: S3 tag key %q uses the reserved %q prefix", errUtils.ErrArtifactConfigInvalid, key, reservedTagHead)) + case len(tags[key]) > maxS3TagValLen: + errs = append(errs, fmt.Errorf("%w: S3 tag %q value exceeds %d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagValLen)) + } + } + return errors.Join(errs...) +} + +// RegistryHostOf returns the host of an oci:// URI without credentials, query, or path, and +// whether the URI is well formed. It is used to bind a catalog to its repository's registry. +func RegistryHostOf(uri string) (string, bool) { + defer perf.Track(nil, "artifact.RegistryHostOf")() + + parsed, err := url.Parse(uri) + if err != nil || parsed.Scheme != "oci" || parsed.Host == "" || parsed.User != nil { + return "", false + } + return strings.ToLower(parsed.Host), true +} diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go new file mode 100644 index 00000000000..beed87248e1 --- /dev/null +++ b/pkg/artifact/options_test.go @@ -0,0 +1,200 @@ +package artifact + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinels: a rename of these schema fields must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: "", Options: nil, Packaging: "", Permissions: "", Catalog: ""} + +func TestDecodeOptions_IsStrict(t *testing.T) { + tests := []struct { + name string + raw map[string]any + wantErr string + wantHint bool + }{ + {name: "valid", raw: map[string]any{"bucket": "acme-artifacts", "region": "us-east-2"}}, + {name: "unknown key is a typo", raw: map[string]any{"buckt": "acme-artifacts"}, wantErr: "buckt"}, + {name: "wrong type", raw: map[string]any{"bucket": 42}, wantErr: "bucket"}, + {name: "credential-looking key gets a hint", raw: map[string]any{"bucket": "acme-artifacts", "secret_key": "x"}, wantErr: "secret_key", wantHint: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var options S3Options + err := DecodeOptions(tt.raw, &options) + if tt.wantErr == "" { + require.NoError(t, err) + assert.Equal(t, "acme-artifacts", options.Bucket) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantHint, strings.Contains(err.Error(), "auth identity")) + }) + } +} + +func TestOptionsValidate(t *testing.T) { + tests := []struct { + name string + options Options + wantErr string + }{ + {name: "local dir ok", options: &LocalDirOptions{Path: ".atmos/artifacts"}}, + {name: "local dir missing path", options: &LocalDirOptions{}, wantErr: `"path" is required`}, + {name: "local dir NUL", options: &LocalDirOptions{Path: "a\x00b"}, wantErr: "NUL"}, + {name: "relative path with traversal is a legitimate destination", options: &LocalDirOptions{Path: "../shared/artifacts"}}, + {name: "archive missing path", options: &LocalArchiveOptions{}, wantErr: `"path" is required`}, + {name: "layout missing path", options: &OCILayoutOptions{}, wantErr: `"path" is required`}, + {name: "s3 ok", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos", Region: "us-east-2", Tags: map[string]string{"purpose": "backup"}}}, + {name: "s3 missing bucket", options: &S3Options{}, wantErr: `"bucket" is required`}, + {name: "s3 bad bucket", options: &S3Options{Bucket: "Bad_Bucket"}, wantErr: "not a valid S3 bucket name"}, + {name: "s3 bad region", options: &S3Options{Bucket: "acme-artifacts", Region: "US East"}, wantErr: "not a valid region"}, + {name: "s3 prefix traversal", options: &S3Options{Bucket: "acme-artifacts", Prefix: "a/../b"}, wantErr: "prefix"}, + {name: "s3 absolute prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "/atmos"}, wantErr: "prefix"}, + {name: "s3 trailing slash prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos/"}, wantErr: "prefix"}, + {name: "s3 reserved prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos/.atmos-artifacts"}, wantErr: "reserved"}, + {name: "s3 reserved tag prefix", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"aws:foo": "x"}}, wantErr: "reserved"}, + {name: "s3 oversized tag value", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("x", 257)}}, wantErr: "value exceeds"}, + {name: "s3 too many tags", options: &S3Options{Bucket: "acme-artifacts", Tags: elevenTags()}, wantErr: "exceed the S3 limit"}, + {name: "s3 bad key pattern", options: &S3Options{Bucket: "acme-artifacts", KeyPattern: "../x"}, wantErr: "segment"}, + {name: "registry ok", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/atmos"}}, + {name: "registry with port", options: &OCIRegistryOptions{Registry: "localhost:5000"}}, + {name: "registry missing", options: &OCIRegistryOptions{}, wantErr: `"registry" is required`}, + {name: "registry with scheme", options: &OCIRegistryOptions{Registry: "https://ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry with credentials", options: &OCIRegistryOptions{Registry: "user:pw@ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry uppercase prefix", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "Acme/Atmos"}, wantErr: "not a valid OCI repository path"}, + {name: "registry prefix traversal", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/../x"}, wantErr: "prefix"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func elevenTags() map[string]string { + tags := make(map[string]string) + for _, key := range strings.Split("a b c d e f g h i j k", " ") { + tags[key] = "v" + } + return tags +} + +func TestValidateKeyPattern(t *testing.T) { + prd := `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz` + tests := []struct { + name string + pattern string + packaging Packaging + wantErr string + wantUnsafe bool + }{ + {name: "empty is fine", pattern: ""}, + {name: "PRD example", pattern: prd, packaging: PackagingTgz}, + {name: "tar.gz extension for tgz", pattern: "{{ .Name }}/{{ .Version }}.tar.gz", packaging: PackagingTgz}, + {name: "extension conflicts with packaging", pattern: "{{ .Name }}/{{ .Version }}.zip", packaging: PackagingTgz, wantErr: "does not match packaging"}, + {name: "archive extension with directory packaging", pattern: "{{ .Name }}/{{ .Version }}.tgz", packaging: PackagingNone, wantErr: "does not match packaging"}, + {name: "extension not checked without packaging", pattern: "{{ .Name }}.zip"}, + {name: "date layout slashes inside action are not segments", pattern: `{{ .CreatedAt | date "2006/01/02" }}/x`}, + {name: "unknown field", pattern: "{{ .Owner }}/x", wantErr: "unknown field .Owner"}, + {name: "unbalanced open", pattern: "{{ .Name /x", wantErr: "unbalanced"}, + {name: "unbalanced close", pattern: ".Name }}/x", wantErr: "unbalanced"}, + {name: "absolute", pattern: "/{{ .Name }}", wantErr: "relative", wantUnsafe: true}, + {name: "traversal", pattern: "{{ .Name }}/../x", wantErr: "segment", wantUnsafe: true}, + {name: "empty segment", pattern: "a//b", wantErr: "segment", wantUnsafe: true}, + {name: "reserved namespace", pattern: ".atmos-artifacts/{{ .Name }}", wantErr: "reserved", wantUnsafe: true}, + {name: "backslash", pattern: `a\b`, wantErr: "backslash", wantUnsafe: true}, + {name: "field name inside string literal is not a field", pattern: `{{ printf ".Owner" }}/{{ .Name }}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateKeyPattern(tt.pattern, tt.packaging) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Equal(t, tt.wantUnsafe, strings.Contains(err.Error(), errUtils.ErrArtifactUnsafePath.Error())) + }) + } +} + +func TestValidatePrefix(t *testing.T) { + require.NoError(t, ValidatePrefix("")) + require.NoError(t, ValidatePrefix("acme/atmos")) + for _, bad := range []string{"/abs", "a/./b", "a//b", "..", `a\b`, "a/.atmos-artifacts"} { + err := ValidatePrefix(bad) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath, bad) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, bad) + } +} + +func TestRegistryHostOf(t *testing.T) { + host, ok := RegistryHostOf("oci://GHCR.io/acme/atmos/catalog:stable") + require.True(t, ok) + assert.Equal(t, "ghcr.io", host) + for _, bad := range []string{"https://ghcr.io/x", "oci:///x", "oci://user:pw@ghcr.io/x", "::"} { + _, ok := RegistryHostOf(bad) + assert.False(t, ok, bad) + } +} + +// TestBuiltinKindsMatchTypedOptions keeps each descriptor's path options and typed options struct in step. +func TestBuiltinKindsMatchTypedOptions(t *testing.T) { + require.NotEmpty(t, BuiltinKinds()) + for _, kind := range BuiltinKinds() { + desc, ok := BuiltinKind(kind) + require.True(t, ok, kind) + require.NotNil(t, desc.NewOptions, kind) + tags := mapstructureTags(desc.NewOptions()) + for _, path := range desc.PathOptions { + assert.Contains(t, tags, path, "%s declares path option %q that its typed options do not decode", kind, path) + } + assert.True(t, desc.Capabilities.HasPackaging(desc.DefaultPackaging), "%s default packaging must be supported", kind) + assert.NotEmpty(t, desc.Capabilities.LockScopes, kind) + } +} + +func mapstructureTags(options Options) []string { + rt := reflect.TypeOf(options).Elem() + tags := make([]string, 0, rt.NumField()) + for i := range rt.NumField() { + tags = append(tags, rt.Field(i).Tag.Get("mapstructure")) + } + return tags +} + +// TestValidationCreatesNothing proves validation is free of filesystem side effects. +func TestValidationCreatesNothing(t *testing.T) { + root := t.TempDir() + cfg := &schema.ArtifactRepositoryConfig{ + Kind: RepositoryKindLocalArchive, + Permissions: "0600", + Options: map[string]any{"path": filepath.Join(root, "not", "created", "yet"), "key_pattern": "{{ .Name }}/{{ .Version }}.tgz"}, + } + desc, _ := BuiltinKind(RepositoryKindLocalArchive) + require.NoError(t, ValidateRepositoryConfig(desc, cfg)) + + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries, "validation must not create destination directories") +} diff --git a/pkg/artifact/permissions.go b/pkg/artifact/permissions.go new file mode 100644 index 00000000000..34c8f80429c --- /dev/null +++ b/pkg/artifact/permissions.go @@ -0,0 +1,47 @@ +package artifact + +import ( + "fmt" + "io/fs" + "regexp" + "strconv" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultArchivePermissions is the mode of a local archive file when permissions are not configured. +const DefaultArchivePermissions fs.FileMode = 0o600 + +const ( + // The numeric base of permission strings. + octalBase = 8 + + // The bit size of a parsed permission value. + permissionBitSize = 32 + + // The largest accepted mode: ordinary permission bits without setuid, setgid, or sticky. + maxPermissionBits = 0o777 +) + +// permissionsPattern accepts three or four octal digits, for example "600" or "0600". +var permissionsPattern = regexp.MustCompile(`^[0-7]{3,4}$`) + +// ParsePermissions parses a quoted octal permission string such as "0600". Only ordinary +// permission bits (at most 0777) are accepted: privileged bits (setuid, setgid, sticky) are +// rejected, as are "preserve" and non-octal forms such as "0o600". +func ParsePermissions(value string) (fs.FileMode, error) { + defer perf.Track(nil, "artifact.ParsePermissions")() + + if !permissionsPattern.MatchString(value) { + return 0, fmt.Errorf("%w: permissions %q must be a quoted octal mode such as \"0600\"", errUtils.ErrArtifactConfigInvalid, value) + } + parsed, err := strconv.ParseUint(value, octalBase, permissionBitSize) + if err != nil { + return 0, fmt.Errorf("%w: permissions %q: %w", errUtils.ErrArtifactConfigInvalid, value, err) + } + if parsed > maxPermissionBits { + return 0, fmt.Errorf("%w: permissions %q must not set privileged mode bits (maximum \"0777\")", errUtils.ErrArtifactConfigInvalid, value) + } + return fs.FileMode(parsed), nil +} diff --git a/pkg/artifact/permissions_test.go b/pkg/artifact/permissions_test.go new file mode 100644 index 00000000000..d7c602e7ac8 --- /dev/null +++ b/pkg/artifact/permissions_test.go @@ -0,0 +1,46 @@ +package artifact + +import ( + "io/fs" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +func TestParsePermissions(t *testing.T) { + tests := []struct { + in string + want fs.FileMode + wantErr bool + }{ + {in: "0600", want: 0o600}, + {in: "600", want: 0o600}, + {in: "0644", want: 0o644}, + {in: "0777", want: 0o777}, + {in: "0000", want: 0}, + {in: "", wantErr: true}, + {in: "preserve", wantErr: true}, + {in: "0o600", wantErr: true}, + {in: "0900", wantErr: true}, + {in: "4755", wantErr: true}, + {in: "1777", wantErr: true}, + {in: "06000", wantErr: true}, + {in: "-0600", wantErr: true}, + {in: " 0600", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + got, err := ParsePermissions(tt.in) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + assert.Equal(t, fs.FileMode(0o600), DefaultArchivePermissions) +} diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go new file mode 100644 index 00000000000..fff2e4f96a8 --- /dev/null +++ b/pkg/artifact/registry.go @@ -0,0 +1,247 @@ +package artifact + +import ( + "context" + "fmt" + "sort" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// IdentityResolver lazily resolves credentials for a named Atmos auth identity. Backends call it +// only when an operation needs credentials; constructing a repository, validating configuration, +// describing configuration, and dry runs never call it. The concrete credential type is owned by +// the backend that asked for it. +type IdentityResolver func(ctx context.Context, identity string) (any, error) + +// RepositoryLookup returns another configured repository by alias, used for delegated locking. +type RepositoryLookup func(alias string) (Repository, error) + +// RepositoryOptions are the construction options a factory receives. +type RepositoryOptions struct { + // ResolveIdentity lazily resolves the repository's identity. It may be nil. + ResolveIdentity IdentityResolver + + // LookupRepository finds another configured repository (for example a lock delegate). It may be nil. + LookupRepository RepositoryLookup +} + +// RepositoryOption configures construction of a repository. +type RepositoryOption func(*RepositoryOptions) + +// WithIdentityResolver supplies the lazy identity resolver. +func WithIdentityResolver(resolver IdentityResolver) RepositoryOption { + defer perf.Track(nil, "artifact.WithIdentityResolver")() + + return func(o *RepositoryOptions) { o.ResolveIdentity = resolver } +} + +// WithRepositoryLookup supplies the lookup used to find delegated repositories. +func WithRepositoryLookup(lookup RepositoryLookup) RepositoryOption { + defer perf.Track(nil, "artifact.WithRepositoryLookup")() + + return func(o *RepositoryOptions) { o.LookupRepository = lookup } +} + +// ApplyRepositoryOptions folds construction options into a RepositoryOptions value for factories. +func ApplyRepositoryOptions(opts ...RepositoryOption) RepositoryOptions { + defer perf.Track(nil, "artifact.ApplyRepositoryOptions")() + + var applied RepositoryOptions + for _, opt := range opts { + if opt != nil { + opt(&applied) + } + } + return applied +} + +// RepositoryFactory constructs a repository. It validates nothing the registry has not already +// validated, must not touch the network, resolve credentials, or create directories, and returns +// a repository whose Capabilities() is available immediately. +type RepositoryFactory func(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) + +type registeredKind struct { + descriptor KindDescriptor + factory RepositoryFactory +} + +// RepositoryRegistry maps repository kinds to descriptors and factories. There is no central +// backend switch: backend packages register their own kind from a composition root. It is safe +// for concurrent use. +type RepositoryRegistry struct { + mu sync.RWMutex + kinds map[string]registeredKind +} + +// NewRepositoryRegistry returns an empty registry. +func NewRepositoryRegistry() *RepositoryRegistry { + defer perf.Track(nil, "artifact.NewRepositoryRegistry")() + + return &RepositoryRegistry{kinds: make(map[string]registeredKind)} +} + +// Register adds a kind. It rejects an empty kind, a nil factory, and a kind that is already registered. +func (r *RepositoryRegistry) Register(desc *KindDescriptor, factory RepositoryFactory) error { + defer perf.Track(nil, "artifact.RepositoryRegistry.Register")() + + if desc == nil || desc.Kind == "" { + return fmt.Errorf("%w: repository kind must not be empty", errUtils.ErrArtifactConfigInvalid) + } + if err := ValidateKind(desc.Kind); err != nil { + return fmt.Errorf("%w: repository kind %q: %w", errUtils.ErrArtifactConfigInvalid, desc.Kind, err) + } + if factory == nil { + return fmt.Errorf("%w: factory for repository kind %q must not be nil", errUtils.ErrArtifactConfigInvalid, desc.Kind) + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.kinds[desc.Kind]; exists { + return fmt.Errorf("%w: repository kind %q is already registered", errUtils.ErrArtifactConflict, desc.Kind) + } + r.kinds[desc.Kind] = registeredKind{descriptor: *desc, factory: factory} + return nil +} + +// Lookup returns the descriptor of a kind: the registered descriptor when a backend registered +// the kind, otherwise the built-in V1 descriptor. The boolean is false for an unknown kind. +func (r *RepositoryRegistry) Lookup(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.RepositoryRegistry.Lookup")() + + r.mu.RLock() + entry, ok := r.kinds[kind] + r.mu.RUnlock() + if ok { + desc := entry.descriptor + return &desc, true + } + return BuiltinKind(kind) +} + +// IsRegistered reports whether a backend registered a factory for the kind. +func (r *RepositoryRegistry) IsRegistered(kind string) bool { + defer perf.Track(nil, "artifact.RepositoryRegistry.IsRegistered")() + + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.kinds[kind] + return ok +} + +// Kinds returns the sorted names of every known kind: the registered ones plus the built-in V1 set. +func (r *RepositoryRegistry) Kinds() []string { + defer perf.Track(nil, "artifact.RepositoryRegistry.Kinds")() + + seen := make(map[string]bool) + for _, kind := range BuiltinKinds() { + seen[kind] = true + } + r.mu.RLock() + for kind := range r.kinds { + seen[kind] = true + } + r.mu.RUnlock() + names := make([]string, 0, len(seen)) + for kind := range seen { + names = append(names, kind) + } + sort.Strings(names) + return names +} + +// New validates the repository configuration against its kind and constructs the repository +// through the registered factory. An unknown kind is a configuration error; a known V1 kind +// whose backend is not registered in this build is an unsupported-capability error. +func (r *RepositoryRegistry) New(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + defer perf.Track(nil, "artifact.RepositoryRegistry.New")() + + if cfg == nil { + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + } + desc, known := r.Lookup(cfg.Kind) + if !known { + return nil, fmt.Errorf("%w: repository %q has unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, alias, cfg.Kind, r.Kinds()) + } + r.mu.RLock() + entry, registered := r.kinds[cfg.Kind] + r.mu.RUnlock() + if !registered { + return nil, NewUnsupportedCapability(alias, "construct", "kind/"+cfg.Kind, "no backend is registered for this repository kind in this build") + } + if err := ValidateRepositoryConfig(desc, cfg); err != nil { + return nil, fmt.Errorf("repository %q: %w", alias, err) + } + repo, err := entry.factory(ctx, alias, cfg, opts...) + if err != nil { + return nil, fmt.Errorf("repository %q: %w", alias, err) + } + return repo, nil +} + +// Reset removes every registered kind. It exists for tests that register fake backends. +func (r *RepositoryRegistry) Reset() { + defer perf.Track(nil, "artifact.RepositoryRegistry.Reset")() + + r.mu.Lock() + defer r.mu.Unlock() + r.kinds = make(map[string]registeredKind) +} + +// defaultRepositoryRegistry is the process-wide registry that backend packages register into. +var defaultRepositoryRegistry = NewRepositoryRegistry() + +// DefaultRepositoryRegistry returns the process-wide registry. +func DefaultRepositoryRegistry() *RepositoryRegistry { + defer perf.Track(nil, "artifact.DefaultRepositoryRegistry")() + + return defaultRepositoryRegistry +} + +// RegisterRepositoryKind registers a backend kind in the process-wide registry. Backend packages +// call it from a composition root. Duplicate registration is an error. +func RegisterRepositoryKind(desc *KindDescriptor, factory RepositoryFactory) error { + defer perf.Track(nil, "artifact.RegisterRepositoryKind")() + + return defaultRepositoryRegistry.Register(desc, factory) +} + +// MustRegisterRepositoryKind is RegisterRepositoryKind for init functions: it panics on error. +func MustRegisterRepositoryKind(desc *KindDescriptor, factory RepositoryFactory) { + defer perf.Track(nil, "artifact.MustRegisterRepositoryKind")() + + if err := RegisterRepositoryKind(desc, factory); err != nil { + panic(err) + } +} + +// LookupRepositoryKind looks up a kind in the process-wide registry. +func LookupRepositoryKind(kind string) (*KindDescriptor, bool) { + defer perf.Track(nil, "artifact.LookupRepositoryKind")() + + return defaultRepositoryRegistry.Lookup(kind) +} + +// RepositoryKinds lists the known kinds of the process-wide registry. +func RepositoryKinds() []string { + defer perf.Track(nil, "artifact.RepositoryKinds")() + + return defaultRepositoryRegistry.Kinds() +} + +// NewRepository constructs a repository from the process-wide registry. +func NewRepository(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + defer perf.Track(nil, "artifact.NewRepository")() + + return defaultRepositoryRegistry.New(ctx, alias, cfg, opts...) +} + +// ResetRepositoryKindsForTesting clears the process-wide registry. Tests that register fake +// backends call it from t.Cleanup so registrations do not leak between tests. +func ResetRepositoryKindsForTesting() { + defer perf.Track(nil, "artifact.ResetRepositoryKindsForTesting")() + + defaultRepositoryRegistry.Reset() +} diff --git a/pkg/artifact/registry_test.go b/pkg/artifact/registry_test.go new file mode 100644 index 00000000000..40093c875b2 --- /dev/null +++ b/pkg/artifact/registry_test.go @@ -0,0 +1,215 @@ +package artifact + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// fakeKind registers a kind whose factory records its inputs and returns a mock repository. +type fakeKind struct { + calls int + alias string + options RepositoryOptions + repo *MockRepository +} + +func (f *fakeKind) factory(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + f.calls++ + f.alias = alias + f.options = ApplyRepositoryOptions(opts...) + return f.repo, nil +} + +func localDirConfig(path string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{Kind: RepositoryKindLocalDir, Options: map[string]any{"path": path}} +} + +func TestRepositoryRegistry_RegisterRejectsInvalidAndDuplicateKinds(t *testing.T) { + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + desc, ok := BuiltinKind(RepositoryKindLocalDir) + require.True(t, ok) + + tests := []struct { + name string + desc *KindDescriptor + factory RepositoryFactory + wantErr error + }{ + {name: "empty kind", desc: &KindDescriptor{}, factory: fake.factory, wantErr: errUtils.ErrArtifactConfigInvalid}, + {name: "malformed kind", desc: &KindDescriptor{Kind: "Not A Kind"}, factory: fake.factory, wantErr: errUtils.ErrArtifactConfigInvalid}, + {name: "nil factory", desc: desc, factory: nil, wantErr: errUtils.ErrArtifactConfigInvalid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := NewRepositoryRegistry().Register(tt.desc, tt.factory) + require.ErrorIs(t, err, tt.wantErr) + }) + } + + registry := NewRepositoryRegistry() + require.NoError(t, registry.Register(desc, fake.factory)) + err := registry.Register(desc, fake.factory) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict, "a duplicate kind must be rejected") + assert.Contains(t, err.Error(), RepositoryKindLocalDir) +} + +func TestRepositoryRegistry_LookupFallsBackToBuiltinAndPrefersRegistered(t *testing.T) { + registry := NewRepositoryRegistry() + + builtin, ok := registry.Lookup(RepositoryKindS3) + require.True(t, ok, "V1 kinds validate before any backend registers") + assert.False(t, registry.IsRegistered(RepositoryKindS3)) + assert.Equal(t, PackagingTgz, builtin.DefaultPackaging) + + _, ok = registry.Lookup("ftp/server") + assert.False(t, ok) + + refined := *builtin + refined.DefaultPackaging = PackagingZip + require.NoError(t, registry.Register(&refined, (&fakeKind{}).factory)) + got, ok := registry.Lookup(RepositoryKindS3) + require.True(t, ok) + assert.Equal(t, PackagingZip, got.DefaultPackaging, "a registered descriptor takes precedence") + assert.True(t, registry.IsRegistered(RepositoryKindS3)) +} + +func TestRepositoryRegistry_KindsListsBuiltinAndRegisteredSorted(t *testing.T) { + registry := NewRepositoryRegistry() + require.NoError(t, registry.Register(&KindDescriptor{Kind: "gcp/gcs"}, (&fakeKind{}).factory)) + assert.Equal(t, []string{"aws/s3", "gcp/gcs", "local/archive", "local/dir", "oci/layout", "oci/registry"}, registry.Kinds()) +} + +func TestRepositoryRegistry_NewConstructsThroughFactoryWithoutNetworkOrCredentials(t *testing.T) { + ctrl := gomock.NewController(t) + repo := NewMockRepository(ctrl) + repo.EXPECT().Capabilities().Return(Capabilities{Read: true, Write: true, Packagings: []Packaging{PackagingNone}}).AnyTimes() + fake := &fakeKind{repo: repo} + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, fake.factory)) + + resolved := 0 + resolver := func(context.Context, string) (any, error) { resolved++; return nil, nil } + lookup := func(string) (Repository, error) { return nil, nil } + got, err := registry.New(context.Background(), "local", localDirConfig(t.TempDir()), WithIdentityResolver(resolver), WithRepositoryLookup(lookup)) + require.NoError(t, err) + assert.Same(t, repo, got) + assert.Equal(t, 1, fake.calls) + assert.Equal(t, "local", fake.alias) + assert.NotNil(t, fake.options.ResolveIdentity) + assert.NotNil(t, fake.options.LookupRepository) + assert.Zero(t, resolved, "constructing a repository must not resolve credentials") + assert.True(t, got.Capabilities().Write, "capabilities are available immediately") +} + +func TestRepositoryRegistry_NewRejectsUnknownUnregisteredAndInvalid(t *testing.T) { + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, fake.factory)) + + _, err := registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: "ftp/server"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "unknown kind") + + _, err = registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: RepositoryKindS3, Options: map[string]any{"bucket": "acme-artifacts"}}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "a known V1 kind without a registered backend is unsupported, not unknown") + + _, err = registry.New(context.Background(), "x", &schema.ArtifactRepositoryConfig{Kind: RepositoryKindLocalDir}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid, "missing required options fail before the factory runs") + + _, err = registry.New(context.Background(), "x", nil) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Zero(t, fake.calls, "invalid configuration never reaches the factory") +} + +func TestRepositoryRegistry_NewWrapsFactoryErrors(t *testing.T) { + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + boom := errors.New("boom") + require.NoError(t, registry.Register(desc, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...RepositoryOption) (Repository, error) { + return nil, boom + })) + _, err := registry.New(context.Background(), "local", localDirConfig(t.TempDir())) + require.ErrorIs(t, err, boom) + assert.Contains(t, err.Error(), `repository "local"`) +} + +func TestRepositoryRegistry_ResetClearsRegistrations(t *testing.T) { + registry := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, registry.Register(desc, (&fakeKind{}).factory)) + registry.Reset() + assert.False(t, registry.IsRegistered(RepositoryKindLocalDir)) + require.NoError(t, registry.Register(desc, (&fakeKind{}).factory), "a reset registry accepts the kind again") +} + +func TestRepositoryRegistry_ConcurrentRegistrationIsSafe(t *testing.T) { + registry := NewRepositoryRegistry() + var wg sync.WaitGroup + errs := make([]error, 8) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + // Every goroutine races for the same kind; exactly one must win. + errs[i] = registry.Register(&KindDescriptor{Kind: "race/kind"}, (&fakeKind{}).factory) + _ = registry.Kinds() + _, _ = registry.Lookup("race/kind") + }() + } + wg.Wait() + winners := 0 + for _, err := range errs { + if err == nil { + winners++ + continue + } + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + } + assert.Equal(t, 1, winners) +} + +func TestDefaultRegistry_RegisterNewAndReset(t *testing.T) { + t.Cleanup(ResetRepositoryKindsForTesting) + ResetRepositoryKindsForTesting() + ctrl := gomock.NewController(t) + fake := &fakeKind{repo: NewMockRepository(ctrl)} + desc, _ := BuiltinKind(RepositoryKindLocalDir) + + require.NoError(t, RegisterRepositoryKind(desc, fake.factory)) + require.ErrorIs(t, RegisterRepositoryKind(desc, fake.factory), errUtils.ErrArtifactConflict) + assert.Panics(t, func() { MustRegisterRepositoryKind(desc, fake.factory) }) + + got, err := NewRepository(context.Background(), "local", localDirConfig(t.TempDir())) + require.NoError(t, err) + assert.Same(t, fake.repo, got) + _, ok := LookupRepositoryKind(RepositoryKindLocalDir) + assert.True(t, ok) + assert.Contains(t, RepositoryKinds(), RepositoryKindLocalDir) + + ResetRepositoryKindsForTesting() + _, err = NewRepository(context.Background(), "local", localDirConfig(t.TempDir())) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestApplyRepositoryOptions_IgnoresNilOptions(t *testing.T) { + applied := ApplyRepositoryOptions(nil, WithRepositoryLookup(func(alias string) (Repository, error) { + return nil, fmt.Errorf("lookup %s", alias) + })) + require.NotNil(t, applied.LookupRepository) + _, err := applied.LookupRepository("x") + require.EqualError(t, err, "lookup x") +} diff --git a/pkg/config/artifacts.go b/pkg/config/artifacts.go new file mode 100644 index 00000000000..f2b0679f9b4 --- /dev/null +++ b/pkg/config/artifacts.go @@ -0,0 +1,217 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/spf13/viper" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/merge" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// artifactsSectionKey is the top-level atmos.yaml key of the artifact configuration. +const artifactsSectionKey = "artifacts" + +// artifactOrigins records, per setting, the configuration file that supplied the final value. It is +// tracked independently of the merged map because deep merging fuses the fields of several files +// into one repository: kind from one file and options.path from another must each keep their own +// declaring file. The last source that declares a setting wins, exactly like the value merge. +type artifactOrigins struct { + repoPath map[string]schema.ArtifactConfigOrigin + repoKeys map[string]schema.ArtifactConfigOrigin + globalKeys schema.ArtifactConfigOrigin +} + +func newArtifactOrigins() *artifactOrigins { + return &artifactOrigins{ + repoPath: make(map[string]schema.ArtifactConfigOrigin), + repoKeys: make(map[string]schema.ArtifactConfigOrigin), + } +} + +// resolveArtifactsConfig rebuilds atmosConfig.Artifacts from the retained configuration sources. +// +// Viper cannot be used for this section: it lowercases keys (repository aliases, label keys, and +// S3 tag keys are case-sensitive), splits keys on dots, and, once files are merged, no longer knows +// which file supplied a value. So the section is parsed from each retained source in merge order, +// merged as raw data, and decoded strictly once. While doing so the declaring file of each +// path-valued setting is recorded and relative local paths are anchored to that file's directory +// before the origin is lost. A relative path declared only by a remote import is left unresolved +// with its origin marked remote; artifactconfig validation then requires a local or absolute +// override. The key_pattern templates are plain strings here and are never rendered. +func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfiguration) error { + defer perf.Track(atmosConfig, "config.resolveArtifactsConfig")() + + merged := map[string]any{} + origins := newArtifactOrigins() + declared := false + for _, source := range collectConfigSourcesForCasePreservation(v, v.ConfigFileUsed()) { + section, ok, err := artifactsSectionFromSource(source) + if err != nil { + return err + } + if !ok { + continue + } + declared = true + origins.record(section, source) + combined, err := merge.MergeWithOptions(nil, []map[string]any{merged, section}, false, false) + if err != nil { + return fmt.Errorf("%w: merge artifacts from %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + merged = combined + } + if !declared { + return nil + } + cfg, err := artifactconfig.Decode(merged) + if err != nil { + return err + } + origins.apply(&cfg) + atmosConfig.Artifacts = cfg + return nil +} + +// artifactsSectionFromSource parses the `artifacts` section of one retained source with its +// original key spelling. The boolean is false when the source declares no such section. +func artifactsSectionFromSource(source mergedConfigSource) (map[string]any, bool, error) { + var document yaml.Node + if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { + return nil, false, fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + node := policyMappingValue(&document, artifactsSectionKey) + if node == nil { + return nil, false, nil + } + value, err := decodeNodeWithYamlFunctionsForFile(node, source.path) + if err != nil { + return nil, false, fmt.Errorf("%w: artifacts in %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + if value == nil { + return nil, false, nil + } + section, ok := value.(map[string]any) + if !ok { + return nil, false, fmt.Errorf("%w: artifacts in %s must be a mapping, got %T", errUtils.ErrArtifactConfigInvalid, source.path, value) + } + return section, true, nil +} + +// record notes which source declares each path-valued setting present in section. +func (o *artifactOrigins) record(section map[string]any, source mergedConfigSource) { + origin := artifactconfig.NewOrigin(source.path, source.ephemeral) + if keysDeclared(mapOf(mapValue(section, "verification"))) { + o.globalKeys = origin + } + repos, _ := mapValue(section, "repositories").(map[string]any) + for alias, raw := range repos { + repo, ok := raw.(map[string]any) + if !ok { + continue + } + if _, isString := mapValue(mapOf(mapValue(repo, "options")), artifact.OptionPath).(string); isString { + o.repoPath[alias] = origin + } + if keysDeclared(mapOf(mapValue(repo, "verification"))) { + o.repoKeys[alias] = origin + } + } +} + +// apply stores the origins on the decoded configuration and anchors relative local paths. +func (o *artifactOrigins) apply(cfg *schema.ArtifactConfig) { + if cfg.Verification != nil { + cfg.Verification.KeysOrigin = o.globalKeys + cfg.Verification.TrustedKeys = anchorKeys(cfg.Verification.TrustedKeys, o.globalKeys) + } + for alias := range cfg.Repositories { + repo := cfg.Repositories[alias] + repo.PathOrigin = o.repoPath[alias] + repo.Options = anchorPathOptions(&repo) + if repo.Verification != nil { + repo.Verification.KeysOrigin = o.repoKeys[alias] + repo.Verification.TrustedKeys = anchorKeys(repo.Verification.TrustedKeys, o.repoKeys[alias]) + } + cfg.Repositories[alias] = repo + } +} + +// anchorPathOptions returns the repository options with relative local destination paths made +// absolute against the declaring file. Options are copied, never mutated in place. A path declared +// by a remote import, or with no known declaring file, stays as written so validation can reject it. +func anchorPathOptions(repo *schema.ArtifactRepositoryConfig) map[string]any { + desc, ok := artifact.LookupRepositoryKind(repo.Kind) + if !ok || len(desc.PathOptions) == 0 || repo.PathOrigin.Remote || repo.PathOrigin.Dir == "" { + return repo.Options + } + out := make(map[string]any, len(repo.Options)) + for key, value := range repo.Options { + out[key] = value + } + for _, option := range desc.PathOptions { + anchorOption(out, option, repo.PathOrigin) + } + return out +} + +// anchorOption rewrites one relative path option of options in place to its anchored form. +func anchorOption(options map[string]any, option string, origin schema.ArtifactConfigOrigin) { + path, isString := options[option].(string) + if !isString || path == "" || filepath.IsAbs(path) { + return + } + if anchored, err := artifactconfig.AnchorPath(origin, path); err == nil { + options[option] = anchored + } +} + +// anchorKeys makes relative trusted-key paths absolute against the file that declared them. +func anchorKeys(keys []string, origin schema.ArtifactConfigOrigin) []string { + if len(keys) == 0 || origin.Remote || origin.Dir == "" { + return keys + } + out := make([]string, len(keys)) + for i, key := range keys { + out[i] = key + if key == "" || filepath.IsAbs(key) { + continue + } + if anchored, err := artifactconfig.AnchorPath(origin, key); err == nil { + out[i] = anchored + } + } + return out +} + +// keysDeclared reports whether a verification mapping declares a trusted_keys list. +func keysDeclared(verification map[string]any) bool { + _, ok := mapValue(verification, "trusted_keys").([]any) + return ok +} + +// mapValue looks up a key case-insensitively, matching how the loader decodes struct fields. +func mapValue(m map[string]any, key string) any { + if value, ok := m[key]; ok { + return value + } + for candidate, value := range m { + if strings.EqualFold(candidate, key) { + return value + } + } + return nil +} + +// mapOf returns value as a mapping, or nil when it is not one. +func mapOf(value any) map[string]any { + m, _ := value.(map[string]any) + return m +} diff --git a/pkg/config/artifacts_test.go b/pkg/config/artifacts_test.go new file mode 100644 index 00000000000..22563d64f3d --- /dev/null +++ b/pkg/config/artifacts_test.go @@ -0,0 +1,418 @@ +package config + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/schema" +) + +// prdRepositories is the YAML from the artifacts PRD "Public configuration" section, verbatim. +const prdRepositories = `artifacts: + repositories: + shared: + kind: oci/registry + identity: registry-publisher + options: + registry: ghcr.io + prefix: acme/atmos + defaults: + metadata: + labels: + team: platform + + backups: + kind: aws/s3 + identity: backup-writer + packaging: tgz + options: + bucket: acme-artifacts + prefix: atmos + region: us-east-2 + tags: + purpose: backup + + local: + kind: local/dir + options: + path: .atmos/artifacts + + archives: + kind: local/archive + packaging: tgz + permissions: "0600" + options: + path: .atmos/backups + key_pattern: '{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz' + + offline: + kind: oci/layout + options: + path: .atmos/oci +` + +// artifactsTestDir creates a config root whose atmos.yaml is main, isolated from the host. +func artifactsTestDir(t *testing.T, main string) string { + t.Helper() + setupTestAdapters() + dir := t.TempDir() + t.Setenv("TEST_GIT_ROOT", dir) + t.Setenv("ATMOS_XDG_CACHE_HOME", filepath.Join(dir, "cache")) + writeArtifactsFile(t, filepath.Join(dir, "atmos.yaml"), "base_path: ./\n"+main) + return dir +} + +func writeArtifactsFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func loadArtifactsDir(t *testing.T, dir string) (schema.AtmosConfiguration, error) { + t.Helper() + return LoadConfig(&schema.ConfigAndStacksInfo{AtmosConfigDirsFromArg: []string{dir}}) +} + +func TestLoadConfig_ArtifactsPRDExampleLoadsUnchanged(t *testing.T) { + dir := artifactsTestDir(t, prdRepositories) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repos := cfg.Artifacts.Repositories + require.Len(t, repos, 5) + + shared := repos["shared"] + assert.Equal(t, artifact.RepositoryKindOCIRegistry, shared.Kind) + assert.Equal(t, "registry-publisher", shared.Identity) + assert.Equal(t, map[string]any{"registry": "ghcr.io", "prefix": "acme/atmos"}, shared.Options) + require.NotNil(t, shared.Defaults) + require.NotNil(t, shared.Defaults.Metadata) + assert.Equal(t, map[string]string{"team": "platform"}, shared.Defaults.Metadata.Labels) + + backups := repos["backups"] + assert.Equal(t, "tgz", backups.Packaging) + assert.Equal(t, map[string]any{"purpose": "backup"}, backups.Options["tags"]) + + // Relative local roots are anchored to the declaring file. + assert.Equal(t, filepath.Join(dir, ".atmos", "artifacts"), repos["local"].Options["path"]) + assert.Equal(t, artifact.RepositoryKindOCILayout, repos["offline"].Kind) + assert.Equal(t, filepath.Join(dir, ".atmos", "oci"), repos["offline"].Options["path"]) + + archives := repos["archives"] + assert.Equal(t, "0600", archives.Permissions) + assert.Equal(t, filepath.Join(dir, ".atmos", "backups"), archives.Options["path"]) + // The key_pattern template survives config loading verbatim; it is never rendered here. + assert.Equal(t, `{{ .CreatedAt | date "2006/01/02" }}/{{ .Name }}/{{ .Version }}.tgz`, archives.Options["key_pattern"]) + + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsCatalogAndVerificationExamplesLoad(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + verification: + require_signature: true + trusted_identities: + - identity: https://github.com/acme/images/.github/workflows/release.yml@refs/heads/main + issuer: https://token.actions.githubusercontent.com + trusted_keys: [] + repositories: + shared: + kind: oci/registry + identity: registry-reader + options: + registry: ghcr.io + prefix: acme/atmos + catalog: oci://ghcr.io/acme/atmos/catalog:stable + locking: + required_scope: repository + repository: coordination + coordination: + kind: aws/s3 + identity: lock-writer + options: + bucket: acme-artifact-coordination +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) + + require.NotNil(t, cfg.Artifacts.Verification) + require.NotNil(t, cfg.Artifacts.Verification.RequireSignature) + assert.True(t, *cfg.Artifacts.Verification.RequireSignature) + assert.Equal(t, "oci://ghcr.io/acme/atmos/catalog:stable", cfg.Artifacts.Repositories["shared"].Catalog) + assert.Equal(t, "coordination", cfg.Artifacts.Repositories["shared"].Locking.Repository) +} + +func TestLoadConfig_ArtifactsPreserveKeyCase(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + Prod.US: + kind: aws/s3 + options: + bucket: acme-artifacts + tags: + Purpose: Backup + defaults: + metadata: + labels: + Team.Name: Platform +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo, ok := cfg.Artifacts.Repositories["Prod.US"] + require.True(t, ok, "repository alias keeps its case and dot, got %v", cfg.Artifacts.Repositories) + assert.Equal(t, map[string]any{"Purpose": "Backup"}, repo.Options["tags"]) + assert.Equal(t, map[string]string{"Team.Name": "Platform"}, repo.Defaults.Metadata.Labels) +} + +func TestLoadConfig_ArtifactsImportedPathsResolveAgainstDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./first/artifacts.yaml\n - ./second/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "first", "artifacts.yaml"), `artifacts: + repositories: + one: + kind: local/dir + options: + path: store-first + two: + kind: local/archive + identity: declared-in-first +`) + writeArtifactsFile(t, filepath.Join(dir, "second", "artifacts.yaml"), `artifacts: + repositories: + one: + identity: declared-in-second + two: + options: + path: store-second +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + one := cfg.Artifacts.Repositories["one"] + two := cfg.Artifacts.Repositories["two"] + // Merged fields come from different files, and each path uses the file that supplied that path. + assert.Equal(t, artifact.RepositoryKindLocalDir, one.Kind) + assert.Equal(t, "declared-in-second", one.Identity) + assert.Equal(t, filepath.Join(dir, "first", "store-first"), one.Options["path"]) + assert.Equal(t, filepath.Join(dir, "first"), one.PathOrigin.Dir) + + assert.Equal(t, artifact.RepositoryKindLocalArchive, two.Kind) + assert.Equal(t, "declared-in-first", two.Identity) + assert.Equal(t, filepath.Join(dir, "second", "store-second"), two.Options["path"]) + assert.Equal(t, filepath.Join(dir, "second"), two.PathOrigin.Dir) + assert.False(t, two.PathOrigin.Remote) + + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsLaterFileOverridesPathAndOrigin(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./base/artifacts.yaml\n"+`artifacts: + repositories: + local: + options: + path: override +`) + writeArtifactsFile(t, filepath.Join(dir, "base", "artifacts.yaml"), `artifacts: + repositories: + local: + kind: local/dir + options: + path: from-base +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "override"), cfg.Artifacts.Repositories["local"].Options["path"]) + assert.Equal(t, dir, cfg.Artifacts.Repositories["local"].PathOrigin.Dir) +} + +func TestLoadConfig_ArtifactsAnchorTrustedKeys(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./trust/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "artifacts.yaml"), `artifacts: + verification: + trusted_keys: + - keys/cosign.pub +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, []string{filepath.Join(dir, "trust", "keys", "cosign.pub")}, cfg.Artifacts.Verification.TrustedKeys) +} + +func TestLoadConfig_ArtifactsRejectsUnquotedNumericPermissions(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + archives: + kind: local/archive + permissions: 0600 + options: + path: out +`) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "quoted octal string") +} + +func TestLoadConfig_ArtifactsRejectsUnknownField(t *testing.T) { + dir := artifactsTestDir(t, `artifacts: + repositories: + local: + kind: local/dir + pakaging: none +`) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "pakaging") +} + +func TestLoadConfig_ArtifactsAbsentLeavesZeroValue(t *testing.T) { + dir := artifactsTestDir(t, "") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Empty(t, cfg.Artifacts.Repositories) + assert.Nil(t, cfg.Artifacts.Verification) +} + +// remoteTestAdapter simulates a remote import: it copies a canned document into the import temp +// directory, exactly as the go-getter adapter downloads into an ephemeral location. +type remoteTestAdapter struct { + documents map[string]string +} + +func (a *remoteTestAdapter) Schemes() []string { return []string{"remote-test://"} } + +func (a *remoteTestAdapter) Resolve(_ context.Context, importPath, _, tempDir string, _, _ int, _ *schema.AtmosConfiguration) ([]ResolvedPaths, error) { + name := strings.TrimPrefix(importPath, "remote-test://") + target := filepath.Join(tempDir, name+".yaml") + if err := os.WriteFile(target, []byte(a.documents[name]), 0o600); err != nil { + return nil, err + } + return []ResolvedPaths{{FilePath: target, ImportPaths: importPath, ImportType: REMOTE}}, nil +} + +func registerRemoteArtifacts(documents map[string]string) { + RegisterImportAdapter(&remoteTestAdapter{documents: documents}) +} + +const remoteRelativeDocument = `artifacts: + repositories: + remote: + kind: local/dir + options: + path: relative-from-remote +` + +func TestLoadConfig_ArtifactsRemoteRelativePathIsNeverAnchoredToDownloadDir(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - remote-test://shared\n") + registerRemoteArtifacts(map[string]string{"shared": remoteRelativeDocument}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, "relative-from-remote", repo.Options["path"], "the ephemeral download directory must not become a write root") + assert.True(t, repo.PathOrigin.Remote) + + err = artifactconfig.Validate(&cfg.Artifacts) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), "remote import") +} + +func TestLoadConfig_ArtifactsRemoteDeclaredPathAcceptsLocalOverride(t *testing.T) { + dir := artifactsTestDir(t, `import: + - remote-test://shared +artifacts: + repositories: + remote: + options: + path: local-override +`) + registerRemoteArtifacts(map[string]string{"shared": remoteRelativeDocument}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, filepath.Join(dir, "local-override"), repo.Options["path"]) + assert.False(t, repo.PathOrigin.Remote) + assert.Equal(t, artifact.RepositoryKindLocalDir, repo.Kind, "the remote file still supplies the other fields") + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsRemoteAbsolutePathIsAccepted(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - remote-test://shared\n") + absolute := filepath.Join(t.TempDir(), "store") + registerRemoteArtifacts(map[string]string{"shared": strings.Replace(remoteRelativeDocument, "relative-from-remote", filepath.ToSlash(absolute), 1)}) + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, absolute, cfg.Artifacts.Repositories["remote"].Options["path"]) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsLocalImportIsNotEphemeral(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./local/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "local", "artifacts.yaml"), remoteRelativeDocument) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.False(t, cfg.Artifacts.Repositories["remote"].PathOrigin.Remote) + assert.Equal(t, filepath.Join(dir, "local", "relative-from-remote"), cfg.Artifacts.Repositories["remote"].Options["path"]) +} + +func TestLoadConfig_ArtifactsAtmosDFragmentAnchorsToItsOwnDirectory(t *testing.T) { + dir := artifactsTestDir(t, "") + writeArtifactsFile(t, filepath.Join(dir, ".atmos.d", "artifacts.yaml"), `artifacts: + repositories: + fragment: + kind: local/dir + options: + path: store +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, ".atmos.d", "store"), cfg.Artifacts.Repositories["fragment"].Options["path"]) +} + +// TestLoadConfig_ArtifactsDescribeRoundTrip covers `atmos describe config`: the loaded section +// serializes under `artifacts` and reads back as the same values, without leaking load-time origins. +func TestLoadConfig_ArtifactsDescribeRoundTrip(t *testing.T) { + dir := artifactsTestDir(t, prdRepositories) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + jsonBytes, err := json.Marshal(cfg) + require.NoError(t, err) + var describedJSON struct { + Artifacts schema.ArtifactConfig `json:"artifacts"` + } + require.NoError(t, json.Unmarshal(jsonBytes, &describedJSON)) + + yamlBytes, err := yaml.Marshal(cfg) + require.NoError(t, err) + var describedYAML struct { + Artifacts schema.ArtifactConfig `yaml:"artifacts"` + } + require.NoError(t, yaml.Unmarshal(yamlBytes, &describedYAML)) + + expected := cfg.Artifacts + for alias, repo := range expected.Repositories { + repo.PathOrigin = schema.ArtifactConfigOrigin{} + expected.Repositories[alias] = repo + } + assert.Equal(t, expected, describedJSON.Artifacts) + assert.Equal(t, expected, describedYAML.Artifacts) + assert.NotContains(t, string(jsonBytes), "PathOrigin") +} diff --git a/pkg/config/global_viper.go b/pkg/config/global_viper.go index d5974afe39f..264b52fc68f 100644 --- a/pkg/config/global_viper.go +++ b/pkg/config/global_viper.go @@ -98,6 +98,9 @@ type mergedFilesTracker struct { type mergedConfigSource struct { path string content string + // ephemeral marks a source downloaded from a remote import. Its local path is a temporary + // location, so relative destinations declared in it must never anchor to that directory. + ephemeral bool } // track appends a source in effective merge order, including repeated paths. @@ -110,6 +113,17 @@ func (t *mergedFilesTracker) track(source mergedConfigSource) { } } +// markEphemeral flags every tracked merge of path as a remote (downloaded) source. +func (t *mergedFilesTracker) markEphemeral(path string) { + t.mu.Lock() + defer t.mu.Unlock() + for i := range t.sources { + if t.sources[i].path == path { + t.sources[i].ephemeral = true + } + } +} + // snapshot returns an independent slice of immutable source records. func (t *mergedFilesTracker) snapshot() []mergedConfigSource { t.mu.Lock() @@ -193,6 +207,16 @@ func (r *mergedFilesRegistry) track(v *viper.Viper, path string, content []byte) } } +// markEphemeral flags a downloaded import source for the load v belongs to. +func (r *mergedFilesRegistry) markEphemeral(v *viper.Viper, path string) { + r.mu.Lock() + tracker := r.trackers[v] + r.mu.Unlock() + if tracker != nil { + tracker.markEphemeral(path) + } +} + // snapshot returns the sources associated with this load, or nil if unregistered. func (r *mergedFilesRegistry) snapshot(v *viper.Viper) []mergedConfigSource { r.mu.Lock() diff --git a/pkg/config/imports.go b/pkg/config/imports.go index a356e7ce683..a72edc9f382 100644 --- a/pkg/config/imports.go +++ b/pkg/config/imports.go @@ -149,6 +149,10 @@ func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs fi continue } log.Trace("successfully merged config from import", "import", SanitizeImport(resolvedPath.ImportPaths), keyFilePath, resolvedPath.FilePath) + if resolvedPath.ImportType != LOCAL { + // Downloaded imports live in a temporary directory that is removed after loading. + mergedFilesReg.markEphemeral(dst, resolvedPath.FilePath) + } content, readErr := fs.ReadFile(resolvedPath.FilePath) if readErr != nil { log.Warn("error re-reading resolved import for base_path detection", "import", SanitizeImport(resolvedPath.ImportPaths), keyFilePath, resolvedPath.FilePath, "error", readErr) diff --git a/pkg/config/load.go b/pkg/config/load.go index 1beff1cc317..5c82186186f 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -485,6 +485,11 @@ func finalizeLoadedConfig(v *viper.Viper, atmosConfig *schema.AtmosConfiguration if err := resolveScaffoldConfig(v, atmosConfig); err != nil { return err } + // Rebuild artifacts from the retained sources: Viper lowercases keys and forgets which + // file declared options.path once imports are merged. + if err := resolveArtifactsConfig(v, &atmosConfig); err != nil { + return atmosConfig, err + } // Post-process to preserve case-sensitive map keys. // Viper lowercases all YAML map keys, but we need to preserve original case diff --git a/pkg/datafetcher/schema/atmos/config/1.0.json b/pkg/datafetcher/schema/atmos/config/1.0.json index 9fbe3b80bc9..c3b610e3e53 100644 --- a/pkg/datafetcher/schema/atmos/config/1.0.json +++ b/pkg/datafetcher/schema/atmos/config/1.0.json @@ -1581,6 +1581,383 @@ "type": "object", "description": "Ansible defines configuration for Ansible components." }, + "ArtifactConfig": { + "properties": { + "repositories": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRepositoryConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ] + }, + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Repositories maps a repository alias to its definition. The alias is case-sensitive and is\nwhat `--repository` and the artifact step `repository` field select." + }, + "verification": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactVerificationConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Verification is the global signature verification policy. Repository policies can add\nrequirements but never weaken it." + } + }, + "type": "object", + "description": "ArtifactConfig is the `artifacts` section of atmos.yaml. It holds the single list of artifact\nrepositories (destinations that are also discovery sources) and the optional global signature\nverification policy." + }, + "ArtifactLockingConfig": { + "properties": { + "required_scope": { + "anyOf": [ + { + "type": "string", + "enum": [ + "", + "host", + "repository" + ] + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "RequiredScope is the minimum lock guarantee mutations need: host (the default) or repository." + }, + "repository": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Repository delegates locking to another configured repository that advertises repository scope.\nThe delegate keeps its own identity and storage root. Cycles are rejected." + } + }, + "type": "object", + "description": "ArtifactLockingConfig configures mutation coordination for a repository." + }, + "ArtifactMetadataDefaults": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Description is the default artifact description." + }, + "labels": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Labels are default exact key/value labels for newly published artifact versions." + } + }, + "type": "object", + "description": "ArtifactMetadataDefaults is the default description and labels of newly published artifacts.\nLabels belong to each published artifact version, not to the repository itself." + }, + "ArtifactRepositoryConfig": { + "properties": { + "kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Kind selects the backend: local/dir, local/archive, aws/s3, oci/registry, or oci/layout." + }, + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity names an Atmos auth identity used for this repository. Credentials never belong in\noptions. The identity is resolved lazily, only when an operation needs credentials." + }, + "options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Options are backend-specific settings, for example `path` for local kinds, `bucket`, `prefix`,\n`region`, `tags`, and `key_pattern` for aws/s3, or `registry` and `prefix` for oci/registry.\nThe `key_pattern` template is never expanded while the configuration is loaded." + }, + "packaging": { + "anyOf": [ + { + "type": "string", + "enum": [ + "", + "none", + "tar", + "tgz", + "zip" + ] + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Packaging selects how file payloads are stored: none, tar, tgz, or zip. The supported values\nand the default depend on the repository kind." + }, + "permissions": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-7][0-7][0-7][0-7]?$" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Permissions is the quoted octal mode of the resulting local archive file, for example \"0600\".\nIt is accepted only for local/archive repositories and must be a quoted string." + }, + "catalog": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Catalog is the optional OCI catalog URI of this repository, for example\noci://ghcr.io/acme/atmos/catalog:stable. It is valid only for oci/registry repositories and\nmust name the repository's own registry host." + }, + "defaults": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRepositoryDefaults" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Defaults supplies defaults for newly published artifacts." + }, + "verification": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactVerificationConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Verification is this repository's signature verification policy. It is combined with the\nglobal policy: both must be satisfied." + }, + "locking": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactLockingConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Locking configures mutation coordination for this repository." + } + }, + "type": "object", + "description": "ArtifactRepositoryConfig defines one artifact repository: a collection root that holds many named\nartifacts of possibly different kinds." + }, + "ArtifactRepositoryDefaults": { + "properties": { + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactMetadataDefaults" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Metadata supplies default artifact metadata. Explicit publication metadata overrides matching keys." + } + }, + "type": "object", + "description": "ArtifactRepositoryDefaults holds defaults applied to artifacts newly published to a repository." + }, + "ArtifactTrustedIdentity": { + "properties": { + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity is the expected certificate identity, for example a workflow URL or an email address." + }, + "issuer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Issuer is the expected OIDC issuer URL, for example https://token.actions.githubusercontent.com." + } + }, + "type": "object", + "description": "ArtifactTrustedIdentity is a trusted signer certificate identity and the OIDC issuer that vouches for it." + }, + "ArtifactVerificationConfig": { + "properties": { + "require_signature": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "RequireSignature makes a missing or invalid signature a failure. Unset means false for new\ngeneral artifacts. A repository cannot set it to false when the global policy requires it." + }, + "trusted_identities": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactTrustedIdentity" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ] + }, + "type": "array" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "TrustedIdentities are certificate identity and OIDC issuer pairs. Any one of them satisfies\nthis policy level's trust set." + }, + "trusted_keys": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + }, + { + "type": "string" + } + ], + "description": "TrustedKeys are operator-managed public-key file paths. Relative paths resolve against the\nconfiguration file that declared them." + } + }, + "type": "object", + "description": "ArtifactVerificationConfig is a signature verification policy, allowed globally under\n`artifacts.verification` and per repository." + }, "Artifacts": { "properties": { "paths": { @@ -18845,6 +19222,20 @@ } ], "description": "LSP settings." + }, + "artifacts": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactConfig" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Artifacts configures artifact repositories and signature verification policy." } }, "type": "object", diff --git a/pkg/schema/artifacts.go b/pkg/schema/artifacts.go new file mode 100644 index 00000000000..06667ed2e84 --- /dev/null +++ b/pkg/schema/artifacts.go @@ -0,0 +1,137 @@ +package schema + +// Developer note: these types describe configuration only. Backend-specific `options` stay a +// generic map so this package never depends on a backend; typed decoding and validation live in +// pkg/artifact and pkg/artifact/config. This package must not import pkg/artifact (pkg/perf +// imports pkg/schema and pkg/artifact imports pkg/perf). + +// ArtifactConfig is the `artifacts` section of atmos.yaml. It holds the single list of artifact +// repositories (destinations that are also discovery sources) and the optional global signature +// verification policy. +type ArtifactConfig struct { + // Repositories maps a repository alias to its definition. The alias is case-sensitive and is + // what `--repository` and the artifact step `repository` field select. + Repositories map[string]ArtifactRepositoryConfig `yaml:"repositories,omitempty" json:"repositories,omitempty" mapstructure:"repositories"` + + // Verification is the global signature verification policy. Repository policies can add + // requirements but never weaken it. + Verification *ArtifactVerificationConfig `yaml:"verification,omitempty" json:"verification,omitempty" mapstructure:"verification"` +} + +// ArtifactRepositoryConfig defines one artifact repository: a collection root that holds many named +// artifacts of possibly different kinds. +type ArtifactRepositoryConfig struct { + // Kind selects the backend: local/dir, local/archive, aws/s3, oci/registry, or oci/layout. + Kind string `yaml:"kind,omitempty" json:"kind,omitempty" mapstructure:"kind"` + + // Identity names an Atmos auth identity used for this repository. Credentials never belong in + // options. The identity is resolved lazily, only when an operation needs credentials. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` + + // Options are backend-specific settings, for example `path` for local kinds, `bucket`, `prefix`, + // `region`, `tags`, and `key_pattern` for aws/s3, or `registry` and `prefix` for oci/registry. + // The `key_pattern` template is never expanded while the configuration is loaded. + Options map[string]any `yaml:"options,omitempty" json:"options,omitempty" mapstructure:"options"` + + // Packaging selects how file payloads are stored: none, tar, tgz, or zip. The supported values + // and the default depend on the repository kind. + Packaging string `yaml:"packaging,omitempty" json:"packaging,omitempty" mapstructure:"packaging" jsonschema:"enum=,enum=none,enum=tar,enum=tgz,enum=zip"` + + // Permissions is the quoted octal mode of the resulting local archive file, for example "0600". + // It is accepted only for local/archive repositories and must be a quoted string. + Permissions string `yaml:"permissions,omitempty" json:"permissions,omitempty" mapstructure:"permissions" jsonschema:"pattern=^[0-7][0-7][0-7][0-7]?$"` + + // Catalog is the optional OCI catalog URI of this repository, for example + // oci://ghcr.io/acme/atmos/catalog:stable. It is valid only for oci/registry repositories and + // must name the repository's own registry host. + Catalog string `yaml:"catalog,omitempty" json:"catalog,omitempty" mapstructure:"catalog"` + + // Defaults supplies defaults for newly published artifacts. + Defaults *ArtifactRepositoryDefaults `yaml:"defaults,omitempty" json:"defaults,omitempty" mapstructure:"defaults"` + + // Verification is this repository's signature verification policy. It is combined with the + // global policy: both must be satisfied. + Verification *ArtifactVerificationConfig `yaml:"verification,omitempty" json:"verification,omitempty" mapstructure:"verification"` + + // Locking configures mutation coordination for this repository. + Locking *ArtifactLockingConfig `yaml:"locking,omitempty" json:"locking,omitempty" mapstructure:"locking"` + + // PathOrigin records the configuration file that declared options.path. It is populated by the + // configuration loader, is never read from or written to YAML, and is tracked independently of + // the other (possibly merged) repository fields. + PathOrigin ArtifactConfigOrigin `yaml:"-" json:"-" mapstructure:"-"` +} + +// ArtifactRepositoryDefaults holds defaults applied to artifacts newly published to a repository. +type ArtifactRepositoryDefaults struct { + // Metadata supplies default artifact metadata. Explicit publication metadata overrides matching keys. + Metadata *ArtifactMetadataDefaults `yaml:"metadata,omitempty" json:"metadata,omitempty" mapstructure:"metadata"` +} + +// ArtifactMetadataDefaults is the default description and labels of newly published artifacts. +// Labels belong to each published artifact version, not to the repository itself. +type ArtifactMetadataDefaults struct { + // Description is the default artifact description. + Description string `yaml:"description,omitempty" json:"description,omitempty" mapstructure:"description"` + + // Labels are default exact key/value labels for newly published artifact versions. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty" mapstructure:"labels"` +} + +// ArtifactVerificationConfig is a signature verification policy, allowed globally under +// `artifacts.verification` and per repository. +type ArtifactVerificationConfig struct { + // RequireSignature makes a missing or invalid signature a failure. Unset means false for new + // general artifacts. A repository cannot set it to false when the global policy requires it. + RequireSignature *bool `yaml:"require_signature,omitempty" json:"require_signature,omitempty" mapstructure:"require_signature"` + + // TrustedIdentities are certificate identity and OIDC issuer pairs. Any one of them satisfies + // this policy level's trust set. + TrustedIdentities []ArtifactTrustedIdentity `yaml:"trusted_identities,omitempty" json:"trusted_identities,omitempty" mapstructure:"trusted_identities"` + + // TrustedKeys are operator-managed public-key file paths. Relative paths resolve against the + // configuration file that declared them. + TrustedKeys []string `yaml:"trusted_keys,omitempty" json:"trusted_keys,omitempty" mapstructure:"trusted_keys"` + + // KeysOrigin records the configuration file that declared trusted_keys. It is populated by the + // configuration loader and is never read from or written to YAML. + KeysOrigin ArtifactConfigOrigin `yaml:"-" json:"-" mapstructure:"-"` +} + +// ArtifactTrustedIdentity is a trusted signer certificate identity and the OIDC issuer that vouches for it. +type ArtifactTrustedIdentity struct { + // Identity is the expected certificate identity, for example a workflow URL or an email address. + Identity string `yaml:"identity" json:"identity" mapstructure:"identity"` + + // Issuer is the expected OIDC issuer URL, for example https://token.actions.githubusercontent.com. + Issuer string `yaml:"issuer" json:"issuer" mapstructure:"issuer"` +} + +// ArtifactLockingConfig configures mutation coordination for a repository. +type ArtifactLockingConfig struct { + // RequiredScope is the minimum lock guarantee mutations need: host (the default) or repository. + RequiredScope string `yaml:"required_scope,omitempty" json:"required_scope,omitempty" mapstructure:"required_scope" jsonschema:"enum=,enum=host,enum=repository"` + + // Repository delegates locking to another configured repository that advertises repository scope. + // The delegate keeps its own identity and storage root. Cycles are rejected. + Repository string `yaml:"repository,omitempty" json:"repository,omitempty" mapstructure:"repository"` +} + +// ArtifactConfigOrigin records which configuration file declared a path-valued setting, so that +// relative paths resolve against the declaring file even after imports have been merged. +type ArtifactConfigOrigin struct { + // File is the absolute path of the declaring configuration file. Empty when unknown. + File string `yaml:"-" json:"-" mapstructure:"-"` + + // Dir is the absolute directory of the declaring file. Empty when unknown. + Dir string `yaml:"-" json:"-" mapstructure:"-"` + + // Remote reports that the value was declared by a remote (downloaded) import, whose local + // path is ephemeral and must never anchor a relative destination. + Remote bool `yaml:"-" json:"-" mapstructure:"-"` +} + +// IsZero reports whether no declaring file is known. +func (o ArtifactConfigOrigin) IsZero() bool { + return o.File == "" && o.Dir == "" && !o.Remote +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index aa820039a15..2f88b3f1f0e 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -222,6 +222,8 @@ type AtmosConfiguration struct { MCP MCPSettings `yaml:"mcp,omitempty" json:"mcp,omitempty" mapstructure:"mcp"` // LSP settings. LSP LSPSettings `yaml:"lsp,omitempty" json:"lsp,omitempty" mapstructure:"lsp"` + // Artifacts configures artifact repositories and signature verification policy. + Artifacts ArtifactConfig `yaml:"artifacts,omitempty" json:"artifacts,omitempty" mapstructure:"artifacts"` } // CastConfig configures Atmos cast recording behavior. From faa07ae2d1e1d276e37491ab872bf2a0265833ff Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 8 Nov 2026 16:27:24 -0600 Subject: [PATCH 03/32] fix(artifacts): repair L2 CI failures in snapshots, fixtures, and Windows tests - Regenerate the describe-config golden snapshots for the new empty "artifacts" config key. - Commit the digest-pinned legacy CI plan archive. The repository-wide "*.tar" ignore rule excluded it, so fresh checkouts could not read the fixture; add a negative ignore rule for compat testdata archives. - Exclude the byte-exact compat fixtures from the CI "atmos validate --affected" step, matching scripts/checks/checks.star, which already excludes them locally. - Build the PRD config test paths from t.TempDir() so they are absolute on Windows. - Compare the remote-import artifacts path in slash form, which is how the loader keeps it by design. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test.yml | 4 +++- .gitignore | 2 ++ ...6789abcdef0123456789abcdef01234567.tfplan.tar | Bin 0 -> 3072 bytes pkg/artifact/config/decode_test.go | 13 +++++++++++-- pkg/config/artifacts_test.go | 3 ++- ...ICommands_atmos_describe_config.stdout.golden | 3 ++- ...secrets-masking_describe_config.stdout.golden | 3 ++- 7 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ea3821ae37..e1c73a821ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -311,7 +311,9 @@ jobs: # builds silently drop that list in the aggregate `validate --affected` path. # Exclude README.md: it's machine-generated by cloudposse/build-harness, so its # formatting isn't ours to enforce. - run: atmos validate --affected --exclude 'tests/fixtures/**' --exclude '**/*.go' --exclude 'specs/**' --exclude 'README.md' --format rich + # pkg/artifact/compat/testdata/** holds frozen, digest-pinned fixtures that must stay + # byte-exact (no trailing newline); this mirrors scripts/checks/checks.star. + run: atmos validate --affected --exclude 'tests/fixtures/**' --exclude 'pkg/artifact/compat/testdata/**' --exclude '**/*.go' --exclude 'specs/**' --exclude 'README.md' --format rich # Committed website casts and screengrabs are generated artifacts, but a # change can also edit or hand-record them directly, bypassing the pipeline diff --git a/.gitignore b/.gitignore index 7f52759e452..1cfe9270714 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,8 @@ tools/gomodcheck/.gomodcheck .DS_Store /variant *.tar +# Frozen compatibility fixtures must be committed even when they are archives. +!pkg/artifact/compat/testdata/**/*.tar *.gz /bin diff --git a/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar b/pkg/artifact/compat/testdata/legacy-ci/plat-ue2-dev/vpc/0123456789abcdef0123456789abcdef01234567.tfplan.tar new file mode 100644 index 0000000000000000000000000000000000000000..22022820a524e66e817866acda15e4eb73f694e4 GIT binary patch literal 3072 zcmeHHOAf*y5bc^%OzK*sAM47)V5Kb)r5UiMG2Y!`V?vB^(P-*|*$jDv`MlvxtN4l9 zsvTVPD`SHdc^)}G*sEtOEuPqTpCnnHMZkVlkRSJKEXN_ek4-s0TJxHl5@UupO4PjU zmu~)*RARa7tuSPULPE+}7@$@l@=D{8DqS9EQ>w4m;iU0N|4Ej<=sztso1p)%58_S0 z1&kX2-1Nb%0MerFl@CjnpUDLcFGovY6b$ S4tc|NfC2-7fxy83FmMOtOi#i9 literal 0 HcmV?d00001 diff --git a/pkg/artifact/config/decode_test.go b/pkg/artifact/config/decode_test.go index dc1f1743d9d..46028d87933 100644 --- a/pkg/artifact/config/decode_test.go +++ b/pkg/artifact/config/decode_test.go @@ -2,6 +2,8 @@ package config import ( "encoding/json" + "path/filepath" + "strings" "testing" "github.com/go-viper/mapstructure/v2" @@ -239,8 +241,15 @@ repositories: } func TestDecodedPRDDocumentValidatesWithAnchoredPaths(t *testing.T) { - cfg, _ := decodeYAMLDocument(t, prdDocument) - origin := NewOrigin("/project/atmos.yaml", false) + // A rooted Unix path such as /etc/atmos has no volume on Windows, so it is not absolute there. + // Build the trusted key path and the declaring file from a platform-absolute directory instead. + dir := t.TempDir() + keyPath := filepath.ToSlash(filepath.Join(dir, "cosign.pub")) + document := strings.Replace(prdDocument, "/etc/atmos/cosign.pub", keyPath, 1) + require.NotEqual(t, prdDocument, document, "the PRD document must contain the trusted key path being replaced") + + cfg, _ := decodeYAMLDocument(t, document) + origin := NewOrigin(filepath.Join(dir, "atmos.yaml"), false) for alias, repo := range cfg.Repositories { repo.PathOrigin = origin cfg.Repositories[alias] = repo diff --git a/pkg/config/artifacts_test.go b/pkg/config/artifacts_test.go index 22563d64f3d..689a9ae1f57 100644 --- a/pkg/config/artifacts_test.go +++ b/pkg/config/artifacts_test.go @@ -359,7 +359,8 @@ func TestLoadConfig_ArtifactsRemoteAbsolutePathIsAccepted(t *testing.T) { cfg, err := loadArtifactsDir(t, dir) require.NoError(t, err) - assert.Equal(t, absolute, cfg.Artifacts.Repositories["remote"].Options["path"]) + // A path declared by a remote import stays as written, which is slash form here. + assert.Equal(t, filepath.ToSlash(absolute), cfg.Artifacts.Repositories["remote"].Options["path"]) require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) } diff --git a/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden b/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden index 94d5d061aab..3c78ed4fe89 100644 --- a/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden +++ b/tests/snapshots/TestCLICommands_atmos_describe_config.stdout.golden @@ -404,5 +404,6 @@ "mcp": { "routing": {} }, - "lsp": {} + "lsp": {}, + "artifacts": {} } diff --git a/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden b/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden index 8516cad17b1..6f83d1cad71 100644 --- a/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden +++ b/tests/snapshots/TestCLICommands_secrets-masking_describe_config.stdout.golden @@ -413,5 +413,6 @@ "mcp": { "routing": {} }, - "lsp": {} + "lsp": {}, + "artifacts": {} } From e29bc8deebfc7a585b3613c6803b2b9848807fbb Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 15:54:09 -0600 Subject: [PATCH 04/32] fix(artifacts): address foundation review findings --- pkg/artifact/canonicaljson.go | 4 +++ pkg/artifact/canonicaljson_test.go | 3 ++ pkg/artifact/compat/compat_test.go | 13 ++++++++ pkg/artifact/compat/doc.go | 2 +- pkg/artifact/kinddata.go | 3 ++ pkg/artifact/kinddata_test.go | 10 ++++++ pkg/artifact/options.go | 20 ++++++++++-- pkg/artifact/options_test.go | 3 ++ pkg/artifact/registry.go | 12 +++++-- pkg/artifact/registry_test.go | 33 ++++++++++++++++++-- pkg/config/artifacts.go | 10 ++++++ pkg/config/artifacts_test.go | 14 +++++++++ pkg/config/imports.go | 17 ++++++++-- pkg/config/schema/overrides.go | 13 ++++++++ pkg/datafetcher/schema/atmos/config/1.0.json | 3 -- 15 files changed, 147 insertions(+), 13 deletions(-) diff --git a/pkg/artifact/canonicaljson.go b/pkg/artifact/canonicaljson.go index 8c277a4c637..822957be00c 100644 --- a/pkg/artifact/canonicaljson.go +++ b/pkg/artifact/canonicaljson.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "math" "sort" "strconv" "strings" @@ -247,6 +248,9 @@ func writeJSONString(out *bytes.Buffer, s string) { func ReadBounded(r io.Reader, limit int64) ([]byte, error) { defer perf.Track(nil, "artifact.ReadBounded")() + if limit < 0 || limit == math.MaxInt64 { + return nil, fmt.Errorf("%w: invalid read limit %d", errUtils.ErrArtifactTooLarge, limit) + } data, err := io.ReadAll(io.LimitReader(r, limit+1)) if err != nil { return nil, err diff --git a/pkg/artifact/canonicaljson_test.go b/pkg/artifact/canonicaljson_test.go index bc4f7dd4b64..3f430a04478 100644 --- a/pkg/artifact/canonicaljson_test.go +++ b/pkg/artifact/canonicaljson_test.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "io" + "math" "strings" "testing" @@ -71,6 +72,8 @@ func TestCanonicalizeJSONRejects(t *testing.T) { } func TestReadBounded(t *testing.T) { + _, limitErr := ReadBounded(strings.NewReader("x"), math.MaxInt64) + require.Error(t, limitErr) data, err := ReadBounded(strings.NewReader("12345"), 5) require.NoError(t, err) assert.Equal(t, "12345", string(data)) diff --git a/pkg/artifact/compat/compat_test.go b/pkg/artifact/compat/compat_test.go index 2683546ece6..ec81d680c89 100644 --- a/pkg/artifact/compat/compat_test.go +++ b/pkg/artifact/compat/compat_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "runtime" "slices" "strings" "testing" @@ -128,6 +129,13 @@ func TestLegacyCIFixtureStillReadsThroughCurrentStore(t *testing.T) { assert.Equal(t, map[string]string{"plan.tfplan": legacyPlanBody, ".terraform.lock.hcl": legacyLockBody}, bodies) assert.Equal(t, legacyMetadata().Stack, metadata.Stack) assert.Equal(t, legacyMetadata().SHA, metadata.SHA) + assert.Equal(t, legacyMetadata().Component, metadata.Component) + assert.Equal(t, legacyMetadata().BaseSHA, metadata.BaseSHA) + assert.Equal(t, legacyMetadata().Branch, metadata.Branch) + assert.Equal(t, legacyMetadata().RunID, metadata.RunID) + assert.Equal(t, legacyMetadata().Repository, metadata.Repository) + assert.Equal(t, legacyMetadata().AtmosVersion, metadata.AtmosVersion) + assert.True(t, legacyMetadata().CreatedAt.Equal(metadata.CreatedAt)) assert.Equal(t, 42, metadata.PRNumber) assert.Equal(t, legacyTarSHA256, metadata.SHA256) assert.Equal(t, map[string]string{"env": "dev", "zone": "a"}, metadata.Custom) @@ -192,6 +200,11 @@ func TestPluginReleaseFixtureDigestsAreStable(t *testing.T) { got, err := os.ReadFile(filepath.Join(destination, "bin", "plugin")) require.NoError(t, err) assert.Equal(t, want, got, platform) + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(destination, "bin", "plugin")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + } } }) } diff --git a/pkg/artifact/compat/doc.go b/pkg/artifact/compat/doc.go index d6fe0290b1a..2ca7f71c3db 100644 --- a/pkg/artifact/compat/doc.go +++ b/pkg/artifact/compat/doc.go @@ -4,5 +4,5 @@ // // It lives apart from pkg/artifact so the core domain package never imports pkg/ci or pkg/plugin. // Fixture bytes under testdata are committed and must never be regenerated to hide an -// incompatible change; see compat_test.go for the generation rules. +// incompatible change; see fixtures_test.go for the generation rules. package compat diff --git a/pkg/artifact/kinddata.go b/pkg/artifact/kinddata.go index b567a043515..9ba4cd7080a 100644 --- a/pkg/artifact/kinddata.go +++ b/pkg/artifact/kinddata.go @@ -27,6 +27,9 @@ func NewKindData(v any) (KindData, error) { if err != nil { return nil, fmt.Errorf("%w: kind data: %w", errUtils.ErrArtifactRecordInvalid, err) } + if string(raw) == "null" { + return nil, nil + } canonical, err := CanonicalizeJSON(raw) if err != nil { return nil, err diff --git a/pkg/artifact/kinddata_test.go b/pkg/artifact/kinddata_test.go index 58850747a6c..d983e7f5660 100644 --- a/pkg/artifact/kinddata_test.go +++ b/pkg/artifact/kinddata_test.go @@ -11,6 +11,16 @@ import ( ) func TestKindDataJSONBehavior(t *testing.T) { + data, err := NewKindData(nil) + require.NoError(t, err) + assert.Empty(t, data) + record := vectorRecordB(t) + record.KindData = data + raw, err := record.Canonical() + require.NoError(t, err) + decoded, err := DecodeRecord(raw) + require.NoError(t, err) + assert.Empty(t, decoded.KindData) t.Run("empty kind data marshals to null and cannot be decoded", func(t *testing.T) { raw, err := KindData(nil).MarshalJSON() require.NoError(t, err) diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index e75944f9343..1eb3f9eaa2e 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -3,9 +3,11 @@ package artifact import ( "errors" "fmt" + "net" "net/url" "regexp" "sort" + "strconv" "strings" "github.com/go-viper/mapstructure/v2" @@ -151,7 +153,7 @@ func (o *S3Options) Validate() error { var errs []error if o.Bucket == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "bucket")) - } else if !s3BucketPattern.MatchString(o.Bucket) { + } else if !s3BucketPattern.MatchString(o.Bucket) || net.ParseIP(o.Bucket) != nil { errs = append(errs, fmt.Errorf("%w: bucket %q is not a valid S3 bucket name", errUtils.ErrArtifactConfigInvalid, o.Bucket)) } if o.Region != "" && !regionPattern.MatchString(o.Region) { @@ -186,7 +188,7 @@ func (o *OCIRegistryOptions) Validate() error { var errs []error if o.Registry == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) - } else if !registryHostPattern.MatchString(o.Registry) { + } else if !validRegistryHost(o.Registry) { errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, o.Registry)) } errs = append(errs, validatePrefix(o.Prefix)) @@ -201,6 +203,20 @@ func (o *OCIRegistryOptions) Validate() error { return errors.Join(errs...) } +const registryPortBits = 16 + +func validRegistryHost(host string) bool { + if !registryHostPattern.MatchString(host) { + return false + } + _, port, found := strings.Cut(host, ":") + if !found { + return true + } + _, err := strconv.ParseUint(port, decimalBase, registryPortBits) + return err == nil +} + // OCILayoutOptions are the options of an oci/layout repository. type OCILayoutOptions struct { // Path is the OCI image layout directory. diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index beed87248e1..a028dbe94b3 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -59,6 +59,7 @@ func TestOptionsValidate(t *testing.T) { {name: "layout missing path", options: &OCILayoutOptions{}, wantErr: `"path" is required`}, {name: "s3 ok", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos", Region: "us-east-2", Tags: map[string]string{"purpose": "backup"}}}, {name: "s3 missing bucket", options: &S3Options{}, wantErr: `"bucket" is required`}, + {name: "s3 IP bucket", options: &S3Options{Bucket: "192.168.0.1"}, wantErr: "not a valid S3 bucket name"}, {name: "s3 bad bucket", options: &S3Options{Bucket: "Bad_Bucket"}, wantErr: "not a valid S3 bucket name"}, {name: "s3 bad region", options: &S3Options{Bucket: "acme-artifacts", Region: "US East"}, wantErr: "not a valid region"}, {name: "s3 prefix traversal", options: &S3Options{Bucket: "acme-artifacts", Prefix: "a/../b"}, wantErr: "prefix"}, @@ -71,6 +72,8 @@ func TestOptionsValidate(t *testing.T) { {name: "s3 bad key pattern", options: &S3Options{Bucket: "acme-artifacts", KeyPattern: "../x"}, wantErr: "segment"}, {name: "registry ok", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/atmos"}}, {name: "registry with port", options: &OCIRegistryOptions{Registry: "localhost:5000"}}, + {name: "registry max port", options: &OCIRegistryOptions{Registry: "localhost:65535"}}, + {name: "registry port overflow", options: &OCIRegistryOptions{Registry: "localhost:65536"}, wantErr: "without a scheme"}, {name: "registry missing", options: &OCIRegistryOptions{}, wantErr: `"registry" is required`}, {name: "registry with scheme", options: &OCIRegistryOptions{Registry: "https://ghcr.io"}, wantErr: "without a scheme"}, {name: "registry with credentials", options: &OCIRegistryOptions{Registry: "user:pw@ghcr.io"}, wantErr: "without a scheme"}, diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go index fff2e4f96a8..b844e59e5d7 100644 --- a/pkg/artifact/registry.go +++ b/pkg/artifact/registry.go @@ -102,10 +102,18 @@ func (r *RepositoryRegistry) Register(desc *KindDescriptor, factory RepositoryFa if _, exists := r.kinds[desc.Kind]; exists { return fmt.Errorf("%w: repository kind %q is already registered", errUtils.ErrArtifactConflict, desc.Kind) } - r.kinds[desc.Kind] = registeredKind{descriptor: *desc, factory: factory} + r.kinds[desc.Kind] = registeredKind{descriptor: cloneKindDescriptor(desc), factory: factory} return nil } +func cloneKindDescriptor(source *KindDescriptor) KindDescriptor { + desc := *source + desc.PathOptions = append([]string(nil), desc.PathOptions...) + desc.Capabilities.Packagings = append([]Packaging(nil), desc.Capabilities.Packagings...) + desc.Capabilities.LockScopes = append([]LockScope(nil), desc.Capabilities.LockScopes...) + return desc +} + // Lookup returns the descriptor of a kind: the registered descriptor when a backend registered // the kind, otherwise the built-in V1 descriptor. The boolean is false for an unknown kind. func (r *RepositoryRegistry) Lookup(kind string) (*KindDescriptor, bool) { @@ -115,7 +123,7 @@ func (r *RepositoryRegistry) Lookup(kind string) (*KindDescriptor, bool) { entry, ok := r.kinds[kind] r.mu.RUnlock() if ok { - desc := entry.descriptor + desc := cloneKindDescriptor(&entry.descriptor) return &desc, true } return BuiltinKind(kind) diff --git a/pkg/artifact/registry_test.go b/pkg/artifact/registry_test.go index 40093c875b2..d1919095922 100644 --- a/pkg/artifact/registry_test.go +++ b/pkg/artifact/registry_test.go @@ -17,13 +17,16 @@ import ( // fakeKind registers a kind whose factory records its inputs and returns a mock repository. type fakeKind struct { + ctx context.Context + cfg *schema.ArtifactRepositoryConfig calls int alias string options RepositoryOptions repo *MockRepository } -func (f *fakeKind) factory(_ context.Context, alias string, _ *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { +func (f *fakeKind) factory(ctx context.Context, alias string, cfg *schema.ArtifactRepositoryConfig, opts ...RepositoryOption) (Repository, error) { + f.ctx, f.cfg = ctx, cfg f.calls++ f.alias = alias f.options = ApplyRepositoryOptions(opts...) @@ -102,9 +105,14 @@ func TestRepositoryRegistry_NewConstructsThroughFactoryWithoutNetworkOrCredentia resolved := 0 resolver := func(context.Context, string) (any, error) { resolved++; return nil, nil } lookup := func(string) (Repository, error) { return nil, nil } - got, err := registry.New(context.Background(), "local", localDirConfig(t.TempDir()), WithIdentityResolver(resolver), WithRepositoryLookup(lookup)) + type contextKey struct{} + ctx := context.WithValue(context.Background(), contextKey{}, "test") + cfg := localDirConfig(t.TempDir()) + got, err := registry.New(ctx, "local", cfg, WithIdentityResolver(resolver), WithRepositoryLookup(lookup)) require.NoError(t, err) assert.Same(t, repo, got) + assert.Same(t, ctx, fake.ctx) + assert.Same(t, cfg, fake.cfg) assert.Equal(t, 1, fake.calls) assert.Equal(t, "local", fake.alias) assert.NotNil(t, fake.options.ResolveIdentity) @@ -213,3 +221,24 @@ func TestApplyRepositoryOptions_IgnoresNilOptions(t *testing.T) { _, err := applied.LookupRepository("x") require.EqualError(t, err, "lookup x") } + +func TestRepositoryRegistryDescriptorIsolation(t *testing.T) { + r := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, r.Register(desc, (&fakeKind{}).factory)) + desc.PathOptions[0] = "changed" + desc.Capabilities.Packagings[0] = PackagingZip + desc.Capabilities.LockScopes[0] = "changed" + got, ok := r.Lookup(desc.Kind) + require.True(t, ok) + assert.Equal(t, OptionPath, got.PathOptions[0]) + assert.Equal(t, PackagingNone, got.Capabilities.Packagings[0]) + assert.NotEqual(t, LockScope("changed"), got.Capabilities.LockScopes[0]) + got.PathOptions[0] = "changed" + got.Capabilities.Packagings[0] = PackagingZip + got.Capabilities.LockScopes[0] = "changed" + again, _ := r.Lookup(desc.Kind) + assert.Equal(t, OptionPath, again.PathOptions[0]) + assert.Equal(t, PackagingNone, again.Capabilities.Packagings[0]) + assert.NotEqual(t, LockScope("changed"), again.Capabilities.LockScopes[0]) +} diff --git a/pkg/config/artifacts.go b/pkg/config/artifacts.go index f2b0679f9b4..616b43e1657 100644 --- a/pkg/config/artifacts.go +++ b/pkg/config/artifacts.go @@ -53,6 +53,16 @@ func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfigurati origins := newArtifactOrigins() declared := false for _, source := range collectConfigSourcesForCasePreservation(v, v.ConfigFileUsed()) { + var document yaml.Node + if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { + return fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + if node := policyMappingValue(&document, artifactsSectionKey); node != nil && node.Tag == "!unset" { + merged = map[string]any{} + origins = newArtifactOrigins() + declared = true + continue + } section, ok, err := artifactsSectionFromSource(source) if err != nil { return err diff --git a/pkg/config/artifacts_test.go b/pkg/config/artifacts_test.go index 689a9ae1f57..878d9237f03 100644 --- a/pkg/config/artifacts_test.go +++ b/pkg/config/artifacts_test.go @@ -417,3 +417,17 @@ func TestLoadConfig_ArtifactsDescribeRoundTrip(t *testing.T) { assert.Equal(t, expected, describedYAML.Artifacts) assert.NotContains(t, string(jsonBytes), "PathOrigin") } + +func TestLoadConfig_ArtifactsSectionUnsetClearsPriorSources(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./base.yaml\nartifacts: !unset\n") + writeArtifactsFile(t, filepath.Join(dir, "base.yaml"), "artifacts:\n repositories:\n old:\n kind: local/dir\n options:\n path: old\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Empty(t, cfg.Artifacts.Repositories) +} + +func TestPathInsideAnyRoot(t *testing.T) { + root := t.TempDir() + assert.True(t, pathInsideAnyRoot(filepath.Join(root, "nested", "file.yaml"), []string{root})) + assert.False(t, pathInsideAnyRoot(root+"-other/file.yaml", []string{root})) +} diff --git a/pkg/config/imports.go b/pkg/config/imports.go index a72edc9f382..d8ad1cfe149 100644 --- a/pkg/config/imports.go +++ b/pkg/config/imports.go @@ -124,7 +124,7 @@ func processConfigImportsWithFSAndBasePathSource(source *schema.AtmosConfigurati log.Debug("processConfigImports resolved paths", "count", len(resolvedPaths)) - return mergeResolvedImports(resolvedPaths, dst, fs) + return mergeResolvedImports(resolvedPaths, dst, fs, tempDir) } // mergeResolvedImports merges each resolved import file into dst and returns the @@ -132,7 +132,7 @@ func processConfigImportsWithFSAndBasePathSource(source *schema.AtmosConfigurati // An optional import (SkipIfMissing) that fails to merge or re-read is skipped so a single // bad import does not abort the whole configuration load; a required import that fails to // merge returns errUtils.ErrRequiredImportFailed (fail closed). -func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs filesystem.FileSystem) (string, error) { +func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs filesystem.FileSystem, temporaryRoots ...string) (string, error) { basePathSourceDir := "" for _, resolvedPath := range resolvedPaths { // Trace: log what we're about to merge (sanitized). @@ -149,7 +149,7 @@ func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs fi continue } log.Trace("successfully merged config from import", "import", SanitizeImport(resolvedPath.ImportPaths), keyFilePath, resolvedPath.FilePath) - if resolvedPath.ImportType != LOCAL { + if resolvedPath.ImportType != LOCAL || pathInsideAnyRoot(resolvedPath.FilePath, temporaryRoots) { // Downloaded imports live in a temporary directory that is removed after loading. mergedFilesReg.markEphemeral(dst, resolvedPath.FilePath) } @@ -166,6 +166,17 @@ func mergeResolvedImports(resolvedPaths []ResolvedPaths, dst *viper.Viper, fs fi return basePathSourceDir, nil } +// pathInsideAnyRoot identifies downloaded files by their actual location, not import ancestry. +func pathInsideAnyRoot(path string, roots []string) bool { + for _, root := range roots { + rel, err := filepath.Rel(root, path) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) { + return true + } + } + return false +} + // ProcessImportsFromAdapter is the public entry point for adapters to process nested imports. // Adapters should call this when they discover nested import statements in resolved configs. // The atmosConfig parameter is threaded through so nested remote imports honor auth settings diff --git a/pkg/config/schema/overrides.go b/pkg/config/schema/overrides.go index 3d20a36ebcd..318ed68bbef 100644 --- a/pkg/config/schema/overrides.go +++ b/pkg/config/schema/overrides.go @@ -130,6 +130,19 @@ func applyOverrides(r *jsonschema.Reflector, root *jsonschema.Schema) { overrideSchemasSection(r, root) applyPolymorphicOverrides(root) allowYamlFunctions(root) + // Artifact trusted keys are decoded strictly and do not use the generic + // string-to-slice coercion that other configuration fields support. + if def := root.Definitions["ArtifactVerificationConfig"]; def != nil { + if keys, ok := def.Properties.Get("trusted_keys"); ok { + filtered := keys.AnyOf[:0] + for _, branch := range keys.AnyOf { + if branch.Type != typeString { + filtered = append(filtered, branch) + } + } + keys.AnyOf = filtered + } + } } // applyPolymorphicOverrides relaxes shapes that custom unmarshalers accept diff --git a/pkg/datafetcher/schema/atmos/config/1.0.json b/pkg/datafetcher/schema/atmos/config/1.0.json index c3b610e3e53..a7dbe774c6e 100644 --- a/pkg/datafetcher/schema/atmos/config/1.0.json +++ b/pkg/datafetcher/schema/atmos/config/1.0.json @@ -1947,9 +1947,6 @@ }, { "$ref": "#/$defs/yamlFunction" - }, - { - "type": "string" } ], "description": "TrustedKeys are operator-managed public-key file paths. Relative paths resolve against the\nconfiguration file that declared them." From f9f26cf5508534295a0190a2fabd36e9a9245805 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:06:31 -0600 Subject: [PATCH 05/32] fix(artifacts): preserve evaluated configuration origins --- pkg/config/artifacts.go | 75 ++++++++++++- pkg/config/artifacts_test.go | 102 ++++++++++++++++++ pkg/config/global_viper.go | 44 +++++--- pkg/config/load.go | 49 ++++++++- pkg/config/process_yaml_include_eval.go | 5 +- pkg/config/yaml_decode.go | 50 ++++++++- .../scenarios/artifacts-describe/atmos.yaml | 11 ++ tests/test-cases/artifacts-describe.yaml | 21 ++++ 8 files changed, 333 insertions(+), 24 deletions(-) create mode 100644 tests/fixtures/scenarios/artifacts-describe/atmos.yaml create mode 100644 tests/test-cases/artifacts-describe.yaml diff --git a/pkg/config/artifacts.go b/pkg/config/artifacts.go index 616b43e1657..3131ae0ef75 100644 --- a/pkg/config/artifacts.go +++ b/pkg/config/artifacts.go @@ -36,6 +36,58 @@ func newArtifactOrigins() *artifactOrigins { } } +// evaluatedArtifacts retains the first evaluation and the file declaring each path-valued setting. +type evaluatedArtifacts struct { + section map[string]any + origins map[string]artifactFileOrigin +} + +type artifactFileOrigin struct { + file string + remote bool +} + +// evaluateArtifactsForLoad resolves tags once before Viper processes the section. +func evaluateArtifactsForLoad(node *yaml.Node, sourceFile string) (*evaluatedArtifacts, error) { + result := &evaluatedArtifacts{origins: make(map[string]artifactFileOrigin)} + decoder := configYamlDecoder{includes: make(map[string]bool)} + decoder.origin = result.recordOrigin + value, err := decoder.decode(node, sourceFile) + if err != nil { + return nil, err + } + if value == nil { + return result, nil + } + section, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("%w: artifacts in %s must be a mapping, got %T", errUtils.ErrArtifactConfigInvalid, sourceFile, value) + } + result.section = section + return result, nil +} + +func (result *evaluatedArtifacts) recordOrigin(path []string, file string, remote bool) { + origin := artifactFileOrigin{file: file, remote: remote} + if len(path) == 2 && strings.EqualFold(path[0], "verification") && strings.EqualFold(path[1], "trusted_keys") { + result.origins["global"] = origin + return + } + if len(path) != 4 || !strings.EqualFold(path[0], "repositories") { + return + } + switch strings.ToLower(path[2]) { + case "options": + if strings.EqualFold(path[3], "path") { + result.origins["path\x00"+path[1]] = origin + } + case "verification": + if strings.EqualFold(path[3], "trusted_keys") { + result.origins["keys\x00"+path[1]] = origin + } + } +} + // resolveArtifactsConfig rebuilds atmosConfig.Artifacts from the retained configuration sources. // // Viper cannot be used for this section: it lowercases keys (repository aliases, label keys, and @@ -63,7 +115,14 @@ func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfigurati declared = true continue } - section, ok, err := artifactsSectionFromSource(source) + var section map[string]any + var ok bool + var err error + if source.artifacts != nil { + section, ok = source.artifacts.section, source.artifacts.section != nil + } else { + section, ok, err = artifactsSectionFromSource(source) + } if err != nil { return err } @@ -72,6 +131,20 @@ func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfigurati } declared = true origins.record(section, source) + if source.artifacts != nil { + for key, file := range source.artifacts.origins { + origin := artifactconfig.NewOrigin(file.file, source.ephemeral || file.remote) + kind, alias, found := strings.Cut(key, "\x00") + switch { + case key == "global": + origins.globalKeys = origin + case found && kind == "path": + origins.repoPath[alias] = origin + case found && kind == "keys": + origins.repoKeys[alias] = origin + } + } + } combined, err := merge.MergeWithOptions(nil, []map[string]any{merged, section}, false, false) if err != nil { return fmt.Errorf("%w: merge artifacts from %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) diff --git a/pkg/config/artifacts_test.go b/pkg/config/artifacts_test.go index 878d9237f03..02ad6106a5e 100644 --- a/pkg/config/artifacts_test.go +++ b/pkg/config/artifacts_test.go @@ -3,11 +3,15 @@ package config import ( "context" "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v3" @@ -431,3 +435,101 @@ func TestPathInsideAnyRoot(t *testing.T) { assert.True(t, pathInsideAnyRoot(filepath.Join(root, "nested", "file.yaml"), []string{root})) assert.False(t, pathInsideAnyRoot(root+"-other/file.yaml", []string{root})) } + +func TestLoadConfig_ArtifactsIncludesRetainDeclaringFiles(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include ./included/artifacts.yaml | eval\n") + writeArtifactsFile(t, filepath.Join(dir, "included", "artifacts.yaml"), `verification: + trusted_keys: [global.pub] +repositories: + local: !include ./nested/repository.yaml | eval +`) + writeArtifactsFile(t, filepath.Join(dir, "included", "nested", "repository.yaml"), `kind: local/dir +options: + path: store +verification: + trusted_keys: [repo.pub] +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{filepath.Join(dir, "included", "global.pub")}, cfg.Artifacts.Verification.TrustedKeys) + repo := cfg.Artifacts.Repositories["local"] + assert.Equal(t, filepath.Join(dir, "included", "nested", "store"), repo.Options["path"]) + require.NotNil(t, repo.Verification) + assert.Equal(t, []string{filepath.Join(dir, "included", "nested", "repo.pub")}, repo.Verification.TrustedKeys) +} + +func TestLoadConfig_ArtifactsExecEvaluatedOnce(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + config := fmt.Sprintf("artifacts:\n repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", shellQuote(os.Args[0]), shellQuote(countPath)) + root := artifactsTestDir(t, config) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(root, "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestProfileImportArtifactsExecEvaluatedOnce(t *testing.T) { + setupTestAdapters() + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + writeArtifactsFile(t, filepath.Join(dir, "import.yaml"), "logs:\n level: Info\n") + profile := filepath.Join(dir, "profile.yaml") + writeArtifactsFile(t, profile, fmt.Sprintf("base_path: %s\nimport: [./import.yaml]\nartifacts:\n repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", dir, shellQuote(os.Args[0]), shellQuote(countPath))) + v := viper.New() + v.SetConfigType("yaml") + resetMergedConfigFiles(v) + defer mergedFilesReg.finish(v) + require.NoError(t, mergeConfigFileWithImports(profile, v)) + var cfg schema.AtmosConfiguration + require.NoError(t, resolveArtifactsConfig(v, &cfg)) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(dir, "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestLoadConfig_ArtifactsPlainIncludeRetainsDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include ./included/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "included", "artifacts.yaml"), "repositories:\n local:\n kind: local/dir\n options:\n path: store\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, "included", "store"), cfg.Artifacts.Repositories["local"].Options["path"]) +} + +func TestLoadConfig_ArtifactsRemoteIncludeDoesNotAnchorPaths(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, "repositories:\n remote:\n kind: local/dir\n options:\n path: remote-store\n") + })) + defer server.Close() + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts: !include "+server.URL+"/artifacts.yaml"+suffix+"\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo := cfg.Artifacts.Repositories["remote"] + assert.Equal(t, "remote-store", repo.Options["path"]) + assert.True(t, repo.PathOrigin.Remote) + }) + } +} + +func TestLoadConfig_ArtifactsIncludedExecEvaluatedOnce(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + root := artifactsTestDir(t, "artifacts: !include ./sub/artifacts.yaml | eval\n") + contents := fmt.Sprintf("repositories:\n local:\n kind: local/dir\n options:\n path: !exec >-\n %s -test.run=^TestMergeConfigFileExecHelper$ -- %s\n", shellQuote(os.Args[0]), shellQuote(countPath)) + writeArtifactsFile(t, filepath.Join(root, "sub", "artifacts.yaml"), contents) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count)) + assert.Equal(t, filepath.Join(root, "sub", "resolved-description"), cfg.Artifacts.Repositories["local"].Options["path"]) +} diff --git a/pkg/config/global_viper.go b/pkg/config/global_viper.go index 264b52fc68f..1c6e01e534e 100644 --- a/pkg/config/global_viper.go +++ b/pkg/config/global_viper.go @@ -89,30 +89,22 @@ func GlobalViper() *SafeViper { // mergedFilesTracker tracks the config files merged during a single LoadConfig // call, guarded by its own mutex. type mergedFilesTracker struct { - mu sync.Mutex - sources []mergedConfigSource + mu sync.Mutex + sources []mergedConfigSource + pendingArtifacts map[string]*evaluatedArtifacts } // mergedConfigSource retains the YAML even after a downloaded import is removed. // Content is immutable so snapshots cannot mutate another reader's source data. type mergedConfigSource struct { - path string - content string + path string + content string + artifacts *evaluatedArtifacts // ephemeral marks a source downloaded from a remote import. Its local path is a temporary // location, so relative destinations declared in it must never anchor to that directory. ephemeral bool } -// track appends a source in effective merge order, including repeated paths. -func (t *mergedFilesTracker) track(source mergedConfigSource) { - t.mu.Lock() - defer t.mu.Unlock() - if source.path != "" { - // Keep repeated merges: their position determines precedence. - t.sources = append(t.sources, source) - } -} - // markEphemeral flags every tracked merge of path as a remote (downloaded) source. func (t *mergedFilesTracker) markEphemeral(path string) { t.mu.Lock() @@ -203,8 +195,30 @@ func (r *mergedFilesRegistry) track(v *viper.Viper, path string, content []byte) tracker := r.trackers[v] r.mu.Unlock() if tracker != nil { - tracker.track(mergedConfigSource{path: path, content: string(content)}) + tracker.mu.Lock() + artifacts := tracker.pendingArtifacts[path] + delete(tracker.pendingArtifacts, path) + if path != "" { + tracker.sources = append(tracker.sources, mergedConfigSource{path: path, content: string(content), artifacts: artifacts}) + } + tracker.mu.Unlock() + } +} + +// rememberArtifacts retains a source's evaluated, case-preserving artifact section until it is tracked. +func (r *mergedFilesRegistry) rememberArtifacts(v *viper.Viper, path string, value *evaluatedArtifacts) { + r.mu.Lock() + tracker := r.trackers[v] + r.mu.Unlock() + if tracker == nil { + return + } + tracker.mu.Lock() + defer tracker.mu.Unlock() + if tracker.pendingArtifacts == nil { + tracker.pendingArtifacts = make(map[string]*evaluatedArtifacts) } + tracker.pendingArtifacts[path] = value } // markEphemeral flags a downloaded import source for the load v belongs to. diff --git a/pkg/config/load.go b/pkg/config/load.go index 5c82186186f..7debfb56293 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -487,8 +487,8 @@ func finalizeLoadedConfig(v *viper.Viper, atmosConfig *schema.AtmosConfiguration } // Rebuild artifacts from the retained sources: Viper lowercases keys and forgets which // file declared options.path once imports are merged. - if err := resolveArtifactsConfig(v, &atmosConfig); err != nil { - return atmosConfig, err + if err := resolveArtifactsConfig(v, atmosConfig); err != nil { + return err } // Post-process to preserve case-sensitive map keys. @@ -2073,6 +2073,10 @@ func mergeConfigFile( // top-level `commands` key, which extractCommandsWithYamlFunctionsForFile decodes with its own // include scope. Git hook steps use the same step-aware decoder before generic processing. func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourceFile string) error { + return preprocessConfigYAML(content, v, sourceFile, false) +} + +func preprocessConfigYAML(content []byte, v *viper.Viper, sourceFile string, skipArtifacts bool) error { var root goyaml.Node if err := goyaml.Unmarshal(content, &root); err != nil { return err @@ -2092,6 +2096,10 @@ func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourc } } } + evaluated, err := prepareArtifactsNode(&root, v, sourceFile, skipArtifacts) + if err != nil { + return err + } // Process the parsed tree directly: aliases can refer to anchors under commands, // so serializing the filtered tree would leave dangling aliases on the next parse. if err := preprocessGitHookSteps(&root, v, sourceFile); err != nil { @@ -2101,7 +2109,38 @@ func preprocessAtmosYamlFuncExceptCommands(content []byte, v *viper.Viper, sourc if err := prepareDevcontainerKeys(&root, v); err != nil { return err } - return processConfigNodeWithAuthSource(&root, v, sourceFile) + if err := processConfigNodeWithAuthSource(&root, v, sourceFile); err != nil { + return err + } + if evaluated != nil { + mergedFilesReg.rememberArtifacts(v, sourceFile, evaluated) + } + return nil +} + +// prepareArtifactsNode evaluates the artifact section once and removes its function tags. +func prepareArtifactsNode(root *goyaml.Node, v *viper.Viper, sourceFile string, skip bool) (*evaluatedArtifacts, error) { + node := findYAMLMappingPath(root, artifactsSectionKey) + if node == nil { + return nil, nil + } + if skip { + // The profile section was already evaluated when it was merged. + *node = goyaml.Node{Kind: goyaml.ScalarNode, Tag: "!!null", Value: "null"} + return nil, nil + } + if node.Tag == "!unset" { + return nil, nil + } + evaluated, err := evaluateArtifactsForLoad(node, sourceFile) + if err != nil { + return nil, err + } + if err := node.Encode(evaluated.section); err != nil { + return nil, err + } + v.Set(artifactsSectionKey, evaluated.section) + return evaluated, nil } // processConfigNodeWithAuthSource retains identities introduced by a whole-auth include. @@ -2203,7 +2242,7 @@ func mergeConfigFileWithImports(path string, v *viper.Viper) error { if len(imports) == 0 { return nil } - if err = preprocessAtmosYamlFuncExceptCommands(content, tempViper, path); err != nil { + if err = preprocessConfigYAML(content, tempViper, path, true); err != nil { return err } overlayProfileSettings(v, tempViper.AllSettings(), "") @@ -2238,7 +2277,7 @@ func applyProfileImports(v, tempViper *viper.Viper, path string, content []byte, func overlayProfileSettings(v *viper.Viper, settings map[string]any, prefix string) { for key, value := range settings { - if key == "import" || key == commandsKey { + if key == "import" || key == commandsKey || (prefix == "" && key == artifactsSectionKey) { continue } path := key diff --git a/pkg/config/process_yaml_include_eval.go b/pkg/config/process_yaml_include_eval.go index 271f80cf2fd..0ebff5902f8 100644 --- a/pkg/config/process_yaml_include_eval.go +++ b/pkg/config/process_yaml_include_eval.go @@ -12,7 +12,10 @@ const maxConfigIncludeDepth = 100 // configYamlDecoder keeps recursive include evaluation within one config read. // The shared include resolver supplies the correct file context for each subtree. type configYamlDecoder struct { - includes map[string]bool + includes map[string]bool + path []string + origin func([]string, string, bool) + remoteSource string } // evaluateInclude resolves an included subtree using its file context and diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 02c4e281f8a..a59c33787f3 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -59,7 +59,16 @@ func (d *configYamlDecoder) decodeMappingNodeWithYamlFunctions(node *yaml.Node, for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] + d.path = append(d.path, keyNode.Value) value, err := d.decode(valueNode, sourceFile) + if err == nil && d.origin != nil { + originFile := sourceFile + if d.remoteSource != "" { + originFile = d.remoteSource + } + d.origin(d.path, originFile, d.remoteSource != "") + } + d.path = d.path[:len(d.path)-1] if err != nil { return nil, err } @@ -128,7 +137,8 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour // Resolve the path against the config file and the project base path, never the working // directory, so the result is the same wherever Atmos runs from. scope := commandIncludeScope(sourceFile) - value := scope.Resolve(node.Value).Value + include := scope.Resolve(node.Value) + value := include.Value resolved := legacyyaml.Node{ Kind: legacyyaml.ScalarNode, Tag: node.Tag, @@ -145,10 +155,22 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour BasePathAbsolute: scope.BasePath, } var err error + var walked bool if node.Tag == u.AtmosYamlFuncIncludeRaw { err = u.ProcessIncludeRawTag(atmosConfig, &resolved, value, sourceFile) + } else if d.origin == nil { + walked, err = u.ProcessIncludeTagWalked(atmosConfig, &resolved, value, sourceFile, d.evaluateInclude) } else { - _, err = u.ProcessIncludeTagWalked(atmosConfig, &resolved, value, sourceFile, d.evaluateInclude) + walked, err = u.ProcessIncludeTagWalkedFrom(atmosConfig, &resolved, value, sourceFile, u.IncludeWalk{ + Local: d.evaluateInclude, + Remote: func(child *legacyyaml.Node, file, source string) error { + previous := d.remoteSource + d.remoteSource = source + defer func() { d.remoteSource = previous }() + return d.evaluateInclude(child, file) + }, + Source: d.remoteSource, + }) } if err != nil { return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) @@ -157,9 +179,33 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour if err := resolved.Decode(&decoded); err != nil { return nil, err } + if !walked && d.origin != nil { + file, remote := include.LocalPath, false + if file == "" { + file, remote = node.Value, true + } + if d.remoteSource != "" { + file, remote = d.remoteSource, true + } + d.recordIncludedOrigins(decoded, file, remote) + } return decoded, nil } +// recordIncludedOrigins records plain, non-evaluated mappings without running their tags. +func (d *configYamlDecoder) recordIncludedOrigins(value any, file string, remote bool) { + mapping, ok := value.(map[string]any) + if !ok { + return + } + for key, child := range mapping { + d.path = append(d.path, key) + d.recordIncludedOrigins(child, file, remote) + d.origin(d.path, file, remote) + d.path = d.path[:len(d.path)-1] + } +} + // includeBasePathForSourceFile returns the directory a config file's bare include paths anchor // to when it declares no base path: the config file's own directory, or the parent of the // nearest enclosing `atmos.d` or `.atmos.d` directory for files loaded from one (at any depth). diff --git a/tests/fixtures/scenarios/artifacts-describe/atmos.yaml b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml new file mode 100644 index 00000000000..04fc65bcbbc --- /dev/null +++ b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml @@ -0,0 +1,11 @@ +base_path: . +artifacts: + repositories: + shared: + kind: aws/s3 + options: + bucket: example-artifacts + verification: + require_signature: true + trusted_keys: + - /keys/release.pub diff --git a/tests/test-cases/artifacts-describe.yaml b/tests/test-cases/artifacts-describe.yaml new file mode 100644 index 00000000000..7275093e322 --- /dev/null +++ b/tests/test-cases/artifacts-describe.yaml @@ -0,0 +1,21 @@ +# yaml-language-server: $schema=schema.json +tests: + - name: describe config with artifacts + enabled: true + snapshot: false + workdir: "fixtures/scenarios/artifacts-describe" + command: "atmos" + env: + ATMOS_PAGER: false + args: + - describe + - config + expect: + valid: + - json + stdout: + - '"shared"' + - '"bucket": "example-artifacts"' + - '"require_signature": true' + - '"/keys/release.pub"' + exit_code: 0 From 89c529df4ba9de009c87df12d418e14548f1a9d7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 16:30:30 -0600 Subject: [PATCH 06/32] refactor(config): simplify artifact source replay --- pkg/config/artifacts.go | 104 ++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/pkg/config/artifacts.go b/pkg/config/artifacts.go index 3131ae0ef75..af81492b6bd 100644 --- a/pkg/config/artifacts.go +++ b/pkg/config/artifacts.go @@ -17,7 +17,10 @@ import ( ) // artifactsSectionKey is the top-level atmos.yaml key of the artifact configuration. -const artifactsSectionKey = "artifacts" +const ( + artifactsSectionKey = "artifacts" + verificationSectionKey = "verification" +) // artifactOrigins records, per setting, the configuration file that supplied the final value. It is // tracked independently of the merged map because deep merging fuses the fields of several files @@ -69,7 +72,7 @@ func evaluateArtifactsForLoad(node *yaml.Node, sourceFile string) (*evaluatedArt func (result *evaluatedArtifacts) recordOrigin(path []string, file string, remote bool) { origin := artifactFileOrigin{file: file, remote: remote} - if len(path) == 2 && strings.EqualFold(path[0], "verification") && strings.EqualFold(path[1], "trusted_keys") { + if len(path) == 2 && strings.EqualFold(path[0], verificationSectionKey) && strings.EqualFold(path[1], "trusted_keys") { result.origins["global"] = origin return } @@ -81,7 +84,7 @@ func (result *evaluatedArtifacts) recordOrigin(path []string, file string, remot if strings.EqualFold(path[3], "path") { result.origins["path\x00"+path[1]] = origin } - case "verification": + case verificationSectionKey: if strings.EqualFold(path[3], "trusted_keys") { result.origins["keys\x00"+path[1]] = origin } @@ -105,51 +108,26 @@ func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfigurati origins := newArtifactOrigins() declared := false for _, source := range collectConfigSourcesForCasePreservation(v, v.ConfigFileUsed()) { - var document yaml.Node - if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { - return fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + section, state, err := artifactSectionForReplay(source) + if err != nil { + return err } - if node := policyMappingValue(&document, artifactsSectionKey); node != nil && node.Tag == "!unset" { + if state == artifactSectionUnset { merged = map[string]any{} origins = newArtifactOrigins() declared = true continue } - var section map[string]any - var ok bool - var err error - if source.artifacts != nil { - section, ok = source.artifacts.section, source.artifacts.section != nil - } else { - section, ok, err = artifactsSectionFromSource(source) - } - if err != nil { - return err - } - if !ok { + if state == artifactSectionAbsent { continue } declared = true origins.record(section, source) - if source.artifacts != nil { - for key, file := range source.artifacts.origins { - origin := artifactconfig.NewOrigin(file.file, source.ephemeral || file.remote) - kind, alias, found := strings.Cut(key, "\x00") - switch { - case key == "global": - origins.globalKeys = origin - case found && kind == "path": - origins.repoPath[alias] = origin - case found && kind == "keys": - origins.repoKeys[alias] = origin - } - } - } - combined, err := merge.MergeWithOptions(nil, []map[string]any{merged, section}, false, false) + origins.recordEvaluated(source) + merged, err = merge.MergeWithOptions(nil, []map[string]any{merged, section}, false, false) if err != nil { return fmt.Errorf("%w: merge artifacts from %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) } - merged = combined } if !declared { return nil @@ -163,6 +141,58 @@ func resolveArtifactsConfig(v *viper.Viper, atmosConfig *schema.AtmosConfigurati return nil } +type artifactSectionState uint8 + +const ( + artifactSectionAbsent artifactSectionState = iota + artifactSectionPresent + artifactSectionUnset +) + +// artifactSectionForReplay returns the evaluated section or an explicit removal. +func artifactSectionForReplay(source mergedConfigSource) (map[string]any, artifactSectionState, error) { + var document yaml.Node + if err := yaml.Unmarshal([]byte(source.content), &document); err != nil { + return nil, artifactSectionAbsent, fmt.Errorf("%w: parse %s: %w", errUtils.ErrArtifactConfigInvalid, source.path, err) + } + if node := policyMappingValue(&document, artifactsSectionKey); node != nil && node.Tag == "!unset" { + return nil, artifactSectionUnset, nil + } + if source.artifacts != nil { + if source.artifacts.section == nil { + return nil, artifactSectionAbsent, nil + } + return source.artifacts.section, artifactSectionPresent, nil + } + section, present, err := artifactsSectionFromSource(source) + if err != nil { + return nil, artifactSectionAbsent, err + } + if !present { + return nil, artifactSectionAbsent, nil + } + return section, artifactSectionPresent, nil +} + +// recordEvaluated applies the origins of values declared inside included files. +func (o *artifactOrigins) recordEvaluated(source mergedConfigSource) { + if source.artifacts == nil { + return + } + for key, file := range source.artifacts.origins { + origin := artifactconfig.NewOrigin(file.file, source.ephemeral || file.remote) + kind, alias, found := strings.Cut(key, "\x00") + switch { + case key == "global": + o.globalKeys = origin + case found && kind == "path": + o.repoPath[alias] = origin + case found && kind == "keys": + o.repoKeys[alias] = origin + } + } +} + // artifactsSectionFromSource parses the `artifacts` section of one retained source with its // original key spelling. The boolean is false when the source declares no such section. func artifactsSectionFromSource(source mergedConfigSource) (map[string]any, bool, error) { @@ -191,7 +221,7 @@ func artifactsSectionFromSource(source mergedConfigSource) (map[string]any, bool // record notes which source declares each path-valued setting present in section. func (o *artifactOrigins) record(section map[string]any, source mergedConfigSource) { origin := artifactconfig.NewOrigin(source.path, source.ephemeral) - if keysDeclared(mapOf(mapValue(section, "verification"))) { + if keysDeclared(mapOf(mapValue(section, verificationSectionKey))) { o.globalKeys = origin } repos, _ := mapValue(section, "repositories").(map[string]any) @@ -203,7 +233,7 @@ func (o *artifactOrigins) record(section map[string]any, source mergedConfigSour if _, isString := mapValue(mapOf(mapValue(repo, "options")), artifact.OptionPath).(string); isString { o.repoPath[alias] = origin } - if keysDeclared(mapOf(mapValue(repo, "verification"))) { + if keysDeclared(mapOf(mapValue(repo, verificationSectionKey))) { o.repoKeys[alias] = origin } } From 998183dd02fdcb6725bc7298871e840a6332c7d9 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Wed, 11 Nov 2026 20:54:17 -0600 Subject: [PATCH 07/32] test(artifacts): handle Windows trusted key paths --- tests/fixtures/scenarios/artifacts-describe/atmos.yaml | 2 +- tests/test-cases/artifacts-describe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/scenarios/artifacts-describe/atmos.yaml b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml index 04fc65bcbbc..36b0cb18133 100644 --- a/tests/fixtures/scenarios/artifacts-describe/atmos.yaml +++ b/tests/fixtures/scenarios/artifacts-describe/atmos.yaml @@ -8,4 +8,4 @@ artifacts: verification: require_signature: true trusted_keys: - - /keys/release.pub + - keys/release.pub diff --git a/tests/test-cases/artifacts-describe.yaml b/tests/test-cases/artifacts-describe.yaml index 7275093e322..f0178c4ad54 100644 --- a/tests/test-cases/artifacts-describe.yaml +++ b/tests/test-cases/artifacts-describe.yaml @@ -17,5 +17,5 @@ tests: - '"shared"' - '"bucket": "example-artifacts"' - '"require_signature": true' - - '"/keys/release.pub"' + - '"trusted_keys":\s*\[\s*"[^"]*(?:/|\\\\)keys(?:/|\\\\)release\.pub"' exit_code: 0 From d07011eab9c3d4f98541f0df3af8f97245e4e48e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 18:57:44 -0600 Subject: [PATCH 08/32] fix(artifacts): carry static sentinels on stream and kind-data failures Reader, close, and cleanup failures now wrap ErrArtifactStreamFailed and non-empty kind-data decode failures wrap ErrArtifactRecordInvalid, keeping the underlying cause for errors.Is and errors.As. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 4 ++++ pkg/artifact/canonicaljson.go | 2 +- pkg/artifact/canonicaljson_test.go | 6 ++++++ pkg/artifact/contracts_test.go | 5 ++++- pkg/artifact/kinddata.go | 5 ++++- pkg/artifact/kinddata_test.go | 27 +++++++++++++++++++++++++++ pkg/artifact/prepared.go | 6 +++++- pkg/artifact/repository_types.go | 4 +++- 8 files changed, 54 insertions(+), 5 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 309b3ec3ea6..dcb294b94d5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -2115,6 +2115,10 @@ var ( // ErrArtifactTooLarge is returned when a record, document, or evidence blob exceeds its bounded-read limit. ErrArtifactTooLarge = errors.New("artifact data exceeds the size limit") + + // ErrArtifactStreamFailed is returned when reading or closing an artifact stream fails. The + // underlying I/O error is wrapped alongside it for diagnostics. + ErrArtifactStreamFailed = errors.New("artifact stream failed") ) // ExitCodeError is a typed error that preserves subcommand exit codes. diff --git a/pkg/artifact/canonicaljson.go b/pkg/artifact/canonicaljson.go index 822957be00c..72044dd930e 100644 --- a/pkg/artifact/canonicaljson.go +++ b/pkg/artifact/canonicaljson.go @@ -253,7 +253,7 @@ func ReadBounded(r io.Reader, limit int64) ([]byte, error) { } data, err := io.ReadAll(io.LimitReader(r, limit+1)) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: reading: %w", errUtils.ErrArtifactStreamFailed, err) } if int64(len(data)) > limit { return nil, fmt.Errorf("%w: more than %d bytes", errUtils.ErrArtifactTooLarge, limit) diff --git a/pkg/artifact/canonicaljson_test.go b/pkg/artifact/canonicaljson_test.go index 3f430a04478..c17dbf46062 100644 --- a/pkg/artifact/canonicaljson_test.go +++ b/pkg/artifact/canonicaljson_test.go @@ -84,6 +84,12 @@ func TestReadBounded(t *testing.T) { boom := errors.New("boom") _, err = ReadBounded(io.MultiReader(strings.NewReader("1"), errReader{boom}), 5) require.ErrorIs(t, err, boom) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "reader failures carry the artifact sentinel") + assert.NotErrorIs(t, err, errUtils.ErrArtifactTooLarge) + + _, _, err = ReadRecord(io.MultiReader(strings.NewReader("{"), errReader{boom})) + require.ErrorIs(t, err, boom) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "ReadRecord surfaces the sentinel") } type errReader struct{ err error } diff --git a/pkg/artifact/contracts_test.go b/pkg/artifact/contracts_test.go index 6d5006f1fce..8c6d8c4381d 100644 --- a/pkg/artifact/contracts_test.go +++ b/pkg/artifact/contracts_test.go @@ -74,6 +74,7 @@ func TestPreparedArtifactCloseRunsCleanupOnce(t *testing.T) { boom := errors.New("cleanup failed") prepared := &PreparedArtifact{Cleanup: func() error { calls++; return boom }} require.ErrorIs(t, prepared.Close(), boom) + require.ErrorIs(t, prepared.Close(), errUtils.ErrArtifactStreamFailed, "cleanup failures carry the artifact sentinel") require.ErrorIs(t, prepared.Close(), boom, "later calls return the first result") assert.Equal(t, 1, calls) @@ -96,7 +97,9 @@ func TestContentCloseClosesEveryStream(t *testing.T) { {Reader: nil}, {Reader: trackingCloser{Reader: strings.NewReader("b"), closed: &closed, err: errors.New("second")}}, }} - require.ErrorIs(t, content.Close(), first) + err := content.Close() + require.ErrorIs(t, err, first) + require.ErrorIs(t, err, errUtils.ErrArtifactStreamFailed, "close failures carry the artifact sentinel") assert.Equal(t, 2, closed, "an early failure does not leak later streams") } diff --git a/pkg/artifact/kinddata.go b/pkg/artifact/kinddata.go index 9ba4cd7080a..db2f1b1a363 100644 --- a/pkg/artifact/kinddata.go +++ b/pkg/artifact/kinddata.go @@ -44,7 +44,10 @@ func (k KindData) Decode(v any) error { if len(k) == 0 { return fmt.Errorf("%w: record has no kind data", errUtils.ErrArtifactRecordInvalid) } - return json.Unmarshal(k, v) + if err := json.Unmarshal(k, v); err != nil { + return fmt.Errorf("%w: decoding kind data: %w", errUtils.ErrArtifactRecordInvalid, err) + } + return nil } // MarshalJSON implements json.Marshaler, emitting the canonical bytes. diff --git a/pkg/artifact/kinddata_test.go b/pkg/artifact/kinddata_test.go index d983e7f5660..febd318823d 100644 --- a/pkg/artifact/kinddata_test.go +++ b/pkg/artifact/kinddata_test.go @@ -41,6 +41,33 @@ func TestKindDataJSONBehavior(t *testing.T) { require.ErrorIs(t, json.Unmarshal([]byte(`{"a":1.5}`), &data), errUtils.ErrArtifactRecordInvalid) }) + t.Run("decode failures carry the artifact sentinel", func(t *testing.T) { + type target struct { + Count int `json:"count"` + } + tests := []struct { + name string + data KindData + want int + }{ + {"string where an integer is required", KindData(`{"count":"x"}`), 0}, + {"array where an object is required", KindData(`[1]`), 0}, + {"matching data decodes", KindData(`{"count":7}`), 7}, + } + for _, tt := range tests { + var got target + err := tt.data.Decode(&got) + if tt.want == 0 { + require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid, tt.name) + var typeErr *json.UnmarshalTypeError + require.ErrorAs(t, err, &typeErr, "the cause is preserved") + continue + } + require.NoError(t, err, tt.name) + assert.Equal(t, tt.want, got.Count) + } + }) + t.Run("marshal failure surfaces as an invalid record", func(t *testing.T) { _, err := NewKindData(make(chan int)) require.ErrorIs(t, err, errUtils.ErrArtifactRecordInvalid) diff --git a/pkg/artifact/prepared.go b/pkg/artifact/prepared.go index f3d880a0172..741d5e30417 100644 --- a/pkg/artifact/prepared.go +++ b/pkg/artifact/prepared.go @@ -2,6 +2,7 @@ package artifact import ( "context" + "fmt" "io" "sync" "time" @@ -9,6 +10,7 @@ import ( "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" ) @@ -81,7 +83,9 @@ func (p *PreparedArtifact) Close() error { p.closeOnce.Do(func() { if p.Cleanup != nil { - p.closeErr = p.Cleanup() + if err := p.Cleanup(); err != nil { + p.closeErr = fmt.Errorf("%w: cleanup: %w", errUtils.ErrArtifactStreamFailed, err) + } } }) return p.closeErr diff --git a/pkg/artifact/repository_types.go b/pkg/artifact/repository_types.go index 9e92982da29..3fa92a29840 100644 --- a/pkg/artifact/repository_types.go +++ b/pkg/artifact/repository_types.go @@ -2,11 +2,13 @@ package artifact import ( "context" + "fmt" "io" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/perf" ) @@ -98,7 +100,7 @@ func (c *Content) Close() error { continue } if err := reader.Close(); err != nil && first == nil { - first = err + first = fmt.Errorf("%w: closing stream %d: %w", errUtils.ErrArtifactStreamFailed, index, err) } } return first From 3e73225fe1561b2725106c831263a4a588373110 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:00:09 -0600 Subject: [PATCH 09/32] fix(artifacts): never echo credentials from rejected URLs and identities Add a shared RedactURL helper that strips userinfo and replaces values it cannot prove credential-free (parse failures, scheme-less user:secret@host) with a fixed placeholder. Use it for catalog references, trusted issuers, and OCI registry hosts, and stop echoing identity values that look like credentials. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/catalog.go | 16 ++--------- pkg/artifact/config/validate.go | 4 +-- pkg/artifact/config/validate_test.go | 5 ++++ pkg/artifact/config/verification.go | 2 +- pkg/artifact/config/verification_test.go | 3 ++ pkg/artifact/options.go | 2 +- pkg/artifact/options_test.go | 3 ++ pkg/artifact/redact.go | 36 ++++++++++++++++++++++++ pkg/artifact/redact_test.go | 33 ++++++++++++++++++++++ 9 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 pkg/artifact/redact.go create mode 100644 pkg/artifact/redact_test.go diff --git a/pkg/artifact/config/catalog.go b/pkg/artifact/config/catalog.go index 1dc4773b921..ea29fb7f5a6 100644 --- a/pkg/artifact/config/catalog.go +++ b/pkg/artifact/config/catalog.go @@ -38,13 +38,13 @@ func validateCatalog(desc *artifact.KindDescriptor, repo *schema.ArtifactReposit func parseCatalogURI(raw string) (*url.URL, error) { parsed, err := url.Parse(raw) if err != nil || parsed.Scheme != "oci" || parsed.Host == "" { - return nil, fmt.Errorf("%w: catalog %q must be an oci:///: reference", errUtils.ErrArtifactConfigInvalid, sanitizedCatalog(raw)) + return nil, fmt.Errorf("%w: catalog %q must be an oci:///: reference", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw)) } if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { - return nil, fmt.Errorf("%w: catalog %q must not contain credentials, a query, or a fragment", errUtils.ErrArtifactConfigInvalid, sanitizedCatalog(raw)) + return nil, fmt.Errorf("%w: catalog %q must not contain credentials, a query, or a fragment", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw)) } if err := validateCatalogReference(strings.TrimPrefix(parsed.Path, "/")); err != nil { - return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, raw, err) + return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw), err) } return parsed, nil } @@ -68,13 +68,3 @@ func validateCatalogReference(reference string) error { } return artifact.ValidateVersion(tag) } - -// sanitizedCatalog hides userinfo so a mistaken credential is not echoed back in the error. -func sanitizedCatalog(raw string) string { - parsed, err := url.Parse(raw) - if err != nil || parsed.User == nil { - return raw - } - parsed.User = nil - return parsed.String() -} diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go index b76bb3ae160..83c76a5d275 100644 --- a/pkg/artifact/config/validate.go +++ b/pkg/artifact/config/validate.go @@ -149,9 +149,9 @@ func (v *Validator) validateIdentity(identity string) error { case len(identity) > maxIdentityLength: return fmt.Errorf("%w: identity exceeds %d characters", errUtils.ErrArtifactConfigInvalid, maxIdentityLength) case strings.IndexFunc(identity, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) >= 0: - return fmt.Errorf("%w: identity %q must be an auth identity name without whitespace", errUtils.ErrArtifactConfigInvalid, identity) + return fmt.Errorf("%w: identity must be an auth identity name without whitespace or control characters", errUtils.ErrArtifactConfigInvalid) case strings.Contains(identity, "://"): - return fmt.Errorf("%w: identity %q looks like a credential or URL; reference an Atmos auth identity name", errUtils.ErrArtifactConfigInvalid, identity) + return fmt.Errorf("%w: identity looks like a credential or URL; reference an Atmos auth identity name", errUtils.ErrArtifactConfigInvalid) case v.identityExists != nil && !v.identityExists(identity): return fmt.Errorf("%w: identity %q is not defined under auth.identities", errUtils.ErrArtifactConfigInvalid, identity) } diff --git a/pkg/artifact/config/validate_test.go b/pkg/artifact/config/validate_test.go index 167d254ae36..e59cebc1f58 100644 --- a/pkg/artifact/config/validate_test.go +++ b/pkg/artifact/config/validate_test.go @@ -61,6 +61,7 @@ func TestValidate_RepositoryBasics(t *testing.T) { {name: "credentials in options", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["access_key"] = "AKIA" }, wantErr: "auth identity"}, {name: "identity with whitespace", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "a b" }, wantErr: "without whitespace"}, {name: "identity that is a URL", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "https://u:p@host" }, wantErr: "credential or URL"}, + {name: "identity with whitespace and a secret", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "user:hunter2@host extra" }, wantErr: "without whitespace"}, {name: "permissions on a directory", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Permissions = "0600" }, wantErr: "does not accept permissions"}, {name: "packaging outside the matrix", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Packaging = "tgz" }, wantErr: "does not support packaging"}, } @@ -77,6 +78,8 @@ func TestValidate_RepositoryBasics(t *testing.T) { } require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "credential-looking values are never echoed") + assert.NotContains(t, err.Error(), "u:p@", "credential-looking values are never echoed") }) } } @@ -204,6 +207,8 @@ func TestValidate_CatalogReferences(t *testing.T) { {name: "bad tag", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog:-bad" }, wantErr: "tag syntax"}, {name: "bad digest", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/acme/catalog@sha256:xyz" }, wantErr: "digest"}, {name: "uppercase path", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://ghcr.io/Acme/catalog:stable" }, wantErr: "lowercase"}, + {name: "malformed escape with credentials", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/acme/%zz:stable" }, wantErr: "must be an oci://"}, + {name: "bad path with credentials is rejected before the path check", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Catalog = "oci://user:hunter2@ghcr.io/Acme/catalog:stable" }, wantErr: "credentials"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/artifact/config/verification.go b/pkg/artifact/config/verification.go index c3b21a43536..428afa4c7fa 100644 --- a/pkg/artifact/config/verification.go +++ b/pkg/artifact/config/verification.go @@ -42,7 +42,7 @@ func validateTrustedIdentity(trusted schema.ArtifactTrustedIdentity) error { } issuer, err := url.Parse(trusted.Issuer) if trusted.Issuer == "" || err != nil || issuer.Scheme != "https" || issuer.Host == "" || issuer.User != nil { - errs = append(errs, fmt.Errorf("%w: issuer %q must be an https URL without credentials", errUtils.ErrArtifactConfigInvalid, trusted.Issuer)) + errs = append(errs, fmt.Errorf("%w: issuer %q must be an https URL without credentials", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(trusted.Issuer))) } return errors.Join(errs...) } diff --git a/pkg/artifact/config/verification_test.go b/pkg/artifact/config/verification_test.go index 25422eefc17..8815f0c4571 100644 --- a/pkg/artifact/config/verification_test.go +++ b/pkg/artifact/config/verification_test.go @@ -39,6 +39,7 @@ func TestValidate_VerificationShape(t *testing.T) { {name: "issuer without https", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "http://issuer.example"}}}, wantErr: "https URL"}, {name: "missing issuer", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me"}}}, wantErr: "https URL"}, {name: "issuer with credentials", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "https://u:p@issuer.example"}}}, wantErr: "https URL"}, + {name: "issuer with credentials and a malformed escape", policy: &schema.ArtifactVerificationConfig{TrustedIdentities: []schema.ArtifactTrustedIdentity{{Identity: "me", Issuer: "https://u:hunter2@issuer.example/%zz"}}}, wantErr: "https URL"}, {name: "absolute key", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{filepath.Join(keyDir, "cosign.pub")}}}, {name: "empty key path", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{""}}, wantErr: "non-empty file path"}, {name: "relative key without a declaring file", policy: &schema.ArtifactVerificationConfig{TrustedKeys: []string{"cosign.pub"}}, wantErr: "no declaring configuration file"}, @@ -54,6 +55,8 @@ func TestValidate_VerificationShape(t *testing.T) { } require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "u:p@", "rejected issuer credentials are never echoed") + assert.NotContains(t, err.Error(), "hunter2", "rejected issuer credentials are never echoed") }) } } diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 1eb3f9eaa2e..7e27c6caf3a 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -189,7 +189,7 @@ func (o *OCIRegistryOptions) Validate() error { if o.Registry == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "registry")) } else if !validRegistryHost(o.Registry) { - errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, o.Registry)) + errs = append(errs, fmt.Errorf("%w: registry %q must be a host with an optional port, without a scheme, path, or credentials", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Registry))) } errs = append(errs, validatePrefix(o.Prefix)) if o.Prefix != "" { diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index a028dbe94b3..8c90ab2729d 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -77,6 +77,7 @@ func TestOptionsValidate(t *testing.T) { {name: "registry missing", options: &OCIRegistryOptions{}, wantErr: `"registry" is required`}, {name: "registry with scheme", options: &OCIRegistryOptions{Registry: "https://ghcr.io"}, wantErr: "without a scheme"}, {name: "registry with credentials", options: &OCIRegistryOptions{Registry: "user:pw@ghcr.io"}, wantErr: "without a scheme"}, + {name: "registry with credentials and a scheme", options: &OCIRegistryOptions{Registry: "https://user:hunter2@ghcr.io"}, wantErr: "without a scheme"}, {name: "registry uppercase prefix", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "Acme/Atmos"}, wantErr: "not a valid OCI repository path"}, {name: "registry prefix traversal", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/../x"}, wantErr: "prefix"}, } @@ -89,6 +90,8 @@ func TestOptionsValidate(t *testing.T) { } require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "hunter2", "rejected registry credentials are never echoed") + assert.NotContains(t, err.Error(), "user:pw", "rejected registry credentials are never echoed") }) } } diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go new file mode 100644 index 00000000000..50f9b6941d4 --- /dev/null +++ b/pkg/artifact/redact.go @@ -0,0 +1,36 @@ +package artifact + +import ( + "net/url" + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// RedactedValue replaces a value that must not be echoed because it may carry credentials. +const RedactedValue = "" + +// RedactURL returns raw with any userinfo removed, so that a mistakenly pasted credential is not +// repeated in an error message. Strings that cannot be proven free of userinfo are replaced by +// [RedactedValue] instead of being returned verbatim: a value that fails to parse, or that carries +// an "@" outside a URL authority (for example "user:secret@registry.example"), is never echoed. +func RedactURL(raw string) string { + defer perf.Track(nil, "artifact.RedactURL")() + + if !strings.Contains(raw, "@") { + return raw + } + parsed, err := url.Parse(raw) + if err != nil { + return RedactedValue + } + if parsed.User != nil { + parsed.User = nil + return parsed.String() + } + if parsed.Host == "" { + // An "@" without an authority is scheme-less userinfo such as "user:secret@host". + return RedactedValue + } + return raw +} diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go new file mode 100644 index 00000000000..4ee484163d1 --- /dev/null +++ b/pkg/artifact/redact_test.go @@ -0,0 +1,33 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRedactURL(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + {name: "empty", raw: "", want: ""}, + {name: "no userinfo", raw: "oci://ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, + {name: "digest reference keeps its at sign", raw: "oci://ghcr.io/acme/catalog@sha256:abc", want: "oci://ghcr.io/acme/catalog@sha256:abc"}, + {name: "user and password", raw: "oci://user:hunter2@ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, + {name: "token as user", raw: "https://hunter2@issuer.example", want: "https://issuer.example"}, + {name: "userinfo and digest", raw: "oci://user:hunter2@ghcr.io/acme/catalog@sha256:abc", want: "oci://ghcr.io/acme/catalog@sha256:abc"}, + {name: "malformed escape with userinfo", raw: "oci://user:hunter2@ghcr.io/acme/%zz:stable", want: RedactedValue}, + {name: "scheme-less userinfo", raw: "user:hunter2@ghcr.io", want: RedactedValue}, + {name: "scheme-less userinfo and port", raw: "user:hunter2@ghcr.io:5000", want: RedactedValue}, + {name: "malformed without an at sign is not credential bearing", raw: "oci://ghcr.io/%zz", want: "oci://ghcr.io/%zz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := RedactURL(tt.raw) + assert.Equal(t, tt.want, got) + assert.NotContains(t, got, "hunter2") + }) + } +} From 978c86c301c923ed278b0463a92c5585d30e1956 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:01:30 -0600 Subject: [PATCH 10/32] fix(artifacts): validate default packaging against the capability matrix An omitted packaging now has its effective default checked against the kind's supported packagings, and a descriptor whose default is outside its own matrix is rejected at registration. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/kinds.go | 9 ++++++-- pkg/artifact/kinds_test.go | 39 +++++++++++++++++++++++++++++++++++ pkg/artifact/registry.go | 3 +++ pkg/artifact/registry_test.go | 26 +++++++++++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/kinds.go b/pkg/artifact/kinds.go index 6525624529d..950d1bbad2c 100644 --- a/pkg/artifact/kinds.go +++ b/pkg/artifact/kinds.go @@ -170,11 +170,16 @@ func ValidateRepositoryConfig(desc *KindDescriptor, cfg *schema.ArtifactReposito // validatePackagingFor checks the packaging against the kind's matrix and returns the effective packaging. func validatePackagingFor(desc *KindDescriptor, configured string) (Packaging, error) { packaging := desc.EffectivePackaging(configured) - if configured == "" { + if configured == "" && packaging == "" { + // A descriptor without a default has nothing to check until the user chooses one. return packaging, nil } if !desc.Capabilities.HasPackaging(packaging) { - return packaging, fmt.Errorf("%w: %w: kind %q does not support packaging %q (supported: %v)", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, configured, desc.Capabilities.Packagings) + source := "packaging" + if configured == "" { + source = "default packaging" + } + return packaging, fmt.Errorf("%w: %w: kind %q does not support %s %q (supported: %v)", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, source, packaging, desc.Capabilities.Packagings) } return packaging, nil } diff --git a/pkg/artifact/kinds_test.go b/pkg/artifact/kinds_test.go index f4ee61c65c5..b503ffcd3e8 100644 --- a/pkg/artifact/kinds_test.go +++ b/pkg/artifact/kinds_test.go @@ -54,6 +54,45 @@ func TestValidateRepositoryConfig_PackagingMatrix(t *testing.T) { } } +// TestValidateRepositoryConfig_DefaultPackagingMustBeSupported covers a registered descriptor whose +// default packaging is outside its own capability matrix: an omitted packaging must be rejected +// exactly like the same value written explicitly. +func TestValidateRepositoryConfig_DefaultPackagingMustBeSupported(t *testing.T) { + builtin, ok := BuiltinKind(RepositoryKindLocalArchive) + require.True(t, ok) + zipOnly := *builtin + zipOnly.Capabilities.Packagings = []Packaging{PackagingZip} + zipOnly.DefaultPackaging = PackagingTgz + + noDefault := zipOnly + noDefault.DefaultPackaging = "" + + tests := []struct { + name string + desc KindDescriptor + packaging string + valid bool + }{ + {name: "default outside the matrix is rejected", desc: zipOnly, packaging: ""}, + {name: "explicit value outside the matrix is rejected", desc: zipOnly, packaging: "tgz"}, + {name: "explicit supported value overrides a bad default", desc: zipOnly, packaging: "zip", valid: true}, + {name: "descriptor without a default accepts an omitted packaging", desc: noDefault, packaging: "", valid: true}, + {name: "built-in default is supported", desc: *builtin, packaging: "", valid: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &schema.ArtifactRepositoryConfig{Kind: tt.desc.Kind, Packaging: tt.packaging, Options: validOptionsFor(tt.desc.Kind)} + err := ValidateRepositoryConfig(&tt.desc, cfg) + if tt.valid { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + } +} + func validOptionsFor(kind string) map[string]any { switch kind { case RepositoryKindS3: diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go index b844e59e5d7..a27b61876c4 100644 --- a/pkg/artifact/registry.go +++ b/pkg/artifact/registry.go @@ -97,6 +97,9 @@ func (r *RepositoryRegistry) Register(desc *KindDescriptor, factory RepositoryFa if factory == nil { return fmt.Errorf("%w: factory for repository kind %q must not be nil", errUtils.ErrArtifactConfigInvalid, desc.Kind) } + if desc.DefaultPackaging != "" && !desc.Capabilities.HasPackaging(desc.DefaultPackaging) { + return fmt.Errorf("%w: %w: repository kind %q default packaging %q is not in its supported packagings %v", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactPackagingInvalid, desc.Kind, desc.DefaultPackaging, desc.Capabilities.Packagings) + } r.mu.Lock() defer r.mu.Unlock() if _, exists := r.kinds[desc.Kind]; exists { diff --git a/pkg/artifact/registry_test.go b/pkg/artifact/registry_test.go index d1919095922..c6ae19d232b 100644 --- a/pkg/artifact/registry_test.go +++ b/pkg/artifact/registry_test.go @@ -87,6 +87,32 @@ func TestRepositoryRegistry_LookupFallsBackToBuiltinAndPrefersRegistered(t *test assert.True(t, registry.IsRegistered(RepositoryKindS3)) } +func TestRepositoryRegistry_RegisterRejectsDefaultPackagingOutsideTheMatrix(t *testing.T) { + tests := []struct { + name string + desc KindDescriptor + want bool + }{ + {name: "default outside the matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingTgz, Capabilities: Capabilities{Packagings: []Packaging{PackagingZip}}}}, + {name: "default with an empty matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingTgz}}, + {name: "default inside the matrix", desc: KindDescriptor{Kind: "gcp/gcs", DefaultPackaging: PackagingZip, Capabilities: Capabilities{Packagings: []Packaging{PackagingZip}}}, want: true}, + {name: "no default", desc: KindDescriptor{Kind: "gcp/gcs"}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry := NewRepositoryRegistry() + err := registry.Register(&tt.desc, (&fakeKind{}).factory) + if tt.want { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + assert.False(t, registry.IsRegistered("gcp/gcs"), "a rejected descriptor is not registered") + }) + } +} + func TestRepositoryRegistry_KindsListsBuiltinAndRegisteredSorted(t *testing.T) { registry := NewRepositoryRegistry() require.NoError(t, registry.Register(&KindDescriptor{Kind: "gcp/gcs"}, (&fakeKind{}).factory)) From aa895265f9266c0272fe97b7f9f252552f36790c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:01:51 -0600 Subject: [PATCH 11/32] fix(artifacts): measure S3 tag and identity limits in characters S3 limits tag keys to 128 and values to 256 Unicode characters, so multibyte values are no longer rejected by a UTF-8 byte count. The identity length check, which already reported characters, is measured the same way. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/validate.go | 3 ++- pkg/artifact/config/validate_test.go | 2 ++ pkg/artifact/options.go | 5 +++-- pkg/artifact/options_test.go | 7 +++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go index 83c76a5d275..cf5ebeb9549 100644 --- a/pkg/artifact/config/validate.go +++ b/pkg/artifact/config/validate.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "unicode" + "unicode/utf8" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -146,7 +147,7 @@ func (v *Validator) validateIdentity(identity string) error { return nil } switch { - case len(identity) > maxIdentityLength: + case utf8.RuneCountInString(identity) > maxIdentityLength: return fmt.Errorf("%w: identity exceeds %d characters", errUtils.ErrArtifactConfigInvalid, maxIdentityLength) case strings.IndexFunc(identity, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) >= 0: return fmt.Errorf("%w: identity must be an auth identity name without whitespace or control characters", errUtils.ErrArtifactConfigInvalid) diff --git a/pkg/artifact/config/validate_test.go b/pkg/artifact/config/validate_test.go index e59cebc1f58..aee0505e74b 100644 --- a/pkg/artifact/config/validate_test.go +++ b/pkg/artifact/config/validate_test.go @@ -61,6 +61,8 @@ func TestValidate_RepositoryBasics(t *testing.T) { {name: "credentials in options", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Options["access_key"] = "AKIA" }, wantErr: "auth identity"}, {name: "identity with whitespace", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "a b" }, wantErr: "without whitespace"}, {name: "identity that is a URL", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "https://u:p@host" }, wantErr: "credential or URL"}, + {name: "identity at the character limit", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = strings.Repeat("é", 256) }}, + {name: "identity over the character limit", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = strings.Repeat("é", 257) }, wantErr: "exceeds 256 characters"}, {name: "identity with whitespace and a secret", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Identity = "user:hunter2@host extra" }, wantErr: "without whitespace"}, {name: "permissions on a directory", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Permissions = "0600" }, wantErr: "does not accept permissions"}, {name: "packaging outside the matrix", alias: "x", mutate: func(r *schema.ArtifactRepositoryConfig) { r.Packaging = "tgz" }, wantErr: "does not support packaging"}, diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 7e27c6caf3a..13f408b2dc3 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "unicode/utf8" "github.com/go-viper/mapstructure/v2" @@ -402,11 +403,11 @@ func validateS3Tags(tags map[string]string) error { var errs []error for _, key := range keys { switch { - case key == "" || len(key) > maxS3TagKeyLen: + case key == "" || utf8.RuneCountInString(key) > maxS3TagKeyLen: errs = append(errs, fmt.Errorf("%w: S3 tag key %q must be 1-%d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagKeyLen)) case strings.HasPrefix(strings.ToLower(key), reservedTagHead): errs = append(errs, fmt.Errorf("%w: S3 tag key %q uses the reserved %q prefix", errUtils.ErrArtifactConfigInvalid, key, reservedTagHead)) - case len(tags[key]) > maxS3TagValLen: + case utf8.RuneCountInString(tags[key]) > maxS3TagValLen: errs = append(errs, fmt.Errorf("%w: S3 tag %q value exceeds %d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagValLen)) } } diff --git a/pkg/artifact/options_test.go b/pkg/artifact/options_test.go index 8c90ab2729d..7225967d632 100644 --- a/pkg/artifact/options_test.go +++ b/pkg/artifact/options_test.go @@ -68,6 +68,13 @@ func TestOptionsValidate(t *testing.T) { {name: "s3 reserved prefix", options: &S3Options{Bucket: "acme-artifacts", Prefix: "atmos/.atmos-artifacts"}, wantErr: "reserved"}, {name: "s3 reserved tag prefix", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"aws:foo": "x"}}, wantErr: "reserved"}, {name: "s3 oversized tag value", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("x", 257)}}, wantErr: "value exceeds"}, + {name: "s3 tag value at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("x", 256)}}}, + {name: "s3 multibyte tag value at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("é", 256)}}}, + {name: "s3 multibyte tag value over the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"k": strings.Repeat("é", 257)}}, wantErr: "value exceeds"}, + {name: "s3 tag key at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("k", 128): "v"}}}, + {name: "s3 multibyte tag key at the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("é", 128): "v"}}}, + {name: "s3 multibyte tag key over the character limit", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{strings.Repeat("é", 129): "v"}}, wantErr: "must be 1-128 characters"}, + {name: "s3 empty tag key", options: &S3Options{Bucket: "acme-artifacts", Tags: map[string]string{"": "v"}}, wantErr: "must be 1-128 characters"}, {name: "s3 too many tags", options: &S3Options{Bucket: "acme-artifacts", Tags: elevenTags()}, wantErr: "exceed the S3 limit"}, {name: "s3 bad key pattern", options: &S3Options{Bucket: "acme-artifacts", KeyPattern: "../x"}, wantErr: "segment"}, {name: "registry ok", options: &OCIRegistryOptions{Registry: "ghcr.io", Prefix: "acme/atmos"}}, From 515cd7446ec7f70afd899ef8287aedc6a834b8d8 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:07:37 -0600 Subject: [PATCH 12/32] fix(artifacts): resolve symlink targets through in-package links A target such as dir/up/../outside cleaned to a contained path even when dir/up pointed at the package root. Inventory validation now walks every target component by component, following in-package links, and rejects an escape or a link loop before the content digest is computed. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/entry.go | 7 ++-- pkg/artifact/entry_symlink.go | 79 +++++++++++++++++++++++++++++++++++ pkg/artifact/record_test.go | 67 +++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 pkg/artifact/entry_symlink.go diff --git a/pkg/artifact/entry.go b/pkg/artifact/entry.go index dff6d0ff251..31917c69c2a 100644 --- a/pkg/artifact/entry.go +++ b/pkg/artifact/entry.go @@ -126,8 +126,9 @@ func (e *Entry) validateSymlink() error { return nil } -// validateInventory checks every entry, rejects duplicate paths, and rejects entries nested under -// a file or symlink entry. +// validateInventory checks every entry, rejects duplicate paths, rejects entries nested under +// a file or symlink entry, and resolves every symlink target through the in-package links so a +// target cannot escape the package or loop. func validateInventory(inventory []Entry) error { types := make(map[string]EntryType, len(inventory)) for _, entry := range inventory { @@ -146,7 +147,7 @@ func validateInventory(inventory []Entry) error { } } } - return nil + return validateSymlinkTargets(inventory) } // Validate checks one document descriptor in isolation. diff --git a/pkg/artifact/entry_symlink.go b/pkg/artifact/entry_symlink.go new file mode 100644 index 00000000000..262fdb108f0 --- /dev/null +++ b/pkg/artifact/entry_symlink.go @@ -0,0 +1,79 @@ +package artifact + +import ( + "fmt" + "path" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// maxSymlinkExpansions bounds how many in-package links one target may traverse. It matches the +// limit that common operating systems apply before reporting a symlink loop. +const maxSymlinkExpansions = 40 + +// validateSymlinkTargets resolves every symlink target against the inventory, so a target that is +// lexically contained but passes through an in-package link cannot leave the package. For example +// "link -> dir/up/../outside" cleans to "dir/outside", yet when "dir/up -> .." is also in the +// inventory, the filesystem follows "dir/up" to the package root before applying "..". +// +// Resolution follows the way a filesystem walks a path: component by component, replacing a link +// with its own target, and rejecting a ".." that would pop above the package root. Paths that name +// nothing in the inventory resolve literally, so dangling targets stay valid. The expansion limit +// rejects link cycles of any shape. +func validateSymlinkTargets(inventory []Entry) error { + links := make(map[string]string) + for _, entry := range inventory { + if entry.Type == EntrySymlink { + links[entry.Path] = entry.Target + } + } + if len(links) == 0 { + return nil + } + for index := range inventory { + if inventory[index].Type != EntrySymlink { + continue + } + if err := resolveSymlink(links, &inventory[index]); err != nil { + return err + } + } + return nil +} + +// resolveSymlink walks the target of one link through the in-package links. +func resolveSymlink(links map[string]string, link *Entry) error { + var resolved []string + if dir := path.Dir(link.Path); dir != "." { + resolved = strings.Split(dir, "/") + } + pending := strings.Split(link.Target, "/") + for expansions := 0; len(pending) > 0; { + component := pending[0] + pending = pending[1:] + switch component { + case "", ".": + continue + case "..": + if len(resolved) == 0 { + return fmt.Errorf("%w: symlink %q target %q escapes the package", errUtils.ErrArtifactUnsafePath, link.Path, link.Target) + } + resolved = resolved[:len(resolved)-1] + continue + } + candidate := strings.Join(append(resolved[:len(resolved):len(resolved)], component), "/") + target, isLink := links[candidate] + if !isLink { + resolved = append(resolved, component) + continue + } + expansions++ + if expansions > maxSymlinkExpansions { + return fmt.Errorf("%w: symlink %q target %q loops or traverses more than %d links", errUtils.ErrArtifactUnsafePath, link.Path, link.Target, maxSymlinkExpansions) + } + // A link is replaced by its target, resolved from the directory that holds the link. + pending = append(strings.Split(target, "/"), pending...) + } + return nil +} diff --git a/pkg/artifact/record_test.go b/pkg/artifact/record_test.go index f70b554d09c..93100003def 100644 --- a/pkg/artifact/record_test.go +++ b/pkg/artifact/record_test.go @@ -58,6 +58,73 @@ func TestEntryValidate(t *testing.T) { } } +// TestValidateInventory_SymlinkResolution resolves every link target through the in-package links, +// component by component, so a ".." after a symlinked directory cannot leave the package. +func TestValidateInventory_SymlinkResolution(t *testing.T) { + file := func(p string) Entry { return Entry{Path: p, Type: EntryFile, Digest: digest.FromString(p)} } + dir := func(p string) Entry { return Entry{Path: p, Type: EntryDirectory} } + link := func(p, target string) Entry { return Entry{Path: p, Type: EntrySymlink, Target: target} } + + tests := []struct { + name string + inventory []Entry + wantErr string + }{ + {name: "no symlinks", inventory: []Entry{file("a"), dir("d"), file("d/b")}}, + {name: "dangling target is allowed", inventory: []Entry{link("l", "missing/file")}}, + {name: "chain of links to a file", inventory: []Entry{file("lib.so.1.2.3"), link("lib.so.1", "lib.so.1.2.3"), link("lib.so", "lib.so.1")}}, + {name: "link to a directory link", inventory: []Entry{dir("real"), file("real/x"), link("alias", "real"), link("via", "alias/x")}}, + {name: "dotdot after a symlinked directory stays inside", inventory: []Entry{dir("dir"), link("dir/up", ".."), link("l", "dir/up/inside")}}, + {name: "dotdot through a link that resolves deeper stays inside", inventory: []Entry{dir("a"), dir("a/b"), link("deep", "a/b"), link("l", "deep/../x")}}, + {name: "link to its own directory", inventory: []Entry{dir("d"), link("d/l", ".")}}, + { + name: "dotdot after a link to the parent escapes", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("link", "dir/up/../outside")}, + wantErr: "escapes the package", + }, + { + name: "escape is independent of entry order", + inventory: []Entry{link("link", "dir/up/../outside"), link("dir/up", ".."), dir("dir")}, + wantErr: "escapes the package", + }, + { + name: "escape through a chain of two links", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("l1", "dir/up"), link("l2", "l1/../out")}, + wantErr: "escapes the package", + }, + { + name: "escape from a deep link through a link to the parent", + inventory: []Entry{dir("a"), dir("a/b"), link("a/b/up", "../.."), link("a/b/l", "up/../out")}, + wantErr: "escapes the package", + }, + { + name: "link resolving to a link that escapes", + inventory: []Entry{dir("dir"), link("dir/up", ".."), link("a/l", "../dir/up/../x")}, + wantErr: "escapes the package", + }, + {name: "absolute target", inventory: []Entry{link("l", "/etc/passwd")}, wantErr: "relative"}, + {name: "lexical escape", inventory: []Entry{link("l", "../x")}, wantErr: "escapes the package"}, + {name: "self loop", inventory: []Entry{link("l", "l")}, wantErr: "loop"}, + {name: "two link loop", inventory: []Entry{link("a", "b"), link("b", "a")}, wantErr: "loop"}, + {name: "loop through a directory component", inventory: []Entry{dir("d"), link("d/l", "../d/l")}, wantErr: "loop"}, + {name: "loop reached through a longer path", inventory: []Entry{link("a", "b/x"), link("b", "a")}, wantErr: "loop"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateInventory(tt.inventory) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) + assert.Contains(t, err.Error(), tt.wantErr) + + _, digestErr := ComputeContentDigest(tt.inventory, nil) + require.ErrorIs(t, digestErr, errUtils.ErrArtifactUnsafePath, "the digest admission gate applies the same check") + }) + } +} + func TestValidateInventoryStructure(t *testing.T) { file := func(p string) Entry { return Entry{Path: p, Type: EntryFile, Digest: digest.FromString(p)} } dir := func(p string) Entry { return Entry{Path: p, Type: EntryDirectory} } From 8f3c0585e5629316d32b377cc3620eaa869a6374 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:12:56 -0600 Subject: [PATCH 13/32] fix(config): evaluate YAML aliases and merge keys and keep include origins Aliases of anchored nodes now reuse the anchor's single evaluation (a tagged !env or !exec behind an anchor runs once), with cycle protection and a deep copy per alias. Merge keys are expanded with YAML precedence and the origins of merged settings are recorded where the merged repository sees them. A value produced by an include of a sequence or scalar, such as trusted_keys, now keeps the included file's origin and remote flag instead of the including file's. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/artifacts_decode_test.go | 258 ++++++++++++++++++++++++ pkg/config/process_yaml_anchor.go | 183 +++++++++++++++++ pkg/config/process_yaml_include_eval.go | 8 + pkg/config/yaml_decode.go | 46 ++++- 4 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 pkg/config/artifacts_decode_test.go create mode 100644 pkg/config/process_yaml_anchor.go diff --git a/pkg/config/artifacts_decode_test.go b/pkg/config/artifacts_decode_test.go new file mode 100644 index 00000000000..244e77813e0 --- /dev/null +++ b/pkg/config/artifacts_decode_test.go @@ -0,0 +1,258 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" +) + +func TestLoadConfig_ArtifactsTaggedAliasIsEvaluated(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + t.Setenv("ARTIFACT_STORE", store) + dir := artifactsTestDir(t, `artifacts: + repositories: + first: + kind: local/dir + options: + path: &store !env ARTIFACT_STORE + second: + kind: local/dir + options: + path: *store +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + assert.Equal(t, store, cfg.Artifacts.Repositories["first"].Options["path"]) + assert.Equal(t, store, cfg.Artifacts.Repositories["second"].Options["path"], "the alias sees the evaluated tag, not the literal variable name") + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsAnchoredExecEvaluatedOnceAcrossAliases(t *testing.T) { + dir := t.TempDir() + countPath := filepath.Join(dir, "count") + t.Setenv(mergeConfigFileExecHelperEnv, "1") + root := artifactsTestDir(t, fmt.Sprintf(`artifacts: + repositories: + first: + kind: local/dir + options: + path: &store !exec >- + %s -test.run=^TestMergeConfigFileExecHelper$ -- %s + second: + kind: local/dir + options: + path: *store +`, shellQuote(os.Args[0]), shellQuote(countPath))) + cfg, err := loadArtifactsDir(t, root) + require.NoError(t, err) + count, err := os.ReadFile(countPath) + require.NoError(t, err) + assert.Equal(t, "1", string(count), "an alias reuses the anchor's result instead of running the command again") + want := filepath.Join(root, "resolved-description") + assert.Equal(t, want, cfg.Artifacts.Repositories["first"].Options["path"]) + assert.Equal(t, want, cfg.Artifacts.Repositories["second"].Options["path"]) +} + +func TestLoadConfig_ArtifactsAliasesAndMergeKeysKeepDeclaringFile(t *testing.T) { + dir := artifactsTestDir(t, "import:\n - ./trust/artifacts.yaml\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "artifacts.yaml"), `artifacts: + repositories: + base: &base + kind: local/dir + options: + path: store + verification: + trusted_keys: &keys [release.pub] + aliased: *base + inherited: + <<: *base + overridden: + <<: *base + options: + path: other + keyed: + kind: local/dir + options: + path: keyed + verification: + trusted_keys: *keys +`) + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + + trust := filepath.Join(dir, "trust") + repos := cfg.Artifacts.Repositories + for _, alias := range []string{"base", "aliased", "inherited"} { + assert.Equal(t, "local/dir", repos[alias].Kind, alias) + assert.Equal(t, filepath.Join(trust, "store"), repos[alias].Options["path"], alias) + require.NotNil(t, repos[alias].Verification, alias) + assert.Equal(t, []string{filepath.Join(trust, "release.pub")}, repos[alias].Verification.TrustedKeys, alias) + } + assert.Equal(t, "local/dir", repos["overridden"].Kind, "kind is inherited through the merge key") + assert.Equal(t, filepath.Join(trust, "other"), repos["overridden"].Options["path"], "the repository's own options win over the merged ones") + assert.Equal(t, []string{filepath.Join(trust, "release.pub")}, repos["keyed"].Verification.TrustedKeys) + require.NoError(t, artifactconfig.Validate(&cfg.Artifacts)) +} + +func TestLoadConfig_ArtifactsTrustedKeysIncludeAnchorsToIncludedFile(t *testing.T) { + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, fmt.Sprintf(`artifacts: + verification: + trusted_keys: !include ./trust/keys.yaml%s + repositories: + local: + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./trust/repo-keys.yaml%s +`, suffix, suffix)) + writeArtifactsFile(t, filepath.Join(dir, "trust", "keys.yaml"), "- release.pub\n") + writeArtifactsFile(t, filepath.Join(dir, "trust", "repo-keys.yaml"), "- repo.pub\n") + // A same-named file beside the including config must not be the one trusted. + writeArtifactsFile(t, filepath.Join(dir, "release.pub"), "wrong\n") + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{filepath.Join(dir, "trust", "release.pub")}, cfg.Artifacts.Verification.TrustedKeys) + assert.Equal(t, filepath.Join(dir, "trust"), cfg.Artifacts.Verification.KeysOrigin.Dir) + assert.False(t, cfg.Artifacts.Verification.KeysOrigin.Remote) + + repo := cfg.Artifacts.Repositories["local"] + require.NotNil(t, repo.Verification) + assert.Equal(t, []string{filepath.Join(dir, "trust", "repo.pub")}, repo.Verification.TrustedKeys) + assert.Equal(t, filepath.Join(dir, "trust"), repo.Verification.KeysOrigin.Dir) + assert.Equal(t, filepath.Join(dir, "store"), repo.Options["path"], "sibling settings keep the including file") + }) + } +} + +func TestLoadConfig_ArtifactsRemoteTrustedKeysIncludeIsNotAnchored(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, "- release.pub\n") + })) + defer server.Close() + for _, suffix := range []string{"", " | eval"} { + t.Run(fmt.Sprintf("eval=%t", suffix != ""), func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n verification:\n trusted_keys: !include "+server.URL+"/keys.yaml"+suffix+"\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, []string{"release.pub"}, cfg.Artifacts.Verification.TrustedKeys, "a remote relative key is never anchored to a local directory") + assert.True(t, cfg.Artifacts.Verification.KeysOrigin.Remote) + + err = artifactconfig.Validate(&cfg.Artifacts) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "remote import") + }) + } +} + +func decodeYAMLForTest(t *testing.T, source string) (any, error) { + t.Helper() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &node)) + return decodeNodeWithYamlFunctions(&node) +} + +func TestDecodeNodeWithYamlFunctions_MergeKeys(t *testing.T) { + tests := []struct { + name string + source string + want map[string]any + wantErr bool + }{ + { + name: "alias of a mapping", + source: "base: &base {a: 1, b: 2}\nchild:\n <<: *base\n b: 3\n c: 4\n", + want: map[string]any{ + "base": map[string]any{"a": 1, "b": 2}, + "child": map[string]any{"a": 1, "b": 3, "c": 4}, + }, + }, + { + name: "own keys win regardless of order", + source: "base: &base {a: 1}\nchild:\n a: 9\n <<: *base\n", + want: map[string]any{"base": map[string]any{"a": 1}, "child": map[string]any{"a": 9}}, + }, + { + name: "inline mapping", + source: "child:\n <<: {a: 1}\n b: 2\n", + want: map[string]any{"child": map[string]any{"a": 1, "b": 2}}, + }, + { + name: "sequence of aliases: earlier sources win", + source: "x: &x {a: 1, b: 1}\ny: &y {b: 2, c: 2}\nchild:\n <<: [*x, *y]\n", + want: map[string]any{ + "x": map[string]any{"a": 1, "b": 1}, + "y": map[string]any{"b": 2, "c": 2}, + "child": map[string]any{"a": 1, "b": 1, "c": 2}, + }, + }, + {name: "scalar merge value", source: "child:\n <<: 5\n", wantErr: true}, + {name: "sequence with a scalar", source: "x: &x {a: 1}\nchild:\n <<: [*x, 5]\n", wantErr: true}, + { + name: "quoted merge key is an ordinary key", + source: "child:\n \"<<\": 1\n", + want: map[string]any{"child": map[string]any{"<<": 1}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeYAMLForTest(t, tt.source) + if tt.wantErr { + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestDecodeNodeWithYamlFunctions_AliasPolicy(t *testing.T) { + t.Run("an alias is a copy: mutating it leaves the anchor intact", func(t *testing.T) { + got, err := decodeYAMLForTest(t, "base: &base {list: [1, 2]}\nref: *base\n") + require.NoError(t, err) + decoded, ok := got.(map[string]any) + require.True(t, ok) + ref, ok := decoded["ref"].(map[string]any) + require.True(t, ok) + ref["list"].([]any)[0] = 99 + ref["extra"] = true + assert.Equal(t, map[string]any{"list": []any{1, 2}}, decoded["base"], "result to source isolation") + + base, ok := decoded["base"].(map[string]any) + require.True(t, ok) + base["list"].([]any)[1] = 77 + assert.Equal(t, []any{99, 2}, ref["list"], "source to result isolation") + }) + + t.Run("an alias to itself is a cycle, not a hang", func(t *testing.T) { + _, err := decodeYAMLForTest(t, "loop: &loop [*loop]\n") + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.Contains(t, err.Error(), "cycle") + }) + + t.Run("an anchor outside the decoded subtree is evaluated on demand", func(t *testing.T) { + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("other: &shared {a: 1}\nsection:\n ref: *shared\n"), &document)) + root := document.Content[0] + section := root.Content[3] + got, err := decodeNodeWithYamlFunctions(section) + require.NoError(t, err) + assert.Equal(t, map[string]any{"ref": map[string]any{"a": 1}}, got) + }) +} diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go new file mode 100644 index 00000000000..06c98e1e3fb --- /dev/null +++ b/pkg/config/process_yaml_anchor.go @@ -0,0 +1,183 @@ +package config + +import ( + "fmt" + + "go.yaml.in/yaml/v3" +) + +// mergeKeyTag is the resolved tag of the YAML merge key "<<". +const mergeKeyTag = "!!merge" + +// includeOrigin is the file an include (or an alias of an anchored include) produced its value from. +type includeOrigin struct { + file string + remote bool +} + +// originRecord is one origin notification, with its path relative to the node that produced it. +type originRecord struct { + rel []string + file string + remote bool +} + +// anchorEntry is the one evaluation of an anchored node. Aliases and merge keys that refer to the +// anchor reuse it instead of evaluating the node's function tags again: an `!exec`, `!env`, or +// `!random` behind an anchor runs once, at the anchor, and every alias sees that result. +type anchorEntry struct { + value any + origins []originRecord + include *includeOrigin +} + +// decodeAnchored evaluates an anchored node once and caches the result for its aliases. +func (d *configYamlDecoder) decodeAnchored(node *yaml.Node, sourceFile string) (any, error) { + entry, err := d.evaluateAnchor(node, sourceFile) + if err != nil { + return nil, err + } + d.nodeOrigin = entry.include + return entry.value, nil +} + +// decodeAlias returns a copy of the anchor's evaluated value. The anchor is evaluated on first use +// when its definition was not visited yet, for example when it sits outside the decoded subtree. +// The origins the anchor recorded are replayed below the alias's own path, so a path-valued setting +// reached through an alias keeps the file that declared it. +func (d *configYamlDecoder) decodeAlias(node *yaml.Node, sourceFile string) (any, error) { + if node.Alias == nil { + return nil, fmt.Errorf("%w: alias %q has no anchor", ErrExecuteYamlFunctions, node.Value) + } + entry, err := d.evaluateAnchor(node.Alias, sourceFile) + if err != nil { + return nil, err + } + d.replayOrigins(entry.origins) + d.nodeOrigin = entry.include + return cloneDecodedValue(entry.value), nil +} + +// evaluateAnchor evaluates an anchored node at most once per decoder. An alias that refers back to +// its own anchor is a cycle and is rejected. +func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) (*anchorEntry, error) { + if entry, ok := d.anchors[node]; ok { + return entry, nil + } + if d.evaluating[node] { + return nil, fmt.Errorf("%w: YAML alias cycle at anchor %q", ErrExecuteYamlFunctions, node.Anchor) + } + if d.anchors == nil { + d.anchors = make(map[*yaml.Node]*anchorEntry) + d.evaluating = make(map[*yaml.Node]bool) + } + d.evaluating[node] = true + defer delete(d.evaluating, node) + + var records []originRecord + previous := d.origin + if previous != nil { + base := len(d.path) + d.origin = func(path []string, file string, remote bool) { + if len(path) > base { + records = append(records, originRecord{rel: append([]string(nil), path[base:]...), file: file, remote: remote}) + } + previous(path, file, remote) + } + defer func() { d.origin = previous }() + } + d.nodeOrigin = nil + value, err := d.decodeNode(node, sourceFile) + if err != nil { + return nil, err + } + entry := &anchorEntry{value: value, origins: records} + if node.Kind == yaml.ScalarNode && isIncludeTag(node.Tag) { + entry.include = d.nodeOrigin + } + d.nodeOrigin = nil + d.anchors[node] = entry + return entry, nil +} + +// replayOrigins reports the origins an anchor recorded, relative to the current path. +func (d *configYamlDecoder) replayOrigins(records []originRecord) { + if d.origin == nil { + return + } + for _, record := range records { + path := append(append([]string(nil), d.path...), record.rel...) + d.origin(path, record.file, record.remote) + } +} + +// isMergeKey reports whether a mapping key is the YAML merge key "<<". +func isMergeKey(key *yaml.Node) bool { + return key.Kind == yaml.ScalarNode && key.Tag == mergeKeyTag +} + +// expandMergeKeys copies the entries named by every "<<" key of the mapping into result. Entries +// the mapping declares itself win over merged ones, and earlier merge sources win over later ones, +// as the YAML merge-key rules require. The merge value is decoded at the mapping's own path, so the +// origins of merged settings are recorded where the merged repository sees them. +func (d *configYamlDecoder) expandMergeKeys(node *yaml.Node, result map[string]interface{}, sourceFile string) error { + for i := 0; i < len(node.Content); i += 2 { + if !isMergeKey(node.Content[i]) { + continue + } + sources, err := d.mergeSources(node.Content[i+1], sourceFile) + if err != nil { + return err + } + for _, source := range sources { + for key, value := range source { + if _, exists := result[key]; !exists { + result[key] = value + } + } + } + } + return nil +} + +// mergeSources decodes the value of a merge key: a mapping, an alias of one, or a sequence of them. +func (d *configYamlDecoder) mergeSources(value *yaml.Node, sourceFile string) ([]map[string]interface{}, error) { + nodes := []*yaml.Node{value} + if value.Kind == yaml.SequenceNode { + nodes = value.Content + } + sources := make([]map[string]interface{}, 0, len(nodes)) + for _, child := range nodes { + decoded, err := d.decode(child, sourceFile) + if err != nil { + return nil, err + } + source, ok := decoded.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("%w: merge key %q requires a mapping or a sequence of mappings, got %T", ErrExecuteYamlFunctions, "<<", decoded) + } + sources = append(sources, source) + } + return sources, nil +} + +// cloneDecodedValue deep-copies the mappings and sequences of a decoded value, so an alias never +// shares mutable structure with its anchor. +func cloneDecodedValue(value any) any { + switch typed := value.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(typed)) + for key, child := range typed { + out[key] = cloneDecodedValue(child) + } + return out + case []interface{}: + out := make([]interface{}, len(typed)) + for index, child := range typed { + out[index] = cloneDecodedValue(child) + } + return out + default: + return value + } +} diff --git a/pkg/config/process_yaml_include_eval.go b/pkg/config/process_yaml_include_eval.go index 0ebff5902f8..04693b45087 100644 --- a/pkg/config/process_yaml_include_eval.go +++ b/pkg/config/process_yaml_include_eval.go @@ -16,6 +16,14 @@ type configYamlDecoder struct { path []string origin func([]string, string, bool) remoteSource string + + // anchors holds each anchored node's single evaluation; evaluating guards against alias cycles. + anchors map[*yaml.Node]*anchorEntry + evaluating map[*yaml.Node]bool + + // nodeOrigin is the file the include or alias just decoded took its value from. The mapping + // entry that owns the value reads it so the setting's origin is the included file. + nodeOrigin *includeOrigin } // evaluateInclude resolves an included subtree using its file context and diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index a59c33787f3..3074474a2b1 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -35,7 +35,17 @@ func (d *configYamlDecoder) decode(node *yaml.Node, sourceFile string) (any, err if node == nil { return nil, nil } + if node.Kind == yaml.AliasNode { + return d.decodeAlias(node, sourceFile) + } + if node.Anchor != "" { + return d.decodeAnchored(node, sourceFile) + } + return d.decodeNode(node, sourceFile) +} +// decodeNode evaluates one node by kind, without consulting the anchor cache. +func (d *configYamlDecoder) decodeNode(node *yaml.Node, sourceFile string) (any, error) { switch node.Kind { case yaml.DocumentNode: if len(node.Content) == 0 { @@ -59,21 +69,25 @@ func (d *configYamlDecoder) decodeMappingNodeWithYamlFunctions(node *yaml.Node, for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] + if isMergeKey(keyNode) { + continue + } d.path = append(d.path, keyNode.Value) + d.nodeOrigin = nil value, err := d.decode(valueNode, sourceFile) if err == nil && d.origin != nil { - originFile := sourceFile - if d.remoteSource != "" { - originFile = d.remoteSource - } - d.origin(d.path, originFile, d.remoteSource != "") + d.recordValueOrigin(valueNode, sourceFile) } + d.nodeOrigin = nil d.path = d.path[:len(d.path)-1] if err != nil { return nil, err } result[keyNode.Value] = value } + if err := d.expandMergeKeys(node, result, sourceFile); err != nil { + return nil, err + } recordScriptSource(node, result, sourceFile) if err := recordCommandFields(node, result); err != nil { return nil, err @@ -81,6 +95,20 @@ func (d *configYamlDecoder) decodeMappingNodeWithYamlFunctions(node *yaml.Node, return result, nil } +// recordValueOrigin reports the file that declared the value just decoded at the current path. A +// value produced by an include, or by an alias of an anchored include, belongs to the included file +// (and is remote when that file is), not to the file that merely references it. +func (d *configYamlDecoder) recordValueOrigin(valueNode *yaml.Node, sourceFile string) { + file, remote := sourceFile, false + if d.remoteSource != "" { + file, remote = d.remoteSource, true + } + if included := d.nodeOrigin; included != nil && (valueNode.Kind == yaml.AliasNode || isIncludeTag(valueNode.Tag)) { + file, remote = included.file, included.remote + } + d.origin(d.path, file, remote) +} + // decodeSequenceNodeWithYamlFunctions evaluates sequence elements in their original order. func (d *configYamlDecoder) decodeSequenceNodeWithYamlFunctions(node *yaml.Node, sourceFile string) (any, error) { result := make([]interface{}, 0, len(node.Content)) @@ -179,7 +207,7 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour if err := resolved.Decode(&decoded); err != nil { return nil, err } - if !walked && d.origin != nil { + if d.origin != nil { file, remote := include.LocalPath, false if file == "" { file, remote = node.Value, true @@ -187,7 +215,11 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour if d.remoteSource != "" { file, remote = d.remoteSource, true } - d.recordIncludedOrigins(decoded, file, remote) + if !walked { + d.recordIncludedOrigins(decoded, file, remote) + } + // Set last, so the outermost include of a nested chain is the one the owning key reads. + d.nodeOrigin = &includeOrigin{file: file, remote: remote} } return decoded, nil } From 40166935205061a8aa9d0e01831a7493f67b3c76 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:13:22 -0600 Subject: [PATCH 14/32] test(artifacts): require the trusted key to anchor to its declaring directory The describe-config pattern accepted any directory ending in keys, so a key anchored to the wrong directory still passed. It now requires the fixture's own directory. Co-Authored-By: Claude Sonnet 5.5 --- tests/test-cases/artifacts-describe.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-cases/artifacts-describe.yaml b/tests/test-cases/artifacts-describe.yaml index f0178c4ad54..fe8d69d7a9c 100644 --- a/tests/test-cases/artifacts-describe.yaml +++ b/tests/test-cases/artifacts-describe.yaml @@ -17,5 +17,6 @@ tests: - '"shared"' - '"bucket": "example-artifacts"' - '"require_signature": true' - - '"trusted_keys":\s*\[\s*"[^"]*(?:/|\\\\)keys(?:/|\\\\)release\.pub"' + # The key is declared by the fixture's atmos.yaml, so it anchors to that directory and no other. + - '"trusted_keys":\s*\[\s*"[^"]*(?:/|\\\\)tests(?:/|\\\\)fixtures(?:/|\\\\)scenarios(?:/|\\\\)artifacts-describe(?:/|\\\\)keys(?:/|\\\\)release\.pub"' exit_code: 0 From 94d24dc3d1860aaab47bce2baa5248fbd3e282f7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 19:19:05 -0600 Subject: [PATCH 15/32] fix(config): redact credentials from include errors An include whose URL carried userinfo repeated it in the error. Include arguments are redacted with the shared RedactURL helper before they are formatted. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/artifacts_decode_test.go | 9 +++++++++ pkg/config/yaml_decode.go | 15 +++++++++++++-- pkg/config/yaml_functions.go | 8 ++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/pkg/config/artifacts_decode_test.go b/pkg/config/artifacts_decode_test.go index 244e77813e0..e5ec5a97143 100644 --- a/pkg/config/artifacts_decode_test.go +++ b/pkg/config/artifacts_decode_test.go @@ -256,3 +256,12 @@ func TestDecodeNodeWithYamlFunctions_AliasPolicy(t *testing.T) { assert.Equal(t, map[string]any{"ref": map[string]any{"a": 1}}, got) }) } + +func TestDecodeNodeWithYamlFunctions_IncludeErrorsDoNotEchoCredentials(t *testing.T) { + dir := t.TempDir() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte("keys: !include https://user:hunter2@127.0.0.1:1/keys.yaml\n"), &node)) + _, err := decodeNodeWithYamlFunctionsForFile(&node, filepath.Join(dir, "atmos.yaml")) + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.NotContains(t, err.Error(), "hunter2", "include URLs are redacted before they reach an error") +} diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 3074474a2b1..7a6593275e8 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "path/filepath" + "strings" "go.yaml.in/yaml/v3" legacyyaml "gopkg.in/yaml.v3" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/function/yamlsource" "github.com/cloudposse/atmos/pkg/policy/admission" "github.com/cloudposse/atmos/pkg/policy/guard" @@ -152,6 +154,15 @@ func (d *configYamlDecoder) processScalarNodeValueForFile(node *yaml.Node, sourc return d.processIncludeNodeValueForFile(node, sourceFile) } +// redactIncludeValue removes credentials from an include argument before it is echoed in an error. +// Only URL-shaped values are touched: a local path or an SCP-style git address keeps its text. +func redactIncludeValue(value string) string { + if !strings.Contains(value, "://") { + return value + } + return artifact.RedactURL(value) +} + func isIncludeTag(tag string) bool { return tag == u.AtmosYamlFuncInclude || tag == u.AtmosYamlFuncIncludeRaw } @@ -174,7 +185,7 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour } key := sourceFile + "\x00" + node.Value if d.includes[key] || len(d.includes) >= maxConfigIncludeDepth { - return nil, fmt.Errorf("%w: recursive include: %s", ErrExecuteYamlFunctions, node.Value) + return nil, fmt.Errorf("%w: recursive include: %s", ErrExecuteYamlFunctions, redactIncludeValue(node.Value)) } d.includes[key] = true defer delete(d.includes, key) @@ -201,7 +212,7 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour }) } if err != nil { - return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } var decoded any if err := resolved.Decode(&decoded); err != nil { diff --git a/pkg/config/yaml_functions.go b/pkg/config/yaml_functions.go index d081438c6be..d227fe8f109 100644 --- a/pkg/config/yaml_functions.go +++ b/pkg/config/yaml_functions.go @@ -43,7 +43,7 @@ func processIncludeTag(nodeTag, nodeValue, strFunc string) (any, error) { includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", "include_data", nodeTag, nodeValue)) if err != nil { log.Debug(failedToProcess, functionKey, strFunc, "error", err) - return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, nodeValue, err) + return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(nodeValue), err) } if includeValue != nil { if data, ok := includeValue[includeDataKey]; ok { @@ -327,7 +327,7 @@ func handleInclude(node *yaml.Node, v *viper.Viper, currentPath string) error { includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", includeDataKey, node.Tag, node.Value)) if err != nil { log.Debug(failedToProcess, functionKey, strFunc, "error", err) - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } if includeValue == nil { log.Debug(emptyValueWarning, functionKey, strFunc) @@ -344,7 +344,7 @@ func handleInclude(node *yaml.Node, v *viper.Viper, currentPath string) error { func storeIncludeValue(node *yaml.Node, v *viper.Viper, currentPath string, data any) error { if err := node.Encode(data); err != nil { - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } encodeDevcontainerKeys(node, currentPath) if err := node.Decode(&data); err != nil { @@ -352,7 +352,7 @@ func storeIncludeValue(node *yaml.Node, v *viper.Viper, currentPath string, data } v.Set(currentPath, data) if err := node.Encode(data); err != nil { - return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, node.Value, err) + return fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } if err := preserveEmptyGitPushRetry(node, v, currentPath); err != nil { return err From 581fb889ac765bbeac0856b90f15d0dfa85faab3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:02:56 -0600 Subject: [PATCH 16/32] fix(config): report merge-key origins only for the keys a merge copies expandMergeKeys replayed every origin record of each merge source before checking whether the key already existed, so a discarded default's trusted_keys origin overwrote an explicit key's origin (and a later source overwrote an earlier one). Hold the origins back while a merge source is decoded and report them only for the keys that are copied. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/process_yaml_anchor.go | 60 +++++++-- pkg/config/process_yaml_origin_test.go | 165 +++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 9 deletions(-) create mode 100644 pkg/config/process_yaml_origin_test.go diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go index 06c98e1e3fb..d0c6b733276 100644 --- a/pkg/config/process_yaml_anchor.go +++ b/pkg/config/process_yaml_anchor.go @@ -116,10 +116,17 @@ func isMergeKey(key *yaml.Node) bool { return key.Kind == yaml.ScalarNode && key.Tag == mergeKeyTag } +// mergeSource is one decoded merge source and the origins its values were decoded with. +type mergeSource struct { + value map[string]interface{} + origins []originRecord +} + // expandMergeKeys copies the entries named by every "<<" key of the mapping into result. Entries // the mapping declares itself win over merged ones, and earlier merge sources win over later ones, -// as the YAML merge-key rules require. The merge value is decoded at the mapping's own path, so the -// origins of merged settings are recorded where the merged repository sees them. +// as the YAML merge-key rules require. The origins of a merge source are held back while it is +// decoded and reported only for the keys that are actually copied, each at the mapping's own path, +// so a discarded default never overwrites the origin of an explicit key or of an earlier source. func (d *configYamlDecoder) expandMergeKeys(node *yaml.Node, result map[string]interface{}, sourceFile string) error { for i := 0; i < len(node.Content); i += 2 { if !isMergeKey(node.Content[i]) { @@ -130,25 +137,40 @@ func (d *configYamlDecoder) expandMergeKeys(node *yaml.Node, result map[string]i return err } for _, source := range sources { - for key, value := range source { - if _, exists := result[key]; !exists { - result[key] = value + for key, value := range source.value { + if _, exists := result[key]; exists { + continue } + result[key] = value + d.replayKeyOrigins(source.origins, key) } } } return nil } +// replayKeyOrigins reports the held-back origins of one merged key, relative to the current path. +func (d *configYamlDecoder) replayKeyOrigins(records []originRecord, key string) { + if d.origin == nil { + return + } + for _, record := range records { + if len(record.rel) > 0 && record.rel[0] == key { + d.replayOrigins([]originRecord{record}) + } + } +} + // mergeSources decodes the value of a merge key: a mapping, an alias of one, or a sequence of them. -func (d *configYamlDecoder) mergeSources(value *yaml.Node, sourceFile string) ([]map[string]interface{}, error) { +// The origins each source reports are collected on the source instead of being forwarded. +func (d *configYamlDecoder) mergeSources(value *yaml.Node, sourceFile string) ([]mergeSource, error) { nodes := []*yaml.Node{value} if value.Kind == yaml.SequenceNode { nodes = value.Content } - sources := make([]map[string]interface{}, 0, len(nodes)) + sources := make([]mergeSource, 0, len(nodes)) for _, child := range nodes { - decoded, err := d.decode(child, sourceFile) + decoded, records, err := d.decodeCollectingOrigins(child, sourceFile) if err != nil { return nil, err } @@ -156,11 +178,31 @@ func (d *configYamlDecoder) mergeSources(value *yaml.Node, sourceFile string) ([ if !ok { return nil, fmt.Errorf("%w: merge key %q requires a mapping or a sequence of mappings, got %T", ErrExecuteYamlFunctions, "<<", decoded) } - sources = append(sources, source) + sources = append(sources, mergeSource{value: source, origins: records}) } return sources, nil } +// decodeCollectingOrigins decodes a node and returns the origins it reported, relative to the +// current path, without forwarding them to the decoder's origin callback. +func (d *configYamlDecoder) decodeCollectingOrigins(node *yaml.Node, sourceFile string) (any, []originRecord, error) { + previous := d.origin + if previous == nil { + value, err := d.decode(node, sourceFile) + return value, nil, err + } + base := len(d.path) + var records []originRecord + d.origin = func(path []string, file string, remote bool) { + if len(path) > base { + records = append(records, originRecord{rel: append([]string(nil), path[base:]...), file: file, remote: remote}) + } + } + defer func() { d.origin = previous }() + value, err := d.decode(node, sourceFile) + return value, records, err +} + // cloneDecodedValue deep-copies the mappings and sequences of a decoded value, so an alias never // shares mutable structure with its anchor. func cloneDecodedValue(value any) any { diff --git a/pkg/config/process_yaml_origin_test.go b/pkg/config/process_yaml_origin_test.go new file mode 100644 index 00000000000..672497dec80 --- /dev/null +++ b/pkg/config/process_yaml_origin_test.go @@ -0,0 +1,165 @@ +package config + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// TestLoadConfig_ArtifactsMergeKeysNeverReplayDiscardedOrigins covers the precedence of origin +// records across merge keys: a key the mapping declares itself keeps its own origin, and a key +// copied from a merge source keeps the origin of the earliest source that supplies it. +func TestLoadConfig_ArtifactsMergeKeysNeverReplayDiscardedOrigins(t *testing.T) { + const bases = ` base: &base + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./defaults/keys.yaml + other: &other + kind: local/dir + options: + path: store + verification: + trusted_keys: !include ./other/keys.yaml + empty: &empty + kind: local/dir + options: + path: store +` + tests := []struct { + name string + // repository is the body of the repository mapping in atmos.yaml. + repository string + // wantDir is the directory, relative to the config, that declares the effective keys. + wantDir string + wantKeys string + }{ + { + name: "explicit keys declared before the merge key", + repository: ` verification: + trusted_keys: [release.pub] + <<: *base +`, + wantDir: ".", + wantKeys: "release.pub", + }, + { + name: "explicit keys declared after the merge key", + repository: ` <<: *base + verification: + trusted_keys: [release.pub] +`, + wantDir: ".", + wantKeys: "release.pub", + }, + { + name: "keys supplied only by the merge source keep the source file", + repository: " <<: *base\n", + wantDir: "defaults", + wantKeys: "default.pub", + }, + { + name: "the earlier of two merge sources supplies the origin", + repository: " <<: [*base, *other]\n", + wantDir: "defaults", + wantKeys: "default.pub", + }, + { + name: "the later merge source supplies keys the earlier one lacks", + repository: " <<: [*empty, *other]\n", + wantDir: "other", + wantKeys: "other.pub", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n repositories:\n"+bases+" r:\n"+tt.repository) + writeArtifactsFile(t, filepath.Join(dir, "defaults", "keys.yaml"), "- default.pub\n") + writeArtifactsFile(t, filepath.Join(dir, "other", "keys.yaml"), "- other.pub\n") + + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + repo := cfg.Artifacts.Repositories["r"] + require.NotNil(t, repo.Verification) + wantDir := filepath.Join(dir, tt.wantDir) + assert.Equal(t, []string{filepath.Join(wantDir, tt.wantKeys)}, repo.Verification.TrustedKeys) + assert.Equal(t, wantDir, repo.Verification.KeysOrigin.Dir) + assert.False(t, repo.Verification.KeysOrigin.Remote) + assert.Equal(t, filepath.Join(dir, "store"), repo.Options["path"]) + }) + } +} + +// TestDecodeNodeWithYamlFunctions_MergeKeyOriginRecords checks the origin records directly: only +// the keys a merge actually copies are reported, and an explicit key is reported exactly once. +func TestDecodeNodeWithYamlFunctions_MergeKeyOriginRecords(t *testing.T) { + tests := []struct { + name string + source string + want []string + }{ + { + name: "explicit key before the merge key", + source: "child:\n a: 1\n <<: {a: 2, b: 3}\n", + want: []string{"child.a", "child.b"}, + }, + { + name: "explicit key after the merge key", + source: "base: &base {a: 2, b: 3}\nchild:\n <<: *base\n a: 1\n", + want: []string{"child.a", "child.b"}, + }, + { + name: "nested mapping of a discarded key reports nothing", + source: "base: &base {n: {x: 1}, m: 2}\nchild:\n <<: *base\n n: 5\n", + want: []string{"child.n", "child.m"}, + }, + { + name: "nested mapping of a copied key is reported", + source: "base: &base {n: {x: 1}}\nchild:\n <<: *base\n", + want: []string{"child.n", "child.n.x"}, + }, + { + name: "earlier merge sources win", + source: "x: &x {a: {p: 1}}\ny: &y {a: {q: 1}, b: 1}\nchild:\n <<: [*x, *y]\n", + want: []string{"child.a", "child.a.p", "child.b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var seen []string + document := mustParseYAML(t, tt.source) + decoder := configYamlDecoder{includes: map[string]bool{}} + decoder.origin = func(path []string, _ string, _ bool) { + if joined := joinPath(path); strings.HasPrefix(joined, "child.") { + seen = append(seen, joined) + } + } + _, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + require.NoError(t, err) + assert.ElementsMatch(t, tt.want, seen, "each key's origin is reported exactly once, by the source that supplied it") + }) + } +} + +func mustParseYAML(t *testing.T, source string) yaml.Node { + t.Helper() + var node yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &node)) + return node +} + +func joinPath(path []string) string { + out := "" + for i, part := range path { + if i > 0 { + out += "." + } + out += part + } + return out +} From ee2bff069aefdde06530d03438a5af6427a9b9a5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:06:37 -0600 Subject: [PATCH 17/32] fix(config): keep remote provenance when a local include wraps a remote include A local include whose result was not a mapping overwrote the origin set by the remote include nested inside it, so the owning key (for example trusted_keys) read the local wrapper as its origin and a remote-supplied relative key was anchored to the local config directory. Track the first remote include evaluated inside an include or an anchor, and keep that origin for non-mapping results. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/process_yaml_anchor.go | 17 ++++- pkg/config/process_yaml_include_eval.go | 5 ++ pkg/config/process_yaml_provenance_test.go | 83 ++++++++++++++++++++++ pkg/config/yaml_decode.go | 16 ++++- 4 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 pkg/config/process_yaml_provenance_test.go diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go index d0c6b733276..65c0745f49e 100644 --- a/pkg/config/process_yaml_anchor.go +++ b/pkg/config/process_yaml_anchor.go @@ -29,6 +29,8 @@ type anchorEntry struct { value any origins []originRecord include *includeOrigin + // remote is the first remote include evaluated while the anchor was evaluated. + remote *includeOrigin } // decodeAnchored evaluates an anchored node once and caches the result for its aliases. @@ -55,6 +57,7 @@ func (d *configYamlDecoder) decodeAlias(node *yaml.Node, sourceFile string) (any } d.replayOrigins(entry.origins) d.nodeOrigin = entry.include + d.noteRemote(entry.remote) return cloneDecodedValue(entry.value), nil } @@ -87,11 +90,16 @@ func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) ( defer func() { d.origin = previous }() } d.nodeOrigin = nil + outerTaint := d.remoteTaint + d.remoteTaint = nil value, err := d.decodeNode(node, sourceFile) + entryTaint := d.remoteTaint + d.remoteTaint = outerTaint + d.noteRemote(entryTaint) if err != nil { return nil, err } - entry := &anchorEntry{value: value, origins: records} + entry := &anchorEntry{value: value, origins: records, remote: entryTaint} if node.Kind == yaml.ScalarNode && isIncludeTag(node.Tag) { entry.include = d.nodeOrigin } @@ -100,6 +108,13 @@ func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) ( return entry, nil } +// noteRemote records the first remote include seen since the taint was last cleared. +func (d *configYamlDecoder) noteRemote(origin *includeOrigin) { + if origin != nil && origin.remote && d.remoteTaint == nil { + d.remoteTaint = origin + } +} + // replayOrigins reports the origins an anchor recorded, relative to the current path. func (d *configYamlDecoder) replayOrigins(records []originRecord) { if d.origin == nil { diff --git a/pkg/config/process_yaml_include_eval.go b/pkg/config/process_yaml_include_eval.go index 04693b45087..2608f60692c 100644 --- a/pkg/config/process_yaml_include_eval.go +++ b/pkg/config/process_yaml_include_eval.go @@ -24,6 +24,11 @@ type configYamlDecoder struct { // nodeOrigin is the file the include or alias just decoded took its value from. The mapping // entry that owns the value reads it so the setting's origin is the included file. nodeOrigin *includeOrigin + + // remoteTaint is the first remote include evaluated since it was last cleared. An include whose + // result is not a mapping (a scalar or a list) has no per-key origin records of its own, so a + // remote include nested anywhere inside it marks the whole result as remote-supplied. + remoteTaint *includeOrigin } // evaluateInclude resolves an included subtree using its file context and diff --git a/pkg/config/process_yaml_provenance_test.go b/pkg/config/process_yaml_provenance_test.go new file mode 100644 index 00000000000..bf3e9502c0a --- /dev/null +++ b/pkg/config/process_yaml_provenance_test.go @@ -0,0 +1,83 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" +) + +// TestLoadConfig_ArtifactsNestedIncludeKeepsRemoteProvenance covers wrapped includes: the +// innermost remote origin survives an outer local wrapper, and a remote wrapper stays remote. +func TestLoadConfig_ArtifactsNestedIncludeKeepsRemoteProvenance(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/keys.yaml": + _, _ = fmt.Fprint(w, "- release.pub\n") + case "/key.yaml": + _, _ = fmt.Fprint(w, "release.pub\n") + case "/wrap-local.yaml": + _, _ = fmt.Fprint(w, "!include ./local-keys.yaml | eval\n") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + tests := []struct { + name string + wrapper string + wantRemote bool + // section replaces the default `trusted_keys: !include ./trust/wrap.yaml` line when set. + section string + // plain reports that the wrapper holds no tags, so it also loads without `| eval`. + plain bool + }{ + {name: "local wrapper around a remote include", wrapper: "!include " + server.URL + "/keys.yaml | eval\n", wantRemote: true}, + {name: "local wrapper around a sequence holding a remote include", wrapper: "- !include " + server.URL + "/key.yaml | eval\n", wantRemote: true}, + {name: "remote wrapper around a local include", wrapper: "!include " + server.URL + "/wrap-local.yaml | eval\n", wantRemote: true}, + { + name: "local mapping wrapper around a remote key list", + wrapper: "trusted_keys: !include " + server.URL + "/keys.yaml | eval\n", + section: "verification: !include ./trust/wrap.yaml | eval\n", + wantRemote: true, + }, + {name: "local wrapper around local content stays local", wrapper: "- release.pub\n", wantRemote: false, plain: true}, + } + for _, tt := range tests { + suffixes := []string{" | eval"} + if tt.plain { + suffixes = append(suffixes, "") + } + for _, suffix := range suffixes { + t.Run(fmt.Sprintf("%s/eval=%t", tt.name, suffix != ""), func(t *testing.T) { + section := " verification:\n trusted_keys: !include ./trust/wrap.yaml" + suffix + "\n" + if tt.section != "" { + section = " " + tt.section + } + dir := artifactsTestDir(t, "artifacts:\n"+section) + writeArtifactsFile(t, filepath.Join(dir, "trust", "wrap.yaml"), tt.wrapper) + writeArtifactsFile(t, filepath.Join(dir, "trust", "local-keys.yaml"), "- release.pub\n") + cfg, err := loadArtifactsDir(t, dir) + require.NoError(t, err) + require.NotNil(t, cfg.Artifacts.Verification) + assert.Equal(t, tt.wantRemote, cfg.Artifacts.Verification.KeysOrigin.Remote) + if !tt.wantRemote { + assert.Equal(t, []string{filepath.Join(dir, "trust", "release.pub")}, cfg.Artifacts.Verification.TrustedKeys) + return + } + assert.Equal(t, []string{"release.pub"}, cfg.Artifacts.Verification.TrustedKeys, "a relative key from remote content is never anchored locally") + err = artifactconfig.Validate(&cfg.Artifacts) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "remote import") + }) + } + } +} diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 7a6593275e8..571bbf50ea3 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -195,6 +195,8 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour } var err error var walked bool + outerTaint := d.remoteTaint + d.remoteTaint = nil if node.Tag == u.AtmosYamlFuncIncludeRaw { err = u.ProcessIncludeRawTag(atmosConfig, &resolved, value, sourceFile) } else if d.origin == nil { @@ -211,6 +213,8 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour Source: d.remoteSource, }) } + walkedRemote := d.remoteTaint + d.remoteTaint = outerTaint if err != nil { return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(node.Value), err) } @@ -229,8 +233,16 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour if !walked { d.recordIncludedOrigins(decoded, file, remote) } - // Set last, so the outermost include of a nested chain is the one the owning key reads. - d.nodeOrigin = &includeOrigin{file: file, remote: remote} + origin := &includeOrigin{file: file, remote: remote} + // A local include never launders remote content: when the result is not a mapping (whose + // keys carry their own origins), the remote include nested inside it keeps its origin. + if _, isMapping := decoded.(map[string]any); !origin.remote && walkedRemote != nil && !isMapping { + origin = walkedRemote + } + d.noteRemote(origin) + // Set last, so the outermost include of a nested chain is the one the owning key reads, + // except that a remote origin is never replaced by a local one. + d.nodeOrigin = origin } return decoded, nil } From 9e0a6ff0a0858150eb07c645cc07bed9c24eaa79 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:11:04 -0600 Subject: [PATCH 18/32] fix(config): redact credentials from every include error path The query-evaluation error for a remote YAML include formatted the raw include argument, so a URL with userinfo leaked its password even though the outer error redacted its own copy. Add utils.RedactIncludeArgument and apply it at every include sink: the splice, parse, OCI and manifest errors in pkg/utils, the debug lines of the config include handlers, and the dotenv include errors, where a URL-shaped target was joined onto the config directory and echoed in the read error. Errors raised while fetching an include now have the credential cut out of their text while keeping the cause in the error chain. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/load.go | 8 ++- pkg/config/process_yaml_redact_test.go | 60 +++++++++++++++++ pkg/config/yaml_decode.go | 7 +- pkg/config/yaml_functions.go | 8 +-- pkg/utils/yaml_include_by_extension.go | 29 ++++---- pkg/utils/yaml_include_redact.go | 80 ++++++++++++++++++++++ pkg/utils/yaml_include_redact_test.go | 91 ++++++++++++++++++++++++++ 7 files changed, 259 insertions(+), 24 deletions(-) create mode 100644 pkg/config/process_yaml_redact_test.go create mode 100644 pkg/utils/yaml_include_redact.go create mode 100644 pkg/utils/yaml_include_redact_test.go diff --git a/pkg/config/load.go b/pkg/config/load.go index 7debfb56293..5afdb2e3e0b 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -1436,8 +1436,10 @@ func resolveDotenvMergeIncludeValue(configFile string, node *yaml.Node) (bool, e func loadDotenvIncludeAsYAMLNode(configFile, includeValue string) (*yaml.Node, error) { includeFile, ok := parseDotenvIncludeFile(includeValue) - if !ok { - return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, includeValue) + // A dotenv include is a local file: a URL-shaped target would be joined onto the config + // directory and echoed, with any credential it carries, in the read error. + if !ok || strings.Contains(includeFile, "://") { + return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, redactIncludeValue(includeValue)) } if !filepath.IsAbs(includeFile) { includeFile = filepath.Join(filepath.Dir(configFile), includeFile) @@ -1460,7 +1462,7 @@ func loadDotenvIncludeAsYAMLNode(configFile, includeValue string) (*yaml.Node, e return nil, err } if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode { - return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, includeValue) + return nil, fmt.Errorf("%w: %s", u.ErrIncludeYamlFunctionInvalidFile, redactIncludeValue(includeValue)) } return doc.Content[0], nil } diff --git a/pkg/config/process_yaml_redact_test.go b/pkg/config/process_yaml_redact_test.go new file mode 100644 index 00000000000..672ec95ae40 --- /dev/null +++ b/pkg/config/process_yaml_redact_test.go @@ -0,0 +1,60 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const includeRedactSecret = "hunter2" + +// TestConfigIncludeErrorsDoNotEchoCredentials covers every config-level include error path that +// formats the include argument: none may repeat a credential carried by a URL-shaped include. +func TestConfigIncludeErrorsDoNotEchoCredentials(t *testing.T) { + const target = "https://user:" + includeRedactSecret + "@127.0.0.1:1/" + tests := []struct { + name string + source string + }{ + {name: "dotenv include", source: "keys: !include " + target + "settings.env\n"}, + {name: "dotenv include with query", source: "keys: !include " + target + "settings.env .a\n"}, + {name: "yaml include with query", source: "keys: !include " + target + "keys.yaml .a\n"}, + {name: "yaml include with eval", source: "keys: !include " + target + "keys.yaml | eval\n"}, + {name: "raw include", source: "keys: !include.raw " + target + "keys.txt\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := artifactsTestDir(t, "artifacts:\n verification:\n "+tt.source) + _, err := loadArtifactsDir(t, dir) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + + _, err = decodeYAMLForTestFile(t, tt.source, filepath.Join(dir, "atmos.yaml")) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + }) + } +} + +func decodeYAMLForTestFile(t *testing.T, source, file string) (any, error) { + t.Helper() + node := mustParseYAML(t, source) + return decodeNodeWithYamlFunctionsForFile(&node, file) +} + +func TestLoadDotenvIncludeErrorsDoNotEchoCredentials(t *testing.T) { + dir := t.TempDir() + for _, value := range []string{ + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.env", + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.env .a", + "https://user:" + includeRedactSecret + "@127.0.0.1:1/settings.yaml", + } { + t.Run(value, func(t *testing.T) { + _, err := loadDotenvIncludeAsYAMLNode(filepath.Join(dir, "atmos.yaml"), value) + require.Error(t, err) + assert.NotContains(t, err.Error(), includeRedactSecret) + }) + } +} diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 571bbf50ea3..7777e17efc8 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -4,12 +4,10 @@ import ( "context" "fmt" "path/filepath" - "strings" "go.yaml.in/yaml/v3" legacyyaml "gopkg.in/yaml.v3" - "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/function/yamlsource" "github.com/cloudposse/atmos/pkg/policy/admission" "github.com/cloudposse/atmos/pkg/policy/guard" @@ -157,10 +155,7 @@ func (d *configYamlDecoder) processScalarNodeValueForFile(node *yaml.Node, sourc // redactIncludeValue removes credentials from an include argument before it is echoed in an error. // Only URL-shaped values are touched: a local path or an SCP-style git address keeps its text. func redactIncludeValue(value string) string { - if !strings.Contains(value, "://") { - return value - } - return artifact.RedactURL(value) + return u.RedactIncludeArgument(value) } func isIncludeTag(tag string) bool { diff --git a/pkg/config/yaml_functions.go b/pkg/config/yaml_functions.go index d227fe8f109..a3db7536c8b 100644 --- a/pkg/config/yaml_functions.go +++ b/pkg/config/yaml_functions.go @@ -39,10 +39,10 @@ func processExecTag(strFunc, nodeValue string) (any, error) { } // processIncludeTag processes the !include tag. -func processIncludeTag(nodeTag, nodeValue, strFunc string) (any, error) { +func processIncludeTag(nodeTag, nodeValue string) (any, error) { includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", "include_data", nodeTag, nodeValue)) if err != nil { - log.Debug(failedToProcess, functionKey, strFunc, "error", err) + log.Debug(failedToProcess, functionKey, nodeTag+" "+redactIncludeValue(nodeValue), "error", err) return nil, fmt.Errorf(errorFormat, ErrExecuteYamlFunctions, u.AtmosYamlFuncInclude, redactIncludeValue(nodeValue), err) } if includeValue != nil { @@ -149,7 +149,7 @@ func processScalarNodeValue(node *yaml.Node) (any, error) { // !literal keeps the value exactly as written. return node.Value, nil case u.AtmosYamlFuncInclude, u.AtmosYamlFuncIncludeRaw: - return processIncludeTag(node.Tag, node.Value, strFunc) + return processIncludeTag(node.Tag, node.Value) default: return processGitScalarNodeValue(node, strFunc) } @@ -323,7 +323,7 @@ func handleExec(node *yaml.Node, v *viper.Viper, currentPath string) error { // handleInclude Process the !include tag and set the value in Viper , returns an error if the processing fails, warns if the value is empty. func handleInclude(node *yaml.Node, v *viper.Viper, currentPath string) error { - strFunc := fmt.Sprintf(tagValueFormat, node.Tag, node.Value) + strFunc := fmt.Sprintf(tagValueFormat, node.Tag, redactIncludeValue(node.Value)) includeValue, err := u.UnmarshalYAML[map[any]any](fmt.Sprintf("%s: %s %s", includeDataKey, node.Tag, node.Value)) if err != nil { log.Debug(failedToProcess, functionKey, strFunc, "error", err) diff --git a/pkg/utils/yaml_include_by_extension.go b/pkg/utils/yaml_include_by_extension.go index 194f0112013..02bd2caab64 100644 --- a/pkg/utils/yaml_include_by_extension.go +++ b/pkg/utils/yaml_include_by_extension.go @@ -164,7 +164,7 @@ func processIncludeTagInternal( data, err := os.ReadFile(localFile) if err != nil { return false, fmt.Errorf("%w: %s, stack manifest: %s, error: %w", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } if err := spliceYAMLText(atmosConfig, node, string(data), includeSplice{query: includeQuery, eval: evalFunctions, val: val, file: file}); err != nil { return false, err @@ -198,7 +198,7 @@ func processIncludeTagInternal( text = string(v) default: return false, fmt.Errorf("%w: %s, stack manifest: %s, error: unexpected remote content type %T", - ErrIncludeYamlFunctionFailedStackManifest, val, file, raw) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), raw) } // Nested paths inside remote content keep the outer manifest as // their context; there is no local directory to resolve against. @@ -296,7 +296,7 @@ func spliceYAMLText(atmosConfig *schema.AtmosConfiguration, node *yaml.Node, yam evaluated, err := evaluateIncludeQuery(atmosConfig, yamlText, query) if err != nil { return fmt.Errorf("%w: %s, stack manifest: %s, error: %w", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } yamlText = evaluated } @@ -518,10 +518,17 @@ func processLocalFile(localFile string, forceRaw bool) (any, error) { return filetype.ParseFileByExtension(os.ReadFile, localFile) } -// processRemoteFile downloads and parses a remote file. +// processRemoteFile downloads and parses a remote file. A credential in the include target is +// removed from the text of any error, because the download layers quote the reference they fail on. func processRemoteFile(atmosConfig *schema.AtmosConfiguration, includeFile string, forceRaw bool) (any, error) { defer perf.Track(atmosConfig, "utils.processRemoteFile")() + res, err := fetchRemoteFile(atmosConfig, includeFile, forceRaw) + return res, redactIncludeError(err, includeFile) +} + +// fetchRemoteFile downloads and parses a remote file. +func fetchRemoteFile(atmosConfig *schema.AtmosConfiguration, includeFile string, forceRaw bool) (any, error) { // oci:// sources need one file picked out of the pulled image, which go-getter's own // "//subdir" handling cannot do (it only copies directories), so the pick happens here // after the shared OCI getter has pulled the image. @@ -571,7 +578,7 @@ func processOCIIncludeFile(atmosConfig *schema.AtmosConfiguration, includeFile s imageRef, subdir := getter.SourceDirSubdir(includeFile) if subdir == "" { - return nil, fmt.Errorf("%w: '%s'", ErrIncludeYamlFunctionOCIMissingPath, includeFile) + return nil, fmt.Errorf("%w: '%s'", ErrIncludeYamlFunctionOCIMissingPath, RedactIncludeArgument(includeFile)) } tempDir, err := os.MkdirTemp("", "atmos-include-oci-*") @@ -587,16 +594,16 @@ func processOCIIncludeFile(atmosConfig *schema.AtmosConfiguration, includeFile s // survive. It applies ociIncludeTimeout itself, so no extra deadline is set here. dl := downloader.NewGoGetterDownloader(atmosConfig).(downloader.ContextFileDownloader) if _, err := dl.FetchWithMetadataContext(context.Background(), imageRef, tempDir, downloader.ClientModeDir, ociIncludeTimeout); err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, includeFile, err) + return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(includeFile), err) } localFile, err := filesystem.SafeJoin(tempDir, subdir) if err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, includeFile, err) + return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), err) } if !FileExists(localFile) { - return nil, fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, imageRef, subdir) + return nil, fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(imageRef), subdir) } return processLocalFile(localFile, forceRaw) @@ -641,15 +648,15 @@ func unmarshalYamlContent(y string, val string, file string) (*yaml.Node, error) var includedNode yaml.Node err := yaml.Unmarshal([]byte(y), &includedNode) if err != nil { - return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: %v", - ErrIncludeYamlFunctionFailedStackManifest, val, file, err) + return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: %w", + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file), err) } // yaml.Unmarshal creates a DocumentNode, we need to use its content if includedNode.Kind == yaml.DocumentNode { if len(includedNode.Content) == 0 { return nil, fmt.Errorf("%w: %s, stack manifest: %s, error: empty document", - ErrIncludeYamlFunctionFailedStackManifest, val, file) + ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(val), RedactIncludeArgument(file)) } return includedNode.Content[0], nil } diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go new file mode 100644 index 00000000000..5b72f1c8cc1 --- /dev/null +++ b/pkg/utils/yaml_include_redact.go @@ -0,0 +1,80 @@ +package utils + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// RedactIncludeArgument removes credentials from an include argument or include target before it is +// echoed in an error or a log line. Only URL-shaped values are touched: a local path or an +// SCP-style git address keeps its text, and a URL that cannot be proven free of userinfo is +// replaced entirely (see artifact.RedactURL). +func RedactIncludeArgument(value string) string { + defer perf.Track(nil, "utils.RedactIncludeArgument")() + + if !strings.Contains(value, "://") { + return value + } + return artifact.RedactURL(value) +} + +// redactedIncludeError is an error whose text has had a credential removed. It keeps the original +// error in its chain so errors.Is and errors.As still see the cause. +type redactedIncludeError struct { + message string + err error +} + +func (e *redactedIncludeError) Error() string { return e.message } + +func (e *redactedIncludeError) Unwrap() error { return e.err } + +// redactIncludeError removes the credential of an include target from the text of an error raised +// while fetching it. Fetch errors from the download and OCI layers quote the reference they were +// given (for example "could not parse reference: user:secret@host/image"), so the include +// argument cannot be redacted once for all of them: the userinfo is cut out of the final text. +func redactIncludeError(err error, target string) error { + if err == nil || !strings.Contains(target, "@") { + return err + } + message := err.Error() + redacted := message + for _, userinfo := range includeUserinfo(target) { + redacted = strings.ReplaceAll(redacted, userinfo.full+"@", "") + if userinfo.secret != "" { + redacted = strings.ReplaceAll(redacted, userinfo.secret, artifact.RedactedValue) + } + } + if redacted == message { + return err + } + return &redactedIncludeError{message: redacted, err: err} +} + +// includeCredential is the userinfo of one URL found in an include target. +type includeCredential struct { + full string + secret string +} + +// includeUserinfo extracts the userinfo of every URL in an include target, including the URL that +// follows a forced-getter prefix such as "git::https://". +func includeUserinfo(target string) []includeCredential { + var found []includeCredential + for _, part := range strings.Split(target, "::") { + _, rest, ok := strings.Cut(part, "://") + if !ok { + continue + } + authority, _, _ := strings.Cut(rest, "/") + userinfo, _, hasUser := strings.Cut(authority, "@") + if !hasUser { + continue + } + _, secret, _ := strings.Cut(userinfo, ":") + found = append(found, includeCredential{full: userinfo, secret: secret}) + } + return found +} diff --git a/pkg/utils/yaml_include_redact_test.go b/pkg/utils/yaml_include_redact_test.go new file mode 100644 index 00000000000..fc665d2edd4 --- /dev/null +++ b/pkg/utils/yaml_include_redact_test.go @@ -0,0 +1,91 @@ +package utils + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/schema" +) + +const redactTestSecret = "hunter2" + +func TestRedactIncludeArgument(t *testing.T) { + tests := []struct { + name string + value string + want string + }{ + {name: "local path", value: "./stacks/base.yaml", want: "./stacks/base.yaml"}, + {name: "local path with query", value: "./base.yaml .a.b | eval", want: "./base.yaml .a.b | eval"}, + {name: "scp style git address", value: "git@github.com:org/repo.git", want: "git@github.com:org/repo.git"}, + {name: "url without credentials", value: "https://example.com/base.yaml .a", want: "https://example.com/base.yaml .a"}, + {name: "url with userinfo", value: "https://user:" + redactTestSecret + "@example.com/base.yaml", want: "https://example.com/base.yaml"}, + {name: "url with userinfo and fragment", value: "https://user:" + redactTestSecret + "@example.com/base.yaml#frag", want: "https://example.com/base.yaml#frag"}, + {name: "forced getter with userinfo", value: "git::https://user:" + redactTestSecret + "@example.com/repo.git", want: ""}, + {name: "unparsable url with userinfo", value: "https://user:" + redactTestSecret + "@exa mple.com/%zz", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := RedactIncludeArgument(tt.value) + assert.Equal(t, tt.want, got) + assert.NotContains(t, got, redactTestSecret) + }) + } +} + +// TestProcessInclude_ErrorsDoNotEchoCredentials runs every error path that formats an include +// argument or target with a credential-bearing URL and checks the credential never reaches the error. +func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/invalid.yaml": + _, _ = fmt.Fprint(w, "value: [\n") + case "/empty.yaml": + _, _ = fmt.Fprint(w, "# only a comment\n---\n") + default: + _, _ = fmt.Fprint(w, "a:\n b: 1\n") + } + })) + defer server.Close() + withUserinfo := strings.Replace(server.URL, "http://", "http://user:"+redactTestSecret+"@", 1) + closed := "http://user:" + redactTestSecret + "@127.0.0.1:1" + + tests := []struct { + name string + include string + // rawOK reports that !include.raw does not parse the body, so it cannot fail on it. + rawOK bool + }{ + {name: "query evaluation failure", include: withUserinfo + "/config.yaml .a[ | eval"}, + {name: "query evaluation failure without eval", include: withUserinfo + "/config.yaml .a["}, + {name: "invalid remote yaml", include: withUserinfo + "/invalid.yaml | eval", rawOK: true}, + {name: "invalid remote yaml without eval", include: withUserinfo + "/invalid.yaml", rawOK: true}, + {name: "fetch failure", include: closed + "/config.yaml"}, + {name: "fetch failure with eval", include: closed + "/config.yaml | eval"}, + {name: "raw fetch failure", include: closed + "/config.txt"}, + {name: "oci include without a file path", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag"}, + {name: "oci pull failure", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag//file.yaml"}, + } + for _, tag := range []string{"!include", "!include.raw"} { + for _, tt := range tests { + t.Run(tag+"/"+tt.name, func(t *testing.T) { + dir := t.TempDir() + config := &schema.AtmosConfiguration{BasePath: dir} + _, err := UnmarshalYAMLFromFile[map[string]any](config, "included: "+tag+" "+tt.include+"\n", filepath.Join(dir, "stack.yaml")) + if tag == "!include.raw" && tt.rawOK { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.NotContains(t, err.Error(), redactTestSecret, "the include credential must not reach the error text") + }) + } + } +} From 51146ff9c92df93f6312376d8ed959f65ee69646 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:12:38 -0600 Subject: [PATCH 19/32] fix(config): bound alias expansion while decoding YAML decodeAlias copied the full expanded value of its anchor on every alias with no limit, so a compact document of nested aliases (ten levels of ten-element sequences, each aliasing the previous) allocated an exponential number of values at config load. Charge the size of each alias against a per-decoder budget of one million values before the copy is made, and reject the document with ErrYamlAliasExpansionLimit when the budget is exceeded. Merge keys go through the same path and are charged the same way. Co-Authored-By: Claude Sonnet 5.5 --- errors/errors.go | 1 + pkg/config/process_yaml_alias_limit_test.go | 94 +++++++++++++++++++++ pkg/config/process_yaml_anchor.go | 47 ++++++++++- pkg/config/process_yaml_include_eval.go | 5 ++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 pkg/config/process_yaml_alias_limit_test.go diff --git a/errors/errors.go b/errors/errors.go index dcb294b94d5..91e322a1422 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -258,6 +258,7 @@ var ( ErrYamlFuncInvalidArguments = errors.New("invalid number of arguments in the Atmos YAML function") ErrYamlFuncTemplateEmbedding = errors.New("a YAML function value cannot be used in this template expression") ErrYamlFuncMaxResolutionDepth = errors.New("Atmos YAML function resolution exceeded the maximum dependency depth (likely an undetected circular dependency)") + ErrYamlAliasExpansionLimit = errors.New("YAML aliases expand to more values than the configuration loader allows") ErrComponentFuncDuringManifestLoad = errors.New("atmos.Component cannot be evaluated while stack manifests are being loaded") ErrDeferredTemplateContextMissing = errors.New("cannot resolve deferred !template value: template context is unavailable because template processing was disabled for this invocation") ErrAwsGetCallerIdentity = errors.New("failed to get AWS caller identity") diff --git a/pkg/config/process_yaml_alias_limit_test.go b/pkg/config/process_yaml_alias_limit_test.go new file mode 100644 index 00000000000..4bd57091512 --- /dev/null +++ b/pkg/config/process_yaml_alias_limit_test.go @@ -0,0 +1,94 @@ +package config + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// billionLaughs builds the classic exponential alias graph: every level is a sequence of +// `fanout` aliases of the previous one, so level n expands to fanout^n leaves. +func billionLaughs(levels, fanout int) string { + var b strings.Builder + b.WriteString("l0: &l0 [" + strings.TrimSuffix(strings.Repeat("x, ", fanout), ", ") + "]\n") + for level := 1; level <= levels; level++ { + aliases := strings.TrimSuffix(strings.Repeat(fmt.Sprintf("*l%d, ", level-1), fanout), ", ") + fmt.Fprintf(&b, "l%d: &l%d [%s]\n", level, level, aliases) + } + return b.String() +} + +func TestDecodeNodeWithYamlFunctions_AliasExpansionIsBounded(t *testing.T) { + t.Run("an exponential alias graph is rejected before it is expanded", func(t *testing.T) { + start := time.Now() + _, err := decodeYAMLForTest(t, billionLaughs(10, 10)) + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + require.ErrorIs(t, err, ErrExecuteYamlFunctions) + assert.Contains(t, err.Error(), "l", "the error names the alias that crossed the limit") + assert.Less(t, time.Since(start), 10*time.Second, "the limit stops the expansion instead of allocating it") + }) + + t.Run("a large but bounded document still loads", func(t *testing.T) { + var b strings.Builder + b.WriteString("base: &base\n") + for i := 0; i < 1000; i++ { + fmt.Fprintf(&b, " - {name: item%d, enabled: true, tags: [a, b]}\n", i) + } + for i := 0; i < 50; i++ { + fmt.Fprintf(&b, "copy%d: *base\n", i) + } + got, err := decodeYAMLForTest(t, b.String()) + require.NoError(t, err) + decoded, ok := got.(map[string]any) + require.True(t, ok) + require.Len(t, decoded, 51) + last, ok := decoded["copy49"].([]any) + require.True(t, ok) + require.Len(t, last, 1000) + assert.Equal(t, map[string]any{"name": "item0", "enabled": true, "tags": []any{"a", "b"}}, last[0]) + assert.Equal(t, map[string]any{"name": "item999", "enabled": true, "tags": []any{"a", "b"}}, last[999]) + }) + + t.Run("the budget is checked against the configured limit exactly", func(t *testing.T) { + // The anchor is a sequence of three scalars: four nodes per alias. + source := "a: &a [1, 2, 3]\nb: *a\nc: *a\nd: *a\n" + tests := []struct { + name string + limit int64 + wantErr bool + }{ + {name: "one below the third alias", limit: 11, wantErr: true}, + {name: "exactly the three aliases", limit: 12, wantErr: false}, + {name: "well above", limit: 1000, wantErr: false}, + {name: "below the first alias", limit: 3, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + document := mustParseYAML(t, source) + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: tt.limit} + got, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + if tt.wantErr { + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + assert.Nil(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, map[string]any{"a": []any{1, 2, 3}, "b": []any{1, 2, 3}, "c": []any{1, 2, 3}, "d": []any{1, 2, 3}}, got) + }) + } + }) + + t.Run("merge keys are charged like aliases", func(t *testing.T) { + document := mustParseYAML(t, "base: &base {a: 1, b: 2}\nx:\n <<: *base\ny:\n <<: *base\n") + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: 5} + _, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + require.ErrorIs(t, err, errUtils.ErrYamlAliasExpansionLimit) + }) +} diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go index 65c0745f49e..66cfe4ec933 100644 --- a/pkg/config/process_yaml_anchor.go +++ b/pkg/config/process_yaml_anchor.go @@ -4,8 +4,17 @@ import ( "fmt" "go.yaml.in/yaml/v3" + + errUtils "github.com/cloudposse/atmos/errors" ) +// maxAliasExpansion bounds the values that every alias in one decoded document may copy, counted as +// mappings, sequences, and scalars. An alias copies the whole value of its anchor, so a compact +// document of nested aliases can otherwise expand exponentially (the "billion laughs" shape). +// A million values is far above any real configuration, which aliases a few hundred values at most, +// and keeps the worst-case allocation of a hostile document to tens of megabytes. +const maxAliasExpansion = 1_000_000 + // mergeKeyTag is the resolved tag of the YAML merge key "<<". const mergeKeyTag = "!!merge" @@ -29,6 +38,8 @@ type anchorEntry struct { value any origins []originRecord include *includeOrigin + // size is the number of values an alias of this anchor copies. + size int64 // remote is the first remote include evaluated while the anchor was evaluated. remote *includeOrigin } @@ -55,12 +66,30 @@ func (d *configYamlDecoder) decodeAlias(node *yaml.Node, sourceFile string) (any if err != nil { return nil, err } + if err := d.chargeAliasExpansion(node.Alias, entry); err != nil { + return nil, err + } d.replayOrigins(entry.origins) d.nodeOrigin = entry.include d.noteRemote(entry.remote) return cloneDecodedValue(entry.value), nil } +// chargeAliasExpansion adds the size of an alias's copy to the decoder's budget before the copy is +// made, so an alias that would cross the limit never allocates. +func (d *configYamlDecoder) chargeAliasExpansion(anchor *yaml.Node, entry *anchorEntry) error { + limit := d.aliasLimit + if limit == 0 { + limit = maxAliasExpansion + } + if entry.size > limit-d.aliasExpanded { + return fmt.Errorf("%w: %w: alias %q copies %d values after %d were already copied, over the limit of %d", + ErrExecuteYamlFunctions, errUtils.ErrYamlAliasExpansionLimit, anchor.Anchor, entry.size, d.aliasExpanded, limit) + } + d.aliasExpanded += entry.size + return nil +} + // evaluateAnchor evaluates an anchored node at most once per decoder. An alias that refers back to // its own anchor is a cycle and is rejected. func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) (*anchorEntry, error) { @@ -99,7 +128,7 @@ func (d *configYamlDecoder) evaluateAnchor(node *yaml.Node, sourceFile string) ( if err != nil { return nil, err } - entry := &anchorEntry{value: value, origins: records, remote: entryTaint} + entry := &anchorEntry{value: value, origins: records, remote: entryTaint, size: countDecodedValues(value)} if node.Kind == yaml.ScalarNode && isIncludeTag(node.Tag) { entry.include = d.nodeOrigin } @@ -218,6 +247,22 @@ func (d *configYamlDecoder) decodeCollectingOrigins(node *yaml.Node, sourceFile return value, records, err } +// countDecodedValues counts the mappings, sequences, and scalars of a decoded value. +func countDecodedValues(value any) int64 { + count := int64(1) + switch typed := value.(type) { + case map[string]interface{}: + for _, child := range typed { + count += countDecodedValues(child) + } + case []interface{}: + for _, child := range typed { + count += countDecodedValues(child) + } + } + return count +} + // cloneDecodedValue deep-copies the mappings and sequences of a decoded value, so an alias never // shares mutable structure with its anchor. func cloneDecodedValue(value any) any { diff --git a/pkg/config/process_yaml_include_eval.go b/pkg/config/process_yaml_include_eval.go index 2608f60692c..272534e8694 100644 --- a/pkg/config/process_yaml_include_eval.go +++ b/pkg/config/process_yaml_include_eval.go @@ -21,6 +21,11 @@ type configYamlDecoder struct { anchors map[*yaml.Node]*anchorEntry evaluating map[*yaml.Node]bool + // aliasLimit overrides maxAliasExpansion when set; aliasExpanded is the number of values the + // aliases decoded so far have copied. + aliasLimit int64 + aliasExpanded int64 + // nodeOrigin is the file the include or alias just decoded took its value from. The mapping // entry that owns the value reads it so the setting's origin is the included file. nodeOrigin *includeOrigin From e496f0300f388a3c567af28d4807ac53f24c34d9 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:14:06 -0600 Subject: [PATCH 20/32] test(config): cover include fetch-error redaction and decoded passwords A fetch error may quote the password in decoded form, so remove both the as-written and the URL-decoded password, and pin the behavior with a table test that also checks the cause stays reachable through errors.Is. Co-Authored-By: Claude Sonnet 5.5 --- pkg/utils/yaml_include_redact.go | 18 ++++++++++++++++-- pkg/utils/yaml_include_redact_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go index 5b72f1c8cc1..40ec95a9620 100644 --- a/pkg/utils/yaml_include_redact.go +++ b/pkg/utils/yaml_include_redact.go @@ -1,6 +1,7 @@ package utils import ( + neturl "net/url" "strings" "github.com/cloudposse/atmos/pkg/artifact" @@ -43,8 +44,8 @@ func redactIncludeError(err error, target string) error { redacted := message for _, userinfo := range includeUserinfo(target) { redacted = strings.ReplaceAll(redacted, userinfo.full+"@", "") - if userinfo.secret != "" { - redacted = strings.ReplaceAll(redacted, userinfo.secret, artifact.RedactedValue) + for _, secret := range userinfo.secrets() { + redacted = strings.ReplaceAll(redacted, secret, artifact.RedactedValue) } } if redacted == message { @@ -59,6 +60,19 @@ type includeCredential struct { secret string } +// secrets returns the password as written and as a URL parser decodes it, since a fetch error may +// quote either form. +func (c includeCredential) secrets() []string { + if c.secret == "" { + return nil + } + secrets := []string{c.secret} + if decoded, err := neturl.PathUnescape(c.secret); err == nil && decoded != c.secret && decoded != "" { + secrets = append(secrets, decoded) + } + return secrets +} + // includeUserinfo extracts the userinfo of every URL in an include target, including the URL that // follows a forced-getter prefix such as "git::https://". func includeUserinfo(target string) []includeCredential { diff --git a/pkg/utils/yaml_include_redact_test.go b/pkg/utils/yaml_include_redact_test.go index fc665d2edd4..704787757fd 100644 --- a/pkg/utils/yaml_include_redact_test.go +++ b/pkg/utils/yaml_include_redact_test.go @@ -1,6 +1,7 @@ package utils import ( + "errors" "fmt" "net/http" "net/http/httptest" @@ -89,3 +90,29 @@ func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { } } } + +func TestRedactIncludeError(t *testing.T) { + cause := errors.New("pull failed") + tests := []struct { + name string + target string + message string + want string + }{ + {name: "no credential", target: "https://example.com/a.yaml", message: "boom at example.com", want: "boom at example.com"}, + {name: "userinfo is cut out", target: "https://user:hunter2@example.com/a", message: "could not parse reference: user:hunter2@example.com/a", want: "could not parse reference: example.com/a"}, + {name: "password quoted on its own", target: "https://user:hunter2@example.com/a", message: "auth failed for hunter2", want: "auth failed for "}, + {name: "encoded password is also removed decoded", target: "https://user:p%40ss@example.com/a", message: "bad password p@ss", want: "bad password "}, + {name: "forced getter prefix", target: "git::https://user:hunter2@example.com/r.git", message: "clone https://user:hunter2@example.com/r.git failed", want: "clone https://example.com/r.git failed"}, + {name: "user name alone is not a password", target: "ssh://git@example.com/r.git", message: "git failed", want: "git failed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := redactIncludeError(fmt.Errorf("%s: %w", tt.message, cause), tt.target) + require.Error(t, got) + assert.Equal(t, tt.want+": pull failed", got.Error()) + assert.ErrorIs(t, got, cause, "the cause stays in the chain") + }) + } + assert.NoError(t, redactIncludeError(nil, "https://user:hunter2@example.com")) +} From 586e100bf41e3951810fff08bdd0d745b3611de5 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:21:20 -0600 Subject: [PATCH 21/32] refactor(config): flatten the include evaluation and origin recording Split processIncludeNodeValueForFile and handleInclude into named helpers so the include walk, the origin recording and the stored include data each have one responsibility. This clears the nestif, gocritic ifElseChain and cyclomatic findings on the lines this layer changed. Behavior is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/process_yaml.go | 3 +- pkg/config/process_yaml_include_origin.go | 68 +++++++++++++++++++++++ pkg/config/yaml_decode.go | 40 +------------ 3 files changed, 72 insertions(+), 39 deletions(-) create mode 100644 pkg/config/process_yaml_include_origin.go diff --git a/pkg/config/process_yaml.go b/pkg/config/process_yaml.go index 647d6d35f36..aee32c8d573 100644 --- a/pkg/config/process_yaml.go +++ b/pkg/config/process_yaml.go @@ -18,7 +18,8 @@ const ( errorFormat = "%w: %v %v error %w" failedToProcess = "failed to process" emptyValueWarning = "execute returned empty value" - includeDataKey = "include_data" + // includeDataKey is the mapping key an include is evaluated under, so its value keeps its type. + includeDataKey = "include_data" ) var ErrExecuteYamlFunctions = errors.New("failed to execute yaml function") diff --git a/pkg/config/process_yaml_include_origin.go b/pkg/config/process_yaml_include_origin.go new file mode 100644 index 00000000000..2277e1da0a9 --- /dev/null +++ b/pkg/config/process_yaml_include_origin.go @@ -0,0 +1,68 @@ +package config + +import ( + legacyyaml "gopkg.in/yaml.v3" + + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" +) + +// includeResult is what the origin of one evaluated include is derived from. +type includeResult struct { + // localPath is the local file the include resolved to, empty for a remote include. + localPath string + // target is the include argument as written. + target string + decoded any + // walked reports that the included content was evaluated with per-key origins already recorded. + walked bool + // walkedRemote is the first remote include evaluated inside this one. + walkedRemote *includeOrigin +} + +// runInclude loads an include into resolved. A raw include is read verbatim. Otherwise the included +// YAML is walked, and when origins are tracked a remote include is walked as remote content. +func (d *configYamlDecoder) runInclude(atmosConfig *schema.AtmosConfiguration, resolved *legacyyaml.Node, tag, value, sourceFile string) (bool, error) { + switch { + case tag == u.AtmosYamlFuncIncludeRaw: + return false, u.ProcessIncludeRawTag(atmosConfig, resolved, value, sourceFile) + case d.origin == nil: + return u.ProcessIncludeTagWalked(atmosConfig, resolved, value, sourceFile, d.evaluateInclude) + default: + return u.ProcessIncludeTagWalkedFrom(atmosConfig, resolved, value, sourceFile, u.IncludeWalk{ + Local: d.evaluateInclude, + Remote: func(child *legacyyaml.Node, file, source string) error { + previous := d.remoteSource + d.remoteSource = source + defer func() { d.remoteSource = previous }() + return d.evaluateInclude(child, file) + }, + Source: d.remoteSource, + }) + } +} + +// recordIncludeOrigin records where an include's value came from, so the owning key reads the +// included file rather than the file that merely references it. +func (d *configYamlDecoder) recordIncludeOrigin(result includeResult) { + file, remote := result.localPath, false + if file == "" { + file, remote = result.target, true + } + if d.remoteSource != "" { + file, remote = d.remoteSource, true + } + if !result.walked { + d.recordIncludedOrigins(result.decoded, file, remote) + } + origin := &includeOrigin{file: file, remote: remote} + // A local include never launders remote content: when the result is not a mapping (whose + // keys carry their own origins), the remote include nested inside it keeps its origin. + if _, isMapping := result.decoded.(map[string]any); !origin.remote && result.walkedRemote != nil && !isMapping { + origin = result.walkedRemote + } + d.noteRemote(origin) + // Set last, so the outermost include of a nested chain is the one the owning key reads, + // except that a remote origin is never replaced by a local one. + d.nodeOrigin = origin +} diff --git a/pkg/config/yaml_decode.go b/pkg/config/yaml_decode.go index 7777e17efc8..32bb108ca62 100644 --- a/pkg/config/yaml_decode.go +++ b/pkg/config/yaml_decode.go @@ -188,26 +188,9 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour BasePath: scope.BasePath, BasePathAbsolute: scope.BasePath, } - var err error - var walked bool outerTaint := d.remoteTaint d.remoteTaint = nil - if node.Tag == u.AtmosYamlFuncIncludeRaw { - err = u.ProcessIncludeRawTag(atmosConfig, &resolved, value, sourceFile) - } else if d.origin == nil { - walked, err = u.ProcessIncludeTagWalked(atmosConfig, &resolved, value, sourceFile, d.evaluateInclude) - } else { - walked, err = u.ProcessIncludeTagWalkedFrom(atmosConfig, &resolved, value, sourceFile, u.IncludeWalk{ - Local: d.evaluateInclude, - Remote: func(child *legacyyaml.Node, file, source string) error { - previous := d.remoteSource - d.remoteSource = source - defer func() { d.remoteSource = previous }() - return d.evaluateInclude(child, file) - }, - Source: d.remoteSource, - }) - } + walked, err := d.runInclude(atmosConfig, &resolved, node.Tag, value, sourceFile) walkedRemote := d.remoteTaint d.remoteTaint = outerTaint if err != nil { @@ -218,26 +201,7 @@ func (d *configYamlDecoder) processIncludeNodeValueForFile(node *yaml.Node, sour return nil, err } if d.origin != nil { - file, remote := include.LocalPath, false - if file == "" { - file, remote = node.Value, true - } - if d.remoteSource != "" { - file, remote = d.remoteSource, true - } - if !walked { - d.recordIncludedOrigins(decoded, file, remote) - } - origin := &includeOrigin{file: file, remote: remote} - // A local include never launders remote content: when the result is not a mapping (whose - // keys carry their own origins), the remote include nested inside it keeps its origin. - if _, isMapping := decoded.(map[string]any); !origin.remote && walkedRemote != nil && !isMapping { - origin = walkedRemote - } - d.noteRemote(origin) - // Set last, so the outermost include of a nested chain is the one the owning key reads, - // except that a remote origin is never replaced by a local one. - d.nodeOrigin = origin + d.recordIncludeOrigin(includeResult{localPath: include.LocalPath, target: node.Value, decoded: decoded, walked: walked, walkedRemote: walkedRemote}) } return decoded, nil } From b0291bf79ff6151201cc1639f783b1e2961c7f31 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:50:09 -0600 Subject: [PATCH 22/32] fix(artifact): find URL userinfo structurally in every redaction helper An include error that quoted a target with a password holding "::" or "@" leaked part of the password: the include redaction split the target on "::" and cut the userinfo at the first "@". artifact.RedactURL had a sibling weakness for a numeric password holding "/" or "#", which parses as a host and port and was echoed whole, and config.SanitizeImport split an https URL at the "::" of its password. All three now share one structural rule. The userinfo ends at the last "@" before the host, as RFC 3986 requires, and an "@" after the authority that does not introduce a digest is treated as the end of a userinfo too, so a password holding "/", "?" or "#" is still removed. The include error scrubber delegates to the new artifact.RedactCredentials, and a forced-getter prefix such as "git::" is kept instead of replacing the whole argument. A table of hostile inputs lives in pkg/artifact/redacttest and runs against all four helpers as one security property: no user or password survives. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/redact.go | 113 ++++++++++++++++++++++++-- pkg/artifact/redact_test.go | 62 ++++++++++++++ pkg/artifact/redacttest/cases.go | 71 ++++++++++++++++ pkg/config/import_test.go | 34 ++++++++ pkg/config/imports.go | 30 +++---- pkg/utils/yaml_include_redact.go | 51 +----------- pkg/utils/yaml_include_redact_test.go | 36 +++++++- 7 files changed, 327 insertions(+), 70 deletions(-) create mode 100644 pkg/artifact/redacttest/cases.go diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go index 50f9b6941d4..0fbdeba4bc7 100644 --- a/pkg/artifact/redact.go +++ b/pkg/artifact/redact.go @@ -2,6 +2,8 @@ package artifact import ( "net/url" + "regexp" + "sort" "strings" "github.com/cloudposse/atmos/pkg/perf" @@ -10,27 +12,126 @@ import ( // RedactedValue replaces a value that must not be echoed because it may carry credentials. const RedactedValue = "" +var ( + // The getter prefix pattern matches the forced-getter prefix of a go-getter address, such as + // "git::" in "git::https://example.com/repo.git". It only matches when a URL scheme follows, so + // a "::" inside scheme-less userinfo ("user::secret@host") is never mistaken for a prefix. + getterPrefixPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*::[A-Za-z][A-Za-z0-9+.-]*://`) + + // The digest pattern matches a digest ("sha256:") that follows the "@" of a digest reference. + // Only well-known hash algorithms qualify, so a host named like an algorithm is never mistaken + // for one. An "@" followed by a digest introduces content, not a credential. + digestPattern = regexp.MustCompile(`^(?:sha1|sha224|sha256|sha384|sha512|md5|blake3):[0-9a-fA-F]+`) +) + // RedactURL returns raw with any userinfo removed, so that a mistakenly pasted credential is not // repeated in an error message. Strings that cannot be proven free of userinfo are replaced by -// [RedactedValue] instead of being returned verbatim: a value that fails to parse, or that carries -// an "@" outside a URL authority (for example "user:secret@registry.example"), is never echoed. +// [RedactedValue] instead of being returned verbatim: a value that fails to parse, that carries +// an "@" outside a URL authority (for example "user:secret@registry.example"), or whose "@" is not +// the start of a digest, is never echoed. The last "@" of the authority ends the userinfo, as +// RFC 3986 requires, so a password containing "@" or "::" is removed as a whole. A forced-getter +// prefix such as "git::" is kept. func RedactURL(raw string) string { defer perf.Track(nil, "artifact.RedactURL")() if !strings.Contains(raw, "@") { return raw } - parsed, err := url.Parse(raw) + prefix, body := splitGetterPrefix(raw) + parsed, err := url.Parse(body) if err != nil { return RedactedValue } if parsed.User != nil { parsed.User = nil - return parsed.String() + return prefix + parsed.String() } - if parsed.Host == "" { - // An "@" without an authority is scheme-less userinfo such as "user:secret@host". + if parsed.Host == "" || len(credentialBoundaries(afterScheme(body))) > 0 { + // An "@" without an authority is scheme-less userinfo such as "user:secret@host", and an "@" + // after the authority that is not a digest may close a password holding "/", "?" or "#". return RedactedValue } return raw } + +// RedactCredentials removes the credential carried by the URL raw from text, an error message that +// may quote raw, its userinfo, or its password on its own. The userinfo and its trailing "@" are +// cut out, and the password is replaced by [RedactedValue] in the form written and in the form a +// URL parser decodes. A user name without a password is not replaced when it stands alone, since it +// is usually a service account such as "git". The credential is found structurally, never by +// splitting on ":" or "::", so a password holding ":", "::", "@", "/", "?" or "#" is removed whole. +func RedactCredentials(text, raw string) string { + defer perf.Track(nil, "artifact.RedactCredentials")() + + if !strings.Contains(raw, "@") { + return text + } + _, body := splitGetterPrefix(raw) + rest := afterScheme(body) + boundaries := credentialBoundaries(rest) + // Longest userinfo first, so a shorter prefix never leaves the tail of a longer password behind. + sort.Sort(sort.Reverse(sort.IntSlice(boundaries))) + for _, boundary := range boundaries { + text = redactUserinfo(text, rest[:boundary]) + } + return text +} + +// redactUserinfo removes one userinfo, written as in a URL, from text. +func redactUserinfo(text, userinfo string) string { + forms := []string{userinfo} + if decoded, err := url.PathUnescape(userinfo); err == nil && decoded != userinfo { + forms = append(forms, decoded) + } + for _, form := range forms { + text = strings.ReplaceAll(text, form+"@", "") + } + for _, form := range forms { + if _, password, ok := strings.Cut(form, ":"); ok && password != "" { + text = strings.ReplaceAll(text, password, RedactedValue) + } + } + return text +} + +// SplitGetterPrefix separates a forced-getter prefix ("git::") from the URL that follows it. The +// prefix is recognized only when a URL scheme follows it, so a "::" inside userinfo is never taken +// for one. The prefix is empty, and body is raw, when there is none. +func SplitGetterPrefix(raw string) (prefix, body string) { + defer perf.Track(nil, "artifact.SplitGetterPrefix")() + + return splitGetterPrefix(raw) +} + +// splitGetterPrefix is [SplitGetterPrefix] without the performance tracking, for internal callers. +func splitGetterPrefix(raw string) (prefix, body string) { + loc := getterPrefixPattern.FindStringIndex(raw) + if loc == nil { + return "", raw + } + cut := strings.Index(raw, "::") + len("::") + return raw[:cut], raw[cut:] +} + +// afterScheme returns the text after the scheme separator "://", after a leading "//" of a +// scheme-relative reference, or s itself when it has neither. +func afterScheme(s string) string { + if _, rest, ok := strings.Cut(s, "://"); ok { + return rest + } + return strings.TrimPrefix(s, "//") +} + +// credentialBoundaries returns the offset of every "@" in rest that may end a userinfo: any "@" +// that is not the start of a digest. A userinfo is cut at the last "@" of the authority, but a +// password may itself hold "/", "?" or "#", which would end the authority early, so every "@" +// that cannot be proven to introduce a digest counts as a boundary. +func credentialBoundaries(rest string) []int { + var boundaries []int + for i := 0; i < len(rest); i++ { + if rest[i] == '@' && !digestPattern.MatchString(rest[i+1:]) { + boundaries = append(boundaries, i) + } + } + return boundaries +} diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go index 4ee484163d1..dca200cabff 100644 --- a/pkg/artifact/redact_test.go +++ b/pkg/artifact/redact_test.go @@ -1,9 +1,12 @@ package artifact import ( + "strings" "testing" "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" ) func TestRedactURL(t *testing.T) { @@ -31,3 +34,62 @@ func TestRedactURL(t *testing.T) { }) } } + +// TestRedactURL_SharedCredentialCases checks the one security property every redaction helper +// shares: no user or password of a credential-bearing input survives redaction. +func TestRedactURL_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + got := RedactURL(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "redacted form of %q", tc.Raw) + } + }) + } +} + +// TestRedactCredentials_SharedCredentialCases quotes each shared case in an error message, as a +// fetch error does, and checks no secret survives in the scrubbed text. +func TestRedactCredentials_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + messages := []string{"fetch " + tc.Raw + " failed: denied"} + for _, quoted := range tc.Quoted { + messages = append(messages, "authentication failed for "+quoted) + messages = append(messages, "could not parse reference: "+tc.Raw+" (password "+quoted+")") + } + for _, message := range messages { + got := RedactCredentials(message, tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "scrubbed form of %q", message) + } + } + }) + } +} + +func TestRedactCredentials(t *testing.T) { + tests := []struct { + name string + text string + raw string + want string + }{ + {name: "no credential", text: "boom at example.com", raw: "https://example.com/a.yaml", want: "boom at example.com"}, + {name: "userinfo is cut out", text: "could not parse reference: user:hunter2@example.com/a", raw: "https://user:hunter2@example.com/a", want: "could not parse reference: example.com/a"}, + {name: "password quoted on its own", text: "auth failed for hunter2", raw: "https://user:hunter2@example.com/a", want: "auth failed for "}, + {name: "encoded password is also removed decoded", text: "bad password p@ss", raw: "https://user:p%40ss@example.com/a", want: "bad password "}, + {name: "double colon is part of the password", text: "bad password pa::ss for https://user:pa::ss@example.com/a", raw: "https://user:pa::ss@example.com/a", want: "bad password for https://example.com/a"}, + {name: "last at sign ends the userinfo", text: "failed: https://user:pa@ss@second@example.com/a", raw: "https://user:pa@ss@second@example.com/a", want: "failed: https://example.com/a"}, + {name: "forced getter prefix", text: "clone https://user:hunter2@example.com/r.git failed", raw: "git::https://user:hunter2@example.com/r.git", want: "clone https://example.com/r.git failed"}, + {name: "user name alone is not a password", text: "git failed", raw: "ssh://git@example.com/r.git", want: "git failed"}, + {name: "digest at sign is not userinfo", text: "pull ghcr.io/a/b@" + sha256Digest + " failed", raw: "oci://ghcr.io/a/b@" + sha256Digest, want: "pull ghcr.io/a/b@" + sha256Digest + " failed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, RedactCredentials(tt.text, tt.raw)) + }) + } +} + +var sha256Digest = "sha256:" + strings.Repeat("0f", 32) diff --git a/pkg/artifact/redacttest/cases.go b/pkg/artifact/redacttest/cases.go new file mode 100644 index 00000000000..6c04dd11d5a --- /dev/null +++ b/pkg/artifact/redacttest/cases.go @@ -0,0 +1,71 @@ +// Package redacttest holds the credential-bearing inputs that every URL redaction helper must +// handle. The helpers in pkg/artifact and pkg/utils share one security property, so they share one +// table: a test feeds the same cases to each helper and asserts that no secret survives. +package redacttest + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// digestBytes is the number of repeated bytes in the sample digest of the OCI cases. +const digestBytes = 32 + +const ( + // User is a distinctive user name, so a test can tell it apart from any host or path. + User = "usr7" + // SecretA and SecretB are distinctive password fragments. + SecretA = "S3cr3tA" + // SecretB is the second password fragment, used when a delimiter splits a password in two. + SecretB = "S3cr3tB" +) + +// Case is one credential-bearing input. +type Case struct { + // Name describes the shape of the input. + Name string + // Raw is the URL, or URL-like address, that carries the credential. + Raw string + // Secrets are the strings that must never appear in the redacted form of Raw: the user, the + // password, and every fragment a delimiter inside the password could leave behind. + Secrets []string + // Quoted are the passwords an error may quote on their own, in any form a parser can decode. + Quoted []string +} + +// Cases returns the shared table of nasty inputs. Each call returns a fresh slice, so a caller may +// not alias or mutate another caller's table. +func Cases() []Case { + defer perf.Track(nil, "redacttest.Cases")() + + digest := "sha256:" + strings.Repeat("ab", digestBytes) + return []Case{ + {Name: "user and password", Raw: "https://usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a colon", Raw: "https://usr7:S3cr3tA:S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA:S3cr3tB"}}, + {Name: "password with a double colon", Raw: "https://usr7:S3cr3tA::S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + {Name: "password with an at sign", Raw: "https://usr7:S3cr3tA@S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA@S3cr3tB"}}, + {Name: "password with two at signs", Raw: "https://usr7:S3cr3tA@S3cr3tB@second@example.com/a", Secrets: []string{User, SecretA, SecretB, "second"}, Quoted: []string{"S3cr3tA@S3cr3tB@second"}}, + {Name: "password with a hash", Raw: "https://usr7:S3cr3tA#S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA#S3cr3tB"}}, + {Name: "password with a question mark", Raw: "https://usr7:S3cr3tA?S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA?S3cr3tB"}}, + {Name: "password with a slash", Raw: "https://usr7:S3cr3tA/S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA/S3cr3tB"}}, + {Name: "numeric password with a hash", Raw: "https://usr7:12345#S3cr3tB@example.com/a", Secrets: []string{User, "12345", SecretB}, Quoted: []string{"12345#S3cr3tB"}}, + {Name: "numeric password with a slash", Raw: "https://usr7:12345/S3cr3tB@example.com/a", Secrets: []string{User, "12345", SecretB}, Quoted: []string{"12345/S3cr3tB"}}, + {Name: "percent-escaped password", Raw: "https://usr7:S3cr3t%40Z@example.com/a", Secrets: []string{User, "S3cr3t%40Z", "S3cr3t@Z"}, Quoted: []string{"S3cr3t%40Z", "S3cr3t@Z"}}, + {Name: "malformed percent escape", Raw: "https://usr7:S3cr3t%zzZ@example.com/a", Secrets: []string{User, "S3cr3t%zzZ"}, Quoted: []string{"S3cr3t%zzZ"}}, + {Name: "empty user", Raw: "https://:S3cr3tA@example.com/a", Secrets: []string{SecretA}, Quoted: []string{SecretA}}, + {Name: "empty password", Raw: "https://usr7:@example.com/a", Secrets: []string{User}}, + {Name: "empty user and password", Raw: "https://:@example.com/a"}, + {Name: "token as user", Raw: "https://S3cr3tA@example.com/a", Secrets: []string{SecretA}}, + {Name: "ipv6 host", Raw: "https://usr7:S3cr3tA@[::1]:8443/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "userinfo with a space", Raw: "https://usr7:S3cr3tA S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA S3cr3tB"}}, + {Name: "upper-case scheme", Raw: "HTTPS://usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "scheme-relative", Raw: "//usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "no scheme", Raw: "usr7:S3cr3tA@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter", Raw: "git::https://usr7:S3cr3tA@example.com/r.git", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter with subdirectory and query", Raw: "git::https://usr7:S3cr3tA@example.com/r.git//sub?ref=v1", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "forced getter with a double colon password", Raw: "git::https://usr7:S3cr3tA::S3cr3tB@example.com/r.git", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + {Name: "oci reference with a digest", Raw: "oci://usr7:S3cr3tA@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "oci reference with a double colon password and a digest", Raw: "oci://usr7:S3cr3tA::S3cr3tB@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + } +} diff --git a/pkg/config/import_test.go b/pkg/config/import_test.go index fdc18d4f290..c85aee2ae58 100644 --- a/pkg/config/import_test.go +++ b/pkg/config/import_test.go @@ -5,12 +5,15 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/redacttest" log "github.com/cloudposse/atmos/pkg/logger" ) @@ -217,6 +220,21 @@ func TestSanitizeImport(t *testing.T) { input: "gcs::https://storage.googleapis.com/bucket/file?access_token=secret", expected: "gcs::https://storage.googleapis.com/bucket/file", }, + { + name: "forced getter with a double colon password", + input: "git::https://user:pa::ss@example.com/repo.git", + expected: "git::https://example.com/repo.git", + }, + { + name: "password with an at sign", + input: "https://user:pa@ss@second@example.com/file.yaml", + expected: "https://example.com/file.yaml", + }, + { + name: "unparsable URL with credentials is replaced", + input: "https://user:hunter2@exa mple.com/%zz", + expected: artifact.RedactedValue, + }, } for _, tt := range tests { @@ -227,6 +245,22 @@ func TestSanitizeImport(t *testing.T) { } } +// TestSanitizeImport_SharedCredentialCases feeds the table shared by every credential redaction +// helper to SanitizeImport and asserts no user or password of a URL-shaped import survives. +func TestSanitizeImport_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + if !strings.Contains(tc.Raw, "://") { + continue + } + t.Run(tc.Name, func(t *testing.T) { + got := SanitizeImport(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "sanitized form of %q", tc.Raw) + } + }) + } +} + // TestProcessImports_OutsideBaseDirectory verifies that imports outside the base directory // work correctly and only log at trace level (not warning level). func TestProcessImports_OutsideBaseDirectory(t *testing.T) { diff --git a/pkg/config/imports.go b/pkg/config/imports.go index d8ad1cfe149..9b7133bb4bb 100644 --- a/pkg/config/imports.go +++ b/pkg/config/imports.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/viper" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/ci" "github.com/cloudposse/atmos/pkg/condition" "github.com/cloudposse/atmos/pkg/filesystem" @@ -28,25 +29,24 @@ import ( // Exported so other config-loading packages (e.g. pkg/config/adapters) can sanitize import // paths before logging them. func SanitizeImport(s string) string { - // Handle go-getter style URLs with :: separator (e.g., git::https://...). - parts := strings.SplitN(s, "::", 2) - var prefix string - urlPart := s - - if len(parts) == 2 { - prefix = parts[0] + "::" - urlPart = parts[1] + // Handle go-getter style URLs with :: separator (e.g., git::https://...). The prefix is only + // recognized when a scheme follows it, so a "::" inside a password never splits the credential. + prefix, urlPart := artifact.SplitGetterPrefix(s) + + // A URL carrying userinfo is redacted structurally, by the helper every include error shares: + // the userinfo ends at the last "@" of the authority, whatever the password holds. + if strings.Contains(urlPart, "@") && (strings.Contains(urlPart, "://") || strings.HasPrefix(urlPart, "//")) { + urlPart = artifact.RedactURL(urlPart) + if urlPart == artifact.RedactedValue { + return urlPart + } } u, err := url.Parse(urlPart) if err != nil { - // Unparsable; return as-is (might be SCP-style git). - return s - } - - // Clear credentials regardless of scheme. - if u.User != nil { - u.User = nil + // Unparsable; keep the text (it might be SCP-style git) but never a query, which may hold tokens. + before, _, _ := strings.Cut(s, "?") + return before } // Clear query params (may contain tokens/credentials). diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go index 40ec95a9620..dab1654cbab 100644 --- a/pkg/utils/yaml_include_redact.go +++ b/pkg/utils/yaml_include_redact.go @@ -1,7 +1,6 @@ package utils import ( - neturl "net/url" "strings" "github.com/cloudposse/atmos/pkg/artifact" @@ -35,60 +34,16 @@ func (e *redactedIncludeError) Unwrap() error { return e.err } // redactIncludeError removes the credential of an include target from the text of an error raised // while fetching it. Fetch errors from the download and OCI layers quote the reference they were // given (for example "could not parse reference: user:secret@host/image"), so the include -// argument cannot be redacted once for all of them: the userinfo is cut out of the final text. +// argument cannot be redacted once for all of them: the credential is cut out of the final text by +// the same helper that redacts URLs, so both agree on where a userinfo ends. func redactIncludeError(err error, target string) error { if err == nil || !strings.Contains(target, "@") { return err } message := err.Error() - redacted := message - for _, userinfo := range includeUserinfo(target) { - redacted = strings.ReplaceAll(redacted, userinfo.full+"@", "") - for _, secret := range userinfo.secrets() { - redacted = strings.ReplaceAll(redacted, secret, artifact.RedactedValue) - } - } + redacted := artifact.RedactCredentials(message, target) if redacted == message { return err } return &redactedIncludeError{message: redacted, err: err} } - -// includeCredential is the userinfo of one URL found in an include target. -type includeCredential struct { - full string - secret string -} - -// secrets returns the password as written and as a URL parser decodes it, since a fetch error may -// quote either form. -func (c includeCredential) secrets() []string { - if c.secret == "" { - return nil - } - secrets := []string{c.secret} - if decoded, err := neturl.PathUnescape(c.secret); err == nil && decoded != c.secret && decoded != "" { - secrets = append(secrets, decoded) - } - return secrets -} - -// includeUserinfo extracts the userinfo of every URL in an include target, including the URL that -// follows a forced-getter prefix such as "git::https://". -func includeUserinfo(target string) []includeCredential { - var found []includeCredential - for _, part := range strings.Split(target, "::") { - _, rest, ok := strings.Cut(part, "://") - if !ok { - continue - } - authority, _, _ := strings.Cut(rest, "/") - userinfo, _, hasUser := strings.Cut(authority, "@") - if !hasUser { - continue - } - _, secret, _ := strings.Cut(userinfo, ":") - found = append(found, includeCredential{full: userinfo, secret: secret}) - } - return found -} diff --git a/pkg/utils/yaml_include_redact_test.go b/pkg/utils/yaml_include_redact_test.go index 704787757fd..ba2611054dd 100644 --- a/pkg/utils/yaml_include_redact_test.go +++ b/pkg/utils/yaml_include_redact_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cloudposse/atmos/pkg/artifact/redacttest" "github.com/cloudposse/atmos/pkg/schema" ) @@ -29,7 +30,7 @@ func TestRedactIncludeArgument(t *testing.T) { {name: "url without credentials", value: "https://example.com/base.yaml .a", want: "https://example.com/base.yaml .a"}, {name: "url with userinfo", value: "https://user:" + redactTestSecret + "@example.com/base.yaml", want: "https://example.com/base.yaml"}, {name: "url with userinfo and fragment", value: "https://user:" + redactTestSecret + "@example.com/base.yaml#frag", want: "https://example.com/base.yaml#frag"}, - {name: "forced getter with userinfo", value: "git::https://user:" + redactTestSecret + "@example.com/repo.git", want: ""}, + {name: "forced getter with userinfo", value: "git::https://user:" + redactTestSecret + "@example.com/repo.git", want: "git::https://example.com/repo.git"}, {name: "unparsable url with userinfo", value: "https://user:" + redactTestSecret + "@exa mple.com/%zz", want: ""}, } for _, tt := range tests { @@ -105,6 +106,10 @@ func TestRedactIncludeError(t *testing.T) { {name: "encoded password is also removed decoded", target: "https://user:p%40ss@example.com/a", message: "bad password p@ss", want: "bad password "}, {name: "forced getter prefix", target: "git::https://user:hunter2@example.com/r.git", message: "clone https://user:hunter2@example.com/r.git failed", want: "clone https://example.com/r.git failed"}, {name: "user name alone is not a password", target: "ssh://git@example.com/r.git", message: "git failed", want: "git failed"}, + {name: "double colon is part of the password", target: "https://user:pa::ss@example.com/a", message: "bad https://user:pa::ss@example.com/a", want: "bad https://example.com/a"}, + {name: "double colon password quoted on its own", target: "https://user:pa::ss@example.com/a", message: "bad pa::ss", want: "bad "}, + {name: "last at sign ends the userinfo", target: "https://user:pa@ss@second@example.com/a", message: "bad https://user:pa@ss@second@example.com/a", want: "bad https://example.com/a"}, + {name: "password with an at sign quoted on its own", target: "https://user:pa@ss@second@example.com/a", message: "bad pa@ss@second", want: "bad "}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -116,3 +121,32 @@ func TestRedactIncludeError(t *testing.T) { } assert.NoError(t, redactIncludeError(nil, "https://user:hunter2@example.com")) } + +// TestRedactIncludeHelpers_SharedCredentialCases feeds the table shared with artifact.RedactURL to +// both include redaction helpers and asserts the one security property: no user or password of the +// input survives, whether the input is echoed as an argument or quoted by a fetch error. +func TestRedactIncludeHelpers_SharedCredentialCases(t *testing.T) { + cause := errors.New("pull failed") + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Raw, "://") { + got := RedactIncludeArgument(tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "argument %q", tc.Raw) + } + } + messages := []string{"fetch " + tc.Raw + " failed"} + for _, quoted := range tc.Quoted { + messages = append(messages, "auth failed for "+quoted) + } + for _, message := range messages { + got := redactIncludeError(fmt.Errorf("%s: %w", message, cause), tc.Raw) + require.Error(t, got) + for _, secret := range tc.Secrets { + assert.NotContains(t, got.Error(), secret, "error %q", message) + } + assert.ErrorIs(t, got, cause) + } + }) + } +} From 6eafe82abf452d295874f1922cd528cba9fdee42 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sat, 14 Nov 2026 22:52:05 -0600 Subject: [PATCH 23/32] fix(config): group merge-source origins by key before replaying them expandMergeKeys replayed the held-back origins of a merge source by scanning the whole record list once for every copied key. An anchored mapping of 20,000 entries merged into 40 mappings made billions of comparisons while loading a configuration, and the alias limit does not stop it because the aliases copy few values. The records are now grouped by their first path component once per merge source, and each copied key replays only its own records, in their original order, so precedence is unchanged. A test merges a 20,000-entry anchor into 40 mappings with origin tracking on and bounds the time generously: the quadratic scan took 20 seconds, the grouped replay half a second. Co-Authored-By: Claude Sonnet 5.5 --- pkg/config/process_yaml_anchor.go | 40 ++++++++++++++------- pkg/config/process_yaml_origin_test.go | 48 ++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/pkg/config/process_yaml_anchor.go b/pkg/config/process_yaml_anchor.go index 66cfe4ec933..a8d03d3d6af 100644 --- a/pkg/config/process_yaml_anchor.go +++ b/pkg/config/process_yaml_anchor.go @@ -181,28 +181,42 @@ func (d *configYamlDecoder) expandMergeKeys(node *yaml.Node, result map[string]i return err } for _, source := range sources { - for key, value := range source.value { - if _, exists := result[key]; exists { - continue - } - result[key] = value - d.replayKeyOrigins(source.origins, key) - } + d.copyMergeSource(source, result) } } return nil } -// replayKeyOrigins reports the held-back origins of one merged key, relative to the current path. -func (d *configYamlDecoder) replayKeyOrigins(records []originRecord, key string) { - if d.origin == nil { - return +// copyMergeSource copies the entries of one merge source that result does not hold yet and reports +// the held-back origins of each copied key. The origins are grouped by key once, so the work is +// linear in the size of the source: replaying a key by scanning every record would be quadratic for +// an anchored mapping that holds thousands of entries. +func (d *configYamlDecoder) copyMergeSource(source mergeSource, result map[string]interface{}) { + var originsByKey map[string][]originRecord + if d.origin != nil { + originsByKey = groupOriginsByKey(source.origins) } + for key, value := range source.value { + if _, exists := result[key]; exists { + continue + } + result[key] = value + d.replayOrigins(originsByKey[key]) + } +} + +// groupOriginsByKey groups origin records by the first component of their relative path, which is +// the key of the mapping entry that produced them. The records of one key keep their order, so the +// precedence of the replay is the one of the original scan. +func groupOriginsByKey(records []originRecord) map[string][]originRecord { + grouped := make(map[string][]originRecord) for _, record := range records { - if len(record.rel) > 0 && record.rel[0] == key { - d.replayOrigins([]originRecord{record}) + if len(record.rel) == 0 { + continue } + grouped[record.rel[0]] = append(grouped[record.rel[0]], record) } + return grouped } // mergeSources decodes the value of a merge key: a mapping, an alias of one, or a sequence of them. diff --git a/pkg/config/process_yaml_origin_test.go b/pkg/config/process_yaml_origin_test.go index 672497dec80..fd9bbd23c46 100644 --- a/pkg/config/process_yaml_origin_test.go +++ b/pkg/config/process_yaml_origin_test.go @@ -1,9 +1,11 @@ package config import ( + "fmt" "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -163,3 +165,49 @@ func joinPath(path []string) string { } return out } + +// TestDecodeNodeWithYamlFunctions_MergeKeyOriginsAreLinear merges one large anchored mapping into +// many mappings with origin tracking on. Replaying the origins of each copied key must not rescan +// the whole origin list of the source, which is quadratic in the size of the anchor: with the +// numbers below a rescan makes billions of comparisons and runs for tens of seconds. +func TestDecodeNodeWithYamlFunctions_MergeKeyOriginsAreLinear(t *testing.T) { + const ( + keys = 20000 + merges = 40 + ) + var source strings.Builder + source.WriteString("base: &base\n") + for i := 0; i < keys; i++ { + fmt.Fprintf(&source, " key%d: %d\n", i, i) + } + for i := 0; i < merges; i++ { + fmt.Fprintf(&source, "child%d:\n <<: *base\n", i) + } + + document := mustParseYAML(t, source.String()) + decoder := configYamlDecoder{includes: map[string]bool{}, aliasLimit: 100_000_000} + reported := map[string]int{} + decoder.origin = func(path []string, _ string, _ bool) { + if len(path) == 2 && strings.HasPrefix(path[0], "child") { + reported[joinPath(path)]++ + } + } + + start := time.Now() + got, err := decoder.decode(&document, filepath.Join(t.TempDir(), "atmos.yaml")) + elapsed := time.Since(start) + require.NoError(t, err) + + decoded, ok := got.(map[string]any) + require.True(t, ok) + last, ok := decoded["child39"].(map[string]any) + require.True(t, ok) + require.Len(t, last, keys) + assert.Equal(t, 0, last["key0"]) + assert.Equal(t, keys-1, last[fmt.Sprintf("key%d", keys-1)]) + + require.Len(t, reported, keys*merges, "every copied key of every mapping reports its origin") + assert.Equal(t, 1, reported["child0.key0"], "each key is reported exactly once") + assert.Equal(t, 1, reported["child39.key19999"], "each key is reported exactly once") + assert.Less(t, elapsed, 15*time.Second, "origin replay for a merge must be linear in the anchor size") +} From 18ef7a30a555e5ffc03a57b50a41602fa7748ece Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 07:50:12 -0600 Subject: [PATCH 24/32] fix(artifact): redact credentials fail-closed through one shared grammar The redaction helpers kept patching one delimiter at a time and still leaked: a password holding "@" and "/" left a fragment behind, a percent-escaped colon in the user name was decoded before the userinfo was split, and a host named sha256:443 was taken for a digest and hid the only credential boundary. Redesign them as one fail-closed implementation. A value passes through only when it provably holds no credential: no "@" at all, or only "@" signs that introduce a complete digest after a host-shaped authority and a lower-case repository path. Userinfo is cut at the last "@" of the authority, and anything else, including any "@" after the authority, scheme-less userinfo, or a host that is not shaped like one, is replaced as a whole by . RedactCredentials cuts the credential of an address out of free text in every form an error quotes it (written, decoded, quoted, escaped, and the fragments a delimiter cuts loose), splits the encoded userinfo at its literal colon before decoding, withholds every URL-shaped word that still carries userinfo, and withholds the whole text if any part of the password survives. RedactError applies it to an error and keeps the cause in the chain, replacing the private copy in pkg/utils. SanitizeImport routes through the same helper and keeps only local paths and SCP-style addresses without a password. The shared table of hostile inputs grows to 40 cases, including the three reported ones, and a generator embeds a random secret in 19 address templates. Every entry point runs the table and the generated inputs, and FuzzRedact, FuzzRedactInclude and FuzzSanitizeImport fuzz them. The fuzzer found two bypasses (a token as the user cut at a "#" by a parse error, and a stray "%" that stopped percent decoding); their minimized inputs are committed under testdata/fuzz. Diagnostic value in error text drops for URLs containing "@": by design a credential is never echoed. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/redact.go | 189 ++++++++-------- pkg/artifact/redact_grammar.go | 183 +++++++++++++++ pkg/artifact/redact_scrub.go | 213 ++++++++++++++++++ pkg/artifact/redact_test.go | 87 +++++-- pkg/artifact/redacttest/cases.go | 19 ++ pkg/artifact/redacttest/generate.go | 132 +++++++++++ .../testdata/fuzz/FuzzRedact/72758857c5de516f | 4 + .../testdata/fuzz/FuzzRedact/b6670d02d88a615c | 4 + pkg/config/import_fuzz_test.go | 31 +++ pkg/config/imports.go | 18 +- pkg/utils/yaml_include_redact.go | 24 +- pkg/utils/yaml_include_redact_fuzz_test.go | 69 ++++++ 12 files changed, 840 insertions(+), 133 deletions(-) create mode 100644 pkg/artifact/redact_grammar.go create mode 100644 pkg/artifact/redact_scrub.go create mode 100644 pkg/artifact/redacttest/generate.go create mode 100644 pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f create mode 100644 pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c create mode 100644 pkg/config/import_fuzz_test.go create mode 100644 pkg/utils/yaml_include_redact_fuzz_test.go diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go index 0fbdeba4bc7..7068e063d75 100644 --- a/pkg/artifact/redact.go +++ b/pkg/artifact/redact.go @@ -2,8 +2,6 @@ package artifact import ( "net/url" - "regexp" - "sort" "strings" "github.com/cloudposse/atmos/pkg/perf" @@ -12,86 +10,130 @@ import ( // RedactedValue replaces a value that must not be echoed because it may carry credentials. const RedactedValue = "" -var ( - // The getter prefix pattern matches the forced-getter prefix of a go-getter address, such as - // "git::" in "git::https://example.com/repo.git". It only matches when a URL scheme follows, so - // a "::" inside scheme-less userinfo ("user::secret@host") is never mistaken for a prefix. - getterPrefixPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*::[A-Za-z][A-Za-z0-9+.-]*://`) - - // The digest pattern matches a digest ("sha256:") that follows the "@" of a digest reference. - // Only well-known hash algorithms qualify, so a host named like an algorithm is never mistaken - // for one. An "@" followed by a digest introduces content, not a credential. - digestPattern = regexp.MustCompile(`^(?:sha1|sha224|sha256|sha384|sha512|md5|blake3):[0-9a-fA-F]+`) -) - // RedactURL returns raw with any userinfo removed, so that a mistakenly pasted credential is not -// repeated in an error message. Strings that cannot be proven free of userinfo are replaced by -// [RedactedValue] instead of being returned verbatim: a value that fails to parse, that carries -// an "@" outside a URL authority (for example "user:secret@registry.example"), or whose "@" is not -// the start of a digest, is never echoed. The last "@" of the authority ends the userinfo, as -// RFC 3986 requires, so a password containing "@" or "::" is removed as a whole. A forced-getter -// prefix such as "git::" is kept. +// repeated in an error message or a log line. It is the one implementation every caller shares. +// +// RedactURL is fail-closed. A value without an "@" cannot hold userinfo and is returned as it is. +// A value whose "@" signs all sit in the authority has its userinfo cut at the last one, as +// RFC 3986 requires, and is returned when what is left is a well-formed host. A value that cannot +// be proven free of a credential is replaced by [RedactedValue] as a whole: an "@" after the +// authority that does not introduce a complete digest (a password that held "/", "?", "#" or "\" +// may have closed there), scheme-less userinfo such as "user:secret@registry.example", a host that +// is not shaped like a host, or an address that does not parse. The diagnostic value of an error +// drops for such inputs by design; a credential is never echoed. A forced-getter prefix such as +// "git::" is kept. func RedactURL(raw string) string { defer perf.Track(nil, "artifact.RedactURL")() if !strings.Contains(raw, "@") { return raw } - prefix, body := splitGetterPrefix(raw) - parsed, err := url.Parse(body) - if err != nil { + a := parseAddress(raw) + if len(a.stray) > 0 { return RedactedValue } - if parsed.User != nil { - parsed.User = nil - return prefix + parsed.String() + if len(a.inside) == 0 { + // Only digest introducers: the address carries no userinfo, but it is still checked. + if a.wellFormed(0) { + return raw + } + return RedactedValue } - if parsed.Host == "" || len(credentialBoundaries(afterScheme(body))) > 0 { - // An "@" without an authority is scheme-less userinfo such as "user:secret@host", and an "@" - // after the authority that is not a digest may close a password holding "/", "?" or "#". + // Scheme-less userinfo cannot be told from a host and a path, so it is never cut out. + if a.scheme == "" { return RedactedValue } - return raw + start := a.inside[len(a.inside)-1] + 1 + if !a.wellFormed(start) { + return RedactedValue + } + return a.prefix + a.scheme + a.rest[start:] } -// RedactCredentials removes the credential carried by the URL raw from text, an error message that -// may quote raw, its userinfo, or its password on its own. The userinfo and its trailing "@" are -// cut out, and the password is replaced by [RedactedValue] in the form written and in the form a -// URL parser decodes. A user name without a password is not replaced when it stands alone, since it -// is usually a service account such as "git". The credential is found structurally, never by -// splitting on ":" or "::", so a password holding ":", "::", "@", "/", "?" or "#" is removed whole. +// wellFormed reports whether the address from offset start of its remainder on has a safe host and +// parses as a URL. +func (a *address) wellFormed(start int) bool { + if !a.hostIsSafe(start) { + return false + } + if a.scheme == "" { + return true + } + _, err := url.Parse(headOf(a.scheme + a.rest[start:])) + return err == nil +} + +// RedactCredentials removes credentials from text, an error message or log line that may quote +// raw, its userinfo, its password on its own, or any other URL. It is fail-closed, in three steps: +// +// 1. The credential carried by raw is cut out of text wherever it is quoted: the userinfo with its +// trailing "@", written as in a URL, decoded, or escaped for a quote, and the password on its +// own together with every fragment of it that a delimiter cut loose. The password is the part +// of the userinfo as written after its first ":", decoded separately. A user name without a +// password is not replaced when it stands alone, since it is usually a service account such as +// "git". +// 2. Every other URL-shaped word of text that still holds an "@" which cannot be proven to +// introduce a digest is replaced by [RedactedValue] as a whole, and so is every word that still +// holds an "@" when raw carried a credential. +// 3. If any part of the password is still present, the whole text is replaced by [RedactedValue]. func RedactCredentials(text, raw string) string { defer perf.Track(nil, "artifact.RedactCredentials")() - if !strings.Contains(raw, "@") { - return text + a := parseAddress(raw) + secret := a.credential() + if !secret.empty() { + text = secret.scrub(text) } - _, body := splitGetterPrefix(raw) - rest := afterScheme(body) - boundaries := credentialBoundaries(rest) - // Longest userinfo first, so a shorter prefix never leaves the tail of a longer password behind. - sort.Sort(sort.Reverse(sort.IntSlice(boundaries))) - for _, boundary := range boundaries { - text = redactUserinfo(text, rest[:boundary]) + text = redactWords(text, !secret.empty()) + if secret.survives(text) { + return RedactedValue } return text } -// redactUserinfo removes one userinfo, written as in a URL, from text. -func redactUserinfo(text, userinfo string) string { - forms := []string{userinfo} - if decoded, err := url.PathUnescape(userinfo); err == nil && decoded != userinfo { - forms = append(forms, decoded) +// RedactError returns err with its message scrubbed of the credentials of every address in raws +// and of any other URL-shaped word that carries userinfo (see [RedactCredentials]). The result keeps +// err in its chain, so errors.Is and errors.As still see the cause, and it is err itself when the +// message needs no change. +func RedactError(err error, raws ...string) error { + defer perf.Track(nil, "artifact.RedactError")() + + if err == nil { + return nil } - for _, form := range forms { - text = strings.ReplaceAll(text, form+"@", "") + message := err.Error() + scrubbed := message + for _, raw := range raws { + scrubbed = RedactCredentials(scrubbed, raw) } - for _, form := range forms { - if _, password, ok := strings.Cut(form, ":"); ok && password != "" { - text = strings.ReplaceAll(text, password, RedactedValue) - } + if len(raws) == 0 { + scrubbed = RedactCredentials(scrubbed, "") } - return text + if scrubbed == message { + return err + } + return &redactedError{message: scrubbed, err: err} +} + +// redactedError is an error whose text has had a credential removed. It keeps the original error in +// its chain so errors.Is and errors.As still see the cause. +type redactedError struct { + message string + err error +} + +//nolint:lintroller // Trivial getter with no I/O. +func (e *redactedError) Error() string { return e.message } + +//nolint:lintroller // Trivial getter with no I/O. +func (e *redactedError) Unwrap() error { return e.err } + +// IsSCPAddress reports whether s is an SCP-style git address such as "git@github.com:org/repo.git". +// Its user carries no password, so it holds no credential. +func IsSCPAddress(s string) bool { + defer perf.Track(nil, "artifact.IsSCPAddress")() + + return scpPattern.MatchString(s) } // SplitGetterPrefix separates a forced-getter prefix ("git::") from the URL that follows it. The @@ -102,36 +144,3 @@ func SplitGetterPrefix(raw string) (prefix, body string) { return splitGetterPrefix(raw) } - -// splitGetterPrefix is [SplitGetterPrefix] without the performance tracking, for internal callers. -func splitGetterPrefix(raw string) (prefix, body string) { - loc := getterPrefixPattern.FindStringIndex(raw) - if loc == nil { - return "", raw - } - cut := strings.Index(raw, "::") + len("::") - return raw[:cut], raw[cut:] -} - -// afterScheme returns the text after the scheme separator "://", after a leading "//" of a -// scheme-relative reference, or s itself when it has neither. -func afterScheme(s string) string { - if _, rest, ok := strings.Cut(s, "://"); ok { - return rest - } - return strings.TrimPrefix(s, "//") -} - -// credentialBoundaries returns the offset of every "@" in rest that may end a userinfo: any "@" -// that is not the start of a digest. A userinfo is cut at the last "@" of the authority, but a -// password may itself hold "/", "?" or "#", which would end the authority early, so every "@" -// that cannot be proven to introduce a digest counts as a boundary. -func credentialBoundaries(rest string) []int { - var boundaries []int - for i := 0; i < len(rest); i++ { - if rest[i] == '@' && !digestPattern.MatchString(rest[i+1:]) { - boundaries = append(boundaries, i) - } - } - return boundaries -} diff --git a/pkg/artifact/redact_grammar.go b/pkg/artifact/redact_grammar.go new file mode 100644 index 00000000000..6e0e0e65d17 --- /dev/null +++ b/pkg/artifact/redact_grammar.go @@ -0,0 +1,183 @@ +package artifact + +import ( + "regexp" + "strings" + "unicode" +) + +// The redaction helpers in this package are fail-closed. They pass a value through only when it +// matches the small grammar below, which they can prove carries no credential, and replace +// everything else, either the whole value or the whole suspicious token, with [RedactedValue]. +// +// An address is "[getter::][scheme://|//]authority[/path][?query][#fragment]". The authority ends +// at the first "/", "?", "#" or "\". Every "@" in the authority ends a userinfo, and the last one +// wins. An "@" after the authority is safe only as the introducer of a complete digest +// ("@sha256:<64 hex digits>") that follows a host and a lower-case repository path, as in +// "ghcr.io/acme/catalog@sha256:...": any other one may close a password that held a "/", "?", "#" +// or "\", so the value cannot be proven free of a credential. +var ( + // The getter prefix pattern matches the forced-getter prefix of a go-getter address, such as + // "git::" in "git::https://example.com/repo.git". It only matches when a URL scheme follows, so + // a "::" inside scheme-less userinfo ("user::secret@host") is never mistaken for a prefix. + getterPrefixPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*::[A-Za-z][A-Za-z0-9+.-]*://`) + + // The scheme pattern matches the "scheme://" that starts an address. + schemePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*://`) + + // The authority pattern is the only shape of host that survives redaction: a name, an address, + // or a bracketed IPv6 literal, with an optional port. Anything else may be a password fragment. + authorityPattern = regexp.MustCompile(`^(?:[A-Za-z0-9_.-]+|\[[0-9A-Za-z:.%]+\])(?::[0-9]*)?$`) + + // The repository path pattern matches the path in front of the "@" of a digest reference: the + // lower-case path of an OCI repository with an optional tag. A path that holds anything else + // might be a password that held a "/". + repositoryPathPattern = regexp.MustCompile(`^/?[a-z0-9]+(?:[._/-]+[a-z0-9]+)*(?::[A-Za-z0-9_][A-Za-z0-9._-]*)?$`) + + // The SCP pattern matches an SCP-style git address, "user@host:path", whose user carries no + // password: a ":" before the "@" would make it a credential. + scpPattern = regexp.MustCompile(`^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:[^@\s]*$`) +) + +const ( + // The authority terminators end the authority of an address. + authorityTerminators = `/?#\` + + // The minimum fragment length is the shortest password fragment that is removed on its own from free + // text. Shorter fragments would mangle ordinary words, and they identify nothing. + minFragmentLength = 4 + + // The scheme separator separates a scheme from its authority. + schemeSeparator = "://" + + // The file scheme is the only scheme whose authority may be empty. + fileScheme = "file://" +) + +// digestHexLengths holds the length in hex digits of each digest algorithm that may follow an "@". +// Only these well-known algorithms qualify, so a host named like an algorithm is never mistaken for +// one. +var digestHexLengths = map[string]int{ + "sha1": 40, + "sha224": 56, + "sha256": 64, + "sha384": 96, + "sha512": 128, + "md5": 32, + "blake3": 64, +} + +// address is an address split at the places that decide whether it carries a credential. +type address struct { + // prefix is the forced-getter prefix, "git::" for example, or empty. + prefix string + // scheme is "scheme://", "//", or empty when the address has none. + scheme string + // rest is the address after the scheme. + rest string + // authorityEnd is the offset in rest where the authority ends. + authorityEnd int + // inside are the offsets in rest of every "@" in the authority. + inside []int + // stray are the offsets in rest of every "@" after the authority that does not introduce a + // digest. + stray []int +} + +// parseAddress splits raw at the getter prefix, the scheme, the authority, and every "@". +func parseAddress(raw string) address { + a := address{} + a.prefix, raw = splitGetterPrefix(raw) + if loc := schemePattern.FindStringIndex(raw); loc != nil { + a.scheme, a.rest = raw[:loc[1]], raw[loc[1]:] + } else if strings.HasPrefix(raw, "//") { + a.scheme, a.rest = "//", raw[2:] + } else { + a.rest = raw + } + a.authorityEnd = strings.IndexAny(a.rest, authorityTerminators) + if a.authorityEnd < 0 { + a.authorityEnd = len(a.rest) + } + for i := 0; i < a.authorityEnd; i++ { + if a.rest[i] == '@' { + a.inside = append(a.inside, i) + } + } + for i := a.authorityEnd; i < len(a.rest); i++ { + if a.rest[i] == '@' && !a.introducesDigest(i) { + a.stray = append(a.stray, i) + } + } + return a +} + +// introducesDigest reports whether the "@" at offset i of the remainder, after the authority, +// introduces a digest: a complete digest follows it, the host in front is shaped like a host, and +// the path in front is the lower-case path of a repository. +func (a *address) introducesDigest(i int) bool { + hostStart := 0 + if len(a.inside) > 0 { + hostStart = a.inside[len(a.inside)-1] + 1 + } + return digestFollows(a.rest[i+1:]) && a.hostIsSafe(hostStart) && repositoryPathPattern.MatchString(a.rest[a.authorityEnd:i]) +} + +// boundaries returns the offset of every "@" that may end a userinfo, in order. +func (a *address) boundaries() []int { + out := make([]int, 0, len(a.inside)+len(a.stray)) + out = append(out, a.inside...) + return append(out, a.stray...) +} + +// digestFollows reports whether s, the text after an "@", starts with a complete digest such as +// "sha256:<64 hex digits>" that is not continued by further letters or digits. +func digestFollows(s string) bool { + algorithm, rest, found := strings.Cut(s, ":") + length, known := digestHexLengths[algorithm] + if !found || !known || len(rest) < length { + return false + } + for i := 0; i < length; i++ { + if !isHexDigit(rest[i]) { + return false + } + } + return len(rest) == length || !isAlphanumeric(rest[length]) +} + +func isHexDigit(c byte) bool { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' +} + +func isAlphanumeric(c byte) bool { + return isHexDigit(c) || c >= 'g' && c <= 'z' || c >= 'G' && c <= 'Z' +} + +// hostIsSafe reports whether the host part of the address, the authority after any userinfo, has +// the shape of a host name or address. A host that has not is not echoed: it may be a password +// fragment that a delimiter cut loose. +func (a *address) hostIsSafe(start int) bool { + host := a.rest[start:a.authorityEnd] + if host == "" { + return a.scheme == fileScheme + } + return authorityPattern.MatchString(host) +} + +// headOf returns s up to its first white space, the part of an include argument that is a URL. +func headOf(s string) string { + if i := strings.IndexFunc(s, unicode.IsSpace); i >= 0 { + return s[:i] + } + return s +} + +// splitGetterPrefix separates a forced-getter prefix ("git::") from the URL that follows it. +func splitGetterPrefix(raw string) (prefix, body string) { + if !getterPrefixPattern.MatchString(raw) { + return "", raw + } + cut := strings.Index(raw, "::") + len("::") + return raw[:cut], raw[cut:] +} diff --git a/pkg/artifact/redact_scrub.go b/pkg/artifact/redact_scrub.go new file mode 100644 index 00000000000..2f426de0e63 --- /dev/null +++ b/pkg/artifact/redact_scrub.go @@ -0,0 +1,213 @@ +package artifact + +import ( + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "unicode" +) + +const ( + // The hex base is the base of the digits of a percent escape. + hexBase = 16 + // The byte bits are the size of the byte a percent escape encodes. + byteBits = 8 + // The escape length is the length of "%XX". + escapeLength = 3 +) + +// wordPattern matches a white-space delimited word of free text. +var wordPattern = regexp.MustCompile(`\S+`) + +// accountPattern matches a plain account name, the only user name that may stand alone in text. +var accountPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{0,64}$`) + +// credentialSet is everything one address can leak into free text. +type credentialSet struct { + // cuts are the userinfos, each with its trailing "@", that are removed from text. + cuts []string + // passwords are the passwords, in every form an error may quote them, and the fragments a + // delimiter inside a password cuts loose. They are replaced by [RedactedValue]. + passwords []string +} + +// empty reports whether the address carried no credential. +func (c credentialSet) empty() bool { + return len(c.cuts) == 0 +} + +// credential collects the credential of the address. Every "@" that may end a userinfo is +// considered, so a password holding "@", "/", "?", "#" or "\" is found whole. +func (a *address) credential() credentialSet { + var set credentialSet + for _, boundary := range a.boundaries() { + userinfo := a.rest[:boundary] + for _, form := range textForms(userinfo) { + set.cuts = append(set.cuts, form+"@") + } + // The userinfo is split at its literal colon first, while it is still encoded, and only the + // password is decoded afterwards: an escaped colon belongs to the user name. + user, password, found := strings.Cut(userinfo, ":") + if !found || password == "" { + // A plain account name such as "git" is not a secret when it stands alone. Anything else + // in the user position is a token, and a delimiter inside it may cut it into fragments. + if !accountPattern.MatchString(user) { + set.passwords = append(set.passwords, fragmentsOf(user)...) + } + continue + } + // A parser may quote the user name followed by the start of the password, with the "@" cut + // off by a delimiter inside the password, so "user:" goes with its password. + for _, form := range textForms(user) { + set.cuts = append(set.cuts, form+":") + } + // Once a password is known the user name is withheld too, wherever it is quoted: it names + // the account the password opens. + set.passwords = append(set.passwords, fragmentsOf(user)...) + set.passwords = append(set.passwords, fragmentsOf(password)...) + } + set.cuts = unique(set.cuts) + set.passwords = unique(set.passwords) + return set +} + +// fragmentsOf returns s in every form an error may quote it, and the fragments of each form that a +// delimiter inside s cuts loose. +func fragmentsOf(s string) []string { + var out []string + for _, form := range textForms(s) { + out = append(out, form) + for _, fragment := range strings.FieldsFunc(form, isDelimiter) { + if len(fragment) >= minFragmentLength { + out = append(out, fragment) + } + } + } + return out +} + +// isDelimiter reports whether r separates the fragments of a password. +func isDelimiter(r rune) bool { + return unicode.IsSpace(r) || strings.ContainsRune(`:@/?#\[]"'`, r) +} + +// textForms returns s as an error message may quote it: as written, decoded, quoted, and escaped. +func textForms(s string) []string { + bases := []string{s, lenientUnescape(s)} + if decoded, err := url.PathUnescape(s); err == nil { + bases = append(bases, decoded) + } + if decoded, err := url.QueryUnescape(s); err == nil { + bases = append(bases, decoded) + } + forms := make([]string, 0, len(bases)*4) + for _, base := range bases { + quoted := strconv.Quote(base) + forms = append(forms, base, quoted[1:len(quoted)-1], url.PathEscape(base), url.QueryEscape(base)) + } + return unique(forms) +} + +// lenientUnescape decodes every valid "%XX" escape of s and leaves anything else as it is, so a +// stray "%" never keeps a decoded form from being tried. A parser may decode what a strict decoder +// rejects, and an error message quotes whatever the parser saw. +func lenientUnescape(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+escapeLength-1 < len(s) && isHexDigit(s[i+1]) && isHexDigit(s[i+2]) { + high, _ := strconv.ParseUint(s[i+1:i+escapeLength], hexBase, byteBits) + b.WriteByte(byte(high)) + i += escapeLength - 1 + continue + } + b.WriteByte(s[i]) + } + return b.String() +} + +// unique drops empty and repeated strings, longest first, so that a longer form is always tried +// before a shorter form it contains. +func unique(in []string) []string { + seen := make(map[string]bool, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if s != "" && !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) }) + return out +} + +// scrub removes the credential from text in a single pass, longest forms first. +func (c credentialSet) scrub(text string) string { + type pair struct{ old, replacement string } + pairs := make([]pair, 0, len(c.cuts)+len(c.passwords)) + for _, cut := range c.cuts { + pairs = append(pairs, pair{cut, ""}) + } + for _, password := range c.passwords { + pairs = append(pairs, pair{password, RedactedValue}) + } + sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].old) > len(pairs[j].old) }) + args := make([]string, 0, len(pairs)*2) + for _, p := range pairs { + args = append(args, p.old, p.replacement) + } + return strings.NewReplacer(args...).Replace(text) +} + +// survives reports whether any part of the credential is still in text, which means it was not +// removed and the whole text must be withheld. The marker that replaced a credential is ignored. +func (c credentialSet) survives(text string) bool { + if c.empty() { + return false + } + text = strings.ReplaceAll(text, RedactedValue, "") + for _, password := range c.passwords { + if strings.Contains(text, password) { + return true + } + } + for _, cut := range c.cuts { + if strings.Contains(text, cut) { + return true + } + } + return false +} + +// redactWords replaces every URL-shaped word of text that holds an "@" which cannot be proven to +// introduce a digest. When bearing is true, because an address that carried a credential was +// scrubbed from the text, any word with such an "@" is replaced, URL-shaped or not. +func redactWords(text string, bearing bool) string { + if !strings.Contains(text, "@") { + return text + } + return wordPattern.ReplaceAllStringFunc(text, func(word string) string { + if !strings.Contains(word, "@") { + return word + } + body := word + if i := strings.Index(word, schemeSeparator); i >= 0 { + for i > 0 && isSchemeByte(word[i-1]) { + i-- + } + body = word[i:] + } else if !bearing { + return word + } + if a := parseAddress(body); len(a.boundaries()) > 0 { + return RedactedValue + } + return word + }) +} + +// isSchemeByte reports whether c may appear in a URL scheme. +func isSchemeByte(c byte) bool { + return isAlphanumeric(c) || c == '+' || c == '.' || c == '-' +} diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go index dca200cabff..5d7dd593fa5 100644 --- a/pkg/artifact/redact_test.go +++ b/pkg/artifact/redact_test.go @@ -1,6 +1,8 @@ package artifact import ( + "errors" + "fmt" "strings" "testing" @@ -17,10 +19,21 @@ func TestRedactURL(t *testing.T) { }{ {name: "empty", raw: "", want: ""}, {name: "no userinfo", raw: "oci://ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, - {name: "digest reference keeps its at sign", raw: "oci://ghcr.io/acme/catalog@sha256:abc", want: "oci://ghcr.io/acme/catalog@sha256:abc"}, + {name: "digest reference keeps its at sign", raw: "oci://ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + {name: "a digest that is not complete is not a digest", raw: "oci://ghcr.io/acme/catalog@sha256:abc", want: RedactedValue}, + {name: "unknown algorithm is not a digest", raw: "oci://ghcr.io/acme/catalog@sha999:" + strings.Repeat("0f", 32), want: RedactedValue}, + {name: "host named like a digest algorithm", raw: "https://user:hunter2@sha256:443/a", want: "https://sha256:443/a"}, + {name: "scheme-less digest reference", raw: "ghcr.io/acme/catalog@" + sha256Digest, want: "ghcr.io/acme/catalog@" + sha256Digest}, + {name: "scheme-less digest of a bare name", raw: "catalog@" + sha256Digest, want: RedactedValue}, + {name: "at sign after the authority is not a digest", raw: "https://user:hunter2/x@example.com/a", want: RedactedValue}, + {name: "at sign in the query", raw: "https://example.com/a?mail=user@example.com", want: RedactedValue}, + {name: "host that is not shaped like a host", raw: "https://user:hunter2@exa mple.com/a", want: RedactedValue}, {name: "user and password", raw: "oci://user:hunter2@ghcr.io/acme/catalog:stable", want: "oci://ghcr.io/acme/catalog:stable"}, {name: "token as user", raw: "https://hunter2@issuer.example", want: "https://issuer.example"}, - {name: "userinfo and digest", raw: "oci://user:hunter2@ghcr.io/acme/catalog@sha256:abc", want: "oci://ghcr.io/acme/catalog@sha256:abc"}, + {name: "userinfo and digest", raw: "oci://user:hunter2@ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + {name: "forced getter prefix is kept", raw: "git::https://user:hunter2@example.com/r.git", want: "git::https://example.com/r.git"}, + {name: "scheme-relative userinfo", raw: "//user:hunter2@example.com/a", want: "//example.com/a"}, + {name: "password with an at sign", raw: "https://user:pa@ss@example.com/a", want: "https://example.com/a"}, {name: "malformed escape with userinfo", raw: "oci://user:hunter2@ghcr.io/acme/%zz:stable", want: RedactedValue}, {name: "scheme-less userinfo", raw: "user:hunter2@ghcr.io", want: RedactedValue}, {name: "scheme-less userinfo and port", raw: "user:hunter2@ghcr.io:5000", want: RedactedValue}, @@ -35,39 +48,77 @@ func TestRedactURL(t *testing.T) { } } -// TestRedactURL_SharedCredentialCases checks the one security property every redaction helper -// shares: no user or password of a credential-bearing input survives redaction. -func TestRedactURL_SharedCredentialCases(t *testing.T) { +// TestRedact_SharedCredentialCases feeds the shared table to every redaction entry point of this +// package and checks the one security property they share: no user or password of a +// credential-bearing input survives, whichever way an error quotes it. +func TestRedact_SharedCredentialCases(t *testing.T) { for _, tc := range redacttest.Cases() { t.Run(tc.Name, func(t *testing.T) { got := RedactURL(tc.Raw) for _, secret := range tc.Secrets { - assert.NotContains(t, got, secret, "redacted form of %q", tc.Raw) + assert.NotContains(t, got, secret, "RedactURL of %q", tc.Raw) } - }) - } -} - -// TestRedactCredentials_SharedCredentialCases quotes each shared case in an error message, as a -// fetch error does, and checks no secret survives in the scrubbed text. -func TestRedactCredentials_SharedCredentialCases(t *testing.T) { - for _, tc := range redacttest.Cases() { - t.Run(tc.Name, func(t *testing.T) { - messages := []string{"fetch " + tc.Raw + " failed: denied"} + messages := []string{"fetch " + tc.Raw + " failed: denied", fmt.Sprintf("fetch %q failed", tc.Raw), "bad: " + got} for _, quoted := range tc.Quoted { messages = append(messages, "authentication failed for "+quoted) messages = append(messages, "could not parse reference: "+tc.Raw+" (password "+quoted+")") + messages = append(messages, fmt.Sprintf("password %q rejected", quoted)) } for _, message := range messages { - got := RedactCredentials(message, tc.Raw) + scrubbed := RedactCredentials(message, tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, scrubbed, secret, "RedactCredentials of %q", message) + } + err := RedactError(fmt.Errorf("%w: %s", errSentinel, message), tc.Raw) for _, secret := range tc.Secrets { - assert.NotContains(t, got, secret, "scrubbed form of %q", message) + assert.NotContains(t, err.Error(), secret, "RedactError of %q", message) } + assert.ErrorIs(t, err, errSentinel, "the cause stays in the chain") } }) } } +var errSentinel = errors.New("fetch failed") + +// TestRedact_GeneratedInputs runs the generated inputs, built from every delimiter of the shared +// table, through every entry point. +func TestRedact_GeneratedInputs(t *testing.T) { + for _, separator := range redacttest.Seeds() { + secret := redacttest.SecretOf("A", separator, "B") + for _, in := range redacttest.Generate(secret) { + checkEntryPoints(t, in) + } + } +} + +// checkEntryPoints runs one generated input through every entry point of this package. +func checkEntryPoints(t testing.TB, in redacttest.Input) { + t.Helper() + redacttest.RequireNoLeak(t, "RedactURL", in, RedactURL(in.Raw)) + for _, message := range in.Messages() { + redacttest.RequireNoLeak(t, "RedactCredentials", in, RedactCredentials(message, in.Raw)) + redacttest.RequireNoLeak(t, "RedactError", in, RedactError(fmt.Errorf("%w: %s", errSentinel, message), in.Raw).Error()) + } +} + +// FuzzRedact embeds a random secret in many address templates and checks that no entry point +// leaks the user name or any fragment of the secret. The seeds are the delimiters of the shared +// table, so every input of the table is a starting point. +func FuzzRedact(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + f.Add("x"+separator+"y", separator, "z"+separator) + } + f.Fuzz(func(t *testing.T, first, separator, second string) { + secret := redacttest.SecretOf(first, separator, second) + for _, in := range redacttest.Generate(secret) { + checkEntryPoints(t, in) + } + }) +} + func TestRedactCredentials(t *testing.T) { tests := []struct { name string diff --git a/pkg/artifact/redacttest/cases.go b/pkg/artifact/redacttest/cases.go index 6c04dd11d5a..c43e4245411 100644 --- a/pkg/artifact/redacttest/cases.go +++ b/pkg/artifact/redacttest/cases.go @@ -19,6 +19,11 @@ const ( SecretA = "S3cr3tA" // SecretB is the second password fragment, used when a delimiter splits a password in two. SecretB = "S3cr3tB" + // SecretC is the third password fragment, used when two delimiters split a password in three. + SecretC = "S3cr3tC" + // Marker starts every secret the generated inputs carry, so a leak of any fragment of a secret, + // however a delimiter cut it, is a leak of the marker. + Marker = "S3cr3t" ) // Case is one credential-bearing input. @@ -65,6 +70,20 @@ func Cases() []Case { {Name: "forced getter", Raw: "git::https://usr7:S3cr3tA@example.com/r.git", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, {Name: "forced getter with subdirectory and query", Raw: "git::https://usr7:S3cr3tA@example.com/r.git//sub?ref=v1", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, {Name: "forced getter with a double colon password", Raw: "git::https://usr7:S3cr3tA::S3cr3tB@example.com/r.git", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, + {Name: "password with an at sign and a slash", Raw: "https://usr7:S3cr3tA@S3cr3tB/S3cr3tC@example.com/a", Secrets: []string{User, SecretA, SecretB, SecretC}, Quoted: []string{"S3cr3tA@S3cr3tB/S3cr3tC"}}, + {Name: "escaped colon in the user name", Raw: "https://usr7%3AS3cr3tB:S3cr3tA%40S3cr3tC@example.com/a", Secrets: []string{User, SecretA, SecretB, SecretC, "S3cr3tA%40S3cr3tC"}, Quoted: []string{"S3cr3tA%40S3cr3tC", "S3cr3tA@S3cr3tC"}}, + {Name: "escaped colon in the user name of a short password", Raw: "https://usr%3Aname:p%40ss@example.com/a", Secrets: []string{"usr%3Aname", "usr:name", "p%40ss", "p@ss"}, Quoted: []string{"p%40ss", "p@ss"}}, + {Name: "host that looks like a digest algorithm and a port", Raw: "https://usr7:S3cr3tA@sha256:443/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "host that looks like a digest", Raw: "https://usr7:S3cr3tA@" + digest + "/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password that ends in a digest", Raw: "https://usr7:S3cr3tA@" + digest + "@example.com/a", Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a slash and a digest after it", Raw: "https://usr7:S3cr3tA/S3cr3tB@" + digest + "@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA/S3cr3tB"}}, + {Name: "credential followed by a digest", Raw: "https://usr7:S3cr3tA@example.com/a@" + digest, Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, + {Name: "password with a backslash", Raw: "https://usr7:S3cr3tA\\S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\\S3cr3tB"}}, + {Name: "password with a bracket", Raw: "https://usr7:S3cr3tA[S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA[S3cr3tB"}}, + {Name: "password with a tab", Raw: "https://usr7:S3cr3tA\tS3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\tS3cr3tB"}}, + {Name: "password with unicode", Raw: "https://usr7:S3cr3tA\u2713S3cr3tB@example.com/a", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA\u2713S3cr3tB"}}, + {Name: "credential repeated in the query", Raw: "https://usr7:S3cr3tA@example.com/a?next=https://usr7:S3cr3tB@other.example/b", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{SecretA}}, + {Name: "credential in the fragment", Raw: "https://usr7:S3cr3tA@example.com/a#https://usr7:S3cr3tB@other.example/b", Secrets: []string{User, SecretA, SecretB}, Quoted: []string{SecretA}}, {Name: "oci reference with a digest", Raw: "oci://usr7:S3cr3tA@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA}, Quoted: []string{SecretA}}, {Name: "oci reference with a double colon password and a digest", Raw: "oci://usr7:S3cr3tA::S3cr3tB@ghcr.io/acme/catalog@" + digest, Secrets: []string{User, SecretA, SecretB}, Quoted: []string{"S3cr3tA::S3cr3tB"}}, } diff --git a/pkg/artifact/redacttest/generate.go b/pkg/artifact/redacttest/generate.go new file mode 100644 index 00000000000..cebb1359a69 --- /dev/null +++ b/pkg/artifact/redacttest/generate.go @@ -0,0 +1,132 @@ +package redacttest + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// Input is one generated credential-bearing address and the ways an error may quote it. +type Input struct { + // Name describes the template the address was built from. + Name string + // Raw is the address. + Raw string + // Password is the password as it was typed into Raw, or empty when Raw has none. + Password string +} + +// Messages returns error texts that quote the input the way fetch, parse, and decode errors do: +// verbatim, quoted with %q, stripped of its scheme, as the password on its own, and as the text of +// a URL parse error. +func (in Input) Messages() []string { + defer perf.Track(nil, "redacttest.Input.Messages")() + + _, withoutScheme, _ := strings.Cut(in.Raw, "://") + messages := []string{ + "fetch " + in.Raw + " failed: denied", + fmt.Sprintf("fetch %q failed: denied", in.Raw), + "could not parse reference: " + withoutScheme, + fmt.Sprintf("Get %q: dial tcp: lookup failed", in.Raw), + } + if _, err := url.Parse(in.Raw); err != nil { + messages = append(messages, err.Error()) + } + if in.Password != "" { + messages = append(messages, "authentication failed for "+in.Password, fmt.Sprintf("password %q rejected", in.Password)) + } + return messages +} + +// Leaks returns what survives in out of the user name and of the secret: every fragment of a +// secret starts with [Marker], so a delimiter inside the secret cannot hide a leak. +func Leaks(out string) []string { + defer perf.Track(nil, "redacttest.Leaks")() + + var leaks []string + for _, needle := range []string{Marker, User} { + if strings.Contains(out, needle) { + leaks = append(leaks, needle) + } + } + return leaks +} + +// RequireNoLeak fails the test when out, the redacted output of the entry point named entry for +// in, still holds the user name or any fragment of the secret. +func RequireNoLeak(t testing.TB, entry string, in Input, out string) { + defer perf.Track(nil, "redacttest.RequireNoLeak")() + + t.Helper() + + if leaks := Leaks(out); len(leaks) > 0 { + t.Fatalf("%s leaked %q for %s input %q: %q", entry, leaks, in.Name, in.Raw, out) + } +} + +// Seeds returns the separators a fuzz run starts from: one for every delimiter the shared table +// splits a password at, so that the corpus covers every input of [Cases] as a generated secret. +func Seeds() []string { + defer perf.Track(nil, "redacttest.Seeds")() + + return []string{"", ":", "::", "@", "@@", "#", "?", "/", "%40", "%zz", " ", "\t", "[", "]", "\\", "\u2713", "@" + digestHex, "://", "@sha256:443/"} +} + +// digestHex is a complete sha256 digest, so a template can place a real digest next to a credential. +var digestHex = "sha256:" + strings.Repeat("0f", digestBytes) + +// Generate embeds secret as the password of many address templates: user information of every +// scheme, with the secret before and after other delimiters, in an escaped form, repeated in the +// query and the fragment, and with hosts named like digests. Each address ends its user +// information with an "@" of its own, so the secret is always a credential and never a host. +// The secret may hold any delimiter, so the same template covers a password that closes the +// authority early. +func Generate(secret string) []Input { + defer perf.Track(nil, "redacttest.Generate")() + + escaped := url.PathEscape(secret) + const ( + https = "https://" + colon = ":" + at = "@" + host = "@example.com/a" + ) + credential := User + colon + secret + templates := []struct{ name, raw, password string }{ + {"https", https + credential + host, secret}, + {"upper-case scheme with a port, a query and a fragment", "HTTPS://" + credential + "@example.com:8443/a?x=1#f", secret}, + {"scheme-relative", "//" + credential + host, secret}, + {"scheme-less", credential + host, secret}, + {"forced getter", "git::" + https + credential + "@example.com/r.git//sub?ref=v1", secret}, + {"oci with a digest", "oci://" + credential + "@ghcr.io/acme/catalog@" + digestHex, secret}, + {"host named like an algorithm with a port", https + credential + "@sha256:443/a", secret}, + {"host named like a digest", https + credential + at + digestHex + "/a", secret}, + {"digest in the user information", https + credential + at + digestHex + host, secret}, + {"ipv6 host", https + credential + "@[::1]:8443/a", secret}, + {"token as the user", https + secret + host, ""}, + {"credential then a digest", https + credential + "@example.com/a@" + digestHex, secret}, + {"credential repeated in the query", https + credential + "@example.com/a?next=" + https + credential + "@other.example/b", secret}, + {"credential repeated in the fragment", https + credential + "@example.com/a#" + https + credential + "@other.example/b", secret}, + {"escaped password", https + User + colon + escaped + host, secret}, + {"escaped colon in the user", https + User + "%3A" + secret + colon + escaped + host, secret}, + {"two userinfos", https + credential + at + secret + host, secret}, + {"ssh", "ssh://" + credential + "@example.com:22/r.git", secret}, + {"no host", https + credential + "@", secret}, + } + inputs := make([]Input, 0, len(templates)) + for _, tmpl := range templates { + inputs = append(inputs, Input{Name: tmpl.name, Raw: tmpl.raw, Password: tmpl.password}) + } + return inputs +} + +// SecretOf builds a secret that carries [Marker] twice, joined by an arbitrary separator, so a +// delimiter inside the secret splits it into fragments that each start with the marker. +func SecretOf(first, separator, second string) string { + defer perf.Track(nil, "redacttest.SecretOf")() + + return Marker + first + separator + Marker + second +} diff --git a/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f b/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f new file mode 100644 index 00000000000..b77f8cb9be1 --- /dev/null +++ b/pkg/artifact/testdata/fuzz/FuzzRedact/72758857c5de516f @@ -0,0 +1,4 @@ +go test fuzz v1 +string("0") +string("0") +string("%0:%") diff --git a/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c b/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c new file mode 100644 index 00000000000..00b2ddd313c --- /dev/null +++ b/pkg/artifact/testdata/fuzz/FuzzRedact/b6670d02d88a615c @@ -0,0 +1,4 @@ +go test fuzz v1 +string("0") +string("\t") +string("#000") diff --git a/pkg/config/import_fuzz_test.go b/pkg/config/import_fuzz_test.go new file mode 100644 index 00000000000..eefb2e762af --- /dev/null +++ b/pkg/config/import_fuzz_test.go @@ -0,0 +1,31 @@ +package config + +import ( + "testing" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" +) + +// TestSanitizeImport_GeneratedInputs runs the generated inputs, built from every delimiter of the +// shared table, through SanitizeImport. +func TestSanitizeImport_GeneratedInputs(t *testing.T) { + for _, separator := range redacttest.Seeds() { + for _, in := range redacttest.Generate(redacttest.SecretOf("A", separator, "B")) { + redacttest.RequireNoLeak(t, "SanitizeImport", in, SanitizeImport(in.Raw)) + } + } +} + +// FuzzSanitizeImport embeds a random secret in many address templates and checks that +// SanitizeImport never leaks the user name or any fragment of the secret. +func FuzzSanitizeImport(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + } + f.Fuzz(func(t *testing.T, first, separator, second string) { + for _, in := range redacttest.Generate(redacttest.SecretOf(first, separator, second)) { + redacttest.RequireNoLeak(t, "SanitizeImport", in, SanitizeImport(in.Raw)) + } + }) +} diff --git a/pkg/config/imports.go b/pkg/config/imports.go index 9b7133bb4bb..c3b806877b7 100644 --- a/pkg/config/imports.go +++ b/pkg/config/imports.go @@ -33,9 +33,11 @@ func SanitizeImport(s string) string { // recognized when a scheme follows it, so a "::" inside a password never splits the credential. prefix, urlPart := artifact.SplitGetterPrefix(s) - // A URL carrying userinfo is redacted structurally, by the helper every include error shares: - // the userinfo ends at the last "@" of the authority, whatever the password holds. - if strings.Contains(urlPart, "@") && (strings.Contains(urlPart, "://") || strings.HasPrefix(urlPart, "//")) { + // An import that holds an "@" is a credential-bearing URL unless it is provably a local path or + // an SCP-style git address. It is redacted by the helper every include error shares: the + // userinfo ends at the last "@" of the authority, and a value that cannot be proven free of a + // credential is replaced as a whole. + if strings.Contains(urlPart, "@") && !isLocalPath(urlPart) && !artifact.IsSCPAddress(urlPart) { urlPart = artifact.RedactURL(urlPart) if urlPart == artifact.RedactedValue { return urlPart @@ -45,7 +47,7 @@ func SanitizeImport(s string) string { u, err := url.Parse(urlPart) if err != nil { // Unparsable; keep the text (it might be SCP-style git) but never a query, which may hold tokens. - before, _, _ := strings.Cut(s, "?") + before, _, _ := strings.Cut(prefix+urlPart, "?") return before } @@ -57,6 +59,14 @@ func SanitizeImport(s string) string { return prefix + u.String() } +// isLocalPath reports whether an import is a file system path, which is never a URL. +func isLocalPath(s string) bool { + if strings.Contains(s, "://") || strings.HasPrefix(s, "//") { + return false + } + return strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") || strings.HasPrefix(s, "~/") +} + type importTypes int const ( diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go index dab1654cbab..3c6790845c7 100644 --- a/pkg/utils/yaml_include_redact.go +++ b/pkg/utils/yaml_include_redact.go @@ -20,30 +20,12 @@ func RedactIncludeArgument(value string) string { return artifact.RedactURL(value) } -// redactedIncludeError is an error whose text has had a credential removed. It keeps the original -// error in its chain so errors.Is and errors.As still see the cause. -type redactedIncludeError struct { - message string - err error -} - -func (e *redactedIncludeError) Error() string { return e.message } - -func (e *redactedIncludeError) Unwrap() error { return e.err } - // redactIncludeError removes the credential of an include target from the text of an error raised // while fetching it. Fetch errors from the download and OCI layers quote the reference they were // given (for example "could not parse reference: user:secret@host/image"), so the include // argument cannot be redacted once for all of them: the credential is cut out of the final text by -// the same helper that redacts URLs, so both agree on where a userinfo ends. +// the helper that redacts URLs, so both agree on where a userinfo ends, and any other URL that +// carries userinfo is withheld as well (see artifact.RedactError). func redactIncludeError(err error, target string) error { - if err == nil || !strings.Contains(target, "@") { - return err - } - message := err.Error() - redacted := artifact.RedactCredentials(message, target) - if redacted == message { - return err - } - return &redactedIncludeError{message: redacted, err: err} + return artifact.RedactError(err, target) } diff --git a/pkg/utils/yaml_include_redact_fuzz_test.go b/pkg/utils/yaml_include_redact_fuzz_test.go new file mode 100644 index 00000000000..800f0204557 --- /dev/null +++ b/pkg/utils/yaml_include_redact_fuzz_test.go @@ -0,0 +1,69 @@ +package utils + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cloudposse/atmos/pkg/artifact/redacttest" +) + +var errIncludeFetch = errors.New("include fetch failed") + +// checkIncludeEntryPoints runs one generated input through every include redaction entry point. +func checkIncludeEntryPoints(t testing.TB, in redacttest.Input) { + t.Helper() + redacttest.RequireNoLeak(t, "RedactIncludeArgument", in, RedactIncludeArgument(in.Raw)) + for _, message := range in.Messages() { + err := redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), in.Raw) + redacttest.RequireNoLeak(t, "redactIncludeError", in, err.Error()) + if !errors.Is(err, errIncludeFetch) { + t.Fatalf("redactIncludeError dropped the cause of %q", message) + } + } +} + +// TestRedactInclude_SharedCredentialCases runs the shared table of hostile inputs through the +// include entry points. A scheme-less input is not URL-shaped, so only its error text is checked. +func TestRedactInclude_SharedCredentialCases(t *testing.T) { + for _, tc := range redacttest.Cases() { + t.Run(tc.Name, func(t *testing.T) { + got := RedactIncludeArgument(tc.Raw) + if strings.Contains(tc.Raw, "://") { + for _, secret := range tc.Secrets { + assert.NotContains(t, got, secret, "RedactIncludeArgument of %q", tc.Raw) + } + } + for _, message := range []string{"fetch " + tc.Raw + " failed", fmt.Sprintf("fetch %q failed", tc.Raw)} { + err := redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), tc.Raw) + for _, secret := range tc.Secrets { + assert.NotContains(t, err.Error(), secret, "redactIncludeError of %q", message) + } + } + }) + } +} + +// FuzzRedactInclude embeds a random secret in many address templates and checks that no include +// entry point leaks the user name or any fragment of the secret. +func FuzzRedactInclude(f *testing.F) { + for _, separator := range redacttest.Seeds() { + f.Add("A", separator, "B") + f.Add("", separator, "") + } + f.Fuzz(func(t *testing.T, first, separator, second string) { + for _, in := range redacttest.Generate(redacttest.SecretOf(first, separator, second)) { + // An argument that is not URL-shaped is a local path and keeps its text by design. + if !strings.Contains(in.Raw, "://") { + for _, message := range in.Messages() { + redacttest.RequireNoLeak(t, "redactIncludeError", in, redactIncludeError(fmt.Errorf("%w: %s", errIncludeFetch, message), in.Raw).Error()) + } + continue + } + checkIncludeEntryPoints(t, in) + } + }) +} From f2afb4d014511927475f43900cee6dc7e290f45e Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 07:50:56 -0600 Subject: [PATCH 25/32] fix(artifact): route every echoed registry, prefix, key pattern, reference, identity and option through the shared redaction Error messages that quoted a user-supplied value still printed it raw: the registry and host of a catalog mismatch, the prefix and key pattern of option validation, each part of a parsed artifact reference, an unknown auth identity, and the text of an options decode error, which quotes keys and values. Pass each through artifact.RedactURL, and the decode, catalog and reference errors through artifact.RedactError with the whole input, since they are validated in pieces and a credential split at its "@" is only visible in the whole. Option keys that look like credentials are redacted in the hint too. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/catalog.go | 4 +-- pkg/artifact/config/validate.go | 2 +- pkg/artifact/options.go | 47 ++++++++++++++++++-------- pkg/artifact/redact_sites_test.go | 55 +++++++++++++++++++++++++++++++ pkg/artifact/ref.go | 26 ++++++++------- 5 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 pkg/artifact/redact_sites_test.go diff --git a/pkg/artifact/config/catalog.go b/pkg/artifact/config/catalog.go index ea29fb7f5a6..6ab0728fc5a 100644 --- a/pkg/artifact/config/catalog.go +++ b/pkg/artifact/config/catalog.go @@ -29,7 +29,7 @@ func validateCatalog(desc *artifact.KindDescriptor, repo *schema.ArtifactReposit } registry, _ := repo.Options["registry"].(string) if registry != "" && !strings.EqualFold(registry, parsed.Host) { - return fmt.Errorf("%w: catalog host %q does not match the repository registry %q; a catalog cannot direct the repository identity to another host", errUtils.ErrArtifactConfigInvalid, parsed.Host, registry) + return fmt.Errorf("%w: catalog host %q does not match the repository registry %q; a catalog cannot direct the repository identity to another host", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(parsed.Host), artifact.RedactURL(registry)) } return nil } @@ -44,7 +44,7 @@ func parseCatalogURI(raw string) (*url.URL, error) { return nil, fmt.Errorf("%w: catalog %q must not contain credentials, a query, or a fragment", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw)) } if err := validateCatalogReference(strings.TrimPrefix(parsed.Path, "/")); err != nil { - return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw), err) + return nil, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(raw), artifact.RedactError(err, raw)) } return parsed, nil } diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go index cf5ebeb9549..f5e616e4ab1 100644 --- a/pkg/artifact/config/validate.go +++ b/pkg/artifact/config/validate.go @@ -154,7 +154,7 @@ func (v *Validator) validateIdentity(identity string) error { case strings.Contains(identity, "://"): return fmt.Errorf("%w: identity looks like a credential or URL; reference an Atmos auth identity name", errUtils.ErrArtifactConfigInvalid) case v.identityExists != nil && !v.identityExists(identity): - return fmt.Errorf("%w: identity %q is not defined under auth.identities", errUtils.ErrArtifactConfigInvalid, identity) + return fmt.Errorf("%w: identity %q is not defined under auth.identities", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(identity)) } return nil } diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 13f408b2dc3..2f38dec586a 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -196,7 +196,7 @@ func (o *OCIRegistryOptions) Validate() error { if o.Prefix != "" { for _, segment := range strings.Split(o.Prefix, "/") { if err := ValidateName(segment); err != nil { - errs = append(errs, fmt.Errorf("%w: prefix %q is not a valid OCI repository path: %w", errUtils.ErrArtifactConfigInvalid, o.Prefix, err)) + errs = append(errs, fmt.Errorf("%w: prefix %q is not a valid OCI repository path: %w", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Prefix), RedactError(err, o.Prefix))) break } } @@ -246,17 +246,38 @@ func DecodeOptions(raw map[string]any, out any) error { return fmt.Errorf("%w: options decoder: %w", errUtils.ErrArtifactConfigInvalid, err) } if err := decoder.Decode(raw); err != nil { - return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, err, credentialHint(raw)) + // A decode error may quote a value, such as a mistyped number that is really a credential, + // so every string key and value is a credential candidate when the message is scrubbed. + return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, RedactError(err, stringValues(raw)...), credentialHint(raw)) } return nil } +// stringValues returns every string held by raw, keys included, at any depth, in no particular order. +func stringValues(raw any) []string { + var out []string + switch v := raw.(type) { + case string: + out = append(out, v) + case map[string]any: + for key, item := range v { + out = append(out, key) + out = append(out, stringValues(item)...) + } + case []any: + for _, item := range v { + out = append(out, stringValues(item)...) + } + } + return out +} + // credentialHint explains where credentials belong when an option key looks like one. func credentialHint(raw map[string]any) string { keys := make([]string, 0, len(raw)) for key := range raw { if credentialOptionPattern.MatchString(key) { - keys = append(keys, key) + keys = append(keys, RedactURL(key)) } } if len(keys) == 0 { @@ -292,14 +313,14 @@ func validatePrefix(prefix string) error { return nil } if strings.HasPrefix(prefix, "/") || strings.Contains(prefix, `\`) || strings.ContainsRune(prefix, 0) { - return fmt.Errorf("%w: %w: prefix %q must be relative and slash-separated", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix) + return fmt.Errorf("%w: %w: prefix %q must be relative and slash-separated", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix)) } for _, segment := range strings.Split(prefix, "/") { switch segment { case "", currentDirSegment, parentDirSegment: - return fmt.Errorf("%w: %w: prefix %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix, currentDirSegment, parentDirSegment) + return fmt.Errorf("%w: %w: prefix %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix), currentDirSegment, parentDirSegment) case ReservedNamespace: - return fmt.Errorf("%w: %w: prefix %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, prefix, ReservedNamespace) + return fmt.Errorf("%w: %w: prefix %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(prefix), ReservedNamespace) } } return nil @@ -322,12 +343,12 @@ func validateKeyPattern(pattern string, packaging Packaging) error { return nil } if strings.ContainsRune(pattern, 0) || strings.Contains(pattern, `\`) { - return fmt.Errorf("%w: %w: key_pattern %q must not contain NUL bytes or backslashes", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern) + return fmt.Errorf("%w: %w: key_pattern %q must not contain NUL bytes or backslashes", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern)) } actions := templateActionPattern.FindAllString(pattern, -1) literal := templateActionPattern.ReplaceAllString(pattern, "\x01") if strings.Contains(literal, "{{") || strings.Contains(literal, "}}") { - return fmt.Errorf("%w: key_pattern %q has unbalanced template delimiters", errUtils.ErrArtifactConfigInvalid, pattern) + return fmt.Errorf("%w: key_pattern %q has unbalanced template delimiters", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern)) } if err := validateKeyPatternFields(pattern, actions); err != nil { return err @@ -343,7 +364,7 @@ func validateKeyPatternFields(pattern string, actions []string) error { stripped := templateStringPattern.ReplaceAllString(action, `""`) for _, match := range templateFieldPattern.FindAllStringSubmatch(stripped, -1) { if !keyPatternFields[match[1]] { - return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, pattern, match[1]) + return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern), match[1]) } } } @@ -352,14 +373,14 @@ func validateKeyPatternFields(pattern string, actions []string) error { func validateKeyPatternSegments(pattern, literal string) error { if strings.HasPrefix(literal, "/") { - return fmt.Errorf("%w: %w: key_pattern %q must be relative to the repository root", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern) + return fmt.Errorf("%w: %w: key_pattern %q must be relative to the repository root", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern)) } for _, segment := range strings.Split(literal, "/") { switch segment { case "", currentDirSegment, parentDirSegment: - return fmt.Errorf("%w: %w: key_pattern %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern, currentDirSegment, parentDirSegment) + return fmt.Errorf("%w: %w: key_pattern %q has an empty, %q, or %q segment", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern), currentDirSegment, parentDirSegment) case ReservedNamespace: - return fmt.Errorf("%w: %w: key_pattern %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, pattern, ReservedNamespace) + return fmt.Errorf("%w: %w: key_pattern %q enters the reserved %s namespace", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern), ReservedNamespace) } } return nil @@ -384,7 +405,7 @@ func validateKeyPatternExtension(pattern, literal string, packaging Packaging) e continue } if packaging != "" && candidate.packaging != packaging { - return fmt.Errorf("%w: key_pattern %q ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, pattern, candidate.suffix, packaging) + return fmt.Errorf("%w: key_pattern %q ends in %q, which does not match packaging %q", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern), candidate.suffix, packaging) } return nil } diff --git a/pkg/artifact/redact_sites_test.go b/pkg/artifact/redact_sites_test.go new file mode 100644 index 00000000000..58290745597 --- /dev/null +++ b/pkg/artifact/redact_sites_test.go @@ -0,0 +1,55 @@ +package artifact + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" +) + +// TestDecodeOptions_ErrorDoesNotEchoCredential checks that an option key or value that holds a +// credential never reaches the decode error. +func TestDecodeOptions_ErrorDoesNotEchoCredential(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com/x" + tests := []struct { + name string + raw map[string]any + }{ + {name: "unknown key", raw: map[string]any{"registry": "ghcr.io", credential: "x"}}, + {name: "credential-like unknown key", raw: map[string]any{"registry": "ghcr.io", "usr7:S3cr3tA@password": "x"}}, + {name: "mistyped value", raw: map[string]any{"registry": map[string]any{"host": credential}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var options OCIRegistryOptions + err := DecodeOptions(tt.raw, &options) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} + +// TestValidationErrors_DoNotEchoCredentials checks the validators that echo a user-supplied +// registry, prefix, key pattern, or reference never repeat a credential in it. +func TestValidationErrors_DoNotEchoCredentials(t *testing.T) { + const credential = "usr7:S3cr3tA@example.com" + errs := map[string]error{ + "registry": (&OCIRegistryOptions{Registry: credential}).Validate(), + "prefix": ValidatePrefix(credential + "/../x"), + "key pattern": ValidateKeyPattern("/"+credential+"/{{.Name}}", PackagingTar), + "reference": func() error { _, err := ParseRef(credential); return err }(), + "name": ValidateName(credential), + "version": ValidateVersion(credential), + } + for name, err := range errs { + t.Run(name, func(t *testing.T) { + require.Error(t, err) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} diff --git a/pkg/artifact/ref.go b/pkg/artifact/ref.go index 667acbf5892..04d3bd78d31 100644 --- a/pkg/artifact/ref.go +++ b/pkg/artifact/ref.go @@ -51,12 +51,14 @@ func ParseRef(s string) (Ref, error) { } else { index := strings.LastIndex(s, ":") if index < 0 { - return Ref{}, fmt.Errorf("%w: %q needs a version (name:version) or digest (name@sha256:...); there is no implicit latest", errUtils.ErrArtifactRefInvalid, s) + return Ref{}, fmt.Errorf("%w: %q needs a version (name:version) or digest (name@sha256:...); there is no implicit latest", errUtils.ErrArtifactRefInvalid, RedactURL(s)) } ref = Ref{Name: s[:index], Version: s[index+1:]} } if err := ref.Validate(); err != nil { - return Ref{}, err + // The reference is validated in pieces, so a credential split at its "@" is only visible in + // the whole. + return Ref{}, RedactError(err, s) } return ref, nil } @@ -70,9 +72,9 @@ func (r Ref) Validate() error { } switch { case r.Version != "" && r.Digest != "": - return fmt.Errorf("%w: %q sets both a version and a digest", errUtils.ErrArtifactRefInvalid, r.Name) + return fmt.Errorf("%w: %q sets both a version and a digest", errUtils.ErrArtifactRefInvalid, RedactURL(r.Name)) case r.Version == "" && r.Digest == "": - return fmt.Errorf("%w: %q needs a version or a digest", errUtils.ErrArtifactRefInvalid, r.Name) + return fmt.Errorf("%w: %q needs a version or a digest", errUtils.ErrArtifactRefInvalid, RedactURL(r.Name)) case r.Version != "": return ValidateVersion(r.Version) default: @@ -107,17 +109,17 @@ func ValidateName(name string) error { return fmt.Errorf("%w: name exceeds %d characters", errUtils.ErrArtifactRefInvalid, MaxNameLength) } if strings.HasPrefix(name, "/") || strings.Contains(name, `\`) { - return fmt.Errorf("%w: %w: %q must be relative and slash-separated", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name) + return fmt.Errorf("%w: %w: %q must be relative and slash-separated", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name)) } for _, segment := range strings.Split(name, "/") { switch segment { case "": - return fmt.Errorf("%w: %w: %q has an empty segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name) + return fmt.Errorf("%w: %w: %q has an empty segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name)) case ".", "..": - return fmt.Errorf("%w: %w: %q has a %q segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, name, segment) + return fmt.Errorf("%w: %w: %q has a %q segment", errUtils.ErrArtifactRefInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(name), segment) } if !nameSegmentPattern.MatchString(segment) { - return fmt.Errorf("%w: segment %q of %q must be lowercase alphanumerics separated by '.', '_', '__', or '-'", errUtils.ErrArtifactRefInvalid, segment, name) + return fmt.Errorf("%w: segment %q of %q must be lowercase alphanumerics separated by '.', '_', '__', or '-'", errUtils.ErrArtifactRefInvalid, RedactURL(segment), RedactURL(name)) } } return nil @@ -131,7 +133,7 @@ func ValidateVersion(version string) error { return fmt.Errorf("%w: version is empty", errUtils.ErrArtifactRefInvalid) } if !versionPattern.MatchString(version) { - return fmt.Errorf("%w: version %q must match the OCI tag syntax [A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", errUtils.ErrArtifactRefInvalid, version) + return fmt.Errorf("%w: version %q must match the OCI tag syntax [A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", errUtils.ErrArtifactRefInvalid, RedactURL(version)) } return nil } @@ -141,10 +143,10 @@ func ValidateDigest(d digest.Digest) error { defer perf.Track(nil, "artifact.ValidateDigest")() if err := d.Validate(); err != nil { - return fmt.Errorf("%w: digest %q: %w", errUtils.ErrArtifactRefInvalid, d, err) + return fmt.Errorf("%w: digest %q: %w", errUtils.ErrArtifactRefInvalid, RedactURL(string(d)), err) } if d.Algorithm() != digest.SHA256 { - return fmt.Errorf("%w: digest %q must use sha256", errUtils.ErrArtifactRefInvalid, d) + return fmt.Errorf("%w: digest %q must use sha256", errUtils.ErrArtifactRefInvalid, RedactURL(string(d))) } return nil } @@ -158,7 +160,7 @@ func ValidateKind(kind string) error { } for _, segment := range strings.Split(kind, "/") { if !kindSegmentPattern.MatchString(segment) { - return fmt.Errorf("%w: kind %q must be slash-separated lowercase segments", errUtils.ErrArtifactRecordInvalid, kind) + return fmt.Errorf("%w: kind %q must be slash-separated lowercase segments", errUtils.ErrArtifactRecordInvalid, RedactURL(kind)) } } return nil From 5f8155d68bac3cd26a38dec96f4cf07d39df03d1 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 10:53:20 -0600 Subject: [PATCH 26/32] fix(artifact): withhold the unknown key_pattern field name when the pattern carries a credential The field name extracted from a template action is a fragment of the pattern, so scrubbing the pattern alone left a password written as a template field (https://user:{{.secret}}@host/) in the error text. The name is now omitted whenever the redaction changes the pattern, and still named otherwise. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/options.go | 14 ++++++++++- pkg/artifact/redact_sites_test.go | 41 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 2f38dec586a..9f8bd638f46 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -364,13 +364,25 @@ func validateKeyPatternFields(pattern string, actions []string) error { stripped := templateStringPattern.ReplaceAllString(action, `""`) for _, match := range templateFieldPattern.FindAllStringSubmatch(stripped, -1) { if !keyPatternFields[match[1]] { - return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, RedactURL(pattern), match[1]) + return unknownKeyPatternField(pattern, match[1]) } } } return nil } +// unknownKeyPatternField reports a template field that key_pattern may not reference. The field +// name is a fragment of the pattern, and a pattern that carries a credential may hold it in the +// userinfo, as in "https://user:{{.secret}}@host/", where scrubbing the pattern does not scrub the +// fragment. The name is therefore withheld whenever the redaction changes the pattern. +func unknownKeyPatternField(pattern, field string) error { + redacted := RedactURL(pattern) + if redacted != pattern { + return fmt.Errorf("%w: key_pattern %q references a field that is not allowed (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, redacted) + } + return fmt.Errorf("%w: key_pattern %q references unknown field .%s (allowed: .Name, .Version, .Kind, .CreatedAt)", errUtils.ErrArtifactConfigInvalid, redacted, field) +} + func validateKeyPatternSegments(pattern, literal string) error { if strings.HasPrefix(literal, "/") { return fmt.Errorf("%w: %w: key_pattern %q must be relative to the repository root", errUtils.ErrArtifactConfigInvalid, errUtils.ErrArtifactUnsafePath, RedactURL(pattern)) diff --git a/pkg/artifact/redact_sites_test.go b/pkg/artifact/redact_sites_test.go index 58290745597..9d25a7fa366 100644 --- a/pkg/artifact/redact_sites_test.go +++ b/pkg/artifact/redact_sites_test.go @@ -53,3 +53,44 @@ func TestValidationErrors_DoNotEchoCredentials(t *testing.T) { }) } } + +// TestValidateKeyPattern_UnknownFieldDoesNotEchoCredential checks that the field name extracted from +// a template action is withheld when the pattern carries a credential: a field name is a fragment +// of the pattern, so scrubbing the pattern alone does not scrub it. +func TestValidateKeyPattern_UnknownFieldDoesNotEchoCredential(t *testing.T) { + tests := []struct { + name string + pattern string + // secrets are what the error must not hold. A field name in the path of the pattern is + // echoed with the rest of the redacted pattern, so only the userinfo is a secret there. + secrets []string + }{ + {name: "password is the field", pattern: "https://usr7:{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t", "usr7"}}, + {name: "token is the field", pattern: "https://{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t"}}, + {name: "scheme-less userinfo", pattern: "usr7:{{.S3cr3tA}}@example.com/{{.Name}}.tar", secrets: []string{"S3cr3t", "usr7"}}, + {name: "field after a literal credential", pattern: "https://usr7:S3cr3tA@example.com/{{.Bogus}}.tar", secrets: []string{"S3cr3tA", "usr7"}}, + {name: "second action holds the field", pattern: "https://usr7:S3cr3tA@example.com/{{.Name}}/{{.Bogus}}.tar", secrets: []string{"S3cr3tA", "usr7"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateKeyPattern(tt.pattern, PackagingTar) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + for _, secret := range tt.secrets { + assert.NotContains(t, err.Error(), secret) + } + assert.NotContains(t, err.Error(), "unknown field .", "the field name is withheld when the pattern carries a credential") + }) + } +} + +// TestValidateKeyPattern_UnknownFieldIsNamedWithoutCredential is the negative path of the test +// above: a pattern that carries no credential still names the unknown field, because the name is +// what lets the user fix the pattern. +func TestValidateKeyPattern_UnknownFieldIsNamedWithoutCredential(t *testing.T) { + err := ValidateKeyPattern("builds/{{.Bogus}}/{{.Name}}.tar", PackagingTar) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, err.Error(), "unknown field .Bogus") + assert.Contains(t, err.Error(), "builds/{{.Bogus}}/{{.Name}}.tar") +} From f1ba0bff344d386119a9f6129efa7bd9d61f1915 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 10:56:43 -0600 Subject: [PATCH 27/32] fix(artifact): withhold an address with too many boundaries before expanding its credential Every "@" of an address is a candidate end of a userinfo, and each candidate expanded into every form an error may quote, so the memory grew with the square of the input: 4096 "@" signs allocated 5.3 GB. An address with more than 64 "@" signs, or longer than 8 KiB when it holds one, is now marked as an overflow while it is parsed, and every entry point withholds it whole (RedactURL the address, RedactCredentials and RedactError the text, the word scan the word). The digest check also stops scanning the whole remainder for its colon. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/redact.go | 15 +++- pkg/artifact/redact_bounds_test.go | 118 +++++++++++++++++++++++++++++ pkg/artifact/redact_grammar.go | 37 ++++++++- pkg/artifact/redact_scrub.go | 2 +- pkg/artifact/redact_test.go | 5 ++ 5 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 pkg/artifact/redact_bounds_test.go diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go index 7068e063d75..d735a87b2e5 100644 --- a/pkg/artifact/redact.go +++ b/pkg/artifact/redact.go @@ -19,7 +19,9 @@ const RedactedValue = "" // be proven free of a credential is replaced by [RedactedValue] as a whole: an "@" after the // authority that does not introduce a complete digest (a password that held "/", "?", "#" or "\" // may have closed there), scheme-less userinfo such as "user:secret@registry.example", a host that -// is not shaped like a host, or an address that does not parse. The diagnostic value of an error +// is not shaped like a host, an address that does not parse, or an address with more "@" signs +// than a real one holds or longer than any real one (see [maxBoundaries] and [maxAddressLength]), +// which is withheld before any work that grows with it starts. The diagnostic value of an error // drops for such inputs by design; a credential is never echoed. A forced-getter prefix such as // "git::" is kept. func RedactURL(raw string) string { @@ -29,7 +31,7 @@ func RedactURL(raw string) string { return raw } a := parseAddress(raw) - if len(a.stray) > 0 { + if a.overflow || len(a.stray) > 0 { return RedactedValue } if len(a.inside) == 0 { @@ -76,10 +78,19 @@ func (a *address) wellFormed(start int) bool { // introduce a digest is replaced by [RedactedValue] as a whole, and so is every word that still // holds an "@" when raw carried a credential. // 3. If any part of the password is still present, the whole text is replaced by [RedactedValue]. +// +// A raw address with more "@" signs than a real one holds, or longer than any real one, has no +// credential that can be listed at a bounded cost, so the whole text is replaced by [RedactedValue] +// without taking any of the steps above. func RedactCredentials(text, raw string) string { defer perf.Track(nil, "artifact.RedactCredentials")() a := parseAddress(raw) + if a.overflow { + // The credential of an address that is past the bounds cannot be listed, so nothing in the + // text can be proven free of it. + return RedactedValue + } secret := a.credential() if !secret.empty() { text = secret.scrub(text) diff --git a/pkg/artifact/redact_bounds_test.go b/pkg/artifact/redact_bounds_test.go new file mode 100644 index 00000000000..bb1ae74bf0f --- /dev/null +++ b/pkg/artifact/redact_bounds_test.go @@ -0,0 +1,118 @@ +package artifact + +import ( + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // The hostile boundaries count is far above any cap a legitimate address needs, so the tests below hold + // whatever the cap is. + hostileBoundaries = 4096 + // The allocation budget is the memory a single hostile input may cost the whole set of entry points. + // The expansion this guards against grows with the square of the input and costs hundreds of + // megabytes at the input size used here, so the budget is generous and still far below it. + allocationBudget = 32 << 20 + // The generous deadline only catches a run that never finishes. It is not a performance assertion. + generousDeadline = 60 * time.Second +) + +// hostileAddress returns an address whose userinfo holds count "@" signs. +func hostileAddress(count int) string { + return "https://usr7:S3cr3tA" + strings.Repeat("@x", count) + "@example.com/a" +} + +// TestRedact_ManyBoundariesFailClosed checks that an address with more "@" signs than any real +// address holds is withheld whole, by every entry point, without expanding a credential candidate +// for each of them. +func TestRedact_ManyBoundariesFailClosed(t *testing.T) { + raw := hostileAddress(hostileBoundaries) + word := "https://usr7:S3cr3tA" + strings.Repeat("@x", hostileBoundaries) + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + start := time.Now() + + assert.Equal(t, RedactedValue, RedactURL(raw), "RedactURL withholds the address") + assert.Equal(t, RedactedValue, RedactCredentials("fetch "+raw+" failed", raw), "RedactCredentials withholds the text") + assert.Equal(t, RedactedValue, RedactCredentials("unrelated text", raw), "the text is withheld even when it does not quote the address") + err := RedactError(errSentinelf(raw), raw) + assert.Equal(t, RedactedValue, err.Error(), "RedactError withholds the message") + assert.ErrorIs(t, err, errSentinel, "the cause stays in the chain") + assert.Equal(t, "see "+RedactedValue+" ok", RedactCredentials("see "+word+" ok", ""), "a word with too many boundaries is withheld on its own") + assert.NotContains(t, RedactCredentials("see "+word+" ok", ""), "S3cr3tA") + + runtime.ReadMemStats(&after) + assert.Less(t, time.Since(start), generousDeadline) + assert.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(allocationBudget), "the work is bounded by the cap, not by the input") +} + +// errSentinelf returns an error that wraps errSentinel and quotes text. +func errSentinelf(text string) error { + return "edError{text: text} +} + +type quotedError struct{ text string } + +func (e *quotedError) Error() string { return "fetch " + e.text + " failed" } +func (e *quotedError) Unwrap() error { return errSentinel } + +// TestRedact_AddressBoundsKeepRealAddresses checks that the bounds do not touch an address that is +// within them, so the credential is still cut out, not the whole address withheld. +func TestRedact_AddressBoundsKeepRealAddresses(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + {name: "user and password", raw: "https://usr7:S3cr3tA@example.com/a", want: "https://example.com/a"}, + {name: "password with a few at signs", raw: hostileAddress(3), want: "https://example.com/a"}, + {name: "digest reference", raw: "oci://ghcr.io/acme/catalog@" + sha256Digest, want: "oci://ghcr.io/acme/catalog@" + sha256Digest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, RedactURL(tt.raw)) + scrubbed := RedactCredentials("fetch "+tt.raw+" failed", tt.raw) + assert.NotContains(t, scrubbed, "S3cr3tA") + assert.NotEqual(t, RedactedValue, scrubbed, "text within the bounds is scrubbed, not withheld") + }) + } + require.Positive(t, hostileBoundaries) +} + +// TestRedact_AddressBoundsEdges checks the two bounds exactly where they are crossed: an address +// at a bound is redacted in parts, and one past it is withheld whole. +func TestRedact_AddressBoundsEdges(t *testing.T) { + longPath := strings.Repeat("a", maxAddressLength) + tests := []struct { + name string + raw string + wantURL string + wantWithheld bool + }{ + {name: "at the boundary cap", raw: hostileAddress(maxBoundaries - 1), wantURL: "https://example.com/a"}, + {name: "past the boundary cap", raw: hostileAddress(maxBoundaries), wantWithheld: true}, + {name: "past the length cap with a credential", raw: "https://usr7:S3cr3tA@example.com/" + longPath, wantWithheld: true}, + {name: "long address without an at sign is not a credential", raw: "https://example.com/" + longPath, wantURL: "https://example.com/" + longPath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scrubbed := RedactCredentials("fetch failed for S3cr3tA at "+tt.raw[:min(len(tt.raw), 40)], tt.raw) + if strings.Contains(tt.raw, "S3cr3tA") { + assert.NotContains(t, scrubbed, "S3cr3tA") + } + if tt.wantWithheld { + assert.Equal(t, RedactedValue, RedactURL(tt.raw)) + assert.Equal(t, RedactedValue, scrubbed) + return + } + assert.Equal(t, tt.wantURL, RedactURL(tt.raw)) + assert.NotEqual(t, RedactedValue, scrubbed) + }) + } +} diff --git a/pkg/artifact/redact_grammar.go b/pkg/artifact/redact_grammar.go index 6e0e0e65d17..c6810018fb4 100644 --- a/pkg/artifact/redact_grammar.go +++ b/pkg/artifact/redact_grammar.go @@ -47,6 +47,18 @@ const ( // text. Shorter fragments would mangle ordinary words, and they identify nothing. minFragmentLength = 4 + // The maximum boundaries is the most "@" signs an address may hold and still be redacted in + // parts. A real address holds a handful: a userinfo, perhaps a password with an "@" in it, and a + // digest. Every "@" is a candidate end of a credential, and each candidate expands into every + // form an error may quote, so the work grows with the number of them. An address with more is + // withheld whole. + maxBoundaries = 64 + + // The maximum address length is the longest address, in bytes, that is redacted in parts when it + // holds an "@". The expansion of a credential candidate is as long as the prefix it came from, so + // the length bounds what each of the [maxBoundaries] candidates costs. + maxAddressLength = 8 << 10 + // The scheme separator separates a scheme from its authority. schemeSeparator = "://" @@ -54,6 +66,9 @@ const ( fileScheme = "file://" ) +// maxDigestAlgorithmLength is the length of the longest name in [digestHexLengths]. +const maxDigestAlgorithmLength = len("sha512") + // digestHexLengths holds the length in hex digits of each digest algorithm that may follow an "@". // Only these well-known algorithms qualify, so a host named like an algorithm is never mistaken for // one. @@ -82,11 +97,21 @@ type address struct { // stray are the offsets in rest of every "@" after the authority that does not introduce a // digest. stray []int + // overflow is set when the address holds more "@" signs than [maxBoundaries] or is longer than + // [maxAddressLength]. The other fields are empty then, and the address must be withheld whole: + // nothing in it can be proven free of a credential at a cost that is bounded. + overflow bool } -// parseAddress splits raw at the getter prefix, the scheme, the authority, and every "@". +// parseAddress splits raw at the getter prefix, the scheme, the authority, and every "@". An +// address that is past the bounds of the grammar is not split at all: it is marked as an overflow +// before any work that grows with the number of its "@" signs starts. func parseAddress(raw string) address { a := address{} + if ats := strings.Count(raw, "@"); ats > maxBoundaries || (ats > 0 && len(raw) > maxAddressLength) { + a.overflow = true + return a + } a.prefix, raw = splitGetterPrefix(raw) if loc := schemePattern.FindStringIndex(raw); loc != nil { a.scheme, a.rest = raw[:loc[1]], raw[loc[1]:] @@ -133,9 +158,15 @@ func (a *address) boundaries() []int { // digestFollows reports whether s, the text after an "@", starts with a complete digest such as // "sha256:<64 hex digits>" that is not continued by further letters or digits. func digestFollows(s string) bool { - algorithm, rest, found := strings.Cut(s, ":") + // The algorithm name ends at the first ":", which is searched for in the few bytes a name can + // take, not in all of s, so the check costs the same however long s is. + colon := strings.IndexByte(s[:min(len(s), maxDigestAlgorithmLength+1)], ':') + if colon < 0 { + return false + } + algorithm, rest := s[:colon], s[colon+1:] length, known := digestHexLengths[algorithm] - if !found || !known || len(rest) < length { + if !known || len(rest) < length { return false } for i := 0; i < length; i++ { diff --git a/pkg/artifact/redact_scrub.go b/pkg/artifact/redact_scrub.go index 2f426de0e63..d89545098b1 100644 --- a/pkg/artifact/redact_scrub.go +++ b/pkg/artifact/redact_scrub.go @@ -200,7 +200,7 @@ func redactWords(text string, bearing bool) string { } else if !bearing { return word } - if a := parseAddress(body); len(a.boundaries()) > 0 { + if a := parseAddress(body); a.overflow || len(a.boundaries()) > 0 { return RedactedValue } return word diff --git a/pkg/artifact/redact_test.go b/pkg/artifact/redact_test.go index 5d7dd593fa5..dc0dae78518 100644 --- a/pkg/artifact/redact_test.go +++ b/pkg/artifact/redact_test.go @@ -111,6 +111,11 @@ func FuzzRedact(f *testing.F) { f.Add("", separator, "") f.Add("x"+separator+"y", separator, "z"+separator) } + // Seeds past the bounds of the grammar: more "@" signs than a real address holds, and an address + // longer than any real one. They must be withheld whole, never expanded. + f.Add("A", strings.Repeat("@", maxBoundaries+8), "B") + f.Add("A", strings.Repeat("@x", hostileBoundaries), "B") + f.Add(strings.Repeat("a", maxAddressLength), "@", "B") f.Fuzz(func(t *testing.T, first, separator, second string) { secret := redacttest.SecretOf(first, separator, second) for _, in := range redacttest.Generate(secret) { From 1003fcae4ad38821062c3dc8588ba520bec408c2 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 10:58:58 -0600 Subject: [PATCH 28/32] fix(utils): redact scheme-less include userinfo and scrub the OCI include errors that quote a fragment An include argument without a scheme that opens with userinfo (user:secret@host/path, //user:secret@host/path) was echoed as written, and so was the target in the missing-file error. The OCI include errors that wrap the pull cause or quote the subdirectory cut at the first double slash now pass through the credential-aware scrub against the whole target. Co-Authored-By: Claude Sonnet 5.5 --- pkg/utils/yaml_include_by_extension.go | 8 +++++--- pkg/utils/yaml_include_not_found.go | 6 +++--- pkg/utils/yaml_include_redact.go | 23 +++++++++++++++++++++-- pkg/utils/yaml_include_redact_test.go | 14 ++++++++++++++ 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pkg/utils/yaml_include_by_extension.go b/pkg/utils/yaml_include_by_extension.go index 02bd2caab64..91f8d5251b6 100644 --- a/pkg/utils/yaml_include_by_extension.go +++ b/pkg/utils/yaml_include_by_extension.go @@ -594,16 +594,18 @@ func processOCIIncludeFile(atmosConfig *schema.AtmosConfiguration, includeFile s // survive. It applies ociIncludeTimeout itself, so no extra deadline is set here. dl := downloader.NewGoGetterDownloader(atmosConfig).(downloader.ContextFileDownloader) if _, err := dl.FetchWithMetadataContext(context.Background(), imageRef, tempDir, downloader.ClientModeDir, ociIncludeTimeout); err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(includeFile), err) + return nil, redactIncludeError(fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionFailedStackManifest, RedactIncludeArgument(includeFile), err), includeFile) } localFile, err := filesystem.SafeJoin(tempDir, subdir) if err != nil { - return nil, fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), err) + return nil, redactIncludeError(fmt.Errorf("%w: '%s': %w", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), err), includeFile) } if !FileExists(localFile) { - return nil, fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(imageRef), subdir) + // The subdirectory is cut from the include target at its first "//", so a password that + // holds one leaves a fragment of itself in it: the whole message is scrubbed against the target. + return nil, redactIncludeError(fmt.Errorf("%w: oci artifact '%s' does not contain '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(imageRef), subdir), includeFile) } return processLocalFile(localFile, forceRaw) diff --git a/pkg/utils/yaml_include_not_found.go b/pkg/utils/yaml_include_not_found.go index bc1ea7e710b..7598b560955 100644 --- a/pkg/utils/yaml_include_not_found.go +++ b/pkg/utils/yaml_include_not_found.go @@ -25,13 +25,13 @@ func includeNotFoundError(tag, args, includeFile, manifestFile string, atmosConf switch { case filepath.IsAbs(includeFile): - return fmt.Errorf("%w: could not find local file '%s'", ErrIncludeYamlFunctionInvalidFile, includeFile) + return fmt.Errorf("%w: could not find local file '%s'", ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile)) case isManifestRelativePath(includeFile): resolved := ResolveRelativePath(includeFile, manifestFile) builder := errUtils.Build(fmt.Errorf( "%w: could not find local file '%s' at '%s' (paths starting with './' or '../' are resolved relative to the including file '%s' only)", - ErrIncludeYamlFunctionInvalidFile, includeFile, resolved, manifestFile, + ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), resolved, manifestFile, )).WithHint("Paths starting with `./` or `../` are relative to the file that contains the `" + tag + "`. Paths without that prefix are relative to `base_path`.") if suggestion := baseRelativeIncludeSuggestion(includeFile, basePath); suggestion != "" { @@ -47,7 +47,7 @@ func includeNotFoundError(tag, args, includeFile, manifestFile string, atmosConf } return errUtils.Build(fmt.Errorf( "%w: could not find local file '%s' (tried relative to base path '%s' and the working directory '%s')", - ErrIncludeYamlFunctionInvalidFile, includeFile, basePath, workingDir, + ErrIncludeYamlFunctionInvalidFile, RedactIncludeArgument(includeFile), basePath, workingDir, )).WithHint("Paths without a `./` or `../` prefix are relative to `base_path`. Start the path with `./` or `../` " + "to resolve it relative to the file that contains the `" + tag + "`.").Err() } diff --git a/pkg/utils/yaml_include_redact.go b/pkg/utils/yaml_include_redact.go index 3c6790845c7..b51aef39fad 100644 --- a/pkg/utils/yaml_include_redact.go +++ b/pkg/utils/yaml_include_redact.go @@ -1,25 +1,44 @@ package utils import ( + "regexp" "strings" "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/perf" ) +// schemeLessUserinfoPattern matches the start of an address that has no scheme but opens with +// userinfo, "user:secret@host/path". The user and the password hold no "/" or white space, and the +// password is closed by the first "@". An SCP-style git address ("git@github.com:org/repo.git") +// has no ":" before its "@" and does not match. +var schemeLessUserinfoPattern = regexp.MustCompile(`^[^\s/@]+:[^\s/@]*@`) + +// windowsDrivePattern matches the drive of a Windows path, "C:\" or "C:/", which is not a user name. +var windowsDrivePattern = regexp.MustCompile(`^[A-Za-z]:[\\/]`) + // RedactIncludeArgument removes credentials from an include argument or include target before it is // echoed in an error or a log line. Only URL-shaped values are touched: a local path or an // SCP-style git address keeps its text, and a URL that cannot be proven free of userinfo is -// replaced entirely (see artifact.RedactURL). +// replaced entirely (see artifact.RedactURL). A value without a scheme that opens with userinfo +// ("user:secret@host/path", "//user:secret@host/path") is URL-shaped too, and is redacted the same way. func RedactIncludeArgument(value string) string { defer perf.Track(nil, "utils.RedactIncludeArgument")() - if !strings.Contains(value, "://") { + if !isURLShaped(value) { return value } return artifact.RedactURL(value) } +// isURLShaped reports whether value may be an address that carries userinfo. +func isURLShaped(value string) bool { + if strings.Contains(value, "://") || strings.HasPrefix(value, "//") { + return true + } + return !windowsDrivePattern.MatchString(value) && schemeLessUserinfoPattern.MatchString(value) +} + // redactIncludeError removes the credential of an include target from the text of an error raised // while fetching it. Fetch errors from the download and OCI layers quote the reference they were // given (for example "could not parse reference: user:secret@host/image"), so the include diff --git a/pkg/utils/yaml_include_redact_test.go b/pkg/utils/yaml_include_redact_test.go index ba2611054dd..fd9d41d4d8c 100644 --- a/pkg/utils/yaml_include_redact_test.go +++ b/pkg/utils/yaml_include_redact_test.go @@ -31,6 +31,13 @@ func TestRedactIncludeArgument(t *testing.T) { {name: "url with userinfo", value: "https://user:" + redactTestSecret + "@example.com/base.yaml", want: "https://example.com/base.yaml"}, {name: "url with userinfo and fragment", value: "https://user:" + redactTestSecret + "@example.com/base.yaml#frag", want: "https://example.com/base.yaml#frag"}, {name: "forced getter with userinfo", value: "git::https://user:" + redactTestSecret + "@example.com/repo.git", want: "git::https://example.com/repo.git"}, + {name: "scheme-less userinfo", value: "user:" + redactTestSecret + "@example.com/base.yaml", want: ""}, + {name: "scheme-less userinfo with a query", value: "user:" + redactTestSecret + "@example.com/base.yaml .a.b | eval", want: ""}, + {name: "scheme-less token as user", value: "forced::user:" + redactTestSecret + "@example.com/repo.git", want: ""}, + {name: "scheme-relative userinfo", value: "//user:" + redactTestSecret + "@example.com/base.yaml", want: "//example.com/base.yaml"}, + {name: "local path holding a colon", value: "./stacks/a:b.yaml", want: "./stacks/a:b.yaml"}, + {name: "windows path holding an at sign", value: `C:\stacks\user@host\base.yaml`, want: `C:\stacks\user@host\base.yaml`}, + {name: "local path holding an at sign", value: "./stacks/user@host/base.yaml", want: "./stacks/user@host/base.yaml"}, {name: "unparsable url with userinfo", value: "https://user:" + redactTestSecret + "@exa mple.com/%zz", want: ""}, } for _, tt := range tests { @@ -64,6 +71,8 @@ func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { include string // rawOK reports that !include.raw does not parse the body, so it cannot fail on it. rawOK bool + // also is a second secret that must not reach the error. + also string }{ {name: "query evaluation failure", include: withUserinfo + "/config.yaml .a[ | eval"}, {name: "query evaluation failure without eval", include: withUserinfo + "/config.yaml .a["}, @@ -73,6 +82,8 @@ func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { {name: "fetch failure with eval", include: closed + "/config.yaml | eval"}, {name: "raw fetch failure", include: closed + "/config.txt"}, {name: "oci include without a file path", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag"}, + {name: "oci pull failure with a double slash in the password", include: "oci://user:" + redactTestSecret + "//hunter3@127.0.0.1:1/org/image:tag//file.yaml", also: "hunter3"}, + {name: "scheme-less userinfo that is not found", include: "user:" + redactTestSecret + "@127.0.0.1:1/config.yaml"}, {name: "oci pull failure", include: "oci://user:" + redactTestSecret + "@127.0.0.1:1/org/image:tag//file.yaml"}, } for _, tag := range []string{"!include", "!include.raw"} { @@ -87,6 +98,9 @@ func TestProcessInclude_ErrorsDoNotEchoCredentials(t *testing.T) { } require.Error(t, err) assert.NotContains(t, err.Error(), redactTestSecret, "the include credential must not reach the error text") + if tt.also != "" { + assert.NotContains(t, err.Error(), tt.also, "no part of the include credential may reach the error text") + } }) } } From cb3cb6f656ce2e32a7794120d78e65ccb2e4b918 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:01:51 -0600 Subject: [PATCH 29/32] fix(artifact): redact every configured name and value an error still quoted The bucket, region, S3 tag keys, label keys, permissions value, repository alias and kind, lock scope, lock delegates and chain, the repository of a capability error, and the keys and values a configuration decode error rejects were echoed as written, so a credential-bearing URL pasted into any of them reached the error text. Each echo now goes through the shared redaction, and plain names are still shown. The options decoder and the artifacts decoder share one RedactDecodeError. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/decode.go | 5 +- pkg/artifact/config/locking.go | 20 ++++-- pkg/artifact/config/redact_values_test.go | 86 +++++++++++++++++++++++ pkg/artifact/config/validate.go | 15 +++- pkg/artifact/errors.go | 4 +- pkg/artifact/labels.go | 16 +++-- pkg/artifact/options.go | 12 ++-- pkg/artifact/permissions.go | 8 ++- pkg/artifact/redact.go | 12 ++++ pkg/artifact/redact_values_test.go | 75 ++++++++++++++++++++ pkg/artifact/registry.go | 8 +-- 11 files changed, 229 insertions(+), 32 deletions(-) create mode 100644 pkg/artifact/config/redact_values_test.go create mode 100644 pkg/artifact/redact_values_test.go diff --git a/pkg/artifact/config/decode.go b/pkg/artifact/config/decode.go index 958823670f3..8158743e66a 100644 --- a/pkg/artifact/config/decode.go +++ b/pkg/artifact/config/decode.go @@ -9,6 +9,7 @@ import ( "github.com/go-viper/mapstructure/v2" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/schema" ) @@ -41,7 +42,7 @@ func Decode(raw map[string]any) (schema.ArtifactConfig, error) { return cfg, fmt.Errorf("%w: decoder: %w", errUtils.ErrArtifactConfigInvalid, err) } if err := decoder.Decode(raw); err != nil { - return cfg, fmt.Errorf("%w: artifacts: %w", errUtils.ErrArtifactConfigInvalid, err) + return cfg, fmt.Errorf("%w: artifacts: %w", errUtils.ErrArtifactConfigInvalid, artifact.RedactDecodeError(err, raw)) } normalize(&cfg) return cfg, nil @@ -102,7 +103,7 @@ func checkPermissionTypes(raw map[string]any) error { continue } if _, isString := value.(string); !isString { - errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T %v (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, alias, value, value)) + errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T %v (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), value, artifact.RedactCredentials(fmt.Sprint(value), ""))) } } return errors.Join(errs...) diff --git a/pkg/artifact/config/locking.go b/pkg/artifact/config/locking.go index 91424172833..f925ce0d3a9 100644 --- a/pkg/artifact/config/locking.go +++ b/pkg/artifact/config/locking.go @@ -32,7 +32,7 @@ func ResolveLockPlan(cfg *schema.ArtifactConfig, alias string) (LockPlan, error) repo, ok := cfg.Repositories[alias] if !ok { - return LockPlan{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, alias) + return LockPlan{}, fmt.Errorf("%w: repository %q is not configured", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias)) } required, err := requiredScope(repo.Locking) if err != nil { @@ -47,10 +47,10 @@ func ResolveLockPlan(cfg *schema.ArtifactConfig, alias string) (LockPlan, error) } next := current.Locking.Repository if _, exists := cfg.Repositories[next]; !exists { - return LockPlan{}, fmt.Errorf("%w: repository %q delegates locking to %q, which is not configured", errUtils.ErrArtifactConfigInvalid, plan.Provider, next) + return LockPlan{}, fmt.Errorf("%w: repository %q delegates locking to %q, which is not configured", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(plan.Provider), artifact.RedactURL(next)) } if visited[next] { - return LockPlan{}, fmt.Errorf("%w: lock delegation cycle: %s -> %s", errUtils.ErrArtifactConfigInvalid, strings.Join(plan.Chain, " -> "), next) + return LockPlan{}, fmt.Errorf("%w: lock delegation cycle: %s -> %s", errUtils.ErrArtifactConfigInvalid, redactedChain(plan.Chain), artifact.RedactURL(next)) } visited[next] = true plan.Chain = append(plan.Chain, next) @@ -67,7 +67,7 @@ func requiredScope(locking *schema.ArtifactLockingConfig) (artifact.LockScope, e case artifact.LockScopeHost, artifact.LockScopeRepository: return scope, nil default: - return "", fmt.Errorf("%w: locking.required_scope %q must be %q or %q", errUtils.ErrArtifactConfigInvalid, locking.RequiredScope, artifact.LockScopeHost, artifact.LockScopeRepository) + return "", fmt.Errorf("%w: locking.required_scope %q must be %q or %q", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(locking.RequiredScope), artifact.LockScopeHost, artifact.LockScopeRepository) } } @@ -93,10 +93,20 @@ func validateLocking(v *Validator, cfg *schema.ArtifactConfig, alias string, rep return errors.Join( errUtils.ErrArtifactConfigInvalid, fmt.Errorf("%w: locking.required_scope %q is not provided by %q (kind %q advertises %v)%s", - errUtils.ErrArtifactUnsupportedCapability, plan.Required, plan.Provider, provider.Kind, desc.Capabilities.LockScopes, delegateHint(plan)), + errUtils.ErrArtifactUnsupportedCapability, plan.Required, artifact.RedactURL(plan.Provider), artifact.RedactURL(provider.Kind), desc.Capabilities.LockScopes, delegateHint(plan)), ) } +// redactedChain joins the aliases of a delegation chain, each redacted: an alias is a name the user +// typed, and it may be a pasted credential-bearing URL. +func redactedChain(chain []string) string { + shown := make([]string, len(chain)) + for i, alias := range chain { + shown[i] = artifact.RedactURL(alias) + } + return strings.Join(shown, " -> ") +} + func delegateHint(plan LockPlan) string { if len(plan.Chain) > 1 { return "" diff --git a/pkg/artifact/config/redact_values_test.go b/pkg/artifact/config/redact_values_test.go new file mode 100644 index 00000000000..973add72ad2 --- /dev/null +++ b/pkg/artifact/config/redact_values_test.go @@ -0,0 +1,86 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// TestEchoedNames_DoNotEchoCredentials checks that a repository alias, kind, lock scope or lock +// delegate that is a pasted credential-bearing URL never reaches a validation error. +func TestEchoedNames_DoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + local := schema.ArtifactRepositoryConfig{Kind: "local/archive"} + delegating := func(target string) *schema.ArtifactConfig { + return &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: target}}, + }} + } + _, missingAlias := ResolveLockPlan(&schema.ArtifactConfig{}, credential) + _, missingDelegate := ResolveLockPlan(delegating(credential), "a") + _, scope := ResolveLockPlan(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{RequiredScope: credential}}, + }}, "a") + _, cycle := ResolveLockPlan(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "a": {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: credential}}, + credential: {Kind: "local/archive", Locking: &schema.ArtifactLockingConfig{Repository: "a"}}, + }}, "a") + withAlias := &schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{credential: local}} + + tests := []struct { + name string + err error + }{ + {name: "lock plan of an unknown alias", err: missingAlias}, + {name: "lock delegate that is not configured", err: missingDelegate}, + {name: "lock scope", err: scope}, + {name: "lock delegation cycle", err: cycle}, + {name: "alias that is not valid", err: NewValidator().ValidateRepository(withAlias, credential)}, + {name: "configured aliases listed beside an unknown one", err: NewValidator().ValidateRepository(withAlias, "nope")}, + {name: "unknown kind", err: NewValidator().ValidateRepository(&schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{"a": {Kind: credential}}}, "a")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.ErrorIs(t, tt.err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, tt.err.Error(), "S3cr3tA") + assert.NotContains(t, tt.err.Error(), "usr7") + }) + } +} + +// TestEchoedNames_KeepNamesWithoutCredentials is the negative path of the test above: a plain alias +// is still named in the error. +func TestEchoedNames_KeepNamesWithoutCredentials(t *testing.T) { + err := NewValidator().ValidateRepository(&schema.ArtifactConfig{}, "prod-eu") + require.Error(t, err) + assert.Contains(t, err.Error(), `repository "prod-eu" is not configured`) +} + +// TestDecode_ErrorsDoNotEchoCredentials checks that the keys and values a decode error quotes are +// scrubbed, and that a non-string permissions value is shown without its credential. +func TestDecode_ErrorsDoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + tests := []struct { + name string + raw map[string]any + }{ + {name: "unknown repository field", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"kind": "local/archive", credential: "x"}}}}, + {name: "unknown top-level field", raw: map[string]any{credential: "x"}}, + {name: "credential-like unknown field", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"usr7:S3cr3tA@password": "x"}}}}, + {name: "permissions holding a credential", raw: map[string]any{"repositories": map[string]any{credential: map[string]any{"kind": "local/archive", "permissions": map[string]any{"k": credential}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Decode(tt.raw) + require.Error(t, err) + assert.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + }) + } +} diff --git a/pkg/artifact/config/validate.go b/pkg/artifact/config/validate.go index f5e616e4ab1..4a6bea4546b 100644 --- a/pkg/artifact/config/validate.go +++ b/pkg/artifact/config/validate.go @@ -99,13 +99,13 @@ func (v *Validator) ValidateRepository(cfg *schema.ArtifactConfig, alias string) repo, ok := cfg.Repositories[alias] if !ok { - return fmt.Errorf("%w: repository %q is not configured (configured: %v)", errUtils.ErrArtifactConfigInvalid, alias, sortedAliases(cfg)) + return fmt.Errorf("%w: repository %q is not configured (configured: %v)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), redactedAliases(cfg)) } return v.validateRepository(cfg, alias, &repo) } func (v *Validator) validateRepository(cfg *schema.ArtifactConfig, alias string, repo *schema.ArtifactRepositoryConfig) error { - prefix := fmt.Sprintf("repository %q", alias) + prefix := fmt.Sprintf("repository %q", artifact.RedactURL(alias)) if !aliasPattern.MatchString(alias) { return fmt.Errorf("%w: %s: alias must start with a letter or digit and contain only letters, digits, '_', '.' and '-'", errUtils.ErrArtifactConfigInvalid, prefix) } @@ -136,7 +136,7 @@ func (v *Validator) resolveKind(kind string) (*artifact.KindDescriptor, error) { } desc, ok := v.registry.Lookup(kind) if !ok { - return nil, fmt.Errorf("%w: unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, kind, v.registry.Kinds()) + return nil, fmt.Errorf("%w: unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(kind), v.registry.Kinds()) } return desc, nil } @@ -189,6 +189,15 @@ func validateDefaults(defaults *schema.ArtifactRepositoryDefaults) error { return errors.Join(errs...) } +// redactedAliases returns the configured aliases in order, each redacted, for an error that lists them. +func redactedAliases(cfg *schema.ArtifactConfig) []string { + aliases := sortedAliases(cfg) + for i, alias := range aliases { + aliases[i] = artifact.RedactURL(alias) + } + return aliases +} + func sortedAliases(cfg *schema.ArtifactConfig) []string { aliases := make([]string, 0, len(cfg.Repositories)) for alias := range cfg.Repositories { diff --git a/pkg/artifact/errors.go b/pkg/artifact/errors.go index 0f5c702f371..65cfa7c42fb 100644 --- a/pkg/artifact/errors.go +++ b/pkg/artifact/errors.go @@ -64,13 +64,13 @@ func (e *CapabilityError) Error() string { message := e.sentinel().Error() if e.Repository != "" { - message = fmt.Sprintf("%s: repository %q", message, e.Repository) + message = fmt.Sprintf("%s: repository %q", message, RedactURL(e.Repository)) } if e.Operation != "" { message = fmt.Sprintf("%s: %s", message, e.Operation) } if e.Capability != "" { - message = fmt.Sprintf("%s: capability %s", message, e.Capability) + message = fmt.Sprintf("%s: capability %s", message, RedactURL(e.Capability)) } if e.Reason != "" { message = fmt.Sprintf("%s: %s", message, e.Reason) diff --git a/pkg/artifact/labels.go b/pkg/artifact/labels.go index 58dc9ca41c9..f9a426d0773 100644 --- a/pkg/artifact/labels.go +++ b/pkg/artifact/labels.go @@ -30,23 +30,25 @@ const ( func ValidateLabel(key, value string) error { defer perf.Track(nil, "artifact.ValidateLabel")() + // The key is echoed in every message, and a key may be a pasted credential-bearing URL. + shown := RedactURL(key) switch { case key == "": return fmt.Errorf("%w: label key must not be empty", errUtils.ErrArtifactConfigInvalid) case len(key) > MaxLabelKeyLength: - return fmt.Errorf("%w: label key %q exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, key, MaxLabelKeyLength) + return fmt.Errorf("%w: label key %q exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, shown, MaxLabelKeyLength) case strings.TrimSpace(key) != key: - return fmt.Errorf("%w: label key %q must not have leading or trailing whitespace", errUtils.ErrArtifactConfigInvalid, key) + return fmt.Errorf("%w: label key %q must not have leading or trailing whitespace", errUtils.ErrArtifactConfigInvalid, shown) case strings.ContainsAny(key, "=:,"): - return fmt.Errorf("%w: label key %q must not contain '=', ':' or ',' (they delimit --labels selectors)", errUtils.ErrArtifactConfigInvalid, key) + return fmt.Errorf("%w: label key %q must not contain '=', ':' or ',' (they delimit --labels selectors)", errUtils.ErrArtifactConfigInvalid, shown) case strings.IndexFunc(key, unicode.IsControl) >= 0: - return fmt.Errorf("%w: label key %q must not contain control characters", errUtils.ErrArtifactConfigInvalid, key) + return fmt.Errorf("%w: label key %q must not contain control characters", errUtils.ErrArtifactConfigInvalid, shown) case len(value) > MaxLabelValueLength: - return fmt.Errorf("%w: label %q value exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, key, MaxLabelValueLength) + return fmt.Errorf("%w: label %q value exceeds %d bytes", errUtils.ErrArtifactConfigInvalid, shown, MaxLabelValueLength) case strings.Contains(value, ","): - return fmt.Errorf("%w: label %q value must not contain ',' (it delimits --labels selectors)", errUtils.ErrArtifactConfigInvalid, key) + return fmt.Errorf("%w: label %q value must not contain ',' (it delimits --labels selectors)", errUtils.ErrArtifactConfigInvalid, shown) case strings.IndexFunc(value, unicode.IsControl) >= 0: - return fmt.Errorf("%w: label %q value must not contain control characters", errUtils.ErrArtifactConfigInvalid, key) + return fmt.Errorf("%w: label %q value must not contain control characters", errUtils.ErrArtifactConfigInvalid, shown) } return nil } diff --git a/pkg/artifact/options.go b/pkg/artifact/options.go index 9f8bd638f46..e6f44c02a8f 100644 --- a/pkg/artifact/options.go +++ b/pkg/artifact/options.go @@ -155,10 +155,10 @@ func (o *S3Options) Validate() error { if o.Bucket == "" { errs = append(errs, fmt.Errorf("%w: option %q is required", errUtils.ErrArtifactConfigInvalid, "bucket")) } else if !s3BucketPattern.MatchString(o.Bucket) || net.ParseIP(o.Bucket) != nil { - errs = append(errs, fmt.Errorf("%w: bucket %q is not a valid S3 bucket name", errUtils.ErrArtifactConfigInvalid, o.Bucket)) + errs = append(errs, fmt.Errorf("%w: bucket %q is not a valid S3 bucket name", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Bucket))) } if o.Region != "" && !regionPattern.MatchString(o.Region) { - errs = append(errs, fmt.Errorf("%w: region %q is not a valid region name", errUtils.ErrArtifactConfigInvalid, o.Region)) + errs = append(errs, fmt.Errorf("%w: region %q is not a valid region name", errUtils.ErrArtifactConfigInvalid, RedactURL(o.Region))) } errs = append(errs, validatePrefix(o.Prefix), validateS3Tags(o.Tags), validateKeyPattern(o.KeyPattern, "")) return errors.Join(errs...) @@ -248,7 +248,7 @@ func DecodeOptions(raw map[string]any, out any) error { if err := decoder.Decode(raw); err != nil { // A decode error may quote a value, such as a mistyped number that is really a credential, // so every string key and value is a credential candidate when the message is scrubbed. - return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, RedactError(err, stringValues(raw)...), credentialHint(raw)) + return fmt.Errorf("%w: options: %w%s", errUtils.ErrArtifactConfigInvalid, RedactDecodeError(err, raw), credentialHint(raw)) } return nil } @@ -437,11 +437,11 @@ func validateS3Tags(tags map[string]string) error { for _, key := range keys { switch { case key == "" || utf8.RuneCountInString(key) > maxS3TagKeyLen: - errs = append(errs, fmt.Errorf("%w: S3 tag key %q must be 1-%d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagKeyLen)) + errs = append(errs, fmt.Errorf("%w: S3 tag key %q must be 1-%d characters", errUtils.ErrArtifactConfigInvalid, RedactURL(key), maxS3TagKeyLen)) case strings.HasPrefix(strings.ToLower(key), reservedTagHead): - errs = append(errs, fmt.Errorf("%w: S3 tag key %q uses the reserved %q prefix", errUtils.ErrArtifactConfigInvalid, key, reservedTagHead)) + errs = append(errs, fmt.Errorf("%w: S3 tag key %q uses the reserved %q prefix", errUtils.ErrArtifactConfigInvalid, RedactURL(key), reservedTagHead)) case utf8.RuneCountInString(tags[key]) > maxS3TagValLen: - errs = append(errs, fmt.Errorf("%w: S3 tag %q value exceeds %d characters", errUtils.ErrArtifactConfigInvalid, key, maxS3TagValLen)) + errs = append(errs, fmt.Errorf("%w: S3 tag %q value exceeds %d characters", errUtils.ErrArtifactConfigInvalid, RedactURL(key), maxS3TagValLen)) } } return errors.Join(errs...) diff --git a/pkg/artifact/permissions.go b/pkg/artifact/permissions.go index 34c8f80429c..f1f5acf017e 100644 --- a/pkg/artifact/permissions.go +++ b/pkg/artifact/permissions.go @@ -33,15 +33,17 @@ var permissionsPattern = regexp.MustCompile(`^[0-7]{3,4}$`) func ParsePermissions(value string) (fs.FileMode, error) { defer perf.Track(nil, "artifact.ParsePermissions")() + // The value is echoed, and it may be a pasted credential-bearing URL. + shown := RedactURL(value) if !permissionsPattern.MatchString(value) { - return 0, fmt.Errorf("%w: permissions %q must be a quoted octal mode such as \"0600\"", errUtils.ErrArtifactConfigInvalid, value) + return 0, fmt.Errorf("%w: permissions %q must be a quoted octal mode such as \"0600\"", errUtils.ErrArtifactConfigInvalid, shown) } parsed, err := strconv.ParseUint(value, octalBase, permissionBitSize) if err != nil { - return 0, fmt.Errorf("%w: permissions %q: %w", errUtils.ErrArtifactConfigInvalid, value, err) + return 0, fmt.Errorf("%w: permissions %q: %w", errUtils.ErrArtifactConfigInvalid, shown, err) } if parsed > maxPermissionBits { - return 0, fmt.Errorf("%w: permissions %q must not set privileged mode bits (maximum \"0777\")", errUtils.ErrArtifactConfigInvalid, value) + return 0, fmt.Errorf("%w: permissions %q must not set privileged mode bits (maximum \"0777\")", errUtils.ErrArtifactConfigInvalid, shown) } return fs.FileMode(parsed), nil } diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go index d735a87b2e5..4f7f8ab23f0 100644 --- a/pkg/artifact/redact.go +++ b/pkg/artifact/redact.go @@ -126,6 +126,18 @@ func RedactError(err error, raws ...string) error { return &redactedError{message: scrubbed, err: err} } +// RedactDecodeError is [RedactError] for the error of decoding raw, a configuration value as it was +// parsed: a decode error quotes the keys and values it rejects, so every string held by raw, keys +// included and at any depth, is a credential candidate when the message is scrubbed. +func RedactDecodeError(err error, raw any) error { + defer perf.Track(nil, "artifact.RedactDecodeError")() + + if err == nil { + return nil + } + return RedactError(err, stringValues(raw)...) +} + // redactedError is an error whose text has had a credential removed. It keeps the original error in // its chain so errors.Is and errors.As still see the cause. type redactedError struct { diff --git a/pkg/artifact/redact_values_test.go b/pkg/artifact/redact_values_test.go new file mode 100644 index 00000000000..bdd5f376937 --- /dev/null +++ b/pkg/artifact/redact_values_test.go @@ -0,0 +1,75 @@ +package artifact + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinel: renaming the schema field the registry test sets must fail the build. +var _ = schema.ArtifactRepositoryConfig{Kind: ""} + +// TestEchoedValues_DoNotEchoCredentials checks every validator and error that quotes a configured +// name or value, other than the registry, prefix, key pattern and reference covered above, when +// that value is a pasted credential-bearing URL. +func TestEchoedValues_DoNotEchoCredentials(t *testing.T) { + const credential = "https://usr7:S3cr3tA@example.com" + registry := NewRepositoryRegistry() + _, registryErr := registry.New(context.Background(), credential, &schema.ArtifactRepositoryConfig{Kind: credential}) + _, nilConfigErr := registry.New(context.Background(), credential, nil) + _, permissionsErr := ParsePermissions(credential) + + tests := []struct { + name string + err error + }{ + {name: "s3 bucket", err: (&S3Options{Bucket: credential}).Validate()}, + {name: "s3 region", err: (&S3Options{Bucket: "bucket", Region: credential}).Validate()}, + {name: "s3 reserved tag key", err: validateS3Tags(map[string]string{"aws:" + credential: "v"})}, + {name: "s3 oversized tag value", err: validateS3Tags(map[string]string{credential: strings.Repeat("v", maxS3TagValLen+1)})}, + {name: "label key", err: ValidateLabel(credential, "v")}, + {name: "label key whitespace", err: ValidateLabel(" "+credential, "v")}, + {name: "label value", err: ValidateLabel(credential, strings.Repeat("v", MaxLabelValueLength+1))}, + {name: "label set", err: ValidateLabels(map[string]string{credential: "v"})}, + {name: "permissions", err: permissionsErr}, + {name: "registry unknown kind", err: registryErr}, + {name: "registry nil configuration", err: nilConfigErr}, + {name: "capability repository", err: NewUnsupportedCapability(credential, "push", "kind/"+credential, "not available")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.NotContains(t, tt.err.Error(), "S3cr3tA") + assert.NotContains(t, tt.err.Error(), "usr7") + }) + } +} + +// TestEchoedValues_KeepNamesWithoutCredentials is the negative path of the test above: a plain +// name is still echoed, because the name is what lets the user find the mistake. +func TestEchoedValues_KeepNamesWithoutCredentials(t *testing.T) { + _, registryErr := NewRepositoryRegistry().New(context.Background(), "prod", &schema.ArtifactRepositoryConfig{Kind: "ftp/server"}) + tests := []struct { + name string + err error + want string + }{ + {name: "s3 bucket", err: (&S3Options{Bucket: "Not_A_Bucket"}).Validate(), want: `bucket "Not_A_Bucket"`}, + {name: "s3 region", err: (&S3Options{Bucket: "bucket", Region: "Not A Region"}).Validate(), want: `region "Not A Region"`}, + {name: "label key", err: ValidateLabel("a=b", "v"), want: `label key "a=b"`}, + {name: "registry unknown kind", err: registryErr, want: `repository "prod" has unknown kind "ftp/server"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Error(t, tt.err) + assert.ErrorIs(t, tt.err, errUtils.ErrArtifactConfigInvalid) + assert.Contains(t, tt.err.Error(), tt.want) + }) + } +} diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go index a27b61876c4..14fefc34432 100644 --- a/pkg/artifact/registry.go +++ b/pkg/artifact/registry.go @@ -170,11 +170,11 @@ func (r *RepositoryRegistry) New(ctx context.Context, alias string, cfg *schema. defer perf.Track(nil, "artifact.RepositoryRegistry.New")() if cfg == nil { - return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, alias) + return nil, fmt.Errorf("%w: repository %q has no configuration", errUtils.ErrArtifactConfigInvalid, RedactURL(alias)) } desc, known := r.Lookup(cfg.Kind) if !known { - return nil, fmt.Errorf("%w: repository %q has unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, alias, cfg.Kind, r.Kinds()) + return nil, fmt.Errorf("%w: repository %q has unknown kind %q (known kinds: %v)", errUtils.ErrArtifactConfigInvalid, RedactURL(alias), RedactURL(cfg.Kind), r.Kinds()) } r.mu.RLock() entry, registered := r.kinds[cfg.Kind] @@ -183,11 +183,11 @@ func (r *RepositoryRegistry) New(ctx context.Context, alias string, cfg *schema. return nil, NewUnsupportedCapability(alias, "construct", "kind/"+cfg.Kind, "no backend is registered for this repository kind in this build") } if err := ValidateRepositoryConfig(desc, cfg); err != nil { - return nil, fmt.Errorf("repository %q: %w", alias, err) + return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), err) } repo, err := entry.factory(ctx, alias, cfg, opts...) if err != nil { - return nil, fmt.Errorf("repository %q: %w", alias, err) + return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), err) } return repo, nil } From 53f416d99a85969663fc31c7bf1bcec48208e6f4 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 11:13:01 -0600 Subject: [PATCH 30/32] refactor(artifact): split the address bounds check and boundary scan out of parseAddress The CI lint configuration caps cyclomatic complexity at 10 and parseAddress reached 12 with the bounds check. The check and the scan are now named helpers. RedactDecodeError also scrubs only the distinct strings that hold an at sign. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/redact.go | 17 ++++++++++++++++- pkg/artifact/redact_bounds_test.go | 8 ++++++++ pkg/artifact/redact_grammar.go | 17 +++++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/redact.go b/pkg/artifact/redact.go index 4f7f8ab23f0..09a498cd3dd 100644 --- a/pkg/artifact/redact.go +++ b/pkg/artifact/redact.go @@ -135,7 +135,22 @@ func RedactDecodeError(err error, raw any) error { if err == nil { return nil } - return RedactError(err, stringValues(raw)...) + return RedactError(err, credentialCandidates(stringValues(raw))...) +} + +// credentialCandidates returns the distinct strings of values that hold an "@", the only ones that +// can carry userinfo, so a large configuration costs one scrub per distinct candidate and not one +// per string it holds. +func credentialCandidates(values []string) []string { + seen := make(map[string]bool, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + if strings.Contains(value, "@") && !seen[value] { + seen[value] = true + out = append(out, value) + } + } + return out } // redactedError is an error whose text has had a credential removed. It keeps the original error in diff --git a/pkg/artifact/redact_bounds_test.go b/pkg/artifact/redact_bounds_test.go index bb1ae74bf0f..bc38b306918 100644 --- a/pkg/artifact/redact_bounds_test.go +++ b/pkg/artifact/redact_bounds_test.go @@ -116,3 +116,11 @@ func TestRedact_AddressBoundsEdges(t *testing.T) { }) } } + +// TestCredentialCandidates checks that only distinct strings that can carry userinfo are scrubbed +// for, in the order they were first seen. +func TestCredentialCandidates(t *testing.T) { + got := credentialCandidates([]string{"plain", "a@b", "a@b", "", "c:d@e", "plain"}) + assert.Equal(t, []string{"a@b", "c:d@e"}, got) + assert.Empty(t, credentialCandidates(nil)) +} diff --git a/pkg/artifact/redact_grammar.go b/pkg/artifact/redact_grammar.go index c6810018fb4..3818f7eb014 100644 --- a/pkg/artifact/redact_grammar.go +++ b/pkg/artifact/redact_grammar.go @@ -108,7 +108,7 @@ type address struct { // before any work that grows with the number of its "@" signs starts. func parseAddress(raw string) address { a := address{} - if ats := strings.Count(raw, "@"); ats > maxBoundaries || (ats > 0 && len(raw) > maxAddressLength) { + if exceedsBounds(raw) { a.overflow = true return a } @@ -124,6 +124,20 @@ func parseAddress(raw string) address { if a.authorityEnd < 0 { a.authorityEnd = len(a.rest) } + a.findBoundaries() + return a +} + +// exceedsBounds reports whether raw holds more "@" signs than [maxBoundaries], or is longer than +// [maxAddressLength] while holding one, so it is withheld without being split. +func exceedsBounds(raw string) bool { + ats := strings.Count(raw, "@") + return ats > maxBoundaries || (ats > 0 && len(raw) > maxAddressLength) +} + +// findBoundaries records the offset of every "@" of the remainder: those in the authority, and +// those after it that do not introduce a digest. +func (a *address) findBoundaries() { for i := 0; i < a.authorityEnd; i++ { if a.rest[i] == '@' { a.inside = append(a.inside, i) @@ -134,7 +148,6 @@ func parseAddress(raw string) address { a.stray = append(a.stray, i) } } - return a } // introducesDigest reports whether the "@" at offset i of the remainder, after the authority, From 342bde4924056ab00b2b228293eed62c7f66925c Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 15 Nov 2026 18:04:43 -0600 Subject: [PATCH 31/32] fix(artifact): keep credentials out of decode and factory error text A non-string permissions value was echoed into the decode error, and a scheme-less credential inside it passed through the redactor unchanged; report only the value's type. A repository factory error that quotes the raw alias was wrapped unchanged; scrub it with the same redaction the package applies elsewhere, keeping the cause in the chain. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/config/decode.go | 2 +- pkg/artifact/config/redact_values_test.go | 1 + pkg/artifact/registry.go | 2 +- pkg/artifact/registry_test.go | 18 ++++++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/config/decode.go b/pkg/artifact/config/decode.go index 8158743e66a..2962209755a 100644 --- a/pkg/artifact/config/decode.go +++ b/pkg/artifact/config/decode.go @@ -103,7 +103,7 @@ func checkPermissionTypes(raw map[string]any) error { continue } if _, isString := value.(string); !isString { - errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T %v (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), value, artifact.RedactCredentials(fmt.Sprint(value), ""))) + errs = append(errs, fmt.Errorf("%w: repository %q: permissions must be a quoted octal string such as \"0600\", got %T (an unquoted 0600 is read by YAML as a different number)", errUtils.ErrArtifactConfigInvalid, artifact.RedactURL(alias), value)) } } return errors.Join(errs...) diff --git a/pkg/artifact/config/redact_values_test.go b/pkg/artifact/config/redact_values_test.go index 973add72ad2..1c50130ab33 100644 --- a/pkg/artifact/config/redact_values_test.go +++ b/pkg/artifact/config/redact_values_test.go @@ -73,6 +73,7 @@ func TestDecode_ErrorsDoNotEchoCredentials(t *testing.T) { {name: "unknown top-level field", raw: map[string]any{credential: "x"}}, {name: "credential-like unknown field", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"usr7:S3cr3tA@password": "x"}}}}, {name: "permissions holding a credential", raw: map[string]any{"repositories": map[string]any{credential: map[string]any{"kind": "local/archive", "permissions": map[string]any{"k": credential}}}}}, + {name: "permissions holding a scheme-less credential", raw: map[string]any{"repositories": map[string]any{"a": map[string]any{"kind": "local/archive", "permissions": map[string]any{"k": "usr7:S3cr3tA@example.com"}}}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/artifact/registry.go b/pkg/artifact/registry.go index 14fefc34432..f613e489279 100644 --- a/pkg/artifact/registry.go +++ b/pkg/artifact/registry.go @@ -187,7 +187,7 @@ func (r *RepositoryRegistry) New(ctx context.Context, alias string, cfg *schema. } repo, err := entry.factory(ctx, alias, cfg, opts...) if err != nil { - return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), err) + return nil, fmt.Errorf("repository %q: %w", RedactURL(alias), RedactError(err, alias)) } return repo, nil } diff --git a/pkg/artifact/registry_test.go b/pkg/artifact/registry_test.go index c6ae19d232b..8f00bf4e07e 100644 --- a/pkg/artifact/registry_test.go +++ b/pkg/artifact/registry_test.go @@ -268,3 +268,21 @@ func TestRepositoryRegistryDescriptorIsolation(t *testing.T) { assert.Equal(t, PackagingNone, again.Capabilities.Packagings[0]) assert.NotEqual(t, LockScope("changed"), again.Capabilities.LockScopes[0]) } + +// A factory error that quotes the raw alias must not carry its credential to the caller, and the +// scrubbed error keeps the cause in its chain. +func TestRepositoryRegistryNew_ScrubsFactoryErrorsOfTheAlias(t *testing.T) { + const alias = "https://usr7:S3cr3tA@example.com/repo" + cause := errors.New("dial " + alias + " failed") + r := NewRepositoryRegistry() + desc, _ := BuiltinKind(RepositoryKindLocalDir) + require.NoError(t, r.Register(desc, func(context.Context, string, *schema.ArtifactRepositoryConfig, ...RepositoryOption) (Repository, error) { + return nil, cause + })) + + _, err := r.New(context.Background(), alias, localDirConfig(t.TempDir())) + require.Error(t, err) + assert.NotContains(t, err.Error(), "S3cr3tA") + assert.NotContains(t, err.Error(), "usr7") + assert.ErrorIs(t, err, cause, "the cause stays in the chain") +} From b2d2f2a7fb202f04730f01bc4042d2cc72c4303b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Mon, 16 Nov 2026 17:48:57 -0600 Subject: [PATCH 32/32] fix(config): satisfy include key comment lint after integration --- pkg/config/process_yaml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/process_yaml.go b/pkg/config/process_yaml.go index aee32c8d573..f89bb82183f 100644 --- a/pkg/config/process_yaml.go +++ b/pkg/config/process_yaml.go @@ -18,7 +18,7 @@ const ( errorFormat = "%w: %v %v error %w" failedToProcess = "failed to process" emptyValueWarning = "execute returned empty value" - // includeDataKey is the mapping key an include is evaluated under, so its value keeps its type. + // The includeDataKey mapping key preserves the type of an evaluated include value. includeDataKey = "include_data" )