diff --git a/cmd/artifacts/artifacts.go b/cmd/artifacts/artifacts.go index c53966cdccc..70d379e97a6 100644 --- a/cmd/artifacts/artifacts.go +++ b/cmd/artifacts/artifacts.go @@ -56,6 +56,7 @@ default repository or an implicit latest version.`, newSignCommand(), newVerifyCommand(), newShowCommand(), + newCatalogCommand(), ) registerDocumentation(command, "") return command @@ -76,6 +77,13 @@ var documentationPages = map[string]string{ "show": "show", "show readme": "show", "show changelog": "show", + + "catalog": "catalog", + "catalog build": "catalog-build", + "catalog publish": "catalog-publish", + "catalog add": "catalog-add", + "catalog remove": "catalog-remove", + "catalog sync": "catalog-sync", } // registerDocumentation links every command to its documentation page. diff --git a/cmd/artifacts/catalog.go b/cmd/artifacts/catalog.go new file mode 100644 index 00000000000..36d81e7d8f4 --- /dev/null +++ b/cmd/artifacts/catalog.go @@ -0,0 +1,204 @@ +package artifacts + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/cmd/markdown" + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifactwire" + "github.com/cloudposse/atmos/pkg/flags" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Catalog flag names and environment variables. A catalog is the optional, maintained discovery +// snapshot of one configured repository. +const ( + flagInput = "input" + flagExpectedDigest = "expected-digest" + + envInput = "ATMOS_ARTIFACTS_INPUT" + envExpectedDigest = "ATMOS_ARTIFACTS_EXPECTED_DIGEST" + + catalogBuildPrefix = "artifacts.catalog.build" + catalogPublishPrefix = "artifacts.catalog.publish" + catalogAddPrefix = "artifacts.catalog.add" + catalogRemovePrefix = "artifacts.catalog.remove" + catalogSyncPrefix = "artifacts.catalog.sync" +) + +// CatalogFactory builds the catalog maintenance service for the loaded configuration. Tests replace +// it to inject a fake; production composes the real repositories lazily, so building the service +// performs no network access, credential resolution, or directory creation. +type CatalogFactory func(config *schema.AtmosConfiguration) (catalog.Maintainer, error) + +var newCatalogs CatalogFactory = artifactwire.NewCatalogs + +// UseCatalogFactory replaces the catalog service factory and returns a function that restores the +// previous one. It is the seam command tests use; production code never calls it. +func UseCatalogFactory(factory CatalogFactory) (restore func()) { + previous := newCatalogs + newCatalogs = factory + return func() { newCatalogs = previous } +} + +func init() { + hintRules = append(catalogHintRules(), hintRules...) +} + +// newCatalogCommand builds the `atmos artifacts catalog` command group. +func newCatalogCommand() *cobra.Command { + command := &cobra.Command{ + Use: "catalog", + Short: "Build and maintain the optional discovery catalog of a repository", + Long: `Build, publish, and maintain the optional catalog of a configured repository. A catalog is a +maintained discovery snapshot published as an artifact record of kind catalog. It lets a registry that +cannot enumerate repositories be listed and searched. Catalogs are discoverable data, never a trust grant: +publishing, signing, or deleting an artifact never edits a catalog, and removing an entry only unlists it. +Catalog writes need one externally serialized publisher.`, + Example: markdown.ArtifactsCatalogUsageMarkdown, + } + command.AddCommand(newCatalogBuildCommand(), newCatalogPublishCommand(), newCatalogAddCommand(), newCatalogRemoveCommand(), newCatalogSyncCommand()) + return command +} + +// catalogRun is the operation of one catalog subcommand. +type catalogRun func(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) + +// catalogCommand describes one catalog subcommand; the pipeline around it is shared. +type catalogCommand struct { + action string + parser *flags.StandardParser + prefix string + run catalogRun +} + +func (c *catalogCommand) build(use, short, long, example string, args cobra.PositionalArgs) *cobra.Command { + cmd := &cobra.Command{Use: use, Short: short, Long: long, Example: example, Args: args, RunE: c.execute} + c.parser.RegisterFlags(cmd) + registerCompletions(cmd, resultFormats) + return cmd +} + +// execute binds flags, builds the service, runs the operation, and renders the result. A structured +// format emits its envelope even when the operation fails, and the returned error keeps the exit +// code nonzero. +func (c *catalogCommand) execute(cmd *cobra.Command, args []string) error { + defer perf.Track(atmosConfigPtr, "artifacts.catalogCommand.execute")() + + values := viper.New() + if err := c.parser.BindFlagsToViper(cmd, values); err != nil { + return err + } + in := &input{v: values, prefix: c.prefix, args: args} + format := in.str(flagFormat) + if err := validateFormat(format, resultFormats); err != nil { + return err + } + res, err := c.invoke(commandContext(cmd), in) + if res == nil { + res = &catalogResult{} + } + res.Action, res.Repository = c.action, in.str(flagRepository) + if err != nil { + res.Error = err.Error() + } + if isStructured(format) { + if emitErr := writeStructured(format, res); emitErr != nil && err == nil { + return emitErr + } + } else if err == nil { + if textErr := writeText(res.pairs()); textErr != nil { + return textErr + } + } + return decorate(err, configuredAliases(atmosConfigPtr)) +} + +func (c *catalogCommand) invoke(ctx context.Context, in *input) (*catalogResult, error) { + if atmosConfigPtr == nil { + return nil, errUtils.Build(errUtils.ErrArtifactConfigInvalid). + WithExplanation("Atmos configuration is not loaded."). + WithHint("Run this command from a project directory that contains an `atmos.yaml`, or pass `--config`."). + Err() + } + if _, err := in.repository(); err != nil { + return nil, err + } + maintainer, err := newCatalogs(atmosConfigPtr) + if err != nil { + return nil, err + } + return c.run(ctx, maintainer, in) +} + +// catalogResult is the structured result of every catalog subcommand. +type catalogResult struct { + Action string `json:"action"` + Repository string `json:"repository,omitempty"` + Catalog string `json:"catalog,omitempty"` + Output string `json:"output,omitempty"` + Digest string `json:"digest,omitempty"` + PreviousDigest string `json:"previous_digest,omitempty"` + BaseDigest string `json:"base_digest,omitempty"` + Revision int `json:"revision"` + Listed int `json:"listed"` + Added []string `json:"added,omitempty"` + Updated []string `json:"updated,omitempty"` + Removed []string `json:"removed,omitempty"` + LockScope string `json:"lock_scope,omitempty"` + Error string `json:"error,omitempty"` +} + +// pairs renders the text view; empty values are omitted, never filled in. +func (r *catalogResult) pairs() []keyValue { + return []keyValue{ + {"repository", r.Repository}, + {"catalog", r.Catalog}, + {"output", r.Output}, + {"digest", r.Digest}, + {"previous_digest", r.PreviousDigest}, + {"base_digest", r.BaseDigest}, + {"revision", fmt.Sprint(r.Revision)}, + {"listed", fmt.Sprint(r.Listed)}, + {"added", strings.Join(r.Added, ", ")}, + {"updated", strings.Join(r.Updated, ", ")}, + {"removed", strings.Join(r.Removed, ", ")}, + {"lock_scope", r.LockScope}, + } +} + +// fromMutation maps a service result onto the command result. +func fromMutation(res *catalog.Result) *catalogResult { + if res == nil { + return nil + } + return &catalogResult{ + Catalog: res.Catalog, Digest: res.Digest, PreviousDigest: res.PreviousDigest, Revision: res.Revision, Listed: res.Listed, + Added: res.Added, Updated: res.Updated, Removed: res.Removed, LockScope: string(res.LockScope), + } +} + +// catalogHintRules attach actionable hints to catalog failures. +func catalogHintRules() []hintRule { + return []hintRule{ + {errUtils.ErrArtifactCatalogNotConfigured, "", func([]string) []string { + return []string{"Set `catalog: oci:///:` on the repository in `artifacts.repositories`. The catalog must live under the repository's registry and prefix."} + }}, + {errUtils.ErrArtifactCatalogConflict, "", func([]string) []string { + return []string{"The catalog changed since it was read. Rebuild the snapshot from the latest catalog, or run the command again; run only one catalog writer at a time."} + }}, + {errUtils.ErrArtifactCatalogInvalid, "", func([]string) []string { + return []string{"A catalog lists only artifacts of the repository it is configured on. Check the inventory refs and the catalog's entries."} + }}, + {errUtils.ErrArtifactDiscoveryOffline, "", func([]string) []string { + return []string{"Offline discovery serves only existing snapshots. Run the command online once, or turn off `plugins.policy.offline`."} + }}, + } +} diff --git a/cmd/artifacts/catalog_commands.go b/cmd/artifacts/catalog_commands.go new file mode 100644 index 00000000000..c1150ef2dd7 --- /dev/null +++ b/cmd/artifacts/catalog_commands.go @@ -0,0 +1,207 @@ +package artifacts + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/cloudposse/atmos/cmd/markdown" + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/flags" +) + +func newCatalogBuildCommand() *cobra.Command { + parser := newParser(catalogBuildPrefix, repositoryFlag(), []flags.Option{ + flags.WithStringFlag(flagInput, "", "", "AtmosArtifactInventory manifest listing the releases to resolve"), + flags.WithStringFlag(flagOutput, "", "", "File to write the catalog snapshot to (.json, .yaml, or .yml)"), + flags.WithStringFlag(flagName, "", "", "Catalog manifest name (default: the configured catalog name)"), + flags.WithEnvVars(flagInput, envInput), + flags.WithEnvVars(flagOutput, envOutput), + flags.WithEnvVars(flagName, envName), + }, formatFlag(formatText, resultFormats)) + c := &catalogCommand{action: "catalog-build", parser: parser, prefix: catalogBuildPrefix, run: runCatalogBuild} + return c.build("build", "Build a catalog snapshot from an inventory without publishing it", + `Resolve the releases of an AtmosArtifactInventory in one configured repository and write an +AtmosArtifactCatalog snapshot. Build reads artifact metadata only: no payload is downloaded and nothing is +published. The snapshot records the revision of the catalog it was based on, so publishing it later fails +if the catalog changed in between.`, + markdown.ArtifactsCatalogBuildUsageMarkdown, flagsOnly("atmos artifacts catalog build --repository= --input= --output=")) +} + +func runCatalogBuild(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) { + repository, err := in.repository() + if err != nil { + return nil, err + } + input, err := in.required(flagInput, envInput) + if err != nil { + return nil, err + } + output, err := in.required(flagOutput, envOutput) + if err != nil { + return nil, err + } + refs, err := catalog.ReadInventoryFile(input) + if err != nil { + return nil, err + } + built, err := maintainer.Build(ctx, &catalog.BuildRequest{Repository: repository, Refs: refs, Name: in.str(flagName)}) + if err != nil { + return nil, err + } + if err := catalog.WriteFile(output, built.File); err != nil { + return nil, err + } + return &catalogResult{Output: output, BaseDigest: built.BaseDigest, Revision: built.File.Metadata.Revision, Listed: countListed(built.File)}, nil +} + +// countListed counts the name:version pairs of a snapshot. +func countListed(file *catalog.File) int { + total := 0 + for index := range file.Spec.Entries { + total += len(file.Spec.Entries[index].Versions) + } + return total +} + +func newCatalogPublishCommand() *cobra.Command { + parser := newParser(catalogPublishPrefix, repositoryFlag(), []flags.Option{ + flags.WithStringFlag(flagExpectedDigest, "", "", "Require the published catalog to have this manifest digest"), + flags.WithEnvVars(flagExpectedDigest, envExpectedDigest), + }, formatFlag(formatText, resultFormats)) + c := &catalogCommand{action: "catalog-publish", parser: parser, prefix: catalogPublishPrefix, run: runCatalogPublish} + return c.build("publish ", "Publish a catalog snapshot to the repository's configured catalog", + `Publish an AtmosArtifactCatalog snapshot to the catalog configured on the repository. The command fails +before any write when the repository has no catalog, when the snapshot names a location outside the +repository, or when the catalog changed since the snapshot was built. Run only one catalog writer at a time: +the checks detect stale input but are not an atomic registry lock.`, + markdown.ArtifactsCatalogPublishUsageMarkdown, exactArgs("atmos artifacts catalog publish --repository=")) +} + +func runCatalogPublish(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) { + repository, err := in.repository() + if err != nil { + return nil, err + } + file, err := catalog.ReadFile(in.args[0]) + if err != nil { + return nil, err + } + res, err := maintainer.Publish(ctx, &catalog.PublishRequest{Repository: repository, File: file, ExpectedDigest: in.str(flagExpectedDigest)}) + return fromMutation(res), err +} + +func newCatalogAddCommand() *cobra.Command { + parser := newParser(catalogAddPrefix, repositoryFlag(), nil, formatFlag(formatText, resultFormats)) + c := &catalogCommand{action: "catalog-add", parser: parser, prefix: catalogAddPrefix, run: runCatalogAdd} + return c.build("add ...", "List releases in the repository's published catalog", + `Resolve the releases (name:version or name@sha256:) in the repository and list them in the latest +published catalog, creating the catalog when none exists. Only artifact metadata is read.`, + markdown.ArtifactsCatalogAddUsageMarkdown, minArgs("atmos artifacts catalog add ... --repository=")) +} + +func runCatalogAdd(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) { + repository, err := in.repository() + if err != nil { + return nil, err + } + refs, err := parseRefs(in.args) + if err != nil { + return nil, err + } + res, err := maintainer.Add(ctx, &catalog.AddRequest{Repository: repository, Refs: refs}) + return fromMutation(res), err +} + +func newCatalogRemoveCommand() *cobra.Command { + parser := newParser(catalogRemovePrefix, repositoryFlag(), nil, formatFlag(formatText, resultFormats)) + c := &catalogCommand{action: "catalog-remove", parser: parser, prefix: catalogRemovePrefix, run: runCatalogRemove} + return c.build("remove ", "Unlist a release from the repository's published catalog", + `Unlist a name:version from the latest published catalog. The artifact, its signatures, and every consumer +lock file are untouched: removal only stops the release from being discovered.`, + markdown.ArtifactsCatalogRemoveUsageMarkdown, exactArgs("atmos artifacts catalog remove --repository=")) +} + +func runCatalogRemove(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) { + repository, err := in.repository() + if err != nil { + return nil, err + } + ref, err := artifact.ParseRef(in.args[0]) + if err != nil { + return nil, err + } + res, err := maintainer.Remove(ctx, &catalog.RemoveRequest{Repository: repository, Ref: ref}) + return fromMutation(res), err +} + +func newCatalogSyncCommand() *cobra.Command { + parser := newParser(catalogSyncPrefix, repositoryFlag(), []flags.Option{ + flags.WithStringFlag(flagInput, "", "", "AtmosArtifactInventory manifest listing the releases to merge"), + flags.WithEnvVars(flagInput, envInput), + }, formatFlag(formatText, resultFormats)) + c := &catalogCommand{action: "catalog-sync", parser: parser, prefix: catalogSyncPrefix, run: runCatalogSync} + return c.build("sync", "Merge an inventory's releases into the repository's published catalog", + `Resolve the releases of an AtmosArtifactInventory and merge them into the latest published catalog. Entries +the inventory does not mention are kept. Only artifact metadata is read.`, + markdown.ArtifactsCatalogSyncUsageMarkdown, flagsOnly("atmos artifacts catalog sync --repository= --input=")) +} + +func runCatalogSync(ctx context.Context, maintainer catalog.Maintainer, in *input) (*catalogResult, error) { + repository, err := in.repository() + if err != nil { + return nil, err + } + input, err := in.required(flagInput, envInput) + if err != nil { + return nil, err + } + refs, err := catalog.ReadInventoryFile(input) + if err != nil { + return nil, err + } + res, err := maintainer.Sync(ctx, &catalog.SyncRequest{Repository: repository, Refs: refs}) + return fromMutation(res), err +} + +// parseRefs parses explicit references; there is no implicit latest. +func parseRefs(args []string) ([]artifact.Ref, error) { + refs := make([]artifact.Ref, 0, len(args)) + for _, text := range args { + ref, err := artifact.ParseRef(text) + if err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, nil +} + +// flagsOnly rejects positional arguments. +func flagsOnly(usage string) cobra.PositionalArgs { + return func(_ *cobra.Command, args []string) error { + if len(args) == 0 { + return nil + } + return errUtils.Build(fmt.Errorf("%w: expected no arguments, got %d", errUtils.ErrInvalidPositionalArgs, len(args))). + WithHintf("Usage: `%s`.", usage). + WithExitCode(2). + Err() + } +} + +// minArgs requires at least one positional argument. +func minArgs(usage string) cobra.PositionalArgs { + return func(_ *cobra.Command, args []string) error { + if len(args) >= 1 { + return nil + } + return errUtils.Build(fmt.Errorf("%w: expected at least 1 argument", errUtils.ErrInvalidPositionalArgs)). + WithHintf("Usage: `%s`.", usage). + WithExitCode(2). + Err() + } +} diff --git a/cmd/artifacts/catalog_test.go b/cmd/artifacts/catalog_test.go new file mode 100644 index 00000000000..c26357a1017 --- /dev/null +++ b/cmd/artifacts/catalog_test.go @@ -0,0 +1,231 @@ +package artifacts + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/schema" +) + +const inventoryYAML = `apiVersion: atmos/v1 +kind: AtmosArtifactInventory +metadata: + name: releases +spec: + refs: + - components/terraform/vpc:1.2.0 + - tools/helper:0.1.0 +` + +// catalogHarness runs the catalog commands against a mock maintainer. +func catalogHarness(t *testing.T) (*harness, *catalog.MockMaintainer) { + t.Helper() + h, _ := newHarness(t) + for _, name := range []string{envInput, envExpectedDigest} { + t.Setenv(name, "") + } + maintainer := catalog.NewMockMaintainer(gomock.NewController(t)) + previous := newCatalogs + newCatalogs = func(*schema.AtmosConfiguration) (catalog.Maintainer, error) { return maintainer, nil } + t.Cleanup(func() { newCatalogs = previous }) + return h, maintainer +} + +func writeInventory(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "inventory.yaml") + require.NoError(t, os.WriteFile(path, []byte(inventoryYAML), 0o600)) + return path +} + +func sampleResult() *catalog.Result { + return &catalog.Result{ + Repository: "dir", Catalog: "oci://ghcr.io/acme/atmos/catalog:stable", Digest: "sha256:" + string(manifestDigest)[7:], + PreviousDigest: string(recordDigest), Revision: 3, Listed: 4, Added: []string{"a:1"}, Updated: []string{"b:2"}, Removed: []string{"c:3"}, LockScope: artifact.LockScopeHost, + } +} + +func TestCatalogBuildResolvesTheInventoryAndWritesTheSnapshot(t *testing.T) { + h, maintainer := catalogHarness(t) + output := filepath.Join(t.TempDir(), "catalog.json") + file := catalog.New("catalog") + file.Metadata.Revision = 2 + file.Spec.Entries = []catalog.Entry{{ + Name: "components/terraform/vpc", Kind: "files", Source: "oci://ghcr.io/acme/atmos/components/terraform/vpc", + Versions: []catalog.Version{{Version: "1.2.0", ResolvedDigest: string(manifestDigest), RecordDigest: string(recordDigest), ContentDigest: string(contentDigest)}}, + }} + maintainer.EXPECT().Build(gomock.Any(), &catalog.BuildRequest{ + Repository: "dir", Name: "custom", + Refs: []artifact.Ref{{Name: "components/terraform/vpc", Version: "1.2.0"}, {Name: "tools/helper", Version: "0.1.0"}}, + }).Return(&catalog.BuildResult{File: file, BaseDigest: string(recordDigest)}, nil) + + got := h.run("catalog", "build", "--repository=dir", "--input="+writeInventory(t), "--output="+output, "--name=custom") + require.NoError(t, got.err) + assert.Contains(t, got.stdout, "output:") + assert.Contains(t, got.stdout, string(recordDigest)) + assert.Regexp(t, `revision:\s+2`, got.stdout) + written, err := catalog.ReadFile(output) + require.NoError(t, err) + assert.Equal(t, file.Spec, written.Spec) +} + +func TestCatalogBuildRequiresRepositoryInputAndOutput(t *testing.T) { + h, _ := catalogHarness(t) + inventory := writeInventory(t) + output := filepath.Join(t.TempDir(), "catalog.json") + for name, args := range map[string][]string{ + "repository": {"catalog", "build", "--input=" + inventory, "--output=" + output}, + "input": {"catalog", "build", "--repository=dir", "--output=" + output}, + "output": {"catalog", "build", "--repository=dir", "--input=" + inventory}, + } { + t.Run(name, func(t *testing.T) { + got := h.run(args...) + require.ErrorIs(t, got.err, errUtils.ErrRequiredFlagNotProvided) + }) + } + t.Run("inventory is validated before the service runs", func(t *testing.T) { + bad := filepath.Join(t.TempDir(), "bad.yaml") + require.NoError(t, os.WriteFile(bad, []byte("kind: Nope"), 0o600)) + got := h.run("catalog", "build", "--repository=dir", "--input="+bad, "--output="+output) + require.ErrorIs(t, got.err, errUtils.ErrArtifactCatalogInvalid) + _, statErr := os.Stat(output) + require.ErrorIs(t, statErr, os.ErrNotExist, "nothing is written for an invalid inventory") + }) +} + +func TestCatalogPublishReadsTheSnapshotAndPassesTheExpectedDigest(t *testing.T) { + h, maintainer := catalogHarness(t) + file := catalog.New("catalog") + path := filepath.Join(t.TempDir(), "catalog.yaml") + require.NoError(t, catalog.WriteFile(path, file)) + maintainer.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ any, request *catalog.PublishRequest) (*catalog.Result, error) { + assert.Equal(t, "dir", request.Repository) + assert.Equal(t, "sha256:abc", request.ExpectedDigest) + assert.Equal(t, "catalog", request.File.Metadata.Name) + return sampleResult(), nil + }) + got := h.run("catalog", "publish", path, "--repository=dir", "--expected-digest=sha256:abc") + require.NoError(t, got.err) + for _, want := range []string{`catalog:\s+oci://`, `revision:\s+3`, `listed:\s+4`, `added:\s+a:1`, `updated:\s+b:2`, `removed:\s+c:3`, `lock_scope:\s+host`} { + assert.Regexp(t, want, got.stdout) + } +} + +func TestCatalogPublishFailureKeepsTheExitCodeAndExplainsTheRemedy(t *testing.T) { + h, maintainer := catalogHarness(t) + path := filepath.Join(t.TempDir(), "catalog.json") + require.NoError(t, catalog.WriteFile(path, catalog.New("catalog"))) + maintainer.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(nil, errors.Join(errUtils.ErrArtifactCatalogConflict)).Times(2) + + got := h.run("catalog", "publish", path, "--repository=dir") + require.ErrorIs(t, got.err, errUtils.ErrArtifactCatalogConflict) + assert.Empty(t, got.stdout, "text format prints nothing for a failure") + + got = h.run("catalog", "publish", path, "--repository=dir", "--format=json") + require.ErrorIs(t, got.err, errUtils.ErrArtifactCatalogConflict) + envelope := decodeObject(t, got.stdout) + assert.Equal(t, "catalog-publish", envelope["action"]) + assert.Equal(t, "dir", envelope["repository"]) + assert.Contains(t, envelope["error"], "catalog changed") +} + +func TestCatalogPublishRejectsAnInvalidSnapshotFile(t *testing.T) { + h, _ := catalogHarness(t) + got := h.run("catalog", "publish", filepath.Join(t.TempDir(), "missing.json"), "--repository=dir") + require.ErrorIs(t, got.err, os.ErrNotExist) + bad := filepath.Join(t.TempDir(), "bad.json") + require.NoError(t, os.WriteFile(bad, []byte("{}"), 0o600)) + got = h.run("catalog", "publish", bad, "--repository=dir") + require.ErrorIs(t, got.err, errUtils.ErrArtifactCatalogInvalid) + got = h.run("catalog", "publish", "--repository=dir") + require.ErrorIs(t, got.err, errUtils.ErrInvalidPositionalArgs) +} + +func TestCatalogAddListsEveryReference(t *testing.T) { + h, maintainer := catalogHarness(t) + maintainer.EXPECT().Add(gomock.Any(), &catalog.AddRequest{ + Repository: "dir", Refs: []artifact.Ref{{Name: "a", Version: "1"}, {Name: "b", Digest: manifestDigest}}, + }).Return(sampleResult(), nil) + got := h.run("catalog", "add", "a:1", "b@"+string(manifestDigest), "--repository=dir", "--format=json") + require.NoError(t, got.err) + envelope := decodeObject(t, got.stdout) + assert.Equal(t, "catalog-add", envelope["action"]) + assert.EqualValues(t, 3, envelope["revision"]) + assert.Equal(t, []any{"a:1"}, envelope["added"]) + + require.ErrorIs(t, h.run("catalog", "add", "--repository=dir").err, errUtils.ErrInvalidPositionalArgs) + require.ErrorIs(t, h.run("catalog", "add", "latest", "--repository=dir").err, errUtils.ErrArtifactRefInvalid, "there is no implicit latest") +} + +func TestCatalogRemoveUnlistsOneVersion(t *testing.T) { + h, maintainer := catalogHarness(t) + maintainer.EXPECT().Remove(gomock.Any(), &catalog.RemoveRequest{Repository: "dir", Ref: artifact.Ref{Name: "a", Version: "1"}}).Return(sampleResult(), nil) + require.NoError(t, h.run("catalog", "remove", "a:1", "--repository=dir").err) + require.ErrorIs(t, h.run("catalog", "remove", "a", "--repository=dir").err, errUtils.ErrArtifactRefInvalid) + require.ErrorIs(t, h.run("catalog", "remove", "--repository=dir").err, errUtils.ErrInvalidPositionalArgs) +} + +func TestCatalogSyncMergesTheInventory(t *testing.T) { + h, maintainer := catalogHarness(t) + maintainer.EXPECT().Sync(gomock.Any(), &catalog.SyncRequest{ + Repository: "dir", Refs: []artifact.Ref{{Name: "components/terraform/vpc", Version: "1.2.0"}, {Name: "tools/helper", Version: "0.1.0"}}, + }).Return(sampleResult(), nil) + got := h.run("catalog", "sync", "--repository=dir", "--input="+writeInventory(t), "--format=yaml") + require.NoError(t, got.err) + assert.Contains(t, got.stdout, "action: catalog-sync") + + require.ErrorIs(t, h.run("catalog", "sync", "--repository=dir").err, errUtils.ErrRequiredFlagNotProvided) + require.ErrorIs(t, h.run("catalog", "sync", "extra", "--repository=dir", "--input=x").err, errUtils.ErrInvalidPositionalArgs) +} + +func TestCatalogCommandsRejectUnsupportedFormatsAndMissingConfiguration(t *testing.T) { + h, _ := catalogHarness(t) + require.ErrorIs(t, h.run("catalog", "remove", "a:1", "--repository=dir", "--format=csv").err, errUtils.ErrInvalidFlag) + + previous := atmosConfigPtr + atmosConfigPtr = nil + t.Cleanup(func() { atmosConfigPtr = previous }) + require.ErrorIs(t, h.run("catalog", "remove", "a:1", "--repository=dir").err, errUtils.ErrArtifactConfigInvalid) +} + +func TestCatalogFactoryFailureIsReported(t *testing.T) { + h, _ := catalogHarness(t) + failure := errors.New("cannot compose") + newCatalogs = func(*schema.AtmosConfiguration) (catalog.Maintainer, error) { return nil, failure } + require.ErrorIs(t, h.run("catalog", "remove", "a:1", "--repository=dir").err, failure) + + restore := UseCatalogFactory(func(*schema.AtmosConfiguration) (catalog.Maintainer, error) { return nil, failure }) + require.ErrorIs(t, h.run("catalog", "remove", "a:1", "--repository=dir").err, failure) + restore() +} + +func TestCatalogCommandsAreRegisteredWithDocumentationAndHints(t *testing.T) { + command := NewCommand() + var catalogCmd *cobra.Command + for _, child := range command.Commands() { + if child.Name() == "catalog" { + catalogCmd = child + } + } + require.NotNil(t, catalogCmd, "the catalog group is registered") + var names []string + for _, child := range catalogCmd.Commands() { + names = append(names, child.Name()) + assert.NotEmpty(t, child.Example, "%s has usage examples", child.Name()) + assert.NotEmpty(t, child.Long, child.Name()) + } + assert.ElementsMatch(t, []string{"build", "publish", "add", "remove", "sync"}, names) + + decorated := decorate(errUtils.ErrArtifactCatalogNotConfigured, nil) + assert.Contains(t, errUtils.Format(decorated, errUtils.FormatterConfig{}), "catalog: oci://") +} diff --git a/cmd/artifacts/convenience.go b/cmd/artifacts/convenience.go new file mode 100644 index 00000000000..572630174ea --- /dev/null +++ b/cmd/artifacts/convenience.go @@ -0,0 +1,69 @@ +package artifacts + +import ( + "github.com/spf13/cobra" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/perf" +) + +// KindPublishSpec describes a convenience publish command that supplies its artifact kind, such as +// `atmos scaffold publish`. The command delegates to the same publish pipeline as +// `atmos artifacts publish`: same flags (without --kind), same environment variables, same service, +// same result rendering, so the two can never drift. +type KindPublishSpec struct { + // Use is the command usage line, for example "publish ". + Use string + + // Short is the one-line description. + Short string + + // Long is the long description. + Long string + + // Example is the rendered usage examples. + Example string + + // Kind is the artifact kind the command publishes, for example "scaffold". + Kind string + + // Prefix is the Viper key prefix that keeps this command's values apart from every other + // command's, for example "scaffold.publish". + Prefix string + + // Usage is the argument usage shown when the source argument is missing. + Usage string +} + +// NewKindPublishCommand builds a convenience publish command for one artifact kind. +func NewKindPublishCommand(spec *KindPublishSpec) *cobra.Command { + defer perf.Track(nil, "artifacts.NewKindPublishCommand")() + + parser := newPublishParserFor(spec.Prefix, false) + c := &command{action: artifact.ActionPublish, parser: parser, prefix: spec.Prefix, params: kindPublishParams(spec.Kind), text: textPublish} + cmd := &cobra.Command{ + Use: spec.Use, + Short: spec.Short, + Long: spec.Long, + Example: spec.Example, + Args: exactArgs(spec.Usage), + RunE: c.execute, + } + parser.RegisterFlags(cmd) + registerCompletions(cmd, resultFormats) + return cmd +} + +// kindPublishParams maps the publish flags like `atmos artifacts publish` does and then supplies the +// command's own kind. +func kindPublishParams(kind string) builder { + return func(in *input) (*invoke.Params, error) { + params, err := paramsPublish(in) + if err != nil { + return nil, err + } + params.Kind = kind + return params, nil + } +} diff --git a/cmd/artifacts/copy.go b/cmd/artifacts/copy.go index deb1750f5be..e43e1e2240b 100644 --- a/cmd/artifacts/copy.go +++ b/cmd/artifacts/copy.go @@ -1,18 +1,17 @@ package artifacts import ( - "context" - "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) func newCopyCommand() *cobra.Command { parser := newCopyParser() - c := &command{action: artifact.ActionCopy, parser: parser, prefix: copyPrefix, run: runCopy, text: textCopy} + c := &command{action: artifact.ActionCopy, parser: parser, prefix: copyPrefix, params: paramsCopy, text: textCopy} cmd := &cobra.Command{ Use: "copy ", Short: "Copy an artifact between two repositories", @@ -28,11 +27,7 @@ content are preserved. Both the source and the destination verification policy a return cmd } -func runCopy(ctx context.Context, svc artifact.Service, in *input) (*result, error) { - ref, err := in.ref() - if err != nil { - return nil, err - } +func paramsCopy(in *input) (*invoke.Params, error) { from, err := in.required(flagFrom, envFrom) if err != nil { return nil, err @@ -41,8 +36,7 @@ func runCopy(ctx context.Context, svc artifact.Service, in *input) (*result, err if err != nil { return nil, err } - op, err := svc.Copy(ctx, artifact.CopyRequest{Ref: ref, From: from, To: to, Force: in.flag(flagForce)}) - return operationFrom(op), err + return &invoke.Params{Action: artifact.ActionCopy, Ref: in.args[0], From: from, To: to, Force: in.flag(flagForce)}, nil } func textCopy(res *result) error { diff --git a/cmd/artifacts/delete.go b/cmd/artifacts/delete.go index c1572ed14b1..541130bbf23 100644 --- a/cmd/artifacts/delete.go +++ b/cmd/artifacts/delete.go @@ -1,18 +1,17 @@ package artifacts import ( - "context" - "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) func newDeleteCommand() *cobra.Command { parser := newRepositoryParser(deletePrefix) - c := &command{action: artifact.ActionDelete, parser: parser, prefix: deletePrefix, run: runDelete, text: textDelete} + c := &command{action: artifact.ActionDelete, parser: parser, prefix: deletePrefix, params: paramsDelete, text: textDelete} cmd := &cobra.Command{ Use: "delete ", Short: "Delete one published artifact reference", @@ -28,17 +27,12 @@ command is noninteractive: name exactly the reference you mean.`, return cmd } -func runDelete(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsDelete(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } - op, err := svc.Delete(ctx, artifact.DeleteRequest{Repository: repository, Ref: ref}) - return operationFrom(op), err + return &invoke.Params{Action: artifact.ActionDelete, Repository: repository, Ref: in.args[0]}, nil } func textDelete(res *result) error { diff --git a/cmd/artifacts/deps.go b/cmd/artifacts/deps.go index 943583a09fd..afb8ad1597c 100644 --- a/cmd/artifacts/deps.go +++ b/cmd/artifacts/deps.go @@ -1,28 +1,22 @@ package artifacts import ( - "context" "sort" - "github.com/cloudposse/atmos/pkg/artifact" - "github.com/cloudposse/atmos/pkg/artifact/compose" - "github.com/cloudposse/atmos/pkg/plugin/bootstrap" - pluginsign "github.com/cloudposse/atmos/pkg/plugin/sign" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifactwire" "github.com/cloudposse/atmos/pkg/schema" ) // ServiceOptions are the per-invocation settings that reach the composition root. -type ServiceOptions struct { - // AcceptTransparencyLog accepts Sigstore transparency log publication without prompting. - AcceptTransparencyLog bool -} +type ServiceOptions = invoke.ServiceOptions // ServiceFactory builds the artifact service for the loaded configuration. Tests replace // newService to inject a fake service; production composes the real repositories lazily, so // building the service performs no network access, credential resolution, or directory creation. -type ServiceFactory func(config *schema.AtmosConfiguration, options ServiceOptions) (artifact.Service, error) +type ServiceFactory = invoke.ServiceFactory -var newService ServiceFactory = productionService +var newService ServiceFactory = artifactwire.NewService // UseServiceFactory replaces the service factory and returns a function that restores the previous // one. It is the seam command tests use to run the real commands against injected repositories, @@ -33,22 +27,6 @@ func UseServiceFactory(factory ServiceFactory) (restore func()) { return func() { newService = previous } } -// productionService wires the repositories, the shared verifier, the managed Cosign signer, and the -// plugin-aware registry authentication preparation into one artifact service. -func productionService(config *schema.AtmosConfiguration, options ServiceOptions) (artifact.Service, error) { - composition, err := compose.New( - config, - compose.WithBlobSigner(pluginsign.Cosign{Config: config, Yes: options.AcceptTransparencyLog}), - compose.WithRegistryPreparation(func(ctx context.Context) (func(), error) { - return bootstrap.PrepareAvailable(ctx, config) - }), - ) - if err != nil { - return nil, err - } - return composition.Service, nil -} - // configuredAliases lists the repository aliases in the loaded configuration, sorted. func configuredAliases(config *schema.AtmosConfiguration) []string { if config == nil { diff --git a/cmd/artifacts/flags.go b/cmd/artifacts/flags.go index 77279447ded..b634c0bfccb 100644 --- a/cmd/artifacts/flags.go +++ b/cmd/artifacts/flags.go @@ -108,10 +108,15 @@ func newParser(prefix string, groups ...[]flags.Option) *flags.StandardParser { } func newPublishParser() *flags.StandardParser { - return newParser(publishPrefix, repositoryFlag(), []flags.Option{ + return newPublishParserFor(publishPrefix, true) +} + +// newPublishParserFor builds the publish flag set under a Viper prefix. A convenience command that +// supplies its own artifact kind passes withKind=false, so it has no --kind flag to contradict. +func newPublishParserFor(prefix string, withKind bool) *flags.StandardParser { + options := []flags.Option{ flags.WithStringFlag(flagName, "", "", "Artifact name (default: producer metadata, then the source basename)"), flags.WithStringFlag(flagVersion, "", "", "Artifact version (required unless the producer supplies one)"), - flags.WithStringFlag(flagKind, "", "", "Artifact kind (default: files)"), flags.WithBoolFlag(flagForce, shortForce, false, "Replace an existing artifact with different content"), flags.WithBoolFlag(flagDryRun, shortDryRun, false, "Validate and plan the publication without writing anything"), flags.WithStringArrayFlag(flagLabels, "", nil, "Exact key=value labels (comma-separated and/or repeated)"), @@ -119,14 +124,21 @@ func newPublishParser() *flags.StandardParser { flags.WithStringFlag(flagChangelog, "", "", "CHANGELOG path, relative to the source directory"), flags.WithEnvVars(flagName, envName), flags.WithEnvVars(flagVersion, envVersion), - flags.WithEnvVars(flagKind, envKind), flags.WithEnvVars(flagForce, envForce), flags.WithEnvVars(flagDryRun, envDryRun), flags.WithEnvVars(flagLabels, envLabels), - flags.WithViperKey(flagLabels, publishPrefix+"."+flagLabels), + flags.WithViperKey(flagLabels, prefix+"."+flagLabels), flags.WithEnvVars(flagReadme, envReadme), flags.WithEnvVars(flagChangelog, envChangelog), - }, formatFlag(formatText, resultFormats)) + } + if withKind { + options = append( + options, + flags.WithStringFlag(flagKind, "", "", "Artifact kind (default: files)"), + flags.WithEnvVars(flagKind, envKind), + ) + } + return newParser(prefix, repositoryFlag(), options, formatFlag(formatText, resultFormats)) } func newPullParser() *flags.StandardParser { diff --git a/cmd/artifacts/inspect.go b/cmd/artifacts/inspect.go index 457b6a61a06..ad42acdd8d1 100644 --- a/cmd/artifacts/inspect.go +++ b/cmd/artifacts/inspect.go @@ -1,18 +1,18 @@ package artifacts import ( - "context" "fmt" "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" ) func newInspectCommand() *cobra.Command { parser := newRepositoryParser(inspectPrefix) - c := &command{action: artifact.ActionInspect, parser: parser, prefix: inspectPrefix, run: runInspect, text: textInspect} + c := &command{action: artifact.ActionInspect, parser: parser, prefix: inspectPrefix, params: paramsInspect, text: textInspect} cmd := &cobra.Command{ Use: "inspect ", Short: "Show the metadata of an artifact without downloading its payload", @@ -28,23 +28,12 @@ checksums. Without trust configuration the artifact is reported as unchecked, ne return cmd } -func runInspect(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsInspect(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } - inspected, err := svc.Inspect(ctx, artifact.InspectRequest{Repository: repository, Ref: ref}) - if inspected == nil { - return nil, err - } - res := operationFrom(&inspected.Operation) - res.Inspection = &inspected.Inspection - res.Verification = inspected.Verification - return res, err + return &invoke.Params{Action: artifact.ActionInspect, Repository: repository, Ref: in.args[0]}, nil } func textInspect(res *result) error { diff --git a/cmd/artifacts/list.go b/cmd/artifacts/list.go index 8131a04ac3f..2b6badb018c 100644 --- a/cmd/artifacts/list.go +++ b/cmd/artifacts/list.go @@ -2,15 +2,14 @@ package artifacts import ( "context" - "sort" - "github.com/opencontainers/go-digest" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/cloudposse/atmos/cmd/markdown" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/flags" "github.com/cloudposse/atmos/pkg/list/column" "github.com/cloudposse/atmos/pkg/list/format" @@ -19,14 +18,11 @@ import ( "github.com/cloudposse/atmos/pkg/ui" ) -// collectionQuery runs a list or a search against the service. -type collectionQuery func(ctx context.Context, svc artifact.Service, in *input, repositories []string, labels []artifact.Label) (*artifact.ListResult, error) - // collection describes one collection-producing subcommand (list or search). type collection struct { + action artifact.Action parser *flags.StandardParser prefix string - query collectionQuery } // collectionSpec describes how list and search differ; everything else is shared. @@ -37,12 +33,12 @@ type collectionSpec struct { example string usage string prefix string - query collectionQuery + action artifact.Action } func newCollectionCommand(spec *collectionSpec) *cobra.Command { parser := newListParser(spec.prefix) - c := &collection{parser: parser, prefix: spec.prefix, query: spec.query} + c := &collection{action: spec.action, parser: parser, prefix: spec.prefix} cmd := &cobra.Command{ Use: spec.use, Short: spec.short, @@ -74,9 +70,7 @@ list only its versions. Labels are exact key=value selectors; every selector mus example: markdown.ArtifactsListUsageMarkdown, usage: "atmos artifacts list [name] [--repository=] [--labels=key=value]", prefix: listPrefix, - query: func(ctx context.Context, svc artifact.Service, in *input, repositories []string, labels []artifact.Label) (*artifact.ListResult, error) { - return svc.List(ctx, artifact.ListRequest{Repositories: repositories, Name: firstArg(in), Labels: labels, Refresh: in.flag(flagRefresh)}) - }, + action: artifact.ActionList, }) } @@ -90,9 +84,7 @@ and caches; it never mutates a catalog.`, example: markdown.ArtifactsSearchUsageMarkdown, usage: "atmos artifacts search [text] [--repository=] [--labels=key=value]", prefix: searchPrefix, - query: func(ctx context.Context, svc artifact.Service, in *input, repositories []string, labels []artifact.Label) (*artifact.ListResult, error) { - return svc.Search(ctx, artifact.SearchRequest{Repositories: repositories, Query: firstArg(in), Labels: labels, Refresh: in.flag(flagRefresh)}) - }, + action: artifact.ActionSearch, }) } @@ -111,15 +103,7 @@ func (c *collection) execute(cmd *cobra.Command, args []string) error { return err } - listing, err := c.run(commandContext(cmd), in) - envelope := &listEnvelope{Artifacts: []listedArtifact{}} - if listing != nil { - envelope.Artifacts = listed(listing.Artifacts) - envelope.Repositories = listing.Repositories - } - if err != nil { - envelope.Error = err.Error() - } + envelope, err := c.run(commandContext(cmd), in) if renderErr := renderCollection(outputFormat, envelope, err != nil); renderErr != nil && err == nil { return renderErr } @@ -127,100 +111,45 @@ func (c *collection) execute(cmd *cobra.Command, args []string) error { } // listEnvelope is the structured result of list and search: the `artifacts` array plus -// per-repository outcomes, and the error when the query failed. -type listEnvelope struct { - // Artifacts are the entries across the selected repositories, sorted lexically. - Artifacts []listedArtifact `json:"artifacts"` - - // Repositories reports per-repository outcomes. - Repositories []artifact.RepositoryListing `json:"repositories,omitempty"` +// per-repository outcomes, and the error when the query failed. The type is shared with the +// artifact step (see pkg/artifact/invoke), so the field names are identical everywhere. +type listEnvelope = invoke.Collection - // Error is set when the query failed. - Error string `json:"error,omitempty"` -} - -// listedArtifact is one entry of the `artifacts` array. It carries the named outputs of an -// operation result (ref, resolved_digest, record_digest, content_digest) next to the listing -// fields, plus the pinned reference a consumer lockfile persists. -type listedArtifact struct { - Repository string `json:"repository,omitempty"` - Ref string `json:"ref"` - Name string `json:"name"` - Version string `json:"version"` - Kind string `json:"kind"` - Description string `json:"description,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Documents []artifact.Document `json:"documents,omitempty"` - ResolvedDigest digest.Digest `json:"resolved_digest,omitempty"` - RecordDigest digest.Digest `json:"record_digest,omitempty"` - ContentDigest digest.Digest `json:"content_digest,omitempty"` - Pinned artifact.PinnedReference `json:"pinned"` -} +// listedArtifact is one entry of the `artifacts` array. +type listedArtifact = invoke.Listed -// listed converts service entries into envelope entries. -func listed(entries []artifact.ListEntry) []listedArtifact { - out := make([]listedArtifact, 0, len(entries)) - for index := range entries { - entry := &entries[index] - out = append(out, listedArtifact{ - Repository: entry.Repository, - Ref: artifact.Ref{Name: entry.Name, Version: entry.Version}.String(), - Name: entry.Name, - Version: entry.Version, - Kind: entry.Kind, - Description: entry.Description, - Labels: entry.Labels, - Documents: entry.Documents, - ResolvedDigest: entry.Pinned.ResolvedDigest, - RecordDigest: entry.Pinned.RecordDigest, - ContentDigest: entry.Pinned.ContentDigest, - Pinned: entry.Pinned, - }) +// run maps the flags onto operation parameters and runs the query through the shared +// implementation. The envelope is never nil: a failed query still yields an empty array and the +// error text, while the returned error keeps the exit code nonzero. +func (c *collection) run(ctx context.Context, in *input) (*listEnvelope, error) { + failed := func(err error) (*listEnvelope, error) { + return &listEnvelope{Artifacts: []listedArtifact{}, Error: err.Error()}, err } - return out -} - -func (c *collection) run(ctx context.Context, in *input) (*artifact.ListResult, error) { if atmosConfigPtr == nil { - return nil, errUtils.Build(errUtils.ErrArtifactConfigInvalid). + return failed(errUtils.Build(errUtils.ErrArtifactConfigInvalid). WithExplanation("Atmos configuration is not loaded."). WithHint("Run this command from a project directory that contains an `atmos.yaml`, or pass `--config`."). - Err() + Err()) } - labels, err := selectorLabels(in) + labels, err := in.labels() if err != nil { - return nil, err + return failed(err) } svc, err := newService(atmosConfigPtr, ServiceOptions{}) if err != nil { - return nil, err + return failed(err) } - var repositories []string + params := &invoke.Params{Action: c.action, Labels: labels, Refresh: in.flag(flagRefresh)} if selected := in.v.GetStringSlice(in.key(flagRepository)); len(selected) > 0 { - repositories = selected - } - return c.query(ctx, svc, in, repositories, labels) -} - -// selectorLabels parses --labels into sorted exact key/value selectors. -func selectorLabels(in *input) ([]artifact.Label, error) { - parsed, err := in.labels() - if err != nil { - return nil, err - } - if len(parsed) == 0 { - return nil, nil + params.Repositories = selected } - keys := make([]string, 0, len(parsed)) - for key := range parsed { - keys = append(keys, key) + if c.action == artifact.ActionSearch { + params.Query = firstArg(in) + } else { + params.Name = firstArg(in) } - sort.Strings(keys) - labels := make([]artifact.Label, 0, len(keys)) - for _, key := range keys { - labels = append(labels, artifact.Label{Key: key, Value: parsed[key]}) - } - return labels, nil + outcome, err := invoke.Run(ctx, svc, params) + return outcome.Collection, err } // renderCollection writes a listing. Structured formats always render; table-like formats render @@ -245,9 +174,16 @@ func renderCollection(outputFormat string, envelope *listEnvelope, failed bool) } // reportListingDiagnostics sends per-repository notes (for example a capability that is missing) to stderr. +// A repository that could not be listed is reported by name, because the command still succeeds +// when other repositories were listed and the result must never read as complete. func reportListingDiagnostics(repositories []artifact.RepositoryListing) { for index := range repositories { reportDiagnostics(repositories[index].Diagnostics) + for _, diagnostic := range repositories[index].Diagnostics { + if diagnostic.Severity == artifact.SeverityError { + ui.Warningf("The listing is incomplete: repository `%s` could not be listed: %s", repositories[index].Repository, diagnostic.Message) + } + } } } diff --git a/cmd/artifacts/list_incomplete_test.go b/cmd/artifacts/list_incomplete_test.go new file mode 100644 index 00000000000..03c9b5d49d0 --- /dev/null +++ b/cmd/artifacts/list_incomplete_test.go @@ -0,0 +1,51 @@ +package artifacts + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +func incompleteListing() *artifact.ListResult { + result := listing() + result.Incomplete = true + result.Repositories = append(result.Repositories, artifact.RepositoryListing{ + Repository: "shared", + Diagnostics: []artifact.Diagnostic{{Severity: artifact.SeverityError, Code: "list-failed", Message: "401 unauthorized"}}, + }) + result.Repositories[0].Source, result.Repositories[0].FetchedAt = "cache", "2026-03-04T05:06:07Z" + return result +} + +func TestIncompleteListingIsNeverPresentedAsComplete(t *testing.T) { + h, svc := newHarness(t) + svc.EXPECT().List(gomock.Any(), gomock.Any()).Return(incompleteListing(), nil).Times(2) + + table := h.run("list") + require.NoError(t, table.err, "the readable repositories are still listed") + assert.Contains(t, table.stdout, "team/app") + assert.Contains(t, table.stderr, "incomplete") + assert.Contains(t, table.stderr, "shared") + assert.Contains(t, table.stderr, "401 unauthorized") + + structured := h.run("list", "--format=json") + require.NoError(t, structured.err) + envelope := decodeObject(t, structured.stdout) + assert.Equal(t, true, envelope["incomplete"]) + repositories := envelope["repositories"].([]any) + first := repositories[0].(map[string]any) + assert.Equal(t, "cache", first["source"], "the provenance of a listing is part of the result") + assert.Equal(t, "2026-03-04T05:06:07Z", first["fetched_at"]) +} + +func TestCompleteListingOmitsTheIncompleteMarker(t *testing.T) { + h, svc := newHarness(t) + svc.EXPECT().List(gomock.Any(), gomock.Any()).Return(listing(), nil) + got := h.run("list", "--format=json") + require.NoError(t, got.err) + assert.NotContains(t, decodeObject(t, got.stdout), "incomplete") +} diff --git a/cmd/artifacts/output.go b/cmd/artifacts/output.go index 6237f4edf7a..6b8b701c584 100644 --- a/cmd/artifacts/output.go +++ b/cmd/artifacts/output.go @@ -10,60 +10,14 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/data" "github.com/cloudposse/atmos/pkg/ui" ) -// result is the structured result of one operation. The operation fields flatten into the top level -// with their named-output spelling (ref, location, resolved_digest, record_digest, content_digest, -// representation_digest, size, outcome), exactly like a step result, and the command-specific parts -// follow. JSON and YAML share these names; the display format never changes semantics. -type result struct { - artifact.OperationResult - - // Error is set when the operation failed. The envelope is still emitted, and the exit code is nonzero. - Error string `json:"error,omitempty"` - - // Record is the sealed Record of a publication (set for dry runs and publications). - Record *artifact.Record `json:"record,omitempty"` - - // Inspection is the metadata view of an inspected artifact. - Inspection *artifact.Inspection `json:"inspection,omitempty"` - - // Verification is the authenticated verification result; absent means unchecked. - Verification *artifact.VerificationResult `json:"verification,omitempty"` - - // Subject is the signed or verified subject. - Subject *artifact.SignatureSubject `json:"subject,omitempty"` - - // Evidence lists the evidence stored by sign. - Evidence []artifact.SignatureEvidence `json:"evidence,omitempty"` - - // Document is the descriptor of a shown document. - Document *artifact.Document `json:"document,omitempty"` - - // Content is the document text of show. - Content *string `json:"content,omitempty"` -} - -// failed fills the failure markers of an envelope. The service leaves the outcome empty on failure -// so a failure is never mistaken for success; the envelope reports it explicitly. -func (r *result) failed(err error, action artifact.Action, repository, requested string) { - if err == nil { - return - } - r.Error = err.Error() - r.Outcome = artifact.OutcomeFailed - if r.Action == "" { - r.Action = action - } - if r.Repository == "" { - r.Repository = repository - } - if r.Requested == "" { - r.Requested = requested - } -} +// result is the structured result of one operation. The type is shared with the artifact step and +// hooks (see pkg/artifact/invoke), so every entry point reports identical field names. +type result = invoke.Result // joinValues renders a format list for flag help and errors. func joinValues(values []string) string { @@ -128,23 +82,16 @@ func writeText(pairs []keyValue) error { return data.Write(builder.String()) } -// operationPairs are the named outputs shared by every operation, in a stable order. +// operationPairs are the named outputs shared by every operation, in a stable order, followed by +// the lock scope. The names and values come from the shared invoke package, so the text rendering, +// the structured output, and the step outputs cannot drift apart. func operationPairs(op *artifact.OperationResult) []keyValue { - size := "" - if op.Size > 0 { - size = fmt.Sprintf("%d", op.Size) - } - return []keyValue{ - {"outcome", string(op.Outcome)}, - {"ref", op.Ref}, - {"location", op.Location}, - {"resolved_digest", string(op.ResolvedDigest)}, - {"record_digest", string(op.RecordDigest)}, - {"content_digest", string(op.ContentDigest)}, - {"representation_digest", string(op.RepresentationDigest)}, - {"size", size}, - {"lock_scope", string(op.LockScope)}, + named := invoke.OutputPairs(op) + pairs := make([]keyValue, 0, len(named)+1) + for _, pair := range named { + pairs = append(pairs, keyValue{pair.Name, pair.Value}) } + return append(pairs, keyValue{"lock_scope", string(op.LockScope)}) } // reportDiagnostics sends the non-fatal messages of an operation to the UI channel (stderr). diff --git a/cmd/artifacts/publish.go b/cmd/artifacts/publish.go index 8871f864122..210a203b4fa 100644 --- a/cmd/artifacts/publish.go +++ b/cmd/artifacts/publish.go @@ -1,7 +1,6 @@ package artifacts import ( - "context" "fmt" "strings" @@ -9,19 +8,13 @@ import ( "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" - "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) -// detailedPublisher is implemented by the real service. It returns the sealed Record next to the -// operation result, which a dry run needs in order to show the files it selected. -type detailedPublisher interface { - PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*service.PublishOutcome, error) -} - func newPublishCommand() *cobra.Command { parser := newPublishParser() - c := &command{action: artifact.ActionPublish, parser: parser, prefix: publishPrefix, run: runPublish, text: textPublish} + c := &command{action: artifact.ActionPublish, parser: parser, prefix: publishPrefix, params: paramsPublish, text: textPublish} cmd := &cobra.Command{ Use: "publish ", Short: "Publish a file or directory as a versioned artifact", @@ -37,7 +30,7 @@ no-op, and different content at the same reference needs --force.`, return cmd } -func runPublish(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsPublish(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err @@ -46,29 +39,19 @@ func runPublish(ctx context.Context, svc artifact.Service, in *input) (*result, if err != nil { return nil, err } - request := artifact.PublishRequest{ + return &invoke.Params{ + Action: artifact.ActionPublish, Repository: repository, Source: in.args[0], Name: in.str(flagName), Version: in.str(flagVersion), Kind: in.str(flagKind), - Metadata: artifact.Metadata{Labels: labels}, + Labels: labels, Readme: in.str(flagReadme), Changelog: in.str(flagChangelog), Force: in.flag(flagForce), DryRun: in.flag(flagDryRun), - } - if detailed, ok := svc.(detailedPublisher); ok { - outcome, err := detailed.PublishDetailed(ctx, request) - res := operationFrom(&outcome.Operation) - if outcome.Record.Name != "" { - record := outcome.Record - res.Record = &record - } - return res, err - } - op, err := svc.Publish(ctx, request) - return operationFrom(op), err + }, nil } // textPublish renders a publication. A dry run lists the selected files, because the plan is the diff --git a/cmd/artifacts/pull.go b/cmd/artifacts/pull.go index c582dc6ff39..f732bd3097a 100644 --- a/cmd/artifacts/pull.go +++ b/cmd/artifacts/pull.go @@ -1,18 +1,17 @@ package artifacts import ( - "context" - "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) func newPullCommand() *cobra.Command { parser := newPullParser() - c := &command{action: artifact.ActionPull, parser: parser, prefix: pullPrefix, run: runPull, text: textPull} + c := &command{action: artifact.ActionPull, parser: parser, prefix: pullPrefix, params: paramsPull, text: textPull} cmd := &cobra.Command{ Use: "pull ", Short: "Verify and materialize an artifact into a directory", @@ -29,21 +28,16 @@ existing directory is unavailable; use a new output path.`, return cmd } -func runPull(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsPull(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } destination, err := in.required(flagOutput, envOutput) if err != nil { return nil, err } - op, err := svc.Pull(ctx, artifact.PullRequest{Repository: repository, Ref: ref, Destination: destination, Force: in.flag(flagForce)}) - return operationFrom(op), err + return &invoke.Params{Action: artifact.ActionPull, Repository: repository, Ref: in.args[0], Destination: destination, Force: in.flag(flagForce)}, nil } func textPull(res *result) error { diff --git a/cmd/artifacts/run.go b/cmd/artifacts/run.go index 1ddcb5267d8..6805b44fd03 100644 --- a/cmd/artifacts/run.go +++ b/cmd/artifacts/run.go @@ -10,6 +10,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/flags" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/tags" @@ -53,31 +54,22 @@ func (in *input) required(name, env string) (string, error) { Err() } -// ref parses the first positional argument as an artifact reference. -func (in *input) ref() (artifact.Ref, error) { - ref, err := artifact.ParseRef(in.args[0]) - if err != nil { - return artifact.Ref{}, err - } - return ref, nil -} - // labels parses the repeatable --labels flag with the shared exact key/value label parser. func (in *input) labels() (map[string]string, error) { raw := tags.ReadLabelsFlagKey(in.v, in.key(flagLabels)) return tags.ParseLabelsFlagFrom(strings.Join(raw, ","), "--labels (or "+envLabels+")") } -// handler runs one operation against the service and returns its structured result. A handler -// returns a non-nil result whenever it has one, even alongside an error. -type handler func(ctx context.Context, svc artifact.Service, in *input) (*result, error) +// builder maps the parsed flags and arguments of one subcommand onto the transport-neutral +// operation parameters that the CLI, the artifact step, and hooks share. +type builder func(in *input) (*invoke.Params, error) // command describes one result-producing subcommand. type command struct { action artifact.Action parser *flags.StandardParser prefix string - run handler + params builder text func(*result) error } @@ -101,14 +93,16 @@ func (c *command) execute(cmd *cobra.Command, args []string) error { if res == nil { res = &result{} } - res.failed(err, c.action, in.str(flagRepository), requestedRef(args)) + res.MarkFailed(err, c.action, in.str(flagRepository), requestedRef(args)) if emitErr := emit(format, res, err != nil, c.text); emitErr != nil && err == nil { return emitErr } return decorate(err, configuredAliases(atmosConfigPtr)) } -// invoke builds the service and runs the handler. +// invoke builds the service, maps the flags onto operation parameters, and runs the operation +// through the shared implementation. The result is non-nil whenever the operation produced one, +// even alongside an error. func (c *command) invoke(ctx context.Context, in *input) (*result, error) { if atmosConfigPtr == nil { return nil, errUtils.Build(errUtils.ErrArtifactConfigInvalid). @@ -120,7 +114,12 @@ func (c *command) invoke(ctx context.Context, in *input) (*result, error) { if err != nil { return nil, err } - return c.run(ctx, svc, in) + params, err := c.params(in) + if err != nil { + return nil, err + } + outcome, err := invoke.Run(ctx, svc, params) + return outcome.Result, err } // commandContext returns the command's context, never nil. @@ -164,11 +163,3 @@ func optionalArg(usage string) cobra.PositionalArgs { Err() } } - -// operationFrom builds a result from a plain operation result and the error that came with it. -func operationFrom(op *artifact.OperationResult) *result { - if op == nil { - return &result{} - } - return &result{OperationResult: *op} -} diff --git a/cmd/artifacts/show.go b/cmd/artifacts/show.go index f79772b62b7..e38f7bec8b0 100644 --- a/cmd/artifacts/show.go +++ b/cmd/artifacts/show.go @@ -1,12 +1,11 @@ package artifacts import ( - "context" - "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/data" ) @@ -25,10 +24,8 @@ func newDocumentCommand(role artifact.DocumentRole) *cobra.Command { parser := newRepositoryParser(showPrefix + "." + string(role)) c := &command{ action: artifact.ActionShow, parser: parser, prefix: showPrefix + "." + string(role), - run: func(ctx context.Context, svc artifact.Service, in *input) (*result, error) { - return runShow(ctx, svc, in, role) - }, - text: textDocument, + params: func(in *input) (*invoke.Params, error) { return paramsShow(in, role) }, + text: textDocument, } cmd := &cobra.Command{ Use: string(role) + " ", @@ -44,27 +41,12 @@ func newDocumentCommand(role artifact.DocumentRole) *cobra.Command { return cmd } -func runShow(ctx context.Context, svc artifact.Service, in *input, role artifact.DocumentRole) (*result, error) { +func paramsShow(in *input, role artifact.DocumentRole) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } - document, err := svc.ShowDocument(ctx, artifact.ShowDocumentRequest{Repository: repository, Ref: ref, Role: role}) - if document == nil { - return nil, err - } - res := operationFrom(&document.Operation) - if document.Document.Digest != "" { - descriptor := document.Document - res.Document = &descriptor - content := string(document.Content) - res.Content = &content - } - return res, err + return &invoke.Params{Action: artifact.ActionShow, Repository: repository, Ref: in.args[0], Document: string(role)}, nil } // textDocument prints the document itself, so it can be piped or redirected as-is. diff --git a/cmd/artifacts/sign.go b/cmd/artifacts/sign.go index 7131d53bbe4..949a4b92f08 100644 --- a/cmd/artifacts/sign.go +++ b/cmd/artifacts/sign.go @@ -1,19 +1,19 @@ package artifacts import ( - "context" "fmt" "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) func newSignCommand() *cobra.Command { parser := newSignParser() - c := &command{action: artifact.ActionSign, parser: parser, prefix: signPrefix, run: runSign, text: textSign} + c := &command{action: artifact.ActionSign, parser: parser, prefix: signPrefix, params: paramsSign, text: textSign} cmd := &cobra.Command{ Use: "sign ", Short: "Add signature evidence to an immutable artifact digest", @@ -31,25 +31,12 @@ Signing does not verify the artifact against trust policy: use atmos artifacts v return cmd } -func runSign(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsSign(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } - signed, err := svc.Sign(ctx, artifact.SignRequest{Repository: repository, Ref: ref}) - if signed == nil { - return nil, err - } - res := operationFrom(&signed.Operation) - if signed.Subject.Digest != "" { - res.Subject = &signed.Subject - } - res.Evidence = signed.Evidence - return res, err + return &invoke.Params{Action: artifact.ActionSign, Repository: repository, Ref: in.args[0]}, nil } func textSign(res *result) error { diff --git a/cmd/artifacts/verify.go b/cmd/artifacts/verify.go index f4a3555cd89..112863527d2 100644 --- a/cmd/artifacts/verify.go +++ b/cmd/artifacts/verify.go @@ -1,19 +1,19 @@ package artifacts import ( - "context" "fmt" "github.com/spf13/cobra" "github.com/cloudposse/atmos/cmd/markdown" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" "github.com/cloudposse/atmos/pkg/ui" ) func newVerifyCommand() *cobra.Command { parser := newRepositoryParser(verifyPrefix) - c := &command{action: artifact.ActionVerify, parser: parser, prefix: verifyPrefix, run: runVerify, text: textVerify} + c := &command{action: artifact.ActionVerify, parser: parser, prefix: verifyPrefix, params: paramsVerify, text: textVerify} cmd := &cobra.Command{ Use: "verify ", Short: "Require a trusted, valid signature for an artifact", @@ -30,26 +30,12 @@ instead of reporting success, and the exit code is nonzero on any failure.`, return cmd } -func runVerify(ctx context.Context, svc artifact.Service, in *input) (*result, error) { +func paramsVerify(in *input) (*invoke.Params, error) { repository, err := in.repository() if err != nil { return nil, err } - ref, err := in.ref() - if err != nil { - return nil, err - } - verified, err := svc.Verify(ctx, artifact.VerifyRequest{Repository: repository, Ref: ref}) - if verified == nil { - return nil, err - } - res := operationFrom(&verified.Operation) - verification := verified.Verification - res.Verification = &verification - if verification.Subject.Digest != "" { - res.Subject = &verification.Subject - } - return res, err + return &invoke.Params{Action: artifact.ActionVerify, Repository: repository, Ref: in.args[0]}, nil } func textVerify(res *result) error { diff --git a/cmd/artifacts_cli_test.go b/cmd/artifacts_cli_test.go index 60769eae157..20bbd42750d 100644 --- a/cmd/artifacts_cli_test.go +++ b/cmd/artifacts_cli_test.go @@ -198,13 +198,14 @@ func TestArtifactsCommandsAreRegisteredDocumentedAndConflictFree(t *testing.T) { "atmos artifacts", "atmos artifacts publish", "atmos artifacts pull", "atmos artifacts copy", "atmos artifacts inspect", "atmos artifacts list", "atmos artifacts search", "atmos artifacts delete", "atmos artifacts sign", "atmos artifacts verify", "atmos artifacts show", "atmos artifacts show readme", "atmos artifacts show changelog", + "atmos artifacts catalog", "atmos artifacts catalog build", "atmos artifacts catalog publish", + "atmos artifacts catalog add", "atmos artifacts catalog remove", "atmos artifacts catalog sync", } { command, found := paths[path] require.True(t, found, path) assert.NotEmpty(t, internal.GetCommandMetadata(command).DocumentationURL, "%s links to its documentation page", path) assert.Equal(t, "true", command.Annotations["experimental"], "%s is labeled experimental", path) } - assert.NotContains(t, paths, "atmos artifacts catalog", "catalog commands belong to a later work package") } func TestArtifactsFlagsMatchTheContractTable(t *testing.T) { @@ -221,6 +222,12 @@ func TestArtifactsFlagsMatchTheContractTable(t *testing.T) { "delete": {"repository", "format"}, "show readme": {"repository", "format"}, "show changelog": {"repository", "format"}, + + "catalog build": {"repository", "input", "output", "name", "format"}, + "catalog publish": {"repository", "expected-digest", "format"}, + "catalog add": {"repository", "format"}, + "catalog remove": {"repository", "format"}, + "catalog sync": {"repository", "input", "format"}, } for _, command := range artifactsCommands() { path := strings.TrimPrefix(command.CommandPath(), "atmos artifacts ") @@ -292,7 +299,7 @@ func TestArtifactsLifecycleOnRealLocalRepositories(t *testing.T) { assert.Len(t, yamlListing.Artifacts, 2) noMatch := p.atmosArtifacts("search", "app", "--labels=team=other", "--format=json") require.NoError(t, noMatch.err) - assert.JSONEq(t, `{"artifacts": [], "repositories": [{"repository": "backups", "count": 0}, {"repository": "work", "count": 0}]}`, noMatch.stdout) + assert.JSONEq(t, `{"artifacts": [], "repositories": [{"repository": "backups", "count": 0, "source": "live"}, {"repository": "work", "count": 0, "source": "live"}]}`, noMatch.stdout) table := p.atmosArtifacts("list") require.NoError(t, table.err) assert.Contains(t, table.stdout, "team/app") diff --git a/cmd/scaffold/scaffold_publish.go b/cmd/scaffold/scaffold_publish.go new file mode 100644 index 00000000000..e1ef57f0912 --- /dev/null +++ b/cmd/scaffold/scaffold_publish.go @@ -0,0 +1,41 @@ +package scaffold + +import ( + artifactscmd "github.com/cloudposse/atmos/cmd/artifacts" + "github.com/cloudposse/atmos/cmd/internal" + "github.com/cloudposse/atmos/cmd/markdown" + "github.com/cloudposse/atmos/pkg/artifact/produce" + artifactscaffold "github.com/cloudposse/atmos/pkg/artifact/scaffold" +) + +// scaffoldPublishSpec describes `atmos scaffold publish`. The command shares the flag set, +// environment variables, service wiring, and result rendering of `atmos artifacts publish` and only +// supplies the scaffold kind; the scaffold producer validates the scaffold configuration before it +// packages anything, without generating output or running scaffold hooks. +var scaffoldPublishSpec = artifactscmd.KindPublishSpec{ + Use: "publish ", + Short: "Validate and publish a scaffold as a versioned artifact", + Long: `Validate a scaffold directory and publish it to one configured artifact repository. + +The scaffold configuration (scaffold.yaml at the source root) is validated first, without generating output and without running scaffold hooks. The artifact name and version default to the scaffold manifest's metadata.name and metadata.version; --name and --version override them. + +The root README is ordinary scaffold content. The files .atmos/artifact/README.md and .atmos/artifact/CHANGELOG.md are published as separate package documents and never appear in generated output.`, + Example: markdown.ScaffoldPublishUsageMarkdown, + Kind: produce.KindScaffold, + Prefix: "scaffold.publish", + Usage: "atmos scaffold publish --repository= [--name=] [--version=]", +} + +var scaffoldPublishCmd = artifactscmd.NewKindPublishCommand(&scaffoldPublishSpec) + +func init() { + // The scaffold producer validates through the scaffold manifest loader, which sits above the + // artifact packages in the dependency graph, so the binary registers it here. + artifactscaffold.Register() + + scaffoldCmd.AddCommand(scaffoldPublishCmd) + internal.RegisterCommandMetadata( + scaffoldPublishCmd, + internal.WithDocumentationURL("https://atmos.tools/cli/commands/scaffold/publish"), + ) +} diff --git a/cmd/scaffold/scaffold_publish_test.go b/cmd/scaffold/scaffold_publish_test.go new file mode 100644 index 00000000000..3f0235a4806 --- /dev/null +++ b/cmd/scaffold/scaffold_publish_test.go @@ -0,0 +1,168 @@ +package scaffold + +import ( + "bytes" + "context" + stdio "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + artifactscmd "github.com/cloudposse/atmos/cmd/artifacts" + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/data" + iolib "github.com/cloudposse/atmos/pkg/io" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// Compile-time guard for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: "local/dir"} + +// bufferedStreams is a minimal io.Streams backed by buffers. +type bufferedStreams struct { + stdout, stderr *bytes.Buffer +} + +func (s *bufferedStreams) Input() stdio.Reader { return &bytes.Buffer{} } +func (s *bufferedStreams) Output() stdio.Writer { return s.stdout } +func (s *bufferedStreams) Error() stdio.Writer { return s.stderr } +func (s *bufferedStreams) RawOutput() stdio.Writer { return s.stdout } +func (s *bufferedStreams) RawError() stdio.Writer { return s.stderr } + +// publishHarness runs a fresh scaffold publish command against an injected service. +type publishHarness struct { + t *testing.T + stdout *bytes.Buffer +} + +// newPublishHarness injects the service and a configuration with one repository, clears the +// environment variables the command reads, and captures the output channels. +func newPublishHarness(t *testing.T, service artifact.Service) *publishHarness { + t.Helper() + for _, name := range []string{ + "ATMOS_ARTIFACTS_REPOSITORY", "ATMOS_ARTIFACTS_NAME", "ATMOS_ARTIFACTS_VERSION", "ATMOS_ARTIFACTS_KIND", + "ATMOS_ARTIFACTS_FORCE", "ATMOS_ARTIFACTS_DRY_RUN", "ATMOS_ARTIFACTS_LABELS", "ATMOS_ARTIFACTS_README", + "ATMOS_ARTIFACTS_CHANGELOG", "ATMOS_ARTIFACTS_FORMAT", + } { + t.Setenv(name, "") + } + artifactscmd.SetAtmosConfig(&schema.AtmosConfiguration{Artifacts: schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: "local/dir"}, + }}}) + t.Cleanup(func() { artifactscmd.SetAtmosConfig(nil) }) + restore := artifactscmd.UseServiceFactory(func(*schema.AtmosConfiguration, artifactscmd.ServiceOptions) (artifact.Service, error) { + return service, nil + }) + t.Cleanup(restore) + + out := &bufferedStreams{stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}} + ioCtx, err := iolib.NewContext(iolib.WithStreams(out)) + require.NoError(t, err) + data.InitWriter(ioCtx) + ui.InitFormatter(ioCtx) + t.Cleanup(data.Reset) + return &publishHarness{t: t, stdout: out.stdout} +} + +func (h *publishHarness) run(args ...string) error { + h.t.Helper() + command := artifactscmd.NewKindPublishCommand(&scaffoldPublishSpec) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(stdio.Discard) + command.SetErr(stdio.Discard) + command.SetArgs(args) + return command.ExecuteContext(context.Background()) +} + +func TestScaffoldPublish_IsRegisteredUnderScaffold(t *testing.T) { + found, _, err := scaffoldCmd.Find([]string{"publish"}) + require.NoError(t, err) + assert.Same(t, scaffoldPublishCmd, found) + assert.Equal(t, "scaffold", scaffoldPublishSpec.Kind, "the command supplies the scaffold kind") + assert.NotEmpty(t, scaffoldPublishCmd.Example, "usage examples are embedded") + assert.Nil(t, scaffoldPublishCmd.Flags().Lookup("kind"), "a convenience command supplies its kind and has no --kind flag") + for _, name := range []string{"repository", "name", "version", "labels", "readme", "changelog", "force", "dry-run", "format"} { + assert.NotNil(t, scaffoldPublishCmd.Flags().Lookup(name), "scaffold publish shares the --%s flag with artifacts publish", name) + } +} + +func TestScaffoldPublish_DelegatesToTheGenericPublisherWithTheScaffoldKind(t *testing.T) { + service := artifact.NewMockService(gomock.NewController(t)) + var captured artifact.PublishRequest + service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + captured = request + return &artifact.OperationResult{Action: artifact.ActionPublish, Outcome: artifact.OutcomeCreated, Repository: request.Repository, Ref: "scaffolds/landing-zone:2.0.0"}, nil + }, + ) + h := newPublishHarness(t, service) + + err := h.run("./scaffolds/landing-zone", "--repository=shared", "--name=scaffolds/landing-zone", "--version=2.0.0", + "--labels=team=platform", "--readme=docs/PACKAGE.md", "--changelog=docs/CHANGES.md", "--force", "--format=json") + + require.NoError(t, err) + assert.Equal(t, "scaffold", captured.Kind) + assert.Equal(t, "shared", captured.Repository) + assert.Equal(t, "./scaffolds/landing-zone", captured.Source) + assert.Equal(t, "scaffolds/landing-zone", captured.Name) + assert.Equal(t, "2.0.0", captured.Version) + assert.Equal(t, map[string]string{"team": "platform"}, captured.Metadata.Labels) + assert.Equal(t, "docs/PACKAGE.md", captured.Readme) + assert.Equal(t, "docs/CHANGES.md", captured.Changelog) + assert.True(t, captured.Force) + assert.False(t, captured.DryRun) + assert.Contains(t, h.stdout.String(), `"ref": "scaffolds/landing-zone:2.0.0"`, "structured output matches artifacts publish") +} + +func TestScaffoldPublish_DryRunAndEnvironmentFlags(t *testing.T) { + service := artifact.NewMockService(gomock.NewController(t)) + var captured artifact.PublishRequest + service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + captured = request + return &artifact.OperationResult{Action: artifact.ActionPublish, Outcome: artifact.OutcomeDryRun, Repository: request.Repository}, nil + }, + ) + h := newPublishHarness(t, service) + t.Setenv("ATMOS_ARTIFACTS_REPOSITORY", "shared") + t.Setenv("ATMOS_ARTIFACTS_VERSION", "3.1.0") + + require.NoError(t, h.run("./scaffolds/landing-zone", "--dry-run", "--format=json")) + + assert.True(t, captured.DryRun) + assert.Equal(t, "shared", captured.Repository, "the shared ATMOS_ARTIFACTS_REPOSITORY variable selects the repository") + assert.Equal(t, "3.1.0", captured.Version) + assert.Equal(t, "scaffold", captured.Kind) + assert.Empty(t, captured.Name, "the name is left to the scaffold manifest") +} + +func TestScaffoldPublish_KindEnvironmentVariableCannotOverrideTheKind(t *testing.T) { + service := artifact.NewMockService(gomock.NewController(t)) + var captured artifact.PublishRequest + service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + captured = request + return &artifact.OperationResult{Action: artifact.ActionPublish, Outcome: artifact.OutcomeCreated}, nil + }, + ) + h := newPublishHarness(t, service) + t.Setenv("ATMOS_ARTIFACTS_KIND", "files") + + require.NoError(t, h.run("./s", "--repository=shared", "--version=1.0.0", "--format=json")) + + assert.Equal(t, "scaffold", captured.Kind) +} + +func TestScaffoldPublish_RequiresARepositoryAndASource(t *testing.T) { + h := newPublishHarness(t, artifact.NewMockService(gomock.NewController(t))) + + err := h.run("./scaffolds/landing-zone") + require.ErrorIs(t, err, errUtils.ErrRequiredFlagNotProvided, "no command picks a default repository") + + err = h.run("--repository=shared") + require.ErrorIs(t, err, errUtils.ErrInvalidPositionalArgs) +} diff --git a/internal/exec/workflow_utils.go b/internal/exec/workflow_utils.go index 2542bdf9b31..afbdf17b312 100644 --- a/internal/exec/workflow_utils.go +++ b/internal/exec/workflow_utils.go @@ -1083,6 +1083,13 @@ func ExecuteWorkflow( } break } + if stepPkg.WrapperRetriesStep(commandType) { + // The handler leaves `retry:` to this wrapper; it keeps the operation identity stable between attempts. + err = stepPkg.RetryWithConditions(context.Background(), steps[stepIdx].Retry, func() error { + return executeExtendedStep(context.Background(), &steps[stepIdx], workflowDefinition, stepEnv, extendedOpts) + }) + break + } err = executeExtendedStep(context.Background(), &steps[stepIdx], workflowDefinition, stepEnv, extendedOpts) } if err != nil { diff --git a/pkg/artifact/artifacttest/recorder.go b/pkg/artifact/artifacttest/recorder.go new file mode 100644 index 00000000000..5ae64860b69 --- /dev/null +++ b/pkg/artifact/artifacttest/recorder.go @@ -0,0 +1,119 @@ +package artifacttest + +import ( + "context" + "sync" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Recorder wraps an [artifact.Service] and records every publication it sees: the request, with the +// operation identity the caller supplied, and the typed result. It can also fail the first +// publications, which lets a test drive a wrapper-level retry through a real service. +// +// Entry points that run artifact operations (the CLI, workflow and custom-command steps, hooks) +// share one service seam, so a test injects a Recorder around a real composition and compares what +// every entry point asked for and received. +type Recorder struct { + artifact.Service + + mu sync.Mutex + requests []artifact.PublishRequest + results []artifact.OperationResult + failures int + failure error +} + +var _ artifact.Service = (*Recorder)(nil) + +// NewRecorder wraps the service. +func NewRecorder(inner artifact.Service) *Recorder { + defer perf.Track(nil, "artifacttest.NewRecorder")() + + return &Recorder{Service: inner} +} + +// FailPublications makes the next count publications fail with err before they reach the wrapped +// service. A failed attempt returns a partial result that carries the supplied operation ID. +func (r *Recorder) FailPublications(count int, err error) { + defer perf.Track(nil, "artifacttest.Recorder.FailPublications")() + + r.mu.Lock() + defer r.mu.Unlock() + r.failures, r.failure = count, err +} + +// Publish implements [artifact.Service]. +// +//nolint:gocritic // The request is passed by value because artifact.Service fixes the signature. +func (r *Recorder) Publish(ctx context.Context, request artifact.PublishRequest) (*artifact.OperationResult, error) { + defer perf.Track(nil, "artifacttest.Recorder.Publish")() + + outcome, err := r.PublishDetailed(ctx, request) + return &outcome.Operation, err +} + +// PublishDetailed returns the sealed Record next to the operation result, like the real service, +// which is what a dry run and the structured result need. The wrapped service must provide it; a +// service that does not still works, with an empty Record. +// +//nolint:gocritic // The request is passed by value to mirror Publish. +func (r *Recorder) PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*service.PublishOutcome, error) { + defer perf.Track(nil, "artifacttest.Recorder.PublishDetailed")() + + r.mu.Lock() + r.requests = append(r.requests, request) + if r.failures > 0 { + r.failures-- + failure := r.failure + partial := artifact.OperationResult{OperationID: request.Execution.OperationID, Action: artifact.ActionPublish, Repository: request.Repository} + r.results = append(r.results, partial) + r.mu.Unlock() + return &service.PublishOutcome{Operation: partial}, failure + } + r.mu.Unlock() + + var ( + outcome *service.PublishOutcome + err error + ) + if detailed, ok := r.Service.(interface { + PublishDetailed(context.Context, artifact.PublishRequest) (*service.PublishOutcome, error) + }); ok { + outcome, err = detailed.PublishDetailed(ctx, request) + } else { + var result *artifact.OperationResult + result, err = r.Service.Publish(ctx, request) + outcome = &service.PublishOutcome{} + if result != nil { + outcome.Operation = *result + } + } + if outcome == nil { + outcome = &service.PublishOutcome{} + } + r.mu.Lock() + defer r.mu.Unlock() + r.results = append(r.results, outcome.Operation) + return outcome, err +} + +// Requests returns the publish requests in the order they arrived. +func (r *Recorder) Requests() []artifact.PublishRequest { + defer perf.Track(nil, "artifacttest.Recorder.Requests")() + + r.mu.Lock() + defer r.mu.Unlock() + return append([]artifact.PublishRequest(nil), r.requests...) +} + +// Results returns the publish results in the order they were produced. +func (r *Recorder) Results() []artifact.OperationResult { + defer perf.Track(nil, "artifacttest.Recorder.Results")() + + r.mu.Lock() + defer r.mu.Unlock() + return append([]artifact.OperationResult(nil), r.results...) +} diff --git a/pkg/artifact/compose/compose.go b/pkg/artifact/compose/compose.go index 42560480f90..06263492ece 100644 --- a/pkg/artifact/compose/compose.go +++ b/pkg/artifact/compose/compose.go @@ -8,6 +8,8 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" artifactconfig "github.com/cloudposse/atmos/pkg/artifact/config" + "github.com/cloudposse/atmos/pkg/artifact/discovery" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" "github.com/cloudposse/atmos/pkg/artifact/local" "github.com/cloudposse/atmos/pkg/artifact/oci" "github.com/cloudposse/atmos/pkg/artifact/s3" @@ -28,6 +30,10 @@ type Composition struct { // Registry holds the registered repository kinds. Registry *artifact.RepositoryRegistry + + // Catalogs builds and maintains the optional catalog of a configured repository. It reads + // artifact metadata only and never publishes, signs, or deletes an artifact. + Catalogs *catalog.Service } // New wires the artifact service from the `artifacts` section of the configuration. It validates @@ -64,7 +70,7 @@ func New(atmosConfig *schema.AtmosConfiguration, opts ...Option) (*Composition, } repos := newRepositories(&cfg, &applied) - svcOptions := []service.Option{service.WithImmutabilityPolicy(applied.immutability)} + svcOptions := []service.Option{service.WithImmutabilityPolicy(applied.immutability), service.WithEnumerator(newDiscovery(atmosConfig, &applied))} if applied.signer != nil { svcOptions = append(svcOptions, service.WithSigner(applied.signer)) } @@ -72,7 +78,7 @@ func New(atmosConfig *schema.AtmosConfiguration, opts ...Option) (*Composition, if err != nil { return nil, err } - return &Composition{Service: svc, Repositories: repos, Registry: applied.registry}, nil + return &Composition{Service: svc, Repositories: repos, Registry: applied.registry, Catalogs: catalog.NewService(repos, applied.catalog...)}, nil } // defaults fills every dependency the caller did not inject. @@ -123,3 +129,14 @@ func newRepositories(cfg *schema.ArtifactConfig, applied *settings) *service.Con ) return repos } + +// newDiscovery wires the artifact discovery cache. Discovery is offline under the same setting +// plugins use (plugins.policy.offline) or a repository's own verification policy; nothing in the +// cache scope is a credential. The scope is computed on first use, so building the composition +// stays free of file access. +func newDiscovery(atmosConfig *schema.AtmosConfiguration, applied *settings) *discovery.Discovery { + options := []discovery.Option{discovery.WithOffline(func(info *service.RepositoryInfo) bool { + return atmosConfig.Plugins.Policy.Offline || info.Policy.Offline + })} + return discovery.New(func() (string, error) { return discovery.ScopeOf(atmosConfig) }, append(options, applied.discovery...)...) +} diff --git a/pkg/artifact/compose/options.go b/pkg/artifact/compose/options.go index bce09849cb5..0e8d355e149 100644 --- a/pkg/artifact/compose/options.go +++ b/pkg/artifact/compose/options.go @@ -2,6 +2,8 @@ package compose import ( "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" "github.com/cloudposse/atmos/pkg/artifact/local" "github.com/cloudposse/atmos/pkg/artifact/oci" "github.com/cloudposse/atmos/pkg/artifact/s3" @@ -22,6 +24,8 @@ type settings struct { preparation registryauth.Preparation registry *artifact.RepositoryRegistry service []service.Option + discovery []discovery.Option + catalog []catalog.Option local []local.Option s3 []s3.Option oci []oci.Option @@ -132,3 +136,18 @@ func WithProducers(producers *artifact.ProducerRegistry) Option { return func(s *settings) { s.producers = producers } } + +// WithDiscoveryOptions passes options to the discovery cache, for example a cache directory, a +// clock, or a policy. +func WithDiscoveryOptions(opts ...discovery.Option) Option { + defer perf.Track(nil, "compose.WithDiscoveryOptions")() + + return func(s *settings) { s.discovery = append(s.discovery, opts...) } +} + +// WithCatalogOptions passes options to the catalog service, for example a clock or lock owner. +func WithCatalogOptions(opts ...catalog.Option) Option { + defer perf.Track(nil, "compose.WithCatalogOptions")() + + return func(s *settings) { s.catalog = append(s.catalog, opts...) } +} diff --git a/pkg/artifact/discovery/cache/boundary_test.go b/pkg/artifact/discovery/cache/boundary_test.go new file mode 100644 index 00000000000..321f087a35a --- /dev/null +++ b/pkg/artifact/discovery/cache/boundary_test.go @@ -0,0 +1,46 @@ +package cache_test + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCacheStaysNeutral keeps the shared snapshot mechanics free of any plugin or artifact schema, +// so both plugin discovery and artifact discovery can build on it without a dependency cycle. +func TestCacheStaysNeutral(t *testing.T) { + forbidden := []string{ + "github.com/cloudposse/atmos/cmd", + "github.com/cloudposse/atmos/pkg/plugin", + "github.com/cloudposse/atmos/pkg/artifact/service", + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog", + "github.com/cloudposse/atmos/pkg/schema", + "github.com/cloudposse/atmos/pkg/auth", + "github.com/google/go-containerregistry", + "oras.land/oras-go", + } + files, err := filepath.Glob("*.go") + require.NoError(t, err) + checked := 0 + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + parsed, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly) + require.NoError(t, err) + checked++ + for _, spec := range parsed.Imports { + path, err := strconv.Unquote(spec.Path.Value) + require.NoError(t, err) + for _, prefix := range forbidden { + require.Falsef(t, path == prefix || strings.HasPrefix(path, prefix+"/"), "%s imports %s", file, path) + } + } + } + require.Positive(t, checked, "no source files were checked") +} diff --git a/pkg/artifact/discovery/cache/cache_test.go b/pkg/artifact/discovery/cache/cache_test.go new file mode 100644 index 00000000000..733c54847d1 --- /dev/null +++ b/pkg/artifact/discovery/cache/cache_test.go @@ -0,0 +1,493 @@ +package cache_test + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" +) + +var ( + errOffline = errors.New("offline") + errTooLarge = errors.New("too large") + errFetch = errors.New("fetch failed") +) + +// listing is a minimal snapshot schema: the shared header plus a payload. +type listing struct { + cache.Header + Items []string `json:"items"` +} + +// clock is a deterministic, settable time source. +type clock struct { + mu sync.Mutex + now time.Time +} + +func (c *clock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +var epoch = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +// harness counts fetches and authorizations of one source. +type harness struct { + store *cache.Store + clock *clock + fetches atomic.Int64 + authorized []bool + mu sync.Mutex + items []string + fail error + authorize error +} + +func newHarness(t *testing.T, scope string, opts ...cache.Option) *harness { + t.Helper() + h := &harness{clock: &clock{now: epoch}, items: []string{"a"}} + all := append([]cache.Option{cache.WithClock(h.clock.Now)}, opts...) + h.store = cache.New(filepath.Join(t.TempDir(), "discovery"), scope, all...) + return h +} + +func (h *harness) source(kind string) *cache.Source[listing, *listing] { + const reference = "ref" + return &cache.Source[listing, *listing]{ + Kind: kind, Reference: reference, Version: 1, + Decode: func(raw []byte) (*listing, error) { + var value listing + if err := json.Unmarshal(raw, &value); err != nil { + return nil, err + } + return &value, nil + }, + Fetch: func(_ context.Context, previous *listing) (*listing, error) { + h.fetches.Add(1) + h.mu.Lock() + defer h.mu.Unlock() + if h.fail != nil { + return nil, h.fail + } + _ = previous + return &listing{Items: append([]string(nil), h.items...)}, nil + }, + Authorize: func(_ context.Context, cached bool) error { + h.mu.Lock() + defer h.mu.Unlock() + h.authorized = append(h.authorized, cached) + return h.authorize + }, + ErrOffline: errOffline, ErrTooLarge: errTooLarge, + } +} + +func (h *harness) load(t *testing.T, source *cache.Source[listing, *listing], mode cache.Mode) (*listing, bool, error) { + t.Helper() + return cache.Load(t.Context(), h.store, source, mode) +} + +func TestLoadFreshSnapshotIsServedWithoutFetching(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + first, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.False(t, cached) + assert.Equal(t, []string{"a"}, first.Items) + assert.Equal(t, epoch, first.FetchedAt, "the fetch time comes from the injected clock, in UTC") + + h.clock.advance(cache.DefaultTTL - time.Second) + second, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.True(t, cached) + assert.Equal(t, first.Items, second.Items) + assert.Equal(t, int64(1), h.fetches.Load(), "a fresh snapshot performs no fetch") + assert.Equal(t, []bool{false, true}, h.authorized, "policy is evaluated for the fetch and again for the cached read") +} + +func TestLoadStaleSnapshotRefreshesAtTheTTLBoundary(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + + h.clock.advance(cache.DefaultTTL) + h.mu.Lock() + h.items = []string{"a", "b"} + h.mu.Unlock() + refreshed, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.False(t, cached, "a snapshot exactly one TTL old is stale") + assert.Equal(t, []string{"a", "b"}, refreshed.Items) + assert.Equal(t, int64(2), h.fetches.Load()) +} + +func TestLoadRefreshBypassesFreshness(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + _, cached, err := h.load(t, source, cache.Mode{Refresh: true}) + require.NoError(t, err) + assert.False(t, cached) + assert.Equal(t, int64(2), h.fetches.Load()) +} + +func TestLoadZeroTTLRevalidatesEveryOnlineLoad(t *testing.T) { + h := newHarness(t, "scope", cache.WithTTL(0)) + source := h.source("repository") + for range 3 { + _, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.False(t, cached) + } + assert.Equal(t, int64(3), h.fetches.Load()) +} + +func TestLoadOfflineServesAnySnapshotAgeButNeverFetches(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + h.clock.advance(1000 * time.Hour) + value, cached, err := h.load(t, source, cache.Mode{Offline: true}) + require.NoError(t, err) + assert.True(t, cached) + assert.Equal(t, []string{"a"}, value.Items) + assert.Equal(t, int64(1), h.fetches.Load()) +} + +func TestLoadOfflineDoesNotCreateLock(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + lock := h.store.Path("repository", "ref") + ".lock" + require.NoError(t, os.Remove(lock)) + _, cached, err := h.load(t, source, cache.Mode{Offline: true}) + require.NoError(t, err) + assert.True(t, cached) + _, err = os.Stat(lock) + assert.True(t, os.IsNotExist(err)) +} + +func TestLoadOfflineFailsWithoutAValidSnapshot(t *testing.T) { + t.Run("missing", func(t *testing.T) { + h := newHarness(t, "scope") + _, _, err := h.load(t, h.source("repository"), cache.Mode{Offline: true}) + require.ErrorIs(t, err, errOffline) + assert.Zero(t, h.fetches.Load()) + }) + t.Run("refresh requested", func(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + _, _, err = h.load(t, source, cache.Mode{Offline: true, Refresh: true}) + // Refresh cannot be satisfied while offline, even with an existing snapshot. + require.ErrorIs(t, err, errOffline) + assert.Equal(t, int64(1), h.fetches.Load()) + }) +} + +func TestLoadRejectsCorruptSnapshotsOfflineAndReplacesThemOnline(t *testing.T) { + for name, corrupt := range map[string]string{ + "not json": `{`, + "other version": `{"version":2,"source":"ref","kind":"repository","fetched_at":"2026-03-04T05:06:07Z"}`, + "other source": `{"version":1,"source":"other","kind":"repository","fetched_at":"2026-03-04T05:06:07Z"}`, + "other kind": `{"version":1,"source":"ref","kind":"catalog","fetched_at":"2026-03-04T05:06:07Z"}`, + "oversized": strings.Repeat(" ", 128), + "empty": ``, + "truncated array": `{"version":1,"source":"ref","kind":"repository","items":[`, + } { + t.Run(name, func(t *testing.T) { + h := newHarness(t, "scope", cache.WithMaxSize(100)) + source := h.source("repository") + require.NoError(t, os.MkdirAll(h.store.Directory(), cache.DirectoryMode)) + require.NoError(t, os.WriteFile(h.store.Path("repository", "ref"), []byte(corrupt), cache.FileMode)) + + _, _, err := h.load(t, source, cache.Mode{Offline: true}) + require.ErrorIs(t, err, errOffline, "offline never trusts a corrupt snapshot") + assert.Zero(t, h.fetches.Load()) + + value, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.False(t, cached) + assert.Equal(t, []string{"a"}, value.Items) + _, cached, err = h.load(t, source, cache.Mode{Offline: true}) + require.NoError(t, err, "the replacement is a valid snapshot") + assert.True(t, cached) + }) + } +} + +func TestLoadDecoderRejectionIsTreatedAsAbsent(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + source.Decode = func([]byte) (*listing, error) { return nil, errors.New("content validation failed") } + _, _, err = h.load(t, source, cache.Mode{Offline: true}) + require.ErrorIs(t, err, errOffline) +} + +func TestLoadFailedFetchKeepsThePreviousSnapshot(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + before, err := os.ReadFile(h.store.Path("repository", "ref")) + require.NoError(t, err) + + h.mu.Lock() + h.fail = errFetch + h.mu.Unlock() + _, _, err = h.load(t, source, cache.Mode{Refresh: true}) + require.ErrorIs(t, err, errFetch) + after, err := os.ReadFile(h.store.Path("repository", "ref")) + require.NoError(t, err) + assert.Equal(t, before, after, "a failed refresh never replaces or truncates the snapshot") +} + +func TestLoadFailedFirstFetchCachesNothing(t *testing.T) { + h := newHarness(t, "scope") + h.fail = errFetch + _, _, err := h.load(t, h.source("repository"), cache.Mode{}) + require.ErrorIs(t, err, errFetch) + files, globErr := filepath.Glob(filepath.Join(h.store.Directory(), "*.json")) + require.NoError(t, globErr) + assert.Empty(t, files) +} + +func TestLoadUnavailableOptionalSourceIsNotCached(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("catalog") + source.Fetch = func(context.Context, *listing) (*listing, error) { return nil, nil } + value, cached, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + assert.Nil(t, value) + assert.False(t, cached) + files, globErr := filepath.Glob(filepath.Join(h.store.Directory(), "*.json")) + require.NoError(t, globErr) + assert.Empty(t, files, "an unavailable optional source is never cached as an empty result") +} + +func TestLoadPolicyRunsOnCachedReadsAndCanRejectThem(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + + policy := errors.New("source no longer permitted") + h.mu.Lock() + h.authorize = policy + h.mu.Unlock() + _, _, err = h.load(t, source, cache.Mode{}) + require.ErrorIs(t, err, policy, "a fresh snapshot is not an authorization") + _, _, err = h.load(t, source, cache.Mode{Offline: true}) + require.ErrorIs(t, err, policy, "offline reads re-evaluate the policy too") + assert.Equal(t, int64(1), h.fetches.Load()) +} + +func TestLoadPolicyRejectionBlocksTheFetch(t *testing.T) { + h := newHarness(t, "scope") + h.authorize = errors.New("denied") + _, _, err := h.load(t, h.source("repository"), cache.Mode{}) + require.Error(t, err) + assert.Zero(t, h.fetches.Load(), "the source is not contacted when policy rejects it") +} + +func TestLoadOversizedFetchFailsWithoutReplacingTheFile(t *testing.T) { + h := newHarness(t, "scope", cache.WithMaxSize(200)) + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + before, err := os.ReadFile(h.store.Path("repository", "ref")) + require.NoError(t, err) + + h.mu.Lock() + h.items = []string{strings.Repeat("x", 500)} + h.mu.Unlock() + _, _, err = h.load(t, source, cache.Mode{Refresh: true}) + require.ErrorIs(t, err, errTooLarge) + after, err := os.ReadFile(h.store.Path("repository", "ref")) + require.NoError(t, err) + assert.Equal(t, before, after) +} + +func TestLoadCanceledContextWritesNothing(t *testing.T) { + h := newHarness(t, "scope") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, _, err := cache.Load(ctx, h.store, h.source("repository"), cache.Mode{}) + require.Error(t, err) + files, globErr := filepath.Glob(filepath.Join(h.store.Directory(), "*.json")) + require.NoError(t, globErr) + assert.Empty(t, files) +} + +func TestLoadConcurrentColdLoadsShareOneFetch(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + const workers = 12 + start := make(chan struct{}) + errs := make(chan error, workers) + var group sync.WaitGroup + for range workers { + group.Go(func() { + <-start + _, _, err := cache.Load(t.Context(), h.store, source, cache.Mode{}) + errs <- err + }) + } + close(start) + group.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + assert.Equal(t, int64(1), h.fetches.Load()) +} + +func TestLoadPassesThePreviousSnapshotToFetch(t *testing.T) { + h := newHarness(t, "scope") + source := h.source("repository") + var seen []*listing + inner := source.Fetch + source.Fetch = func(ctx context.Context, previous *listing) (*listing, error) { + seen = append(seen, previous) + return inner(ctx, previous) + } + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + _, _, err = h.load(t, source, cache.Mode{Refresh: true}) + require.NoError(t, err) + require.Len(t, seen, 2) + assert.Nil(t, seen[0]) + require.NotNil(t, seen[1]) + assert.Equal(t, []string{"a"}, seen[1].Items) +} + +func TestScopeSeparatesSnapshotsAndIsDeterministic(t *testing.T) { + first, err := cache.Scope(map[string]string{"realm": "a"}, "x", []string{"h"}) + require.NoError(t, err) + again, err := cache.Scope(map[string]string{"realm": "a"}, "x", []string{"h"}) + require.NoError(t, err) + other, err := cache.Scope(map[string]string{"realm": "b"}, "x", []string{"h"}) + require.NoError(t, err) + assert.Equal(t, first, again) + assert.NotEqual(t, first, other) + + dir := filepath.Join(t.TempDir(), "discovery") + a := cache.New(dir, first) + b := cache.New(dir, other) + assert.NotEqual(t, a.Path("repository", "ref"), b.Path("repository", "ref")) + assert.NotEqual(t, a.Path("repository", "ref"), a.Path("catalog", "ref")) + assert.NotEqual(t, a.Path("repository", "ref"), a.Path("repository", "ref2")) + assert.Equal(t, a.Path("repository", "ref"), cache.New(dir, first).Path("repository", "ref")) + assert.Equal(t, filepath.Join(dir, cache.Key(first, "repository", "ref")+".json"), a.Path("repository", "ref")) + + _, err = cache.Scope(make(chan int)) + require.ErrorIs(t, err, errUtils.ErrCacheMarshal) +} + +func TestSnapshotsInDifferentScopesNeverCrossOver(t *testing.T) { + dir := filepath.Join(t.TempDir(), "discovery") + h := newHarness(t, "scope-a") + h.store = cache.New(dir, "scope-a", cache.WithClock(h.clock.Now)) + source := h.source("repository") + _, _, err := h.load(t, source, cache.Mode{}) + require.NoError(t, err) + + h.store = cache.New(dir, "scope-b", cache.WithClock(h.clock.Now)) + _, _, err = h.load(t, source, cache.Mode{Offline: true}) + require.ErrorIs(t, err, errOffline, "another scope cannot reuse the snapshot") +} + +func TestWriteSnapshotIsAtomicAndSizeBounded(t *testing.T) { + path := filepath.Join(t.TempDir(), "snapshot.json") + original := []byte("previous verified snapshot") + require.NoError(t, os.WriteFile(path, original, cache.FileMode)) + store := cache.New("", "", cache.WithMaxSize(128)) + + require.ErrorIs(t, store.WriteSnapshot(path, listing{Items: []string{strings.Repeat("x", 200)}}, errTooLarge), errTooLarge) + require.ErrorIs(t, store.WriteSnapshot(path, listing{Header: cache.Header{FetchedAt: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}}, errTooLarge), errUtils.ErrCacheMarshal) + current, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, original, current, "a rejected write leaves the previous contents in place") + + require.NoError(t, store.WriteSnapshot(path, listing{Items: []string{"ok"}}, errTooLarge)) + written, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(written), `"items":["ok"]`) + if runtime.GOOS != "windows" { + info, statErr := os.Stat(path) + require.NoError(t, statErr) + assert.Equal(t, os.FileMode(cache.FileMode), info.Mode().Perm()) + } +} + +func TestLoadFailsWhenTheDirectoryCannotBeCreated(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "occupied") + require.NoError(t, os.WriteFile(blocker, []byte("file"), cache.FileMode)) + store := cache.New(filepath.Join(blocker, "discovery"), "scope") + h := newHarness(t, "scope") + _, _, err := cache.Load(t.Context(), store, h.source("repository"), cache.Mode{}) + require.ErrorIs(t, err, errUtils.ErrCacheDir) + assert.Zero(t, h.fetches.Load(), "nothing is fetched when the cache cannot be written") +} + +func TestStoreAccessorsAndHeaderRoundTrip(t *testing.T) { + store := cache.New("dir", "scope", cache.WithTTL(time.Minute)) + assert.Equal(t, "dir", store.Directory()) + assert.Equal(t, time.Minute, store.TTL()) + assert.Equal(t, cache.DefaultTTL, cache.New("dir", "scope").TTL()) + + var header cache.Header + header.SetSnapshotHeader(cache.Header{Version: 3, Source: "s", Kind: "k", FetchedAt: epoch}) + assert.Equal(t, cache.Header{Version: 3, Source: "s", Kind: "k", FetchedAt: epoch}, header.SnapshotHeader()) + assert.Len(t, cache.Hash([]byte("x")), 64) +} + +func TestFreshnessRejectsFutureTimestamps(t *testing.T) { + h := newHarness(t, "scope") + assert.True(t, h.store.Fresh(epoch.Add(-time.Minute))) + assert.True(t, h.store.Fresh(epoch.Add(time.Minute)), "clock skew does not force refresh churn") + assert.False(t, h.store.Fresh(epoch.Add(-cache.DefaultTTL))) +} + +func TestFreshnessZeroTTLIsNeverFreshEvenForFutureTimestamps(t *testing.T) { + h := newHarness(t, "scope", cache.WithTTL(0)) + assert.False(t, h.store.Fresh(epoch.Add(-time.Minute))) + assert.False(t, h.store.Fresh(epoch)) + assert.False(t, h.store.Fresh(epoch.Add(time.Minute)), "a future timestamp must not bypass a zero TTL") +} + +func TestWriteSnapshotWrapsFilesystemFailures(t *testing.T) { + blocker := filepath.Join(t.TempDir(), "occupied") + require.NoError(t, os.WriteFile(blocker, []byte("file"), cache.FileMode)) + store := cache.New(t.TempDir(), "scope") + err := store.WriteSnapshot(filepath.Join(blocker, "snapshot.json"), listing{Items: []string{"ok"}}, errTooLarge) + require.ErrorIs(t, err, errUtils.ErrCacheWrite) +} diff --git a/pkg/artifact/discovery/cache/doc.go b/pkg/artifact/discovery/cache/doc.go new file mode 100644 index 00000000000..29ec7b7ba22 --- /dev/null +++ b/pkg/artifact/discovery/cache/doc.go @@ -0,0 +1,15 @@ +// Package cache provides neutral, source-scoped discovery snapshots shared by plugin discovery and +// artifact discovery. +// +// A [Store] owns one private directory. Each discovery source (a repository listing, a catalog) +// has one snapshot file whose name is derived from a scope hash plus the source kind and +// reference, so snapshots are isolated by effective configuration, authentication realm, and host +// bindings without ever storing a credential value. [Load] applies the shared policy: a fresh +// snapshot is served without contacting the source, an explicit refresh or a stale snapshot fetches +// a new one, offline mode serves only an existing valid snapshot, and a fetched snapshot replaces +// the old file atomically only after the fetch completed. +// +// The package knows nothing about plugin or artifact schemas. Callers embed [Header] first in their +// snapshot struct (so the historical JSON keys stay in order), supply a decoder that validates the +// content, and supply the fetch and policy callbacks. +package cache diff --git a/pkg/artifact/discovery/cache/load.go b/pkg/artifact/discovery/cache/load.go new file mode 100644 index 00000000000..d1b298b745b --- /dev/null +++ b/pkg/artifact/discovery/cache/load.go @@ -0,0 +1,163 @@ +package cache + +import ( + "context" + "fmt" + "os" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/filelock" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Mode selects how [Load] may use the network. +type Mode struct { + // Refresh bypasses snapshot freshness and fetches again. It fails when Offline is set. + Refresh bool + + // Offline serves an existing valid snapshot of any age and never fetches. + Offline bool +} + +// Source describes one discovery source and the callbacks [Load] needs. P is a pointer to the +// caller's snapshot struct T, which embeds [Header]. +type Source[T any, P interface { + *T + Snapshot +}] struct { + // Kind and Reference identify the source and select its snapshot file. + Kind, Reference string + + // Version is the snapshot schema version a readable snapshot must carry. + Version int + + // Decode parses and validates the content of a snapshot file. A snapshot that fails decoding + // is treated as absent: it is replaced online and rejected offline. + Decode func(raw []byte) (P, error) + + // Fetch returns a fresh snapshot, given the previous valid one (nil when none). It may reuse + // the previous snapshot's content when nothing changed. Returning nil without an error means + // the source is unavailable but optional: nothing is cached. + Fetch func(ctx context.Context, previous P) (P, error) + + // Authorize runs the current policy before a snapshot is served from cache (cached true) and + // before the source is contacted (cached false). It may be nil. + Authorize func(ctx context.Context, cached bool) error + + // ErrOffline is returned in offline mode when no valid snapshot exists or a refresh is requested. + ErrOffline error + + // ErrTooLarge is returned when a fetched snapshot exceeds the size bound. + ErrTooLarge error +} + +// Load returns the snapshot of a source and whether it came from the cache. Concurrent loads of the +// same source (in this or another process) are serialized on a lock file, so a cold cache is +// filled once and the others read the result. +func Load[T any, P interface { + *T + Snapshot +}](ctx context.Context, store *Store, source *Source[T, P], mode Mode) (P, bool, error) { + defer perf.Track(nil, "cache.Load")() + + path := store.Path(source.Kind, source.Reference) + if mode.Offline { + return update(ctx, store, source, mode, path) + } + if err := os.MkdirAll(store.directory, DirectoryMode); err != nil { + return nil, false, fmt.Errorf("%w: %w", errUtils.ErrCacheDir, err) + } + var ( + result P + cached bool + ) + err := filelock.New(path+".lock").WithExclusive(ctx, func() error { + var loadErr error + result, cached, loadErr = update(ctx, store, source, mode, path) + return loadErr + }) + if err != nil { + return nil, false, err + } + return result, cached, nil +} + +// update applies freshness, offline, and refresh rules. Online callers hold the source lock. +func update[T any, P interface { + *T + Snapshot +}](ctx context.Context, store *Store, source *Source[T, P], mode Mode, path string) (P, bool, error) { + if mode.Offline && mode.Refresh { + return nil, false, source.ErrOffline + } + previous := readSnapshot(store, source, path) + if previous != nil && (mode.Offline || (!mode.Refresh && store.Fresh(previous.SnapshotHeader().FetchedAt))) { + if err := authorize(ctx, source, true); err != nil { + return nil, true, err + } + return previous, true, nil + } + if mode.Offline { + return nil, false, source.ErrOffline + } + current, err := fetch(ctx, store, source, path, previous) + return current, false, err +} + +// fetch reads the source and, only after the read completed, atomically replaces the snapshot. A +// nil snapshot without an error means the source is unavailable but optional. +func fetch[T any, P interface { + *T + Snapshot +}](ctx context.Context, store *Store, source *Source[T, P], path string, previous P) (P, error) { + if err := authorize(ctx, source, false); err != nil { + return nil, err + } + current, err := source.Fetch(ctx, previous) + if err != nil { + return nil, err + } + if current == nil { + // Do not cache an unavailable optional source as an empty result. + return nil, nil + } + current.SetSnapshotHeader(Header{Version: source.Version, Source: source.Reference, Kind: source.Kind, FetchedAt: store.now().UTC()}) + if err := ctx.Err(); err != nil { + return nil, err + } + if err := store.WriteSnapshot(path, current, source.ErrTooLarge); err != nil { + return nil, err + } + return current, nil +} + +func authorize[T any, P interface { + *T + Snapshot +}](ctx context.Context, source *Source[T, P], cached bool) error { + if source.Authorize == nil { + return nil + } + return source.Authorize(ctx, cached) +} + +// readSnapshot returns the valid snapshot of a source, or nil when it is absent, oversized, +// corrupt, written for another source or schema version, or fails content validation. +func readSnapshot[T any, P interface { + *T + Snapshot +}](store *Store, source *Source[T, P], path string) P { + raw := store.read(path) + if raw == nil { + return nil + } + value, err := source.Decode(raw) + if err != nil || value == nil { + return nil + } + header := value.SnapshotHeader() + if header.Version != source.Version || header.Source != source.Reference || header.Kind != source.Kind { + return nil + } + return value +} diff --git a/pkg/artifact/discovery/cache/store.go b/pkg/artifact/discovery/cache/store.go new file mode 100644 index 00000000000..6d711dc412f --- /dev/null +++ b/pkg/artifact/discovery/cache/store.go @@ -0,0 +1,215 @@ +package cache + +import ( + "crypto" + _ "crypto/sha256" // Registers crypto.SHA256 for Hash. + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/filesystem" + "github.com/cloudposse/atmos/pkg/perf" +) + +const ( + // DefaultTTL is how long a snapshot stays fresh. + DefaultTTL = 15 * time.Minute + + // DefaultMaxSize bounds the bytes read from, and written to, one snapshot file. + DefaultMaxSize = 32 << 20 + + // DirectoryMode is the permission of the cache directory. + DirectoryMode = 0o700 + + // FileMode is the permission of a snapshot file. + FileMode = 0o600 + + // The separator splits the hashed parts of a key; no part may contain it. + separator = "\x00" +) + +// Header is the metadata every snapshot starts with. Embed it first in a snapshot struct so the +// serialized keys keep their order: version, source, kind, fetched_at. +type Header struct { + // Version is the snapshot schema version. + Version int `json:"version"` + + // Source is the reference the snapshot describes. + Source string `json:"source"` + + // Kind is the source kind, for example "repository" or "catalog". + Kind string `json:"kind"` + + // FetchedAt is when the snapshot was fetched, in UTC. + FetchedAt time.Time `json:"fetched_at"` +} + +// Snapshot is implemented by pointers to snapshot structs. A struct that embeds [Header] satisfies +// it through the embedded methods; a struct that keeps the four fields itself implements both +// methods. +type Snapshot interface { + // SnapshotHeader returns the header of the snapshot. + SnapshotHeader() Header + + // SetSnapshotHeader replaces the header of the snapshot. + SetSnapshotHeader(Header) +} + +// SnapshotHeader returns the header itself. +// +//nolint:lintroller // Trivial accessor. +func (h *Header) SnapshotHeader() Header { return *h } + +// SetSnapshotHeader replaces the header. +// +//nolint:lintroller // Trivial accessor. +func (h *Header) SetSnapshotHeader(value Header) { *h = value } + +// Store is one private snapshot directory plus the scope that isolates its files. +type Store struct { + directory string + scope string + ttl time.Duration + maxSize int64 + now func() time.Time +} + +// Option customizes a [Store]. +type Option func(*Store) + +// WithTTL sets the freshness window. Zero makes every online load a refresh. +func WithTTL(ttl time.Duration) Option { + defer perf.Track(nil, "cache.WithTTL")() + + return func(s *Store) { s.ttl = ttl } +} + +// WithMaxSize sets the bound on a snapshot file, in bytes. +func WithMaxSize(size int64) Option { + defer perf.Track(nil, "cache.WithMaxSize")() + + return func(s *Store) { s.maxSize = size } +} + +// WithClock injects the clock; tests use a deterministic one. +func WithClock(now func() time.Time) Option { + defer perf.Track(nil, "cache.WithClock")() + + return func(s *Store) { s.now = now } +} + +// New returns a store rooted at directory. The scope is the hash from [Scope]; it separates the +// snapshots of different effective configurations that share the directory. +func New(directory, scope string, opts ...Option) *Store { + defer perf.Track(nil, "cache.New")() + + store := &Store{directory: directory, scope: scope, ttl: DefaultTTL, maxSize: DefaultMaxSize, now: time.Now} + for _, opt := range opts { + if opt != nil { + opt(store) + } + } + return store +} + +// Directory returns the snapshot directory. +func (s *Store) Directory() string { + defer perf.Track(nil, "cache.Store.Directory")() + + return s.directory +} + +// TTL returns the freshness window. +func (s *Store) TTL() time.Duration { + defer perf.Track(nil, "cache.Store.TTL")() + + return s.ttl +} + +// Path returns the snapshot file of a source. +func (s *Store) Path(kind, reference string) string { + defer perf.Track(nil, "cache.Store.Path")() + + return filepath.Join(s.directory, Key(s.scope, kind, reference)+".json") +} + +// Scope hashes the parts that define a cache scope: the effective configuration, the +// authentication realm, and host bindings. Callers pass configuration, never credential values. +// Equal parts always produce the same scope. +func Scope(parts ...any) (string, error) { + defer perf.Track(nil, "cache.Scope")() + + raw, err := json.Marshal(parts) + if err != nil { + return "", fmt.Errorf("%w: %w", errUtils.ErrCacheMarshal, err) + } + return Hash(raw), nil +} + +// Hash returns the lowercase hexadecimal SHA-256 of raw. The digest only names cache files and +// scopes, and it must stay plain SHA-256 so snapshot files written by the plugin discovery cache +// keep their names. It is not a password hash and never protects a credential. +func Hash(raw []byte) string { + defer perf.Track(nil, "cache.Hash")() + + digest := crypto.SHA256.New() + digest.Write(raw) + return hex.EncodeToString(digest.Sum(nil)) +} + +// Key derives the file stem of a source within a scope. +func Key(scope, kind, reference string) string { + defer perf.Track(nil, "cache.Key")() + + return Hash([]byte(scope + separator + kind + separator + reference)) +} + +// Fresh reports whether a snapshot fetched at the given time is still within the TTL. +func (s *Store) Fresh(fetched time.Time) bool { + defer perf.Track(nil, "cache.Store.Fresh")() + + if s.ttl <= 0 { + return false + } + age := s.now().Sub(fetched) + return age < s.ttl +} + +// read returns the raw bytes of a snapshot file within the size bound, or nil when it is absent, +// unreadable, or oversized. +func (s *Store) read(path string) []byte { + file, err := os.Open(path) + if err != nil { + return nil + } + defer file.Close() + raw, err := io.ReadAll(io.LimitReader(file, s.maxSize+1)) + if err != nil || int64(len(raw)) > s.maxSize { + return nil + } + return raw +} + +// WriteSnapshot serializes value and atomically replaces the file at path; a reader sees the old +// or the new file, never a mix. A value larger than the size bound fails with errTooLarge and +// leaves the previous file untouched. +func (s *Store) WriteSnapshot(path string, value any, errTooLarge error) error { + defer perf.Track(nil, "cache.Store.WriteSnapshot")() + + raw, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("%w: %w", errUtils.ErrCacheMarshal, err) + } + if int64(len(raw)) > s.maxSize { + return errTooLarge + } + if err := filesystem.NewOSFileSystem().WriteFileAtomic(path, raw, FileMode); err != nil { + return fmt.Errorf("%w: %w", errUtils.ErrCacheWrite, err) + } + return nil +} diff --git a/pkg/artifact/discovery/catalog/catalog_test.go b/pkg/artifact/discovery/catalog/catalog_test.go new file mode 100644 index 00000000000..78daba8df91 --- /dev/null +++ b/pkg/artifact/discovery/catalog/catalog_test.go @@ -0,0 +1,488 @@ +package catalog_test + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/oci" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +var now = time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + +// env is one registry repository with a catalog service over it. +type env struct { + t *testing.T + registry *testkit.Registry + repo *oci.Repository + info service.RepositoryInfo + service *catalog.Service + uri string +} + +func newEnv(t *testing.T, configured bool) *env { + t.Helper() + e := &env{t: t, registry: testkit.NewRegistry(t)} + if configured { + e.uri = e.registry.CatalogURI("catalog", "stable") + } + e.repo = e.registry.Repo(t, "shared", e.uri) + e.info = e.registry.Info(e.repo, e.uri) + e.service = catalog.NewService( + &service.StaticRepositories{Repositories: map[string]artifact.Repository{"shared": e.repo}, Info: map[string]service.RepositoryInfo{"shared": e.info}}, + catalog.WithClock(func() time.Time { return now }), catalog.WithOwner("test-owner"), + ) + return e +} + +func (e *env) publish(name, version string, labels map[string]string) *artifact.PublishResult { + e.t.Helper() + return testkit.Publish(e.t, e.repo, testkit.Spec{Name: name, Version: version, Content: name + version, Labels: labels}) +} + +func ref(text string) artifact.Ref { + parsed, err := artifact.ParseRef(text) + if err != nil { + panic(err) + } + return parsed +} + +func refsOf(texts ...string) []artifact.Ref { + out := make([]artifact.Ref, 0, len(texts)) + for _, text := range texts { + out = append(out, ref(text)) + } + return out +} + +// latest reads the published catalog straight from the repository. +func (e *env) latest() *catalog.Loaded { + e.t.Helper() + loaded, err := catalog.Read(context.Background(), e.repo, ref("catalog:stable")) + require.NoError(e.t, err) + return loaded +} + +func listed(file *catalog.File) []string { + var out []string + for _, entry := range file.Spec.Entries { + for _, version := range entry.Versions { + out = append(out, entry.Name+":"+version.Version) + } + } + return out +} + +func TestBuildReadsMetadataOnlyAndNeverPublishes(t *testing.T) { + e := newEnv(t, true) + vpc := e.publish("components/vpc", "1.2.0", map[string]string{"team": "platform"}) + e.publish("tools/helper", "0.1.0", nil) + e.registry.Reset() + + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("components/vpc:1.2.0", "tools/helper:0.1.0")}) + require.NoError(t, err) + assert.Equal(t, []string{"components/vpc:1.2.0", "tools/helper:0.1.0"}, listed(built.File)) + assert.Zero(t, built.File.Metadata.Revision) + assert.Empty(t, built.BaseDigest, "no catalog is published yet") + assert.Equal(t, "catalog", built.File.Metadata.Name) + + entry := built.File.Spec.Entries[0] + assert.Equal(t, "oci://"+e.registry.Location()+"/components/vpc", entry.Source, "the entry is bound to the repository path, not to credentials") + assert.Equal(t, "files", entry.Kind) + assert.Equal(t, "package components/vpc", entry.Description) + version := entry.Versions[0] + assert.Equal(t, vpc.Pinned.ResolvedDigest.String(), version.ResolvedDigest) + assert.Equal(t, vpc.Pinned.RecordDigest.String(), version.RecordDigest) + assert.Equal(t, vpc.Pinned.ContentDigest.String(), version.ContentDigest) + assert.Equal(t, map[string]string{"team": "platform"}, version.Labels) + require.Len(t, version.Documents, 1) + assert.Equal(t, "readme", version.Documents[0].Role) + + for _, request := range e.registry.Requests() { + assert.NotContains(t, request, vpc.Representation.Payload[0].Digest.Encoded(), "build never reads a payload layer: %s", request) + assert.False(t, strings.HasPrefix(request, "PUT ") || strings.HasPrefix(request, "POST "), "build never writes: %s", request) + } + _, err = catalog.Read(context.Background(), e.repo, ref("catalog:stable")) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "build does not publish") +} + +func TestBuildRecordsTheCatalogItWasBasedOn(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + _, err := e.service.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + published := e.latest() + + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0"), Name: "custom"}) + require.NoError(t, err) + assert.Equal(t, 1, built.File.Metadata.Revision) + assert.Equal(t, published.Digest.String(), built.BaseDigest) + assert.Equal(t, "custom", built.File.Metadata.Name) +} + +func TestBuildWithoutCatalogConfigurationStillBuilds(t *testing.T) { + e := newEnv(t, false) + e.publish("app", "1.0.0", nil) + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + assert.Equal(t, []string{"app:1.0.0"}, listed(built.File)) +} + +func TestBuildRejectsUnresolvableAndConflictingReferences(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + _, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:9.9.9")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + _, err = e.service.Build(context.Background(), &catalog.BuildRequest{Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + _, err = e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "missing", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestPublishWritesAGenericCatalogRecord(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", map[string]string{"env": "prod"}) + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + + result, err := e.service.Publish(context.Background(), &catalog.PublishRequest{Repository: "shared", File: built.File}) + require.NoError(t, err) + assert.Equal(t, 1, result.Revision) + assert.Equal(t, 1, result.Listed) + assert.Equal(t, e.uri, result.Catalog) + assert.Equal(t, []string{"app:1.0.0"}, result.Added) + assert.NotEmpty(t, result.Digest) + assert.Empty(t, result.PreviousDigest) + assert.Equal(t, artifact.LockScopeHost, result.LockScope, "the repository coordinator guards the write") + + // The catalog is a generic Record of kind catalog with a direct JSON payload layer. + pinned, err := e.repo.Resolve(context.Background(), ref("catalog:stable")) + require.NoError(t, err) + inspection, err := e.repo.Inspect(context.Background(), pinned) + require.NoError(t, err) + require.NotNil(t, inspection.Record) + assert.Equal(t, artifact.KindCatalog, inspection.Record.Kind) + assert.Equal(t, artifact.PackagingCatalog, inspection.Representation.Packaging) + require.Len(t, inspection.Representation.Payload, 1) + assert.Equal(t, artifact.MediaTypeCatalogPayload, inspection.Representation.Payload[0].MediaType) + + reference, err := name.ParseReference(e.registry.Location() + "/catalog:stable") + require.NoError(t, err) + descriptor, err := remote.Get(reference) + require.NoError(t, err) + var manifest ocispec.Manifest + require.NoError(t, json.Unmarshal(descriptor.Manifest, &manifest)) + assert.Equal(t, artifact.MediaTypeCatalog, manifest.ArtifactType) + assert.Equal(t, artifact.MediaTypeConfig, manifest.Config.MediaType, "the shared Record config media type is retained") + require.Len(t, manifest.Layers, 1) + assert.Equal(t, artifact.MediaTypeCatalogPayload, manifest.Layers[0].MediaType) + + loaded := e.latest() + assert.Equal(t, []string{"app:1.0.0"}, listed(loaded.File)) + assert.Equal(t, now.Format("2006-01-02T15:04:05Z"), loaded.File.Metadata.GeneratedAt) + assert.Equal(t, 1, loaded.File.Metadata.Revision) +} + +func TestMutatingCommandsFailBeforeAnyWriteWithoutACatalog(t *testing.T) { + e := newEnv(t, false) + e.publish("app", "1.0.0", nil) + file := catalog.New("catalog") + e.registry.Reset() + ctx := context.Background() + + _, err := e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: file}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogNotConfigured) + _, err = e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogNotConfigured) + _, err = e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: ref("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogNotConfigured) + _, err = e.service.Sync(ctx, &catalog.SyncRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogNotConfigured) + assert.Zero(t, e.registry.Total(), "configuration is checked before the registry is contacted: %v", e.registry.Requests()) +} + +func TestAddRemoveAndSyncListWithoutTouchingArtifacts(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + vpc := e.publish("components/vpc", "1.2.0", nil) + e.publish("components/vpc", "1.10.0", nil) + e.publish("components/vpc", "1.9.0", nil) + e.publish("tools/helper", "0.1.0", nil) + + added, err := e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("components/vpc:1.2.0", "components/vpc:1.10.0", "components/vpc:1.9.0")}) + require.NoError(t, err) + assert.Len(t, added.Added, 3) + assert.Equal(t, []string{"components/vpc:1.10.0", "components/vpc:1.2.0", "components/vpc:1.9.0"}, listed(e.latest().File), + "versions are opaque labels sorted lexically, not as SemVer") + + synced, err := e.service.Sync(ctx, &catalog.SyncRequest{Repository: "shared", Refs: refsOf("tools/helper:0.1.0")}) + require.NoError(t, err) + assert.Equal(t, []string{"tools/helper:0.1.0"}, synced.Added) + assert.Equal(t, 4, synced.Listed, "entries absent from the inventory are retained") + assert.Equal(t, 2, synced.Revision) + assert.Equal(t, added.Digest, synced.PreviousDigest) + + e.registry.Reset() + removed, err := e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: ref("components/vpc:1.2.0")}) + require.NoError(t, err) + assert.Equal(t, []string{"components/vpc:1.2.0"}, removed.Removed) + assert.Equal(t, []string{"components/vpc:1.10.0", "components/vpc:1.9.0", "tools/helper:0.1.0"}, listed(e.latest().File)) + for _, request := range e.registry.Requests() { + assert.False(t, strings.HasPrefix(request, "DELETE "), "remove only unlists; it deletes nothing: %s", request) + assert.NotContains(t, request, "/acme/atmos/components/vpc/", "remove never touches the artifact's repository: %s", request) + } + pinned, err := e.repo.Resolve(ctx, ref("components/vpc:1.2.0")) + require.NoError(t, err, "the unlisted artifact is still published") + assert.Equal(t, vpc.Pinned.ResolvedDigest, pinned.ResolvedDigest, "its content is unchanged") + + _, err = e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: ref("components/vpc:1.2.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "a version that is not listed cannot be unlisted") + _, err = e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: artifact.Ref{Name: "tools/helper", Digest: vpc.Pinned.ResolvedDigest}}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + + _, err = e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: ref("tools/helper:0.1.0")}) + require.NoError(t, err) + assert.NotContains(t, strings.Join(listed(e.latest().File), ","), "tools/helper", "removing the last version drops the entry") +} + +func TestRemoveWithoutAPublishedCatalogReportsNotFound(t *testing.T) { + e := newEnv(t, true) + _, err := e.service.Remove(context.Background(), &catalog.RemoveRequest{Repository: "shared", Ref: ref("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) +} + +func TestSyncUpdatesAListingWhoseTagMoved(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + e.publish("app", "1.0.0", nil) + _, err := e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + before := e.latest().File.Spec.Entries[0].Versions[0].ResolvedDigest + + moved, err := e.repo.Publish(ctx, testkit.Prepare(t, &testkit.Spec{Name: "app", Version: "1.0.0", Content: "changed"}), artifact.PublishOptions{Force: true, ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digest.Digest(before)}}) + require.NoError(t, err) + result, err := e.service.Sync(ctx, &catalog.SyncRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + assert.Equal(t, []string{"app:1.0.0"}, result.Updated) + assert.Equal(t, moved.Pinned.ResolvedDigest.String(), e.latest().File.Spec.Entries[0].Versions[0].ResolvedDigest) +} + +func TestPublishRejectsStaleRevisionsAndDigests(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + e.publish("app", "1.0.0", nil) + e.publish("app", "1.1.0", nil) + + t.Run("snapshot built before the first catalog existed", func(t *testing.T) { + built, err := e.service.Build(ctx, &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + _, err = e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.1.0")}) + require.NoError(t, err) + _, err = e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: built.File}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) + assert.Equal(t, []string{"app:1.1.0"}, listed(e.latest().File), "the stale snapshot did not overwrite newer metadata") + }) + t.Run("revision moved after the snapshot", func(t *testing.T) { + built, err := e.service.Build(ctx, &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + _, err = e.service.Remove(ctx, &catalog.RemoveRequest{Repository: "shared", Ref: ref("app:1.1.0")}) + require.NoError(t, err) + _, err = e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: built.File}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) + }) + t.Run("expected digest", func(t *testing.T) { + built, err := e.service.Build(ctx, &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + _, err = e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: built.File, ExpectedDigest: "sha256:" + strings.Repeat("0", 64)}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) + fresh, err := e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: built.File, ExpectedDigest: built.BaseDigest}) + require.NoError(t, err) + assert.Equal(t, []string{"app:1.0.0"}, listed(e.latest().File)) + assert.Equal(t, built.File.Metadata.Revision+1, fresh.Revision) + }) +} + +func TestPublishAgainstANewCatalogRejectsAnExpectedDigest(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + _, err = e.service.Publish(context.Background(), &catalog.PublishRequest{Repository: "shared", File: built.File, ExpectedDigest: "sha256:" + strings.Repeat("1", 64)}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) +} + +func TestWriteDetectsACatalogThatChangedAfterItWasRead(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + e.publish("app", "1.0.0", nil) + e.publish("app", "1.1.0", nil) + _, err := e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + stale := e.latest() // A writer reads the latest catalog... + + _, err = e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.1.0")}) // ...another writer publishes... + require.NoError(t, err) + + next := stale.File + next.Metadata.Revision++ + _, err = catalog.Write(ctx, e.repo, ref("catalog:stable"), next, &catalog.WriteOptions{Prior: &stale.Pinned, Now: now, OperationID: "op"}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict, "...so the final prior-reference check rejects the stale write") + assert.Equal(t, []string{"app:1.0.0", "app:1.1.0"}, listed(e.latest().File)) +} + +func TestSerializedWritersReadTheLatestCatalogInsideTheGuard(t *testing.T) { + e := newEnv(t, true) + const writers = 6 + for index := range writers { + e.publish(fmt.Sprintf("app%d", index), "1.0.0", nil) + } + var group sync.WaitGroup + errs := make(chan error, writers) + for index := range writers { + group.Go(func() { + _, err := e.service.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf(fmt.Sprintf("app%d:1.0.0", index))}) + errs <- err + }) + } + group.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + loaded := e.latest() + assert.Len(t, listed(loaded.File), writers, "no concurrent writer's entry was lost") + assert.Equal(t, writers, loaded.File.Metadata.Revision) +} + +func TestPublishingOrDeletingAnArtifactNeverEditsTheCatalog(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + e.publish("app", "1.0.0", nil) + _, err := e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + before := e.latest() + + later := e.publish("app", "2.0.0", nil) + assert.Equal(t, before.Digest, e.latest().Digest, "publishing a new version leaves the catalog unchanged") + _, err = e.repo.Delete(ctx, later.Pinned, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + first, err := e.repo.Resolve(ctx, ref("app:1.0.0")) + require.NoError(t, err) + _, err = e.repo.Delete(ctx, first, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + after := e.latest() + assert.Equal(t, before.Digest, after.Digest, "deleting an artifact leaves the catalog unchanged") + assert.Equal(t, []string{"app:1.0.0"}, listed(after.File)) +} + +func TestPublishRejectsLocationsOutsideTheDestinationRepository(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + for source, label := range map[string]string{ + "oci://evil.example.com/acme/atmos/app": "another registry", + "oci://" + e.registry.Host + "/other/prefix/app": "another prefix", + "oci://" + e.registry.Location() + "/different": "another name", + "oci://user:pass@" + e.registry.Location() + "/app": "credentials in the source", + } { + t.Run(label, func(t *testing.T) { + file := *built.File + file.Spec.Entries = append([]catalog.Entry(nil), built.File.Spec.Entries...) + file.Spec.Entries[0].Source = source + e.registry.Reset() + _, err := e.service.Publish(context.Background(), &catalog.PublishRequest{Repository: "shared", File: &file}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + assert.Zero(t, e.registry.Total(), "validation happens before any access") + }) + } +} + +func TestCatalogLocationMustLieInsideTheRepository(t *testing.T) { + registry := testkit.NewRegistry(t) + location := registry.Location() + good, err := catalog.RefOf(location, "oci://"+strings.ToUpper(registry.Host)+"/"+testkit.Prefix+"/catalog:stable") + require.NoError(t, err) + assert.Equal(t, artifact.Ref{Name: "catalog", Version: "stable"}, good) + byDigest, err := catalog.RefOf(location, "oci://"+location+"/team/catalog@sha256:"+strings.Repeat("a", 64)) + require.NoError(t, err) + assert.Equal(t, "team/catalog", byDigest.Name) + assert.True(t, byDigest.IsDigest()) + + for uri, want := range map[string]string{ + "https://" + location + "/catalog:stable": "oci://", + "oci://other.example.com/acme/atmos/catalog:stable": "outside the repository path", + "oci://" + registry.Host + "/elsewhere/catalog:stable": "outside the repository path", + "oci://" + location + "/catalog": "version", + "oci://" + location + "/": "outside the repository path", + "oci://user:secret@" + location + "/catalog:stable": "outside the repository path", + "oci://user:secret@" + location + "/catalog@sha256:" + strings.Repeat("a", 64): "outside the repository path", + } { + _, err := catalog.RefOf(location, uri) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, uri) + assert.Contains(t, err.Error(), want, uri) + assert.NotContains(t, err.Error(), "secret", "credentials are never echoed") + } +} + +func TestReadRejectsArtifactsThatAreNotCatalogs(t *testing.T) { + e := newEnv(t, true) + e.publish("catalog", "stable", nil) + _, err := catalog.Read(context.Background(), e.repo, ref("catalog:stable")) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, "an ordinary package is not a catalog") +} + +func TestFilesRoundTripInventoriesAndSnapshots(t *testing.T) { + dir := t.TempDir() + inventory := filepath.Join(dir, "inventory.yaml") + require.NoError(t, os.WriteFile(inventory, []byte("apiVersion: atmos/v1\nkind: AtmosArtifactInventory\nmetadata:\n name: releases\nspec:\n refs:\n - tools/helper:0.1.0\n - components/vpc:1.2.0\n"), 0o600)) + refs, err := catalog.ReadInventoryFile(inventory) + require.NoError(t, err) + assert.Equal(t, []string{"components/vpc:1.2.0", "tools/helper:0.1.0"}, []string{refs[0].String(), refs[1].String()}, "inventory refs are sorted") + + _, err = catalog.ReadInventoryFile(filepath.Join(dir, "missing.yaml")) + require.ErrorIs(t, err, os.ErrNotExist) + + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + built, err := e.service.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + for _, name := range []string{"catalog.json", "catalog.yaml", "catalog.yml"} { + path := filepath.Join(dir, name) + require.NoError(t, catalog.WriteFile(path, built.File)) + read, err := catalog.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, built.File.Spec, read.Spec, name) + } + raw, err := os.ReadFile(filepath.Join(dir, "catalog.json")) + require.NoError(t, err) + assert.True(t, json.Valid(raw)) + yamlRaw, err := os.ReadFile(filepath.Join(dir, "catalog.yaml")) + require.NoError(t, err) + assert.False(t, json.Valid(yamlRaw)) + assert.Contains(t, string(yamlRaw), "kind: AtmosArtifactCatalog") + + oversized := filepath.Join(dir, "huge.json") + require.NoError(t, os.WriteFile(oversized, []byte(strings.Repeat(" ", int(artifact.MaxRecordSize)+1)), 0o600)) + _, err = catalog.ReadFile(oversized) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} diff --git a/pkg/artifact/discovery/catalog/doc.go b/pkg/artifact/discovery/catalog/doc.go new file mode 100644 index 00000000000..83074a6f556 --- /dev/null +++ b/pkg/artifact/discovery/catalog/doc.go @@ -0,0 +1,19 @@ +// Package catalog maintains optional artifact catalogs: discovery snapshots published as generic +// artifact Records of kind "catalog" (artifact type application/vnd.atmos.artifact.catalog.v1) with +// one direct JSON payload layer (application/vnd.atmos.artifact.catalog.v1+json). +// +// A catalog is discoverable data, never a trust grant. It lists names, versions, digests, labels, +// and document descriptors of artifacts in the repository it is bound to, so a collection can be +// searched where the registry cannot enumerate repositories. Every entry is bound to the actual +// repository path (never credentials) and is validated against the destination repository before +// any use. Publishing, signing, or deleting an artifact never edits its catalog, and removing a +// catalog entry only unlists it: content, signatures, and consumer lock files are untouched. +// +// The AtmosArtifactCatalog and AtmosArtifactInventory manifests are separate from the plugin +// catalog schema. Version labels are opaque; names and versions sort lexically, with no SemVer +// semantics. Catalog writes require one externally serialized publisher spanning read-latest, +// build, and publish. Revision and digest checks reject already-stale input and the read inside the +// write guard sees the latest catalog, but registries offer no atomic compare-and-swap, so a +// preflight check alone does not prevent races. The repository coordinator (lock) is used when the +// repository provides one. +package catalog diff --git a/pkg/artifact/discovery/catalog/files.go b/pkg/artifact/discovery/catalog/files.go new file mode 100644 index 00000000000..2e05fcd7128 --- /dev/null +++ b/pkg/artifact/discovery/catalog/files.go @@ -0,0 +1,83 @@ +package catalog + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + sigsyaml "sigs.k8s.io/yaml" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/filesystem" + "github.com/cloudposse/atmos/pkg/perf" +) + +// fileMode is the permission of a written catalog snapshot. +const fileMode = 0o644 + +// readBounded reads a local manifest file within the catalog size bound. +func readBounded(path string) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + raw, err := io.ReadAll(io.LimitReader(file, artifact.MaxRecordSize+1)) + if err != nil { + return nil, err + } + if int64(len(raw)) > artifact.MaxRecordSize { + return nil, fmt.Errorf("%w: %s exceeds %d bytes", errUtils.ErrArtifactTooLarge, path, artifact.MaxRecordSize) + } + return raw, nil +} + +// ReadInventoryFile reads and validates an AtmosArtifactInventory manifest from a local file. +func ReadInventoryFile(path string) ([]artifact.Ref, error) { + defer perf.Track(nil, "catalog.ReadInventoryFile")() + + raw, err := readBounded(path) + if err != nil { + return nil, err + } + _, refs, err := ParseInventory(raw) + return refs, err +} + +// ReadFile reads and validates an AtmosArtifactCatalog manifest from a local file. +func ReadFile(path string) (*File, error) { + defer perf.Track(nil, "catalog.ReadFile")() + + raw, err := readBounded(path) + if err != nil { + return nil, err + } + return Parse(raw) +} + +// WriteFile atomically writes a catalog snapshot. The format follows the file extension: YAML for +// .yaml and .yml, JSON otherwise. +func WriteFile(path string, file *File) error { + defer perf.Track(nil, "catalog.WriteFile")() + + Normalize(file) + raw, err := json.MarshalIndent(file, "", " ") + if err != nil { + return err + } + if extension := strings.ToLower(filepath.Ext(path)); extension == ".yaml" || extension == ".yml" { + if raw, err = sigsyaml.JSONToYAML(raw); err != nil { + return err + } + } else { + raw = append(raw, '\n') + } + if _, err := Parse(raw); err != nil { + return err + } + return filesystem.NewOSFileSystem().WriteFileAtomic(path, raw, fileMode) +} diff --git a/pkg/artifact/discovery/catalog/guard.go b/pkg/artifact/discovery/catalog/guard.go new file mode 100644 index 00000000000..c099d9fb4f5 --- /dev/null +++ b/pkg/artifact/discovery/catalog/guard.go @@ -0,0 +1,78 @@ +package catalog + +import ( + "context" + "fmt" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// guard is the repository coordinator held around a read-latest, modify, publish cycle. A guard +// without a handle means the repository offers no coordinator: writers must then be serialized +// externally, which the catalog documentation states. +type guard struct { + lock artifact.LockHandle +} + +func (g *guard) handle() artifact.LockHandle { return g.lock } + +func (g *guard) scope() artifact.LockScope { + if g.lock == nil { + return "" + } + return g.lock.Scope() +} + +// release relinquishes the lock with a context detached from cancellation, so an interrupted +// command never leaks ownership. +func (g *guard) release(ctx context.Context) error { + if g.lock == nil { + return nil + } + if err := g.lock.Release(context.WithoutCancel(ctx)); err != nil { + return fmt.Errorf("release catalog lock %s: operator recovery may be required: %w", g.lock.Resource(), err) + } + return nil +} + +// acquire takes the repository's coordinator when it advertises a scope that satisfies the +// required one. A repository without a coordinator proceeds unguarded; one that has a coordinator +// but cannot grant the lock fails the write instead of proceeding unserialized. +func (s *Service) acquire(ctx context.Context, t *target, repo artifact.Repository, operationID string) (*guard, error) { + required := t.info.RequiredLockScope + if required == "" { + required = artifact.LockScopeHost + } + if !t.info.Capabilities.HasLockScope(required) { + return &guard{}, nil + } + provider, resource, err := s.provider(ctx, t, repo) + if err != nil { + return nil, err + } + handle, err := provider.Lock(ctx, artifact.LockRequest{Resource: resource, Owner: s.owner, OperationID: operationID, Required: required}) + if err != nil { + return nil, fmt.Errorf("lock the catalog %s of repository %q: %w", t.ref, t.alias, err) + } + return &guard{lock: handle}, nil +} + +// provider returns the lock provider and the resource it locks. A repository that delegates +// coordination (locking.repository) is locked through its delegate, opened with its own identity, +// with the resource namespaced by the collection so unrelated collections sharing a coordinator +// never contend. +func (s *Service) provider(ctx context.Context, t *target, repo artifact.Repository) (artifact.LockProvider, string, error) { + resource := "catalog/" + t.ref.String() + if t.info.LockProvider == "" || t.info.LockProvider == t.alias { + return repo, resource, nil + } + delegate, err := s.repos.Open(ctx, t.info.LockProvider) + if err != nil { + return nil, "", fmt.Errorf("lock delegate %q of repository %q: %w", t.info.LockProvider, t.alias, err) + } + namespace := t.alias + if namespacer, ok := repo.(artifact.LockNamespacer); ok && namespacer.LockNamespace() != "" { + namespace = namespacer.LockNamespace() + } + return delegate, namespace + "/" + resource, nil +} diff --git a/pkg/artifact/discovery/catalog/guard_test.go b/pkg/artifact/discovery/catalog/guard_test.go new file mode 100644 index 00000000000..af85f6961ed --- /dev/null +++ b/pkg/artifact/discovery/catalog/guard_test.go @@ -0,0 +1,172 @@ +package catalog_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// lockSpy wraps a repository and observes (or fails) its coordinator. +type lockSpy struct { + artifact.Repository + lockErr error + releaseErr error + locks atomic.Int64 + releases atomic.Int64 + resources []string +} + +func (l *lockSpy) Lock(ctx context.Context, request artifact.LockRequest) (artifact.LockHandle, error) { + l.locks.Add(1) + l.resources = append(l.resources, request.Resource) + if l.lockErr != nil { + return nil, l.lockErr + } + handle, err := l.Repository.Lock(ctx, request) + if err != nil { + return nil, err + } + return &spyHandle{LockHandle: handle, spy: l}, nil +} + +type spyHandle struct { + artifact.LockHandle + spy *lockSpy +} + +func (h *spyHandle) Release(ctx context.Context) error { + h.spy.releases.Add(1) + err := h.LockHandle.Release(ctx) + if h.spy.releaseErr != nil { + return h.spy.releaseErr + } + return err +} + +func withRepos(repos map[string]artifact.Repository, infos map[string]service.RepositoryInfo, opts ...catalog.Option) *catalog.Service { + return catalog.NewService(&service.StaticRepositories{Repositories: repos, Info: infos}, opts...) +} + +func TestRepositoryWithoutACoordinatorProceedsUnguarded(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + info := e.info + info.Capabilities.LockScopes = nil + svc := withRepos(map[string]artifact.Repository{"shared": e.repo}, map[string]service.RepositoryInfo{"shared": info}, catalog.WithClock(func() time.Time { return now })) + result, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + assert.Empty(t, result.LockScope, "no coordinator means external serialization, which the documentation states") +} + +func TestWriteHoldsTheRepositoryLockAndReleasesIt(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + spy := &lockSpy{Repository: e.repo} + svc := withRepos(map[string]artifact.Repository{"shared": spy}, map[string]service.RepositoryInfo{"shared": e.info}, + catalog.WithOwner("ci-runner"), catalog.WithIDGenerator(func() string { return "op-fixed" }), catalog.WithConcurrency(2)) + _, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + assert.Equal(t, int64(1), spy.locks.Load()) + assert.Equal(t, int64(1), spy.releases.Load(), "the lock is released after the write") + assert.Equal(t, []string{"catalog/catalog:stable"}, spy.resources) +} + +func TestLockFailureStopsTheWriteBeforeAnyChange(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + failure := errors.New("lock is held by another writer") + spy := &lockSpy{Repository: e.repo, lockErr: failure} + svc := withRepos(map[string]artifact.Repository{"shared": spy}, map[string]service.RepositoryInfo{"shared": e.info}) + _, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, failure) + _, err = catalog.Read(context.Background(), e.repo, ref("catalog:stable")) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound, "nothing was published") +} + +func TestLockReleaseFailureIsReportedWithRecoveryContext(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + failure := errors.New("release refused") + spy := &lockSpy{Repository: e.repo, releaseErr: failure} + svc := withRepos(map[string]artifact.Repository{"shared": spy}, map[string]service.RepositoryInfo{"shared": e.info}) + result, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, failure) + assert.Contains(t, err.Error(), "operator recovery") + require.NotNil(t, result, "the completed write is still reported") + assert.Equal(t, []string{"app:1.0.0"}, result.Added) +} + +func TestLockDelegationUsesTheDelegateRepository(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + coordinator, err := local.NewDir("locks", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": testkit.LocalDir(t)}}) + require.NoError(t, err) + spy := &lockSpy{Repository: coordinator} + info := e.info + info.LockProvider = "locks" + svc := withRepos( + map[string]artifact.Repository{"shared": e.repo, "locks": spy}, + map[string]service.RepositoryInfo{"shared": info, "locks": {Capabilities: coordinator.Capabilities()}}, + ) + result, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.NoError(t, err) + assert.Equal(t, int64(1), spy.locks.Load(), "the delegate holds the lock") + assert.Equal(t, int64(1), spy.releases.Load()) + require.Len(t, spy.resources, 1) + assert.Contains(t, spy.resources[0], "catalog/catalog:stable") + assert.NotEmpty(t, result.LockScope) + + t.Run("delegate cannot be opened", func(t *testing.T) { + broken := withRepos(map[string]artifact.Repository{"shared": e.repo}, map[string]service.RepositoryInfo{"shared": info}) + _, err := broken.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) +} + +func TestReadAndWriteCapabilitiesAreRequired(t *testing.T) { + e := newEnv(t, true) + e.publish("app", "1.0.0", nil) + readOnly := e.info + readOnly.Capabilities.Write = false + svc := withRepos(map[string]artifact.Repository{"shared": e.repo}, map[string]service.RepositoryInfo{"shared": readOnly}) + _, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) + + writeOnly := e.info + writeOnly.Capabilities.Read = false + svc = withRepos(map[string]artifact.Repository{"shared": e.repo}, map[string]service.RepositoryInfo{"shared": writeOnly}) + _, err = svc.Build(context.Background(), &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} + +func TestCatalogOutsideTheRepositoryPathFailsBeforeAnyWrite(t *testing.T) { + e := newEnv(t, true) + info := e.info + info.Discovery.Catalog = "oci://other.example.com/acme/atmos/catalog:stable" + svc := withRepos(map[string]artifact.Repository{"shared": e.repo}, map[string]service.RepositoryInfo{"shared": info}) + e.registry.Reset() + _, err := svc.Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0")}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + assert.Zero(t, e.registry.Total()) +} + +func TestWriteRejectsDigestReferences(t *testing.T) { + e := newEnv(t, true) + _, err := catalog.Write(context.Background(), e.repo, artifact.Ref{Name: "catalog", Digest: digest.Digest("sha256:" + strings.Repeat("a", 64))}, catalog.New("catalog"), &catalog.WriteOptions{Now: now, OperationID: "op"}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) +} diff --git a/pkg/artifact/discovery/catalog/inventory.go b/pkg/artifact/discovery/catalog/inventory.go new file mode 100644 index 00000000000..9343ff821eb --- /dev/null +++ b/pkg/artifact/discovery/catalog/inventory.go @@ -0,0 +1,60 @@ +package catalog + +import ( + "fmt" + "slices" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/manifest" + "github.com/cloudposse/atmos/pkg/perf" +) + +// InventoryKind is the manifest kind of a catalog inventory: the explicit releases a maintainer +// wants listed. +const InventoryKind = "AtmosArtifactInventory" + +// InventoryFile is the standard Atmos manifest envelope of an inventory. +type InventoryFile = manifest.Manifest[InventorySpec] + +// InventorySpec lists the releases to resolve in the selected configured repository. +type InventorySpec struct { + // Refs are explicit "name:version" or "name@sha256:" references, relative to the selected + // configured repository. There is no implicit latest. + Refs []string `yaml:"refs" json:"refs"` +} + +func init() { manifest.MustRegister(InventoryKind, manifest.DefaultAPIVersion, &InventorySpec{}) } + +// ParseInventory decodes and validates an inventory manifest. Every reference must name a version +// or a digest, and a reference listed twice is rejected. +func ParseInventory(raw []byte) (*InventoryFile, []artifact.Ref, error) { + defer perf.Track(nil, "catalog.ParseInventory")() + + if int64(len(raw)) > artifact.MaxRecordSize { + return nil, nil, fmt.Errorf("%w: the inventory is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(raw), artifact.MaxRecordSize) + } + file, err := manifest.Load[InventorySpec](InventoryKind, raw) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactCatalogInvalid, err) + } + if len(file.Spec.Refs) == 0 { + return nil, nil, invalid("the inventory lists no refs") + } + refs := make([]artifact.Ref, 0, len(file.Spec.Refs)) + seen := make(map[string]bool, len(file.Spec.Refs)) + for _, text := range file.Spec.Refs { + ref, err := artifact.ParseRef(text) + if err != nil { + return nil, nil, fmt.Errorf("%w: inventory ref %q: %w", errUtils.ErrArtifactCatalogInvalid, text, err) + } + if seen[ref.String()] { + return nil, nil, invalid("inventory ref %q is listed twice", text) + } + seen[ref.String()] = true + refs = append(refs, ref) + } + slices.SortFunc(refs, func(a, b artifact.Ref) int { return strings.Compare(a.String(), b.String()) }) + return file, refs, nil +} diff --git a/pkg/artifact/discovery/catalog/maintainer.go b/pkg/artifact/discovery/catalog/maintainer.go new file mode 100644 index 00000000000..c0e217c6c0e --- /dev/null +++ b/pkg/artifact/discovery/catalog/maintainer.go @@ -0,0 +1,25 @@ +package catalog + +import "context" + +//go:generate mockgen -package catalog -destination mock_maintainer.go github.com/cloudposse/atmos/pkg/artifact/discovery/catalog Maintainer + +// Maintainer is the catalog maintenance API the CLI consumes. [Service] implements it. +type Maintainer interface { + // Build resolves references into a catalog snapshot without writing anything. + Build(ctx context.Context, request *BuildRequest) (*BuildResult, error) + + // Publish replaces the configured catalog with a built snapshot. + Publish(ctx context.Context, request *PublishRequest) (*Result, error) + + // Add lists releases in the latest catalog. + Add(ctx context.Context, request *AddRequest) (*Result, error) + + // Remove unlists a version without deleting the artifact. + Remove(ctx context.Context, request *RemoveRequest) (*Result, error) + + // Sync merges inventory releases into the latest catalog and keeps the other entries. + Sync(ctx context.Context, request *SyncRequest) (*Result, error) +} + +var _ Maintainer = (*Service)(nil) diff --git a/pkg/artifact/discovery/catalog/manifest.go b/pkg/artifact/discovery/catalog/manifest.go new file mode 100644 index 00000000000..f4849f03d4e --- /dev/null +++ b/pkg/artifact/discovery/catalog/manifest.go @@ -0,0 +1,378 @@ +package catalog + +import ( + "fmt" + "sort" + "strings" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/manifest" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Kind is the manifest kind of an artifact catalog. +const Kind = "AtmosArtifactCatalog" + +// sourceScheme prefixes the source of every catalog entry. +const sourceScheme = "oci://" + +// File is the standard Atmos manifest envelope of a catalog. +type File = manifest.Manifest[Spec] + +// Spec holds the discoverable artifacts. Catalog inclusion never establishes trust. +type Spec struct { + // Entries are the listed artifact names, sorted lexically. + Entries []Entry `yaml:"entries" json:"entries"` +} + +// Entry lists the versions of one artifact name. It is bound to the repository path the artifact +// lives at, never to credentials. +type Entry struct { + // Name is the logical artifact name, for example "components/terraform/vpc". + Name string `yaml:"name" json:"name"` + + // Kind is the artifact kind. + Kind string `yaml:"kind" json:"kind"` + + // Source is the OCI repository path of the artifact, for example "oci://ghcr.io/acme/atmos/components/terraform/vpc". + Source string `yaml:"source" json:"source"` + + // Description is the description of the newest listed version. + Description string `yaml:"description,omitempty" json:"description,omitempty"` + + // Versions are the listed versions, sorted lexically. Version labels are opaque. + Versions []Version `yaml:"versions" json:"versions"` +} + +// Version pins a listing to immutable digests and carries the published-version metadata that +// label filters apply to. +type Version struct { + // Version is the opaque version label. + Version string `yaml:"version" json:"version"` + + // ResolvedDigest is the OCI manifest digest the version resolved to when it was listed. + ResolvedDigest string `yaml:"resolved_digest" json:"resolved_digest"` + + // RecordDigest is the digest of the canonical Record. + RecordDigest string `yaml:"record_digest" json:"record_digest"` + + // ContentDigest is the portable content digest from the Record. + ContentDigest string `yaml:"content_digest" json:"content_digest"` + + // Labels are the labels published with this version. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` + + // Documents are the README and CHANGELOG descriptors; catalogs never carry the Markdown. + Documents []Document `yaml:"documents,omitempty" json:"documents,omitempty"` +} + +// Document is a README or CHANGELOG descriptor. +type Document struct { + // Role is "readme" or "changelog". + Role string `yaml:"role" json:"role"` + + // SourcePath is the document path relative to the publish source. + SourcePath string `yaml:"source_path" json:"source_path"` + + // MediaType is the document media type. + MediaType string `yaml:"media_type" json:"media_type"` + + // Size is the document size in bytes. + Size int64 `yaml:"size" json:"size"` + + // Digest is the SHA-256 of the document bytes. + Digest string `yaml:"digest" json:"digest"` +} + +func init() { manifest.MustRegister(Kind, manifest.DefaultAPIVersion, &Spec{}) } + +// New creates an empty catalog manifest. +func New(name string) *File { + defer perf.Track(nil, "catalog.New")() + + return &File{APIVersion: manifest.DefaultAPIVersion, Kind: Kind, Metadata: manifest.Metadata{Name: name}, Spec: Spec{Entries: []Entry{}}} +} + +// Parse decodes and validates catalog JSON or YAML without contacting any repository. It rejects +// duplicate names or versions, malformed digests, and oversized input, and returns the entries and +// versions in lexical order. +func Parse(raw []byte) (*File, error) { + defer perf.Track(nil, "catalog.Parse")() + + if int64(len(raw)) > artifact.MaxRecordSize { + return nil, fmt.Errorf("%w: the catalog is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(raw), artifact.MaxRecordSize) + } + file, err := manifest.Load[Spec](Kind, raw) + if err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactCatalogInvalid, err) + } + if err := check(file); err != nil { + return nil, err + } + Normalize(file) + return file, nil +} + +// check validates the content of a catalog manifest. +func check(file *File) error { + if file.Metadata.Name == "" { + return invalid("the catalog needs a metadata name") + } + seen := make(map[string]bool, len(file.Spec.Entries)) + for index := range file.Spec.Entries { + entry := &file.Spec.Entries[index] + if err := artifact.ValidateName(entry.Name); err != nil { + return fmt.Errorf("%w: entry %q: %w", errUtils.ErrArtifactCatalogInvalid, entry.Name, err) + } + if seen[entry.Name] { + return invalid("entry %q is listed twice", entry.Name) + } + seen[entry.Name] = true + if err := checkEntry(entry); err != nil { + return err + } + } + return nil +} + +func checkEntry(entry *Entry) error { + if err := artifact.ValidateKind(entry.Kind); err != nil { + return fmt.Errorf("%w: entry %q: %w", errUtils.ErrArtifactCatalogInvalid, entry.Name, err) + } + if !strings.HasPrefix(entry.Source, sourceScheme) || strings.ContainsAny(entry.Source, "@?#") || strings.Contains(strings.TrimPrefix(entry.Source, sourceScheme), "://") { + return invalid("entry %q has an invalid source; a source is an oci:/// repository path without credentials, tag, or digest", entry.Name) + } + if len(entry.Versions) == 0 { + return invalid("entry %q lists no versions", entry.Name) + } + versions := make(map[string]bool, len(entry.Versions)) + for index := range entry.Versions { + version := &entry.Versions[index] + if err := checkVersion(entry.Name, version); err != nil { + return err + } + if versions[version.Version] { + return invalid("entry %q lists version %q twice", entry.Name, version.Version) + } + versions[version.Version] = true + } + return nil +} + +func checkVersion(name string, version *Version) error { + if err := artifact.ValidateVersion(version.Version); err != nil { + return fmt.Errorf("%w: entry %q: %w", errUtils.ErrArtifactCatalogInvalid, name, err) + } + for _, value := range []string{version.ResolvedDigest, version.RecordDigest, version.ContentDigest} { + if err := artifact.ValidateDigest(digest.Digest(value)); err != nil { + return fmt.Errorf("%w: entry %q version %q: %w", errUtils.ErrArtifactCatalogInvalid, name, version.Version, err) + } + } + for key := range version.Labels { + if key == "" { + return invalid("entry %q version %q has an empty label key", name, version.Version) + } + } + return checkDocuments(name, version) +} + +// checkDocuments validates the README and CHANGELOG descriptors of a version. +func checkDocuments(name string, version *Version) error { + roles := make(map[string]bool, len(version.Documents)) + for index := range version.Documents { + doc := &version.Documents[index] + if doc.Role != string(artifact.DocumentReadme) && doc.Role != string(artifact.DocumentChangelog) || roles[doc.Role] { + return invalid("entry %q version %q has an invalid or repeated document role %q", name, version.Version, doc.Role) + } + if doc.Size < 0 || doc.SourcePath == "" { + return invalid("entry %q version %q has an invalid %s document descriptor", name, version.Version, doc.Role) + } + roles[doc.Role] = true + if err := artifact.ValidateDigest(digest.Digest(doc.Digest)); err != nil { + return fmt.Errorf("%w: entry %q version %q %s: %w", errUtils.ErrArtifactCatalogInvalid, name, version.Version, doc.Role, err) + } + } + return nil +} + +func invalid(format string, args ...any) error { + return fmt.Errorf("%w: %s", errUtils.ErrArtifactCatalogInvalid, fmt.Sprintf(format, args...)) +} + +// Normalize sorts the entries by name, the versions by version label, and the documents by role. +// Sorting is lexical: version labels are opaque and carry no SemVer ordering. +func Normalize(file *File) { + defer perf.Track(nil, "catalog.Normalize")() + + sort.Slice(file.Spec.Entries, func(i, j int) bool { return file.Spec.Entries[i].Name < file.Spec.Entries[j].Name }) + for index := range file.Spec.Entries { + versions := file.Spec.Entries[index].Versions + sort.Slice(versions, func(i, j int) bool { return versions[i].Version < versions[j].Version }) + for position := range versions { + documents := versions[position].Documents + sort.Slice(documents, func(i, j int) bool { return documents[i].Role < documents[j].Role }) + } + } +} + +// Source returns the catalog source of an artifact name below a repository location. +func Source(location, name string) string { + defer perf.Track(nil, "catalog.Source")() + + return sourceScheme + strings.ToLower(location) + "/" + name +} + +// Validate requires every entry to be bound to the repository path of the destination repository: +// its source must be exactly the repository location joined with the entry name. A catalog that +// names another registry, another prefix, or another path is rejected, so catalog data can never +// direct reads, or the credentials bound to the destination repository, anywhere else. +func Validate(file *File, location string) error { + defer perf.Track(nil, "catalog.Validate")() + + if err := check(file); err != nil { + return err + } + for index := range file.Spec.Entries { + entry := &file.Spec.Entries[index] + if want := Source(location, entry.Name); entry.Source != want { + return invalid("entry %q is bound to %q, not to the destination repository path %q", entry.Name, entry.Source, want) + } + } + return nil +} + +// Upsert lists a version, creating its entry when needed. An existing version is replaced, which +// reports whether it differed. The entry keeps the kind recorded first; a different kind for the +// same name is a conflict. +func Upsert(file *File, entry *Entry, version *Version) (replaced bool, err error) { + defer perf.Track(nil, "catalog.Upsert")() + + for index := range file.Spec.Entries { + existing := &file.Spec.Entries[index] + if existing.Name != entry.Name { + continue + } + if existing.Source != entry.Source || existing.Kind != entry.Kind { + return false, fmt.Errorf("%w: %q is already listed from %s as kind %q", errUtils.ErrArtifactCatalogConflict, entry.Name, existing.Source, existing.Kind) + } + existing.Description = entry.Description + for position := range existing.Versions { + if existing.Versions[position].Version == version.Version { + differs := existing.Versions[position].ResolvedDigest != version.ResolvedDigest + existing.Versions[position] = *version + return differs, nil + } + } + existing.Versions = append(existing.Versions, *version) + return false, nil + } + added := *entry + added.Versions = []Version{*version} + file.Spec.Entries = append(file.Spec.Entries, added) + return false, nil +} + +// Unlist removes one version, and the entry when it was the last. It never touches artifact content. +func Unlist(file *File, name, version string) error { + defer perf.Track(nil, "catalog.Unlist")() + + for index := range file.Spec.Entries { + entry := &file.Spec.Entries[index] + if entry.Name != name { + continue + } + for position := range entry.Versions { + if entry.Versions[position].Version != version { + continue + } + entry.Versions = append(entry.Versions[:position], entry.Versions[position+1:]...) + if len(entry.Versions) == 0 { + file.Spec.Entries = append(file.Spec.Entries[:index], file.Spec.Entries[index+1:]...) + } + return nil + } + } + return fmt.Errorf("%w: %s:%s is not listed in the catalog", errUtils.ErrArtifactNotFound, name, version) +} + +// ListEntries converts a validated catalog into list entries. Each entry's pinned reference is bound +// to the destination repository path and carries the listed digests; the listing grants no trust, +// and every later read resolves the reference through the repository again. +func ListEntries(file *File, location, alias string) []artifact.ListEntry { + defer perf.Track(nil, "catalog.ListEntries")() + + var entries []artifact.ListEntry + for index := range file.Spec.Entries { + entry := &file.Spec.Entries[index] + for position := range entry.Versions { + version := &entry.Versions[position] + entries = append(entries, artifact.ListEntry{ + Pinned: artifact.PinnedReference{ + Ref: artifact.Ref{Name: entry.Name, Version: version.Version}, + Repository: strings.ToLower(location) + "/" + entry.Name, + ResolvedDigest: digest.Digest(version.ResolvedDigest), + RecordDigest: digest.Digest(version.RecordDigest), + ContentDigest: digest.Digest(version.ContentDigest), + }, + Name: entry.Name, Version: version.Version, Kind: entry.Kind, Description: entry.Description, + Labels: copyLabels(version.Labels), Documents: artifactDocuments(version.Documents), Repository: alias, + }) + } + } + return entries +} + +func copyLabels(labels map[string]string) map[string]string { + if len(labels) == 0 { + return nil + } + out := make(map[string]string, len(labels)) + for key, value := range labels { + out[key] = value + } + return out +} + +func artifactDocuments(documents []Document) []artifact.Document { + var out []artifact.Document + for index := range documents { + doc := &documents[index] + out = append(out, artifact.Document{ + Role: artifact.DocumentRole(doc.Role), SourcePath: doc.SourcePath, MediaType: doc.MediaType, + Size: doc.Size, Digest: digest.Digest(doc.Digest), + }) + } + return out +} + +// FromInspection builds the entry and version a published artifact contributes to a catalog. It +// reads only the inspection's Record and pinned digests: metadata, never payload. +func FromInspection(location string, inspection *artifact.Inspection) (Entry, Version, error) { + defer perf.Track(nil, "catalog.FromInspection")() + + record := inspection.Record + if record == nil { + return Entry{}, Version{}, invalid("%s is not an Atmos artifact package and cannot be listed", inspection.Pinned.Ref) + } + if inspection.Pinned.RecordDigest == "" { + return Entry{}, Version{}, invalid("%s has no Record digest", inspection.Pinned.Ref) + } + entry := Entry{Name: record.Name, Kind: record.Kind, Source: Source(location, record.Name), Description: record.Metadata.Description} + version := Version{ + Version: record.Version, + ResolvedDigest: inspection.Pinned.ResolvedDigest.String(), + RecordDigest: inspection.Pinned.RecordDigest.String(), + ContentDigest: record.ContentDigest.String(), + Labels: copyLabels(record.Metadata.Labels), + } + for index := range record.Documents { + doc := &record.Documents[index] + version.Documents = append(version.Documents, Document{ + Role: string(doc.Role), SourcePath: doc.SourcePath, MediaType: doc.MediaType, Size: doc.Size, Digest: doc.Digest.String(), + }) + } + return entry, version, nil +} diff --git a/pkg/artifact/discovery/catalog/manifest_test.go b/pkg/artifact/discovery/catalog/manifest_test.go new file mode 100644 index 00000000000..af16100da25 --- /dev/null +++ b/pkg/artifact/discovery/catalog/manifest_test.go @@ -0,0 +1,273 @@ +package catalog_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" +) + +var ( + sha = func(c string) string { return "sha256:" + strings.Repeat(c, 64) } + + // Compile-time guards for the manifest fields these tests configure. + _ = catalog.Entry{Name: "", Kind: "", Source: "", Description: "", Versions: nil} + _ = catalog.Version{Version: "", ResolvedDigest: "", RecordDigest: "", ContentDigest: "", Labels: nil, Documents: nil} + _ = catalog.Document{Role: "", SourcePath: "", MediaType: "", Size: 0, Digest: ""} +) + +const location = "ghcr.io/acme/atmos" + +func entry(name string, versions ...string) catalog.Entry { + out := catalog.Entry{Name: name, Kind: "files", Source: catalog.Source(location, name), Description: "about " + name} + for _, version := range versions { + out.Versions = append(out.Versions, catalog.Version{Version: version, ResolvedDigest: sha("a"), RecordDigest: sha("b"), ContentDigest: sha("c")}) + } + return out +} + +func file(entries ...catalog.Entry) *catalog.File { + built := catalog.New("catalog") + built.Spec.Entries = entries + return built +} + +func marshal(t *testing.T, value any) []byte { + t.Helper() + raw, err := json.Marshal(value) + require.NoError(t, err) + return raw +} + +func TestParseValidatesAndSortsLexically(t *testing.T) { + parsed, err := catalog.Parse(marshal(t, file(entry("zeta", "9", "10", "1.9", "1.10"), entry("alpha", "2")))) + require.NoError(t, err) + var names []string + for _, e := range parsed.Spec.Entries { + names = append(names, e.Name) + } + assert.Equal(t, []string{"alpha", "zeta"}, names) + var versions []string + for _, v := range parsed.Spec.Entries[1].Versions { + versions = append(versions, v.Version) + } + assert.Equal(t, []string{"1.10", "1.9", "10", "9"}, versions, "opaque version labels sort lexically; there is no SemVer ordering") +} + +func TestParseRejectsInvalidCatalogs(t *testing.T) { + good := func() *catalog.File { return file(entry("app", "1.0.0")) } + tests := []struct { + name string + mutate func(*catalog.File) + }{ + {"missing metadata name", func(f *catalog.File) { f.Metadata.Name = "" }}, + {"invalid entry name", func(f *catalog.File) { f.Spec.Entries[0].Name = "Not Valid" }}, + {"duplicate entry", func(f *catalog.File) { f.Spec.Entries = append(f.Spec.Entries, entry("app", "2.0.0")) }}, + {"invalid kind", func(f *catalog.File) { f.Spec.Entries[0].Kind = "BAD KIND" }}, + {"source without scheme", func(f *catalog.File) { f.Spec.Entries[0].Source = "ghcr.io/acme/atmos/app" }}, + {"source with a digest", func(f *catalog.File) { f.Spec.Entries[0].Source += "@sha256:" + strings.Repeat("a", 64) }}, + {"source with credentials", func(f *catalog.File) { f.Spec.Entries[0].Source = "oci://user:pw@ghcr.io/acme/atmos/app" }}, + {"source with a query", func(f *catalog.File) { f.Spec.Entries[0].Source += "?token=x" }}, + {"nested scheme", func(f *catalog.File) { f.Spec.Entries[0].Source = "oci://https://ghcr.io/acme/atmos/app" }}, + {"no versions", func(f *catalog.File) { f.Spec.Entries[0].Versions = nil }}, + {"duplicate version", func(f *catalog.File) { + f.Spec.Entries[0].Versions = append(f.Spec.Entries[0].Versions, f.Spec.Entries[0].Versions[0]) + }}, + {"invalid version", func(f *catalog.File) { f.Spec.Entries[0].Versions[0].Version = "-bad" }}, + {"bad resolved digest", func(f *catalog.File) { f.Spec.Entries[0].Versions[0].ResolvedDigest = "sha256:short" }}, + {"missing record digest", func(f *catalog.File) { f.Spec.Entries[0].Versions[0].RecordDigest = "" }}, + {"bad content digest", func(f *catalog.File) { f.Spec.Entries[0].Versions[0].ContentDigest = "nope" }}, + {"empty label key", func(f *catalog.File) { f.Spec.Entries[0].Versions[0].Labels = map[string]string{"": "x"} }}, + {"unknown document role", func(f *catalog.File) { + f.Spec.Entries[0].Versions[0].Documents = []catalog.Document{{Role: "license", Digest: sha("d")}} + }}, + {"repeated document role", func(f *catalog.File) { + doc := catalog.Document{Role: "readme", SourcePath: "README.md", MediaType: "text/markdown", Size: 1, Digest: sha("d")} + f.Spec.Entries[0].Versions[0].Documents = []catalog.Document{doc, doc} + }}, + {"negative document size", func(f *catalog.File) { + f.Spec.Entries[0].Versions[0].Documents = []catalog.Document{{Role: "readme", SourcePath: "README.md", Size: -1, Digest: sha("d")}} + }}, + {"empty document source", func(f *catalog.File) { + f.Spec.Entries[0].Versions[0].Documents = []catalog.Document{{Role: "readme", Digest: sha("d")}} + }}, + {"bad document digest", func(f *catalog.File) { + f.Spec.Entries[0].Versions[0].Documents = []catalog.Document{{Role: "readme", SourcePath: "README.md", Digest: "x"}} + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + value := good() + tc.mutate(value) + _, err := catalog.Parse(marshal(t, value)) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + }) + } + t.Run("wrong kind", func(t *testing.T) { + value := good() + value.Kind = "AtmosPluginCatalog" + _, err := catalog.Parse(marshal(t, value)) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, "the plugin catalog schema is never accepted") + }) + t.Run("not a manifest", func(t *testing.T) { + _, err := catalog.Parse([]byte("{")) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + }) + t.Run("oversized", func(t *testing.T) { + _, err := catalog.Parse([]byte(strings.Repeat(" ", int(artifact.MaxRecordSize)+1))) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) + }) +} + +func TestValidateBindsEntriesToTheDestinationRepositoryPath(t *testing.T) { + require.NoError(t, catalog.Validate(file(entry("components/vpc", "1.0.0")), location)) + require.NoError(t, catalog.Validate(file(entry("components/vpc", "1.0.0")), strings.ToUpper(location)), "the host compares case-insensitively") + for name, source := range map[string]string{ + "registry": "oci://evil.example.com/acme/atmos/components/vpc", + "prefix": "oci://ghcr.io/other/components/vpc", + "name": "oci://ghcr.io/acme/atmos/components/other", + "prefix substring": "oci://ghcr.io/acme/atmos-evil/components/vpc", + } { + t.Run(name, func(t *testing.T) { + value := file(entry("components/vpc", "1.0.0")) + value.Spec.Entries[0].Source = source + require.ErrorIs(t, catalog.Validate(value, location), errUtils.ErrArtifactCatalogInvalid) + }) + } +} + +func TestUpsertAndUnlist(t *testing.T) { + value := file() + e := entry("app", "1.0.0") + replaced, err := catalog.Upsert(value, &e, &e.Versions[0]) + require.NoError(t, err) + assert.False(t, replaced) + assert.Len(t, value.Spec.Entries, 1) + + second := entry("app", "1.1.0") + replaced, err = catalog.Upsert(value, &second, &second.Versions[0]) + require.NoError(t, err) + assert.False(t, replaced) + assert.Len(t, value.Spec.Entries[0].Versions, 2) + + moved := entry("app", "1.0.0") + moved.Description = "newer description" + moved.Versions[0].ResolvedDigest = sha("9") + replaced, err = catalog.Upsert(value, &moved, &moved.Versions[0]) + require.NoError(t, err) + assert.True(t, replaced, "a version listed at a new digest is reported as replaced") + assert.Len(t, value.Spec.Entries[0].Versions, 2) + assert.Equal(t, "newer description", value.Spec.Entries[0].Description) + + same := entry("app", "1.0.0") + same.Versions[0].ResolvedDigest = sha("9") + replaced, err = catalog.Upsert(value, &same, &same.Versions[0]) + require.NoError(t, err) + assert.False(t, replaced, "an identical digest is not a change") + + foreign := entry("app", "2.0.0") + foreign.Source = "oci://elsewhere.example.com/app" + _, err = catalog.Upsert(value, &foreign, &foreign.Versions[0]) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) + otherKind := entry("app", "2.0.0") + otherKind.Kind = "scaffold" + _, err = catalog.Upsert(value, &otherKind, &otherKind.Versions[0]) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogConflict) + + require.NoError(t, catalog.Unlist(value, "app", "1.0.0")) + assert.Len(t, value.Spec.Entries[0].Versions, 1) + require.NoError(t, catalog.Unlist(value, "app", "1.1.0")) + assert.Empty(t, value.Spec.Entries) + require.ErrorIs(t, catalog.Unlist(value, "app", "1.1.0"), errUtils.ErrArtifactNotFound) +} + +func TestListEntriesKeepPinnedDigestsLabelsAndAlias(t *testing.T) { + e := entry("components/vpc", "1.0.0", "1.1.0") + e.Versions[0].Labels = map[string]string{"team": "platform"} + e.Versions[0].Documents = []catalog.Document{{Role: "readme", SourcePath: "README.md", MediaType: "text/markdown", Size: 3, Digest: sha("d")}} + entries := catalog.ListEntries(file(e), location, "shared") + require.Len(t, entries, 2) + first := entries[0] + assert.Equal(t, "shared", first.Repository) + assert.Equal(t, artifact.Ref{Name: "components/vpc", Version: "1.0.0"}, first.Pinned.Ref) + assert.Equal(t, location+"/components/vpc", first.Pinned.Repository) + assert.Equal(t, digest.Digest(sha("a")), first.Pinned.ResolvedDigest) + assert.Equal(t, digest.Digest(sha("b")), first.Pinned.RecordDigest) + assert.Equal(t, digest.Digest(sha("c")), first.Pinned.ContentDigest) + assert.Equal(t, map[string]string{"team": "platform"}, first.Labels) + require.Len(t, first.Documents, 1) + assert.Equal(t, artifact.DocumentReadme, first.Documents[0].Role) + assert.Equal(t, "files", first.Kind) + + // Isolation in both directions: mutating the result leaves the catalog unchanged and vice versa. + first.Labels["team"] = "changed" + assert.Equal(t, "platform", e.Versions[0].Labels["team"]) + e.Versions[0].Labels["extra"] = "x" + assert.NotContains(t, first.Labels, "extra") +} + +func TestFromInspectionRequiresAnAtmosPackage(t *testing.T) { + _, _, err := catalog.FromInspection(location, &artifact.Inspection{Pinned: artifact.PinnedReference{Ref: artifact.Ref{Name: "img", Version: "1"}}}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, "a foreign image has no Record and cannot be listed") + + record := &artifact.Record{Name: "app", Version: "1.0.0", Kind: "files", ContentDigest: digest.Digest(sha("c")), Metadata: artifact.Metadata{Description: "d"}} + _, _, err = catalog.FromInspection(location, &artifact.Inspection{Record: record, Pinned: artifact.PinnedReference{ResolvedDigest: digest.Digest(sha("a"))}}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, "a record digest is required") + + entry, version, err := catalog.FromInspection(location, &artifact.Inspection{Record: record, Pinned: artifact.PinnedReference{ResolvedDigest: digest.Digest(sha("a")), RecordDigest: digest.Digest(sha("b"))}}) + require.NoError(t, err) + assert.Equal(t, catalog.Source(location, "app"), entry.Source) + assert.Equal(t, sha("c"), version.ContentDigest) +} + +func TestParseInventory(t *testing.T) { + text := func(refs ...string) []byte { + return marshal(t, map[string]any{"apiVersion": "atmos/v1", "kind": "AtmosArtifactInventory", "metadata": map[string]any{"name": "inv"}, "spec": map[string]any{"refs": refs}}) + } + _, refs, err := catalog.ParseInventory(text("b/app:1.0.0", "a/app@"+sha("e"), "a/app:2")) + require.NoError(t, err) + var got []string + for _, r := range refs { + got = append(got, r.String()) + } + assert.Equal(t, []string{"a/app:2", "a/app@" + sha("e"), "b/app:1.0.0"}, got, "refs are sorted for deterministic builds") + + for name, raw := range map[string][]byte{ + "empty": text(), + "no version": text("app"), + "duplicate": text("app:1", "app:1"), + "invalid name": text("BAD:1"), + "wrong kind": marshal(t, file(entry("app", "1"))), + "not a manifest": []byte("{"), + } { + t.Run(name, func(t *testing.T) { + _, _, err := catalog.ParseInventory(raw) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + }) + } + _, _, err = catalog.ParseInventory([]byte(strings.Repeat(" ", int(artifact.MaxRecordSize)+1))) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestParseNeverEchoesAnInvalidEntrySource(t *testing.T) { + for _, source := range []string{ + "oci://user:secret@ghcr.io/acme/atmos/app", + "oci://ghcr.io/acme/atmos/app?token=secret", + "oci://ghcr.io/acme/atmos/app#secret", + "oci://ghcr.io/acme/atmos/app@sha256:" + strings.Repeat("a", 64), + } { + f := file(entry("app", "1.0.0")) + f.Spec.Entries[0].Source = source + _, err := catalog.Parse(marshal(t, f)) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, source) + assert.NotContains(t, err.Error(), "secret", "credentials are never echoed") + assert.NotContains(t, err.Error(), source, "the raw source is never echoed") + } +} diff --git a/pkg/artifact/discovery/catalog/mock_maintainer.go b/pkg/artifact/discovery/catalog/mock_maintainer.go new file mode 100644 index 00000000000..b48d7d0526c --- /dev/null +++ b/pkg/artifact/discovery/catalog/mock_maintainer.go @@ -0,0 +1,116 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/discovery/catalog (interfaces: Maintainer) +// +// Generated by this command: +// +// mockgen -package catalog -destination mock_maintainer.go github.com/cloudposse/atmos/pkg/artifact/discovery/catalog Maintainer +// + +// Package catalog is a generated GoMock package. +package catalog + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockMaintainer is a mock of Maintainer interface. +type MockMaintainer struct { + ctrl *gomock.Controller + recorder *MockMaintainerMockRecorder + isgomock struct{} +} + +// MockMaintainerMockRecorder is the mock recorder for MockMaintainer. +type MockMaintainerMockRecorder struct { + mock *MockMaintainer +} + +// NewMockMaintainer creates a new mock instance. +func NewMockMaintainer(ctrl *gomock.Controller) *MockMaintainer { + mock := &MockMaintainer{ctrl: ctrl} + mock.recorder = &MockMaintainerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMaintainer) EXPECT() *MockMaintainerMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockMaintainer) Add(ctx context.Context, request *AddRequest) (*Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", ctx, request) + ret0, _ := ret[0].(*Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Add indicates an expected call of Add. +func (mr *MockMaintainerMockRecorder) Add(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockMaintainer)(nil).Add), ctx, request) +} + +// Build mocks base method. +func (m *MockMaintainer) Build(ctx context.Context, request *BuildRequest) (*BuildResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Build", ctx, request) + ret0, _ := ret[0].(*BuildResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Build indicates an expected call of Build. +func (mr *MockMaintainerMockRecorder) Build(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockMaintainer)(nil).Build), ctx, request) +} + +// Publish mocks base method. +func (m *MockMaintainer) Publish(ctx context.Context, request *PublishRequest) (*Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, request) + ret0, _ := ret[0].(*Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Publish indicates an expected call of Publish. +func (mr *MockMaintainerMockRecorder) Publish(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockMaintainer)(nil).Publish), ctx, request) +} + +// Remove mocks base method. +func (m *MockMaintainer) Remove(ctx context.Context, request *RemoveRequest) (*Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove", ctx, request) + ret0, _ := ret[0].(*Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Remove indicates an expected call of Remove. +func (mr *MockMaintainerMockRecorder) Remove(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockMaintainer)(nil).Remove), ctx, request) +} + +// Sync mocks base method. +func (m *MockMaintainer) Sync(ctx context.Context, request *SyncRequest) (*Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Sync", ctx, request) + ret0, _ := ret[0].(*Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Sync indicates an expected call of Sync. +func (mr *MockMaintainerMockRecorder) Sync(ctx, request any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sync", reflect.TypeOf((*MockMaintainer)(nil).Sync), ctx, request) +} diff --git a/pkg/artifact/discovery/catalog/operations.go b/pkg/artifact/discovery/catalog/operations.go new file mode 100644 index 00000000000..778720f4d66 --- /dev/null +++ b/pkg/artifact/discovery/catalog/operations.go @@ -0,0 +1,468 @@ +package catalog + +import ( + "context" + "errors" + "fmt" + "sort" + + "golang.org/x/sync/errgroup" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// BuildRequest resolves inventory references into a catalog snapshot. Build never writes to a +// repository and never publishes. +type BuildRequest struct { + // Repository is the configured repository alias the references are relative to. + Repository string + + // Refs are the explicit references to resolve, relative to Repository. + Refs []artifact.Ref + + // Name is the catalog manifest name; empty uses the configured catalog name, or "catalog". + Name string +} + +// BuildResult is the snapshot plus the catalog state it was based on. +type BuildResult struct { + // File is the catalog snapshot. Its revision is the revision of the latest published catalog it + // was based on (zero when none exists or none is configured), so a later publish detects that the + // catalog changed in between. + File *File + + // BaseDigest is the OCI manifest digest of the latest published catalog the snapshot was based on; + // empty when there was none. Pass it to publish as the expected digest. + BaseDigest string +} + +// Build resolves each reference and verifies its Record metadata, reading no payload, and returns a +// catalog snapshot. A repository without a configured catalog still builds; only the base revision +// is then zero. +func (s *Service) Build(ctx context.Context, request *BuildRequest) (*BuildResult, error) { + defer perf.Track(nil, "catalog.Service.Build")() + + t, err := s.describe(request.Repository, false) + if err != nil { + return nil, err + } + if t.info.Discovery.Kind == artifact.RepositoryKindS3 { + return nil, invalid("catalog snapshots do not support S3 repository %q", request.Repository) + } + repo, err := s.repos.Open(ctx, request.Repository) + if err != nil { + return nil, err + } + file := New(nameOf(t, request.Name)) + result := &BuildResult{File: file} + if t.uri != "" { + base, err := latest(ctx, repo, t.ref) + if err != nil { + return nil, err + } + if base != nil { + file.Metadata.Revision, result.BaseDigest = base.File.Metadata.Revision, base.Digest.String() + } + } + listed, err := s.resolve(ctx, repo, t, request.Refs) + if err != nil { + return nil, err + } + for index := range listed { + if _, err := Upsert(file, &listed[index].entry, &listed[index].version); err != nil { + return nil, err + } + } + Normalize(file) + return result, nil +} + +// PublishRequest publishes a built snapshot to the repository's configured catalog. +type PublishRequest struct { + // Repository is the configured repository alias. + Repository string + + // File is the snapshot to publish. It replaces the catalog's entries. + File *File + + // ExpectedDigest, when set, must equal the digest of the catalog currently published. + ExpectedDigest string +} + +// Result describes a catalog mutation. +type Result struct { + // Repository is the configured repository alias. + Repository string `json:"repository"` + + // Catalog is the configured catalog reference. + Catalog string `json:"catalog"` + + // Digest is the OCI manifest digest of the catalog after the write. + Digest string `json:"digest,omitempty"` + + // PreviousDigest is the digest of the catalog before the write; empty when it did not exist. + PreviousDigest string `json:"previous_digest,omitempty"` + + // Revision is the catalog revision after the write. + Revision int `json:"revision"` + + // Listed is the number of name:version pairs the catalog lists after the write. + Listed int `json:"listed"` + + // Added are the references newly listed. + Added []string `json:"added,omitempty"` + + // Updated are the references whose listing now points at a different digest. + Updated []string `json:"updated,omitempty"` + + // Removed are the references unlisted. + Removed []string `json:"removed,omitempty"` + + // LockScope is the actual scope of the lock held during the write; empty when none was available. + LockScope artifact.LockScope `json:"lock_scope,omitempty"` +} + +// Publish replaces the configured catalog with a built snapshot. It fails before any write when the +// repository has no catalog configured, when the snapshot names locations outside the repository, +// or when the catalog changed since the snapshot was built (revision or digest). +func (s *Service) Publish(ctx context.Context, request *PublishRequest) (*Result, error) { + defer perf.Track(nil, "catalog.Service.Publish")() + + t, err := s.describeForWrite(request.Repository) + if err != nil { + return nil, err + } + if err := Validate(request.File, t.info.Discovery.Location); err != nil { + return nil, err + } + return s.mutate(ctx, t, func(_ context.Context, _ artifact.Repository, current *Loaded) (*File, *change, error) { + if err := checkFresh(request.File, request.ExpectedDigest, current); err != nil { + return nil, nil, err + } + next := *request.File + next.Spec.Entries = append([]Entry(nil), request.File.Spec.Entries...) + return &next, diff(current, &next), nil + }) +} + +// checkFresh rejects a snapshot built on a catalog that has since changed. +func checkFresh(file *File, expectedDigest string, current *Loaded) error { + if current == nil { + if file.Metadata.Revision != 0 || expectedDigest != "" { + return fmt.Errorf("%w: the catalog does not exist (the snapshot was based on revision %d); rebuild it", errUtils.ErrArtifactCatalogConflict, file.Metadata.Revision) + } + return nil + } + if current.File.Metadata.Revision != file.Metadata.Revision { + return fmt.Errorf("%w: the snapshot is based on revision %d but the catalog is at revision %d; rebuild it from the latest catalog", errUtils.ErrArtifactCatalogConflict, file.Metadata.Revision, current.File.Metadata.Revision) + } + if expectedDigest != "" && expectedDigest != current.Digest.String() { + return fmt.Errorf("%w: expected catalog digest %s but the catalog is %s", errUtils.ErrArtifactCatalogConflict, expectedDigest, current.Digest) + } + return nil +} + +// AddRequest lists explicit releases in the configured catalog. +type AddRequest struct { + // Repository is the configured repository alias. + Repository string + + // Refs are the releases to list, relative to Repository. + Refs []artifact.Ref +} + +// Add resolves releases and lists them in the latest catalog, creating the catalog when none exists. +func (s *Service) Add(ctx context.Context, request *AddRequest) (*Result, error) { + defer perf.Track(nil, "catalog.Service.Add")() + + t, err := s.describeForWrite(request.Repository) + if err != nil { + return nil, err + } + return s.mutate(ctx, t, func(ctx context.Context, repo artifact.Repository, current *Loaded) (*File, *change, error) { + return s.merge(ctx, repo, t, current, request.Refs) + }) +} + +// SyncRequest merges an inventory's releases into the configured catalog. +type SyncRequest struct { + // Repository is the configured repository alias. + Repository string + + // Refs are the inventory references, relative to Repository. + Refs []artifact.Ref +} + +// Sync merges the verified releases of an inventory into the latest catalog and keeps every entry +// the inventory does not mention. +func (s *Service) Sync(ctx context.Context, request *SyncRequest) (*Result, error) { + defer perf.Track(nil, "catalog.Service.Sync")() + + t, err := s.describeForWrite(request.Repository) + if err != nil { + return nil, err + } + return s.mutate(ctx, t, func(ctx context.Context, repo artifact.Repository, current *Loaded) (*File, *change, error) { + return s.merge(ctx, repo, t, current, request.Refs) + }) +} + +// RemoveRequest unlists one version from the configured catalog. +type RemoveRequest struct { + // Repository is the configured repository alias. + Repository string + + // Ref is the listed name and version. + Ref artifact.Ref +} + +// Remove unlists a version. It never deletes the artifact, its signatures, or a consumer lock file. +func (s *Service) Remove(ctx context.Context, request *RemoveRequest) (*Result, error) { + defer perf.Track(nil, "catalog.Service.Remove")() + + if request.Ref.Version == "" { + return nil, fmt.Errorf("%w: unlisting needs a name:version reference, not a digest", errUtils.ErrArtifactRefInvalid) + } + t, err := s.describeForWrite(request.Repository) + if err != nil { + return nil, err + } + return s.mutate(ctx, t, func(_ context.Context, _ artifact.Repository, current *Loaded) (*File, *change, error) { + if current == nil { + return nil, nil, fmt.Errorf("%w: repository %q has no published catalog to unlist from", errUtils.ErrArtifactNotFound, request.Repository) + } + next := clone(current.File) + if err := Unlist(next, request.Ref.Name, request.Ref.Version); err != nil { + return nil, nil, err + } + return next, &change{removed: []string{request.Ref.String()}}, nil + }) +} + +// describeForWrite resolves the configuration of a mutating command, before any I/O. +func (s *Service) describeForWrite(alias string) (*target, error) { + t, err := s.describe(alias, true) + if err != nil { + return nil, err + } + return t, t.requireWritable() +} + +// change records what a mutation did to the listing. +type change struct { + added, updated, removed []string +} + +// mutator computes the next catalog from the latest one read inside the write guard. +type mutator func(ctx context.Context, repo artifact.Repository, current *Loaded) (*File, *change, error) + +// mutate runs a read-latest, modify, publish cycle inside the repository's write guard. The latest +// catalog is read after the guard is taken, so a serialized writer never builds on a stale copy. +func (s *Service) mutate(ctx context.Context, t *target, apply mutator) (*Result, error) { + repo, err := s.repos.Open(ctx, t.alias) + if err != nil { + return nil, err + } + operationID := s.newID() + guard, err := s.acquire(ctx, t, repo, operationID) + if err != nil { + return nil, err + } + result, err := s.write(ctx, &session{target: t, repo: repo, guard: guard, operationID: operationID}, apply) + return result, errors.Join(err, guard.release(ctx)) +} + +// session is one guarded read-latest, modify, publish cycle. +type session struct { + target *target + repo artifact.Repository + guard *guard + operationID string +} + +func (s *Service) write(ctx context.Context, run *session, apply mutator) (*Result, error) { + t, repo, guard, operationID := run.target, run.repo, run.guard, run.operationID + current, err := latest(ctx, repo, t.ref) + if err != nil { + return nil, err + } + next, changed, err := apply(ctx, repo, current) + if err != nil { + return nil, err + } + next.Metadata.Revision = revisionOf(current) + 1 + next.Metadata.GeneratedAt = s.clock().UTC().Format("2006-01-02T15:04:05Z") + Normalize(next) + if err := Validate(next, t.info.Discovery.Location); err != nil { + return nil, err + } + options := &WriteOptions{Lock: guard.handle(), Now: s.clock(), OperationID: operationID} + result := &Result{Repository: t.alias, Catalog: t.uri, Revision: next.Metadata.Revision, Listed: count(next), LockScope: guard.scope()} + if current != nil { + options.Prior, result.PreviousDigest = ¤t.Pinned, current.Digest.String() + } + published, err := Write(ctx, repo, t.ref, next, options) + if err != nil { + return nil, err + } + result.Digest = published.Pinned.ResolvedDigest.String() + result.Added, result.Updated, result.Removed = changed.added, changed.updated, changed.removed + return result, nil +} + +func revisionOf(current *Loaded) int { + if current == nil { + return 0 + } + return current.File.Metadata.Revision +} + +// listing is one resolved release. +type listing struct { + entry Entry + version Version +} + +// resolve reads the Record metadata of references, bounded and in order. It never downloads a +// payload: it resolves each reference and inspects it. +func (s *Service) resolve(ctx context.Context, repo artifact.Repository, t *target, refs []artifact.Ref) ([]listing, error) { + listed := make([]listing, len(refs)) + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(s.concurrency) + for index := range refs { + group.Go(func() error { + pinned, err := repo.Resolve(groupCtx, refs[index]) + if err != nil { + return fmt.Errorf("resolve %s in repository %q: %w", refs[index], t.alias, err) + } + inspection, err := repo.Inspect(groupCtx, pinned) + if err != nil { + return fmt.Errorf("inspect %s in repository %q: %w", refs[index], t.alias, err) + } + entry, version, err := FromInspection(t.info.Discovery.Location, inspection) + if err != nil { + return err + } + listed[index] = listing{entry: entry, version: version} + return nil + }) + } + if err := group.Wait(); err != nil { + return nil, err + } + return listed, nil +} + +// merge lists resolved releases in the latest catalog and keeps everything else. +func (s *Service) merge(ctx context.Context, repo artifact.Repository, t *target, current *Loaded, refs []artifact.Ref) (*File, *change, error) { + file := New(nameOf(t, "")) + if current != nil { + file = clone(current.File) + } + listed, err := s.resolve(ctx, repo, t, refs) + if err != nil { + return nil, nil, err + } + changed := &change{} + for index := range listed { + known := hasVersion(file, listed[index].entry.Name, listed[index].version.Version) + moved, err := Upsert(file, &listed[index].entry, &listed[index].version) + if err != nil { + return nil, nil, err + } + label := artifact.Ref{Name: listed[index].entry.Name, Version: listed[index].version.Version}.String() + switch { + case moved: + changed.updated = append(changed.updated, label) + case !known: + changed.added = append(changed.added, label) + } + } + return file, changed, nil +} + +func hasVersion(file *File, name, version string) bool { + for index := range file.Spec.Entries { + if file.Spec.Entries[index].Name != name { + continue + } + for position := range file.Spec.Entries[index].Versions { + if file.Spec.Entries[index].Versions[position].Version == version { + return true + } + } + } + return false +} + +// diff reports changes to references in a replacement snapshot. +func diff(current *Loaded, next *File) *change { + previous := make(map[string]string) + if current != nil { + for _, entry := range current.File.Spec.Entries { + for _, version := range entry.Versions { + label := artifact.Ref{Name: entry.Name, Version: version.Version}.String() + previous[label] = version.ResolvedDigest + } + } + } + changed := &change{} + for _, entry := range next.Spec.Entries { + for _, version := range entry.Versions { + label := artifact.Ref{Name: entry.Name, Version: version.Version}.String() + old, exists := previous[label] + switch { + case !exists: + changed.added = append(changed.added, label) + case old != version.ResolvedDigest: + changed.updated = append(changed.updated, label) + } + delete(previous, label) + } + } + for label := range previous { + changed.removed = append(changed.removed, label) + } + sort.Strings(changed.removed) + return changed +} + +func count(file *File) int { + total := 0 + for index := range file.Spec.Entries { + total += len(file.Spec.Entries[index].Versions) + } + return total +} + +// clone deep-copies a catalog so a mutation never aliases the loaded copy. +func clone(file *File) *File { + out := *file + out.Spec.Entries = make([]Entry, len(file.Spec.Entries)) + for index := range file.Spec.Entries { + entry := file.Spec.Entries[index] + entry.Versions = make([]Version, len(file.Spec.Entries[index].Versions)) + for position := range file.Spec.Entries[index].Versions { + version := file.Spec.Entries[index].Versions[position] + version.Labels = copyLabels(version.Labels) + version.Documents = append([]Document(nil), version.Documents...) + entry.Versions[position] = version + } + out.Spec.Entries[index] = entry + } + return &out +} + +// nameOf picks the catalog manifest name. +func nameOf(t *target, requested string) string { + switch { + case requested != "": + return requested + case t.ref.Name != "": + return t.ref.Name + default: + return "catalog" + } +} diff --git a/pkg/artifact/discovery/catalog/publish_changes_test.go b/pkg/artifact/discovery/catalog/publish_changes_test.go new file mode 100644 index 00000000000..c9c18ac4057 --- /dev/null +++ b/pkg/artifact/discovery/catalog/publish_changes_test.go @@ -0,0 +1,49 @@ +package catalog_test + +import ( + "context" + "testing" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPublishReportsReplacementChanges(t *testing.T) { + e := newEnv(t, true) + ctx := context.Background() + e.publish("app", "1.0.0", nil) + e.publish("old", "1.0.0", nil) + e.publish("new", "1.0.0", nil) + _, err := e.service.Add(ctx, &catalog.AddRequest{Repository: "shared", Refs: refsOf("app:1.0.0", "old:1.0.0")}) + require.NoError(t, err) + var before string + for _, entry := range e.latest().File.Spec.Entries { + if entry.Name == "app" { + before = entry.Versions[0].ResolvedDigest + } + } + require.NotEmpty(t, before) + _, err = e.repo.Publish(ctx, testkit.Prepare(t, &testkit.Spec{Name: "app", Version: "1.0.0", Content: "changed"}), artifact.PublishOptions{Force: true, ExpectedPrior: &artifact.PinnedReference{ResolvedDigest: digest.Digest(before)}}) + require.NoError(t, err) + built, err := e.service.Build(ctx, &catalog.BuildRequest{Repository: "shared", Refs: refsOf("app:1.0.0", "new:1.0.0")}) + require.NoError(t, err) + result, err := e.service.Publish(ctx, &catalog.PublishRequest{Repository: "shared", File: built.File}) + require.NoError(t, err) + assert.Equal(t, []string{"new:1.0.0"}, result.Added) + assert.Equal(t, []string{"app:1.0.0"}, result.Updated) + assert.Equal(t, []string{"old:1.0.0"}, result.Removed) +} + +func TestBuildRejectsS3BeforeOpeningRepository(t *testing.T) { + e := newEnv(t, false) + info := service.RepositoryInfo{Discovery: service.DiscoveryInfo{Kind: artifact.RepositoryKindS3, Location: "s3://bucket/prefix"}} + svc := catalog.NewService(&service.StaticRepositories{Repositories: map[string]artifact.Repository{"s3": e.repo}, Info: map[string]service.RepositoryInfo{"s3": info}}) + _, err := svc.Build(context.Background(), &catalog.BuildRequest{Repository: "s3", Refs: refsOf("app:1")}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) +} diff --git a/pkg/artifact/discovery/catalog/service.go b/pkg/artifact/discovery/catalog/service.go new file mode 100644 index 00000000000..84abd44ed6b --- /dev/null +++ b/pkg/artifact/discovery/catalog/service.go @@ -0,0 +1,145 @@ +package catalog + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DefaultConcurrency bounds the metadata reads of one build or sync. +const DefaultConcurrency = artifact.DefaultConcurrency + +// Repositories resolves configured repository aliases. [service.ConfigRepositories] and +// [service.StaticRepositories] implement it. Describe never authenticates; Open may. +type Repositories interface { + // Describe returns the repository description without authenticating. + Describe(alias string) (service.RepositoryInfo, error) + + // Open constructs the repository, resolving credentials as needed. + Open(ctx context.Context, alias string) (artifact.Repository, error) +} + +// Service builds and maintains the optional catalog of configured repositories. It reads artifact +// metadata only: it never downloads a payload, and it never publishes, signs, or deletes an artifact. +type Service struct { + repos Repositories + clock func() time.Time + newID func() string + owner string + concurrency int +} + +// Option customizes a [Service]. +type Option func(*Service) + +// WithClock injects the clock used for generation timestamps and Records. +func WithClock(clock func() time.Time) Option { + defer perf.Track(nil, "catalog.WithClock")() + + return func(s *Service) { s.clock = clock } +} + +// WithIDGenerator injects the operation identifier generator. +func WithIDGenerator(newID func() string) Option { + defer perf.Track(nil, "catalog.WithIDGenerator")() + + return func(s *Service) { s.newID = newID } +} + +// WithOwner sets the lock owner recorded in claims. +func WithOwner(owner string) Option { + defer perf.Track(nil, "catalog.WithOwner")() + + return func(s *Service) { s.owner = owner } +} + +// WithConcurrency bounds concurrent metadata reads. +func WithConcurrency(n int) Option { + defer perf.Track(nil, "catalog.WithConcurrency")() + + return func(s *Service) { s.concurrency = n } +} + +// NewService returns a catalog service over the configured repositories. +func NewService(repos Repositories, opts ...Option) *Service { + defer perf.Track(nil, "catalog.NewService")() + + s := &Service{repos: repos, clock: time.Now, newID: uuid.NewString, concurrency: DefaultConcurrency} + for _, opt := range opts { + if opt != nil { + opt(s) + } + } + if s.concurrency < 1 { + s.concurrency = DefaultConcurrency + } + if s.owner == "" { + host, _ := os.Hostname() + s.owner = fmt.Sprintf("%s/%d", host, os.Getpid()) + } + return s +} + +// target is a repository with its catalog configuration resolved. +type target struct { + alias string + info service.RepositoryInfo + ref artifact.Ref + uri string +} + +// describe resolves the catalog configuration of an alias. It touches neither the network nor any +// credential, so a mutating command fails on missing configuration before any write. +func (s *Service) describe(alias string, requireCatalog bool) (*target, error) { + if alias == "" { + return nil, fmt.Errorf("%w: a repository is required", errUtils.ErrArtifactConfigInvalid) + } + info, err := s.repos.Describe(alias) + if err != nil { + return nil, err + } + found := &target{alias: alias, info: info, uri: info.Discovery.Catalog} + if info.Discovery.Catalog == "" { + if requireCatalog { + return nil, fmt.Errorf("%w: repository %q has no catalog; set `catalog` on its entry in artifacts.repositories", errUtils.ErrArtifactCatalogNotConfigured, alias) + } + return found, nil + } + if !info.Capabilities.Read { + return nil, artifact.NewUnsupportedCapability(alias, "catalog", "read", "the repository is write-only") + } + found.ref, err = RefOf(info.Discovery.Location, info.Discovery.Catalog) + if err != nil { + return nil, err + } + return found, nil +} + +// requireWritable additionally requires the repository to accept writes. +func (t *target) requireWritable() error { + if !t.info.Capabilities.Write { + return artifact.NewUnsupportedCapability(t.alias, "catalog", "write", "the repository is read-only") + } + return nil +} + +// latest reads the current catalog; a missing catalog yields nil. +func latest(ctx context.Context, repo artifact.Repository, ref artifact.Ref) (*Loaded, error) { + loaded, err := Read(ctx, repo, ref) + if err != nil { + if errors.Is(err, errUtils.ErrArtifactNotFound) { + return nil, nil + } + return nil, err + } + return loaded, nil +} diff --git a/pkg/artifact/discovery/catalog/store.go b/pkg/artifact/discovery/catalog/store.go new file mode 100644 index 00000000000..28fc6a38cdb --- /dev/null +++ b/pkg/artifact/discovery/catalog/store.go @@ -0,0 +1,253 @@ +package catalog + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// payloadPath is the inventory path of the catalog document inside its Record. +const payloadPath = "catalog.json" + +// payloadMode is the recorded permission of the catalog document. +const payloadMode = 0o644 + +// catalogDescription describes catalog Records. +const catalogDescription = "Atmos artifact catalog" + +// RefOf returns the repository reference of a configured catalog URI. The URI +// ("oci:///:") must lie inside the destination repository location +// ("[/]"): a catalog outside it cannot be read or published through this +// repository, so it is rejected rather than followed to another path. +func RefOf(location, uri string) (artifact.Ref, error) { + defer perf.Track(nil, "catalog.RefOf")() + + reference := strings.TrimPrefix(uri, sourceScheme) + if reference == uri { + return artifact.Ref{}, invalid("catalog %q must be an oci:///: reference", sanitize(uri)) + } + root := strings.ToLower(location) + "/" + lowered := reference + if separator := strings.Index(reference, "/"); separator > 0 { + lowered = strings.ToLower(reference[:separator]) + reference[separator:] + } + relative, found := strings.CutPrefix(lowered, root) + if !found || relative == "" { + return artifact.Ref{}, invalid("catalog %q is outside the repository path %q; a catalog can only be read from and published to the repository it is configured on", sanitize(uri), location) + } + ref, err := artifact.ParseRef(relative) + if err != nil { + return artifact.Ref{}, fmt.Errorf("%w: catalog %q: %w", errUtils.ErrArtifactCatalogInvalid, sanitize(uri), err) + } + return ref, nil +} + +// sanitize hides userinfo so a mistaken credential is never echoed. +func sanitize(uri string) string { + base := 0 + if i := strings.Index(uri, "://"); i >= 0 { + base = i + len("://") + } + end := len(uri) + if i := strings.IndexAny(uri[base:], "/?#"); i >= 0 { + end = base + i + } + if at := strings.LastIndex(uri[base:end], "@"); at >= 0 && strings.Contains(uri[base:base+at], ":") { + return uri[:base] + "@" + uri[base+at+1:] + } + return uri +} + +// Loaded is a catalog read from a repository. +type Loaded struct { + // File is the validated manifest. + File *File + + // Pinned is the immutable reference the catalog was read at. + Pinned artifact.PinnedReference + + // Digest is the OCI manifest digest, which names this exact catalog revision. + Digest digest.Digest +} + +// Head resolves the catalog without reading its payload, returning the pinned reference whose +// resolved digest names the current catalog. A missing catalog is an ErrArtifactNotFound error. +func Head(ctx context.Context, repo artifact.Repository, ref artifact.Ref) (artifact.PinnedReference, error) { + defer perf.Track(nil, "catalog.Head")() + + pinned, err := repo.Resolve(ctx, ref) + if err != nil { + return artifact.PinnedReference{}, fmt.Errorf("resolve catalog %s: %w", ref, err) + } + return pinned, nil +} + +// Read fetches the catalog at a reference with a bounded, digest-verified read and validates it. +// Only a Record of kind "catalog" with the catalog payload layer is accepted; any other artifact +// found at the reference is rejected. +func Read(ctx context.Context, repo artifact.Repository, ref artifact.Ref) (*Loaded, error) { + defer perf.Track(nil, "catalog.Read")() + + pinned, err := Head(ctx, repo, ref) + if err != nil { + return nil, err + } + return ReadPinned(ctx, repo, &pinned) +} + +// ReadPinned fetches the catalog at an already resolved reference. +func ReadPinned(ctx context.Context, repo artifact.Repository, pinned *artifact.PinnedReference) (*Loaded, error) { + defer perf.Track(nil, "catalog.ReadPinned")() + + content, err := repo.Read(ctx, *pinned) + if err != nil { + return nil, fmt.Errorf("read catalog %s: %w", pinned.Ref, err) + } + defer content.Close() + raw, err := payloadOf(content, pinned.Ref) + if err != nil { + return nil, err + } + file, err := Parse(raw) + if err != nil { + return nil, err + } + return &Loaded{File: file, Pinned: *pinned, Digest: pinned.ResolvedDigest}, nil +} + +// payloadOf extracts the catalog document from the content of a catalog Record. +func payloadOf(content *artifact.Content, ref artifact.Ref) ([]byte, error) { + if content.Record == nil || content.Record.Kind != artifact.KindCatalog { + return nil, invalid("%s is not an artifact catalog; a catalog is a Record of kind %q", ref, artifact.KindCatalog) + } + for index := range content.Streams { + stream := &content.Streams[index] + if stream.Role != artifact.BlobPayload { + continue + } + if stream.Descriptor.MediaType != artifact.MediaTypeCatalogPayload { + return nil, invalid("%s has payload type %q, not %q", ref, stream.Descriptor.MediaType, artifact.MediaTypeCatalogPayload) + } + raw, err := artifact.ReadBounded(stream.Reader, artifact.MaxRecordSize) + if err != nil { + return nil, fmt.Errorf("read catalog %s: %w", ref, err) + } + if err := checkInventory(content.Record, raw); err != nil { + return nil, err + } + return raw, nil + } + return nil, invalid("%s has no catalog payload layer", ref) +} + +// checkInventory requires the Record to describe exactly the document that was read. +func checkInventory(record *artifact.Record, raw []byte) error { + if len(record.Inventory) != 1 || record.Inventory[0].Path != payloadPath || record.Inventory[0].Digest != digest.FromBytes(raw) || record.Inventory[0].Size != int64(len(raw)) { + return fmt.Errorf("%w: the catalog payload does not match its Record", errUtils.ErrArtifactIntegrityFailed) + } + return nil +} + +// WriteOptions control one catalog publication. +type WriteOptions struct { + // Prior is the catalog observed immediately before the write; nil means the catalog must not exist. + Prior *artifact.PinnedReference + + // Lock is the writer lock already held, if any. + Lock artifact.LockHandle + + // Now is the creation time recorded in the Record. + Now time.Time + + // OperationID is the logical operation identifier. + OperationID string +} + +// Write publishes a catalog as a generic Record of kind "catalog". The repository re-checks that the +// prior reference is still current immediately before moving the tag, and a lost race surfaces as +// ErrArtifactCatalogConflict. Without atomic compare-and-swap that check is not a lock, so the caller +// must still serialize catalog writers. +func Write(ctx context.Context, repo artifact.Repository, ref artifact.Ref, file *File, options *WriteOptions) (*artifact.PublishResult, error) { + defer perf.Track(nil, "catalog.Write")() + + prepared, err := prepare(ref, file, options) + if err != nil { + return nil, err + } + defer prepared.Close() + result, err := repo.Publish(ctx, prepared, artifact.PublishOptions{ExpectedPrior: options.Prior, Force: options.Prior != nil, Lock: options.Lock}) + if err != nil { + if errors.Is(err, errUtils.ErrArtifactConflict) { + // The catalog sentinel comes first, so the command hint for a catalog conflict wins over + // the generic one that suggests --force. + return nil, fmt.Errorf("%w: %w", errUtils.ErrArtifactCatalogConflict, err) + } + return nil, err + } + return result, nil +} + +// prepare builds the Record and the staged payload of a catalog. +func prepare(ref artifact.Ref, file *File, options *WriteOptions) (*artifact.PreparedArtifact, error) { + if ref.Version == "" { + return nil, invalid("a catalog is published under a tag; %s names a digest", ref) + } + Normalize(file) + raw, err := json.Marshal(file) + if err != nil { + return nil, err + } + if int64(len(raw)) > artifact.MaxRecordSize { + return nil, fmt.Errorf("%w: the catalog is %d bytes; the limit is %d", errUtils.ErrArtifactTooLarge, len(raw), artifact.MaxRecordSize) + } + if _, err := Parse(raw); err != nil { + return nil, err + } + sum := digest.FromBytes(raw) + record := artifact.Record{ + Name: ref.Name, Version: ref.Version, Kind: artifact.KindCatalog, + CreatedAt: artifact.NewTimestamp(options.Now), + Metadata: artifact.Metadata{Description: catalogDescription}, + Inventory: []artifact.Entry{{Path: payloadPath, Type: artifact.EntryFile, Size: int64(len(raw)), Mode: payloadMode, Digest: sum}}, + Documents: []artifact.Document{}, + } + if err := record.Seal(); err != nil { + return nil, err + } + intent, err := record.IntentDigest() + if err != nil { + return nil, err + } + return &artifact.PreparedArtifact{ + Record: record, IntentDigest: intent, OperationID: options.OperationID, PreparedAt: options.Now.UTC(), + Packaging: artifact.PackagingCatalog, + Blobs: []artifact.StagedBlob{{ + Descriptor: ocispec.Descriptor{MediaType: artifact.MediaTypeCatalogPayload, Digest: sum, Size: int64(len(raw))}, + Role: artifact.BlobPayload, Path: payloadPath, + Open: func(context.Context) (readCloser, error) { return nopCloser{bytes.NewReader(raw)}, nil }, + }}, + }, nil +} + +// readCloser is io.ReadCloser, named locally to keep the staged blob literal short. +type readCloser = io.ReadCloser + +// nopCloser closes nothing: the staged bytes live in memory. +type nopCloser struct{ io.Reader } + +// Close implements io.Closer. +// +//nolint:lintroller // Trivial no-op. +func (nopCloser) Close() error { return nil } diff --git a/pkg/artifact/discovery/catalog_source_test.go b/pkg/artifact/discovery/catalog_source_test.go new file mode 100644 index 00000000000..b0200589af0 --- /dev/null +++ b/pkg/artifact/discovery/catalog_source_test.go @@ -0,0 +1,220 @@ +package discovery_test + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +// catalogEnv is a registry that cannot enumerate repositories, with a catalog configured on it. +func catalogEnv(t *testing.T) *env { + t.Helper() + registry := testkit.NewRegistry(t) + return newEnv(t, envOptions{noCatalog: true, catalog: registry.CatalogURI("catalog", "stable"), registryOverride: registry}) +} + +func (e *env) catalogs() *catalog.Service { + return catalog.NewService( + &service.StaticRepositories{Repositories: map[string]artifact.Repository{"shared": e.writerFor()}, Info: map[string]service.RepositoryInfo{"shared": e.info}}, + catalog.WithClock(func() time.Time { return epoch }), + ) +} + +// writerFor returns a catalog-capable repository over the same registry. +func (e *env) writerFor() artifact.Repository { return e.writer } + +func (e *env) addToCatalog(refs ...string) { + e.t.Helper() + parsed := make([]artifact.Ref, 0, len(refs)) + for _, text := range refs { + ref, err := artifact.ParseRef(text) + require.NoError(e.t, err) + parsed = append(parsed, ref) + } + _, err := e.catalogs().Add(context.Background(), &catalog.AddRequest{Repository: "shared", Refs: parsed}) + require.NoError(e.t, err) +} + +func TestCollectionIsListedFromTheConfiguredCatalogWhenTheRegistryCannotEnumerate(t *testing.T) { + e := catalogEnv(t) + vpc := e.publish("components/vpc", "1.0.0", map[string]string{"team": "platform"}) + e.publish("components/vpc", "1.1.0", nil) + e.publish("unlisted/app", "1.0.0", nil) + e.addToCatalog("components/vpc:1.0.0", "components/vpc:1.1.0") + e.rebuild(envOptions{}) + e.registry.Reset() + + result, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/vpc:1.0.0", "shared|components/vpc:1.1.0"}, refs(result), + "only catalog entries are listed; an artifact published since the catalog was updated is not") + assert.Equal(t, "catalog", result.Repositories[0].Source) + require.NotEmpty(t, result.Repositories[0].Diagnostics) + assert.Equal(t, "catalog", result.Repositories[0].Diagnostics[0].Code) + assert.Equal(t, vpc.Pinned.ResolvedDigest, result.Artifacts[0].Pinned.ResolvedDigest) + assert.Equal(t, vpc.Pinned.RecordDigest, result.Artifacts[0].Pinned.RecordDigest) + assert.Equal(t, map[string]string{"team": "platform"}, result.Artifacts[0].Labels) + assert.Zero(t, e.registry.Count("GET", "/_catalog"), "the registry's repository catalog is not used") + for _, request := range e.registry.Requests() { + assert.NotContains(t, request, "/components/vpc/", "catalog discovery reads the catalog, never the listed artifacts: %s", request) + } + + // Search and label filters apply to the catalog's published-version metadata. + labeled, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}}}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/vpc:1.0.0"}, refs(labeled)) + + // A fresh snapshot answers without any request, and named listing still reads the tags. + e.registry.Reset() + cached, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, refs(result), refs(cached)) + assert.Zero(t, e.registry.Total()) + named, err := e.list(artifact.ListRequest{Name: "unlisted/app"}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|unlisted/app:1.0.0"}, refs(named), "explicit-name listing needs neither a catalog nor registry enumeration") +} + +func TestCatalogRefreshReusesEntriesWhenTheCatalogDigestIsUnchanged(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + e.rebuild(envOptions{}) + e.registry.Reset() + again, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0"}, refs(again)) + assert.Zero(t, e.registry.Count("GET", "/blobs/"), "an unchanged catalog digest reuses the snapshot without reading the payload: %v", e.registry.Requests()) + + e.publish("app", "1.1.0", nil) + e.addToCatalog("app:1.1.0") + e.rebuild(envOptions{}) + changed, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0", "shared|app:1.1.0"}, refs(changed)) +} + +func TestMissingConfiguredCatalogIsAnErrorNotAnEmptyListing(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + result, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactNotFound) + assert.Contains(t, err.Error(), "catalog") + assert.Empty(t, result.Artifacts) + assert.True(t, result.Incomplete) +} + +func TestCatalogEntriesOutsideTheDestinationRepositoryAreRejectedAndNeverCached(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + loaded, err := catalog.Read(context.Background(), e.writer, artifact.Ref{Name: "catalog", Version: "stable"}) + require.NoError(t, err) + + // A catalog that points an entry at another registry is data from an untrusted publisher. + hostile := loaded.File + hostile.Spec.Entries[0].Source = "oci://evil.example.com/acme/atmos/app" + hostile.Metadata.Revision++ + _, err = catalog.Write(context.Background(), e.writer, artifact.Ref{Name: "catalog", Version: "stable"}, hostile, &catalog.WriteOptions{Prior: &loaded.Pinned, Now: epoch, OperationID: "op"}) + require.NoError(t, err) + + e.rebuild(envOptions{}) + e.registry.Reset() + result, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) + assert.Empty(t, result.Artifacts) + assert.Empty(t, e.snapshots(), "a rejected catalog is not cached") + for _, request := range e.registry.Requests() { + assert.NotContains(t, request, "evil.example.com", "no request follows a location named by the catalog") + } +} + +func TestTamperedCatalogSnapshotIsRejectedByPolicyOnRead(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + files := e.snapshots() + require.Len(t, files, 1) + raw, err := os.ReadFile(files[0]) + require.NoError(t, err) + var snapshot map[string]any + require.NoError(t, json.Unmarshal(raw, &snapshot)) + entries := snapshot["entries"].([]any) + pinned := entries[0].(map[string]any)["pinned"].(map[string]any) + pinned["repository"] = "evil.example.com/acme/atmos/app" + tampered, err := json.Marshal(snapshot) + require.NoError(t, err) + require.NoError(t, os.WriteFile(files[0], tampered, 0o600)) + + e.registry.Reset() + _, err = e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid, "the location binding is re-checked when a snapshot is read") + assert.Zero(t, e.registry.Total()) + e.offline.Store(true) + _, err = e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactCatalogInvalid) +} + +func TestCatalogReconfigurationStartsANewSnapshot(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + e.info.Discovery.Catalog = e.registry.CatalogURI("other", "stable") + e.rebuild(envOptions{}) + e.offline.Store(true) + _, err = e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline, "a snapshot of one catalog is never served for another") +} + +func TestDigestOnlyCatalogReferenceListsFromThatRevision(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + pinned, err := e.writer.Resolve(context.Background(), artifact.Ref{Name: "catalog", Version: "stable"}) + require.NoError(t, err) + e.publish("app", "1.1.0", nil) + e.addToCatalog("app:1.1.0") + + e.info.Discovery.Catalog = "oci://" + e.registry.Location() + "/catalog@" + pinned.ResolvedDigest.String() + e.rebuild(envOptions{}) + result, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0"}, refs(result), "a digest-pinned catalog is an immutable revision") +} + +func TestCatalogDiagnosticNamesTheCatalogAndItsDigest(t *testing.T) { + e := catalogEnv(t) + e.publish("app", "1.0.0", nil) + e.addToCatalog("app:1.0.0") + e.rebuild(envOptions{}) + result, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + message := result.Repositories[0].Diagnostics[0].Message + assert.Contains(t, message, e.info.Discovery.Catalog) + assert.Contains(t, message, "sha256:") + assert.True(t, strings.Contains(message, "not included"), message) +} diff --git a/pkg/artifact/discovery/discovery.go b/pkg/artifact/discovery/discovery.go new file mode 100644 index 00000000000..3829d576538 --- /dev/null +++ b/pkg/artifact/discovery/discovery.go @@ -0,0 +1,231 @@ +package discovery + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/xdg" +) + +// Discovery implements [service.Enumerator] over the shared discovery cache and optional catalogs. +type Discovery struct { + scope func() (string, error) + directory string + ttl time.Duration + now func() time.Time + offline func(info *service.RepositoryInfo) bool + policy Policy + maxSize int64 + + once sync.Once + store *cache.Store + err error +} + +var _ service.Enumerator = (*Discovery)(nil) + +// New returns a discovery cache. The scope function is called once, on first use, so building the +// service performs no file or network access; compose supplies [ScopeOf] bound to the configuration. +func New(scope func() (string, error), opts ...Option) *Discovery { + defer perf.Track(nil, "discovery.New")() + + d := &Discovery{ + scope: scope, directory: xdg.LookupXDGCacheDir("artifacts/discovery"), ttl: defaultTTL, now: time.Now, + offline: func(*service.RepositoryInfo) bool { return false }, policy: LocationPolicy{}, maxSize: cache.DefaultMaxSize, + } + for _, opt := range opts { + if opt != nil { + opt(d) + } + } + return d +} + +// open builds the store once. +func (d *Discovery) open() (*cache.Store, error) { + d.once.Do(func() { + scope, err := d.scope() + if err != nil { + d.err = err + return + } + d.store = cache.New(d.directory, scope, cache.WithTTL(d.ttl), cache.WithClock(d.now), cache.WithMaxSize(d.maxSize)) + }) + return d.store, d.err +} + +// Enumerate implements [service.Enumerator]. +func (d *Discovery) Enumerate(ctx context.Context, request *service.EnumerateRequest) (*service.Enumeration, error) { + defer perf.Track(nil, "discovery.Discovery.Enumerate")() + + offline := d.offline(request.Info) + if request.Refresh && offline { + return nil, offlineError(request.Alias, "a refresh needs the network") + } + source := d.sourceOf(request) + if !request.Info.Discovery.Remote() { + return d.live(ctx, request, source) + } + store, err := d.open() + if err != nil { + return nil, err + } + loaded, cached, err := cache.Load(ctx, store, d.spec(request, source), cache.Mode{Refresh: request.Refresh, Offline: offline}) + if err != nil { + return nil, err + } + if loaded == nil { + return nil, fmt.Errorf("%w: repository %q returned no listing", errUtils.ErrArtifactDiscoveryIncomplete, request.Alias) + } + if err := d.policy.AuthorizeEntries(ctx, source, loaded.Entries); err != nil { + return nil, err + } + return d.result(source, loaded, cached), nil +} + +// sourceOf selects what is enumerated: one name's versions, the configured catalog, or the +// collection. +func (d *Discovery) sourceOf(request *service.EnumerateRequest) *SourceCheck { + info := request.Info + check := &SourceCheck{Alias: request.Alias, Info: info, Kind: kindCollection, Reference: info.Discovery.Location} + switch { + case request.Name != "": + check.Kind, check.Reference = kindNamed, info.Discovery.Location+"|"+request.Name + case info.Discovery.Catalog != "": + check.Kind, check.Reference = kindCatalog, info.Discovery.Catalog + } + check.Reference += "#" + fingerprint(info) + return check +} + +// fingerprintLength is how many hexadecimal characters of the settings hash name a source. +const fingerprintLength = 16 + +// fingerprint hashes the repository settings that change what a source contains (options and the +// bound identity name, never a credential), so reconfiguring a repository starts a new snapshot. +func fingerprint(info *service.RepositoryInfo) string { + raw, _ := json.Marshal([]any{info.Discovery.Kind, info.Discovery.Options, info.Discovery.Identity, info.Discovery.Catalog}) + return cache.Hash(raw)[:fingerprintLength] +} + +// live lists a repository that is cheap to read directly: no snapshot, no cache. +func (d *Discovery) live(ctx context.Context, request *service.EnumerateRequest, source *SourceCheck) (*service.Enumeration, error) { + if err := d.policy.AuthorizeSource(ctx, source); err != nil { + return nil, err + } + entries, err := request.Live(ctx, nil) + if err != nil { + return nil, err + } + sortEntries(entries) + if err := d.policy.AuthorizeEntries(ctx, source, entries); err != nil { + return nil, err + } + return &service.Enumeration{Entries: entries, Source: sourceLive}, nil +} + +// Source names reported on a listing. +const ( + sourceLive = "live" + sourceCache = "cache" + sourceCatalog = "catalog" +) + +// result describes where the entries came from. +func (d *Discovery) result(source *SourceCheck, loaded *snapshot, cached bool) *service.Enumeration { + enumeration := &service.Enumeration{Entries: loaded.Entries, Source: sourceLive} + if cached { + enumeration.Source, enumeration.FetchedAt = sourceCache, loaded.FetchedAt + } + if source.Kind == kindCatalog { + enumeration.Source, enumeration.FetchedAt = sourceCatalog, loaded.FetchedAt + enumeration.Diagnostics = append(enumeration.Diagnostics, artifact.Diagnostic{ + Severity: artifact.SeverityInfo, Code: "catalog", + Message: fmt.Sprintf("listed from the catalog %s (%s); artifacts published since the catalog was last updated are not included", source.Info.Discovery.Catalog, loaded.Digest), + }) + } + return enumeration +} + +// spec describes the source to the shared cache. +func (d *Discovery) spec(request *service.EnumerateRequest, source *SourceCheck) *cache.Source[snapshot, *snapshot] { + return &cache.Source[snapshot, *snapshot]{ + Kind: source.Kind, Reference: source.Reference, Version: snapshotVersion, Decode: decodeSnapshot, + Fetch: func(ctx context.Context, previous *snapshot) (*snapshot, error) { + if source.Kind == kindCatalog { + return d.fetchCatalog(ctx, request, previous) + } + return d.fetchRepository(ctx, request, previous) + }, + Authorize: func(ctx context.Context, cached bool) error { + check := *source + check.Cached = cached + return d.policy.AuthorizeSource(ctx, &check) + }, + ErrOffline: offlineError(request.Alias, "there is no valid discovery snapshot for it"), + ErrTooLarge: fmt.Errorf("%w: repository %q", errUtils.ErrArtifactDiscoveryTooLarge, request.Alias), + } +} + +// fetchRepository lists the repository completely. A failure at any page returns the error and +// caches nothing, so the previous snapshot stays authoritative and a disappeared reference is +// removed only by a refresh that completed. +func (d *Discovery) fetchRepository(ctx context.Context, request *service.EnumerateRequest, previous *snapshot) (*snapshot, error) { + entries, err := request.Live(ctx, knownByDigest(previous)) + if err != nil { + return nil, err + } + sortEntries(entries) + return &snapshot{Entries: entries}, nil +} + +// fetchCatalog reads the repository's configured catalog. When the catalog manifest digest equals +// the previous snapshot's, the previous entries are reused without reading the catalog payload. A +// configured catalog that cannot be read is an error, never an empty listing. +func (d *Discovery) fetchCatalog(ctx context.Context, request *service.EnumerateRequest, previous *snapshot) (*snapshot, error) { + location := request.Info.Discovery.Location + ref, err := catalog.RefOf(location, request.Info.Discovery.Catalog) + if err != nil { + return nil, err + } + repo, err := request.Open(ctx) + if err != nil { + return nil, err + } + pinned, err := catalog.Head(ctx, repo, ref) + if err != nil { + return nil, fmt.Errorf("the catalog %s of repository %q is not readable: %w", request.Info.Discovery.Catalog, request.Alias, err) + } + if previous != nil && previous.Digest != "" && digest.Digest(previous.Digest) == pinned.ResolvedDigest { + return &snapshot{Digest: previous.Digest, Entries: previous.Entries}, nil + } + loaded, err := catalog.ReadPinned(ctx, repo, &pinned) + if err != nil { + return nil, err + } + if err := catalog.Validate(loaded.File, location); err != nil { + return nil, err + } + entries := catalog.ListEntries(loaded.File, location, request.Alias) + sortEntries(entries) + return &snapshot{Digest: loaded.Digest.String(), Entries: entries}, nil +} + +// offlineError builds the offline discovery error with its remedy. +func offlineError(alias, reason string) error { + return errUtils.Build(errUtils.ErrArtifactDiscoveryOffline). + WithExplanationf("Repository %q cannot be listed offline: %s.", alias, reason). + WithHint("Run `atmos artifacts list` online first to create a discovery snapshot, or disable offline mode."). + Err() +} diff --git a/pkg/artifact/discovery/discovery_edge_test.go b/pkg/artifact/discovery/discovery_edge_test.go new file mode 100644 index 00000000000..6bc16520bfb --- /dev/null +++ b/pkg/artifact/discovery/discovery_edge_test.go @@ -0,0 +1,94 @@ +package discovery_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/service" +) + +func TestScopeFailureSurfacesBeforeAnyRegistryAccess(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + failure := errors.New("realm is invalid") + e.disc = discovery.New(func() (string, error) { return "", failure }, discovery.WithDirectory(e.dir)) + svc, err := service.New(e.repos, nil, service.WithEnumerator(e.disc)) + require.NoError(t, err) + e.registry.Reset() + _, err = svc.List(context.Background(), artifact.ListRequest{}) + require.ErrorIs(t, err, failure) + assert.Zero(t, e.registry.Total()) +} + +func TestOversizedSnapshotFailsInsteadOfBeingTruncated(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + e.disc = discovery.New(func() (string, error) { return "scope", nil }, discovery.WithDirectory(e.dir), discovery.WithMaxSize(64), discovery.WithTTL(time.Hour)) + svc, err := service.New(e.repos, nil, service.WithEnumerator(e.disc)) + require.NoError(t, err) + result, err := svc.List(context.Background(), artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryTooLarge) + assert.Empty(t, result.Artifacts) + assert.Empty(t, e.snapshots()) +} + +// denyAll rejects every source, including live ones. +type denyAll struct{ discovery.LocationPolicy } + +func (denyAll) AuthorizeSource(context.Context, *discovery.SourceCheck) error { return errPolicy } + +func TestPolicyRejectionAppliesToLiveRepositoriesToo(t *testing.T) { + backups := newLocalRepo(t) + e := newEnv(t, envOptions{ + extraRepos: map[string]artifact.Repository{"backups": backups}, + extraInfo: map[string]service.RepositoryInfo{"backups": {Capabilities: backups.Capabilities(), Discovery: service.DiscoveryInfo{Kind: artifact.RepositoryKindLocalDir, Location: "x"}}}, + policy: denyAll{}, + }) + testkit.Publish(t, backups, testkit.Spec{Name: "app", Version: "1.0.0", Content: "x", Packaging: artifact.PackagingNone}) + _, err := e.list(artifact.ListRequest{Repositories: []string{"backups"}}) + require.ErrorIs(t, err, errPolicy) +} + +func TestTTLOptionControlsFreshness(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + e.disc = discovery.New(func() (string, error) { return "scope", nil }, discovery.WithDirectory(e.dir), discovery.WithClock(e.clock.Now), discovery.WithTTL(time.Minute)) + svc, err := service.New(e.repos, nil, service.WithEnumerator(e.disc)) + require.NoError(t, err) + _, err = svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + e.registry.Reset() + e.clock.advance(30 * time.Second) + _, err = svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + assert.Zero(t, e.registry.Total()) + e.clock.advance(time.Minute) + _, err = svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + assert.Positive(t, e.registry.Total(), "a snapshot older than the configured TTL is revalidated") +} + +func TestListingWithoutAnEnumeratorReadsRepositoriesDirectly(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", map[string]string{"team": "a"}) + e.publish("app", "1.1.0", map[string]string{"team": "b"}) + svc, err := service.New(e.repos, nil) + require.NoError(t, err) + result, err := svc.List(context.Background(), artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "b"}}}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.1.0"}, refs(result)) + assert.Empty(t, result.Repositories[0].Source, "without discovery there is no cache provenance to report") + e.registry.Reset() + _, err = svc.List(context.Background(), artifact.ListRequest{}) + require.NoError(t, err) + assert.Positive(t, e.registry.Total(), "no cache: every listing reads the registry") +} diff --git a/pkg/artifact/discovery/discovery_test.go b/pkg/artifact/discovery/discovery_test.go new file mode 100644 index 00000000000..63e2924685e --- /dev/null +++ b/pkg/artifact/discovery/discovery_test.go @@ -0,0 +1,591 @@ +package discovery_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery" + "github.com/cloudposse/atmos/pkg/artifact/discovery/internal/testkit" + "github.com/cloudposse/atmos/pkg/artifact/local" + "github.com/cloudposse/atmos/pkg/artifact/oci" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guard for the schema fields these tests configure. +var _ = schema.ArtifactRepositoryConfig{Kind: "local/dir", Catalog: "", Identity: ""} + +var epoch = time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + +// clock is a settable time source shared by the cache and the test. +type clock struct{ now atomic.Pointer[time.Time] } + +func newClock() *clock { + c := &clock{} + c.set(epoch) + return c +} + +func (c *clock) Now() time.Time { return *c.now.Load() } +func (c *clock) set(at time.Time) { c.now.Store(&at) } +func (c *clock) advance(d time.Duration) { c.set(c.Now().Add(d)) } + +// countingRepositories counts how often each repository is opened, which is when credentials +// would be resolved. +type countingRepositories struct { + *service.StaticRepositories + opens atomic.Int64 +} + +func (c *countingRepositories) Open(ctx context.Context, alias string) (artifact.Repository, error) { + c.opens.Add(1) + return c.StaticRepositories.Open(ctx, alias) +} + +// env wires one OCI registry repository, a discovery cache, and a service. +type env struct { + t *testing.T + registry *testkit.Registry + repo *oci.Repository + writer *oci.Repository + repos *countingRepositories + disc *discovery.Discovery + svc *service.Service + clock *clock + dir string + scope string + offline atomic.Bool + info service.RepositoryInfo + + repoOptions []oci.Option + catalog string +} + +// envOptions customize the environment. +type envOptions struct { + catalog string + noCatalog bool + policy discovery.Policy + scope string + identity string + extraRepos map[string]artifact.Repository + extraInfo map[string]service.RepositoryInfo + + // registryOverride reuses a registry the caller already started (its catalog URI names it). + registryOverride *testkit.Registry +} + +//nolint:gocritic // Test helper; the options are a small literal. +func newEnv(t *testing.T, options envOptions) *env { + t.Helper() + registry := options.registryOverride + if registry == nil { + registry = testkit.NewRegistry(t) + } + e := &env{t: t, registry: registry, clock: newClock(), dir: filepath.Join(t.TempDir(), "discovery"), scope: options.scope} + if e.scope == "" { + e.scope = "scope-a" + } + var repoOptions []oci.Option + if options.noCatalog { + repoOptions = append(repoOptions, oci.WithoutCollectionEnumeration()) + } + e.writer = e.registry.Repo(t, "writer", options.catalog) + e.repoOptions = repoOptions + e.repo = e.registry.Repo(t, "shared", options.catalog, repoOptions...) + e.info = e.registry.Info(e.repo, options.catalog) + e.info.Discovery.Identity = options.identity + e.catalog = options.catalog + e.rebuild(options) + return e +} + +// rebuild creates a fresh service and discovery over the same directory and registry, like a new +// process invocation. +// +//nolint:gocritic // Test helper; the options are a small literal. +func (e *env) rebuild(options envOptions) { + e.repo = e.registry.Repo(e.t, "shared", e.catalog, e.repoOptions...) + repositories := map[string]artifact.Repository{"shared": e.repo} + infos := map[string]service.RepositoryInfo{"shared": e.info} + for alias, repo := range options.extraRepos { + repositories[alias] = repo + } + for alias := range options.extraInfo { + infos[alias] = options.extraInfo[alias] + } + e.repos = &countingRepositories{StaticRepositories: &service.StaticRepositories{Repositories: repositories, Info: infos}} + scope := e.scope + discOptions := []discovery.Option{ + discovery.WithDirectory(e.dir), discovery.WithClock(e.clock.Now), + discovery.WithOffline(func(*service.RepositoryInfo) bool { return e.offline.Load() }), + } + if options.policy != nil { + discOptions = append(discOptions, discovery.WithPolicy(options.policy)) + } + e.disc = discovery.New(func() (string, error) { return scope, nil }, discOptions...) + svc, err := service.New(e.repos, nil, service.WithEnumerator(e.disc)) + require.NoError(e.t, err) + e.svc = svc +} + +func (e *env) publish(name, version string, labels map[string]string) *artifact.PublishResult { + e.t.Helper() + return testkit.Publish(e.t, e.writer, testkit.Spec{Name: name, Version: version, Content: name + version, Labels: labels}) +} + +func (e *env) list(request artifact.ListRequest) (*artifact.ListResult, error) { + e.t.Helper() + return e.svc.List(context.Background(), request) +} + +func (e *env) snapshots() []string { + files, err := filepath.Glob(filepath.Join(e.dir, "*.json")) + require.NoError(e.t, err) + return files +} + +func refs(result *artifact.ListResult) []string { + var out []string + for index := range result.Artifacts { + out = append(out, result.Artifacts[index].Repository+"|"+result.Artifacts[index].Name+":"+result.Artifacts[index].Version) + } + return out +} + +func TestFreshCacheSearchPerformsZeroRegistryRequests(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("components/vpc", "1.0.0", nil) + e.publish("components/vpc", "1.1.0", nil) + e.publish("tools/helper", "2.0.0", nil) + + first, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/vpc:1.0.0", "shared|components/vpc:1.1.0", "shared|tools/helper:2.0.0"}, refs(first)) + assert.Equal(t, "live", first.Repositories[0].Source) + assert.NotZero(t, e.registry.Total()) + opened := e.repos.opens.Load() + + e.registry.Reset() + e.clock.advance(14 * time.Minute) + found, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "vpc"}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/vpc:1.0.0", "shared|components/vpc:1.1.0"}, refs(found)) + assert.Zero(t, e.registry.Total(), "a fresh snapshot answers a search with zero registry requests: %v", e.registry.Requests()) + assert.Equal(t, opened, e.repos.opens.Load(), "a cache hit never opens the repository or resolves its credentials") + assert.Equal(t, "cache", found.Repositories[0].Source) + assert.Equal(t, epoch.Format(time.RFC3339), found.Repositories[0].FetchedAt) + assert.False(t, found.Incomplete) + + // The snapshot keeps repository alias and pinned digests so results stay unambiguous. + assert.Equal(t, first.Artifacts[0].Pinned, found.Artifacts[0].Pinned) + assert.NotEmpty(t, found.Artifacts[0].Pinned.ResolvedDigest) +} + +func TestStaleSnapshotIsRefreshedAfterTheTTL(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + e.publish("app", "1.1.0", nil) + + e.clock.advance(14 * time.Minute) + cached, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0"}, refs(cached), "the snapshot is served for fifteen minutes; publishing never edits it") + + e.clock.advance(time.Minute) + refreshed, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0", "shared|app:1.1.0"}, refs(refreshed)) +} + +func TestRefreshReusesMetadataByDigestAndReadsNoPayload(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", map[string]string{"team": "platform"}) + e.publish("app", "1.1.0", nil) + first, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + e.rebuild(envOptions{}) // A new process starts with cold in-memory caches. + e.registry.Reset() + again, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Equal(t, first.Artifacts, again.Artifacts) + assert.Equal(t, "live", again.Repositories[0].Source) + assert.Zero(t, e.registry.Count("GET", "/manifests/"), "unchanged digests reuse metadata: no manifest is read: %v", e.registry.Requests()) + assert.Zero(t, e.registry.Count("GET", "/blobs/"), "no Record config or payload blob is read: %v", e.registry.Requests()) + assert.Equal(t, 2, e.registry.Count("HEAD", "/manifests/"), "each reference is still resolved, so a moved tag is noticed") + + // A second tag for an existing manifest must not duplicate its original record. + e.registry.TagExisting(t, "app", "1.0.0", "1.0.1") + e.rebuild(envOptions{}) + aliased, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0", "shared|app:1.1.0"}, refs(aliased)) + + // A changed digest is read again. + e.publish("app", "1.2.0", nil) + e.rebuild(envOptions{}) + e.registry.Reset() + changed, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Len(t, changed.Artifacts, 3) + assert.Positive(t, e.registry.Count("GET", "/blobs/"), "a new reference's Record is read") +} + +func TestDisappearedRefsAreRemovedOnlyAfterACompleteRefresh(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + removed := e.publish("app", "1.1.0", nil) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + pinned := removed.Pinned + _, err = e.writer.Delete(context.Background(), pinned, artifact.DeleteOptions{Scope: artifact.DeleteManifest}) + require.NoError(t, err) + + // An interrupted refresh reports its error and leaves the snapshot, with the old ref, intact. + failure := errors.New("connection reset") + e.registry.FailAfter(2, failure) + _, err = e.list(artifact.ListRequest{Refresh: true}) + require.ErrorIs(t, err, failure) + e.registry.FailAll(nil) + + e.registry.Reset() + stale, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0", "shared|app:1.1.0"}, refs(stale), "a failed refresh never drops refs") + assert.Zero(t, e.registry.Total()) + + complete, err := e.list(artifact.ListRequest{Refresh: true}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0"}, refs(complete), "a complete refresh removes the disappeared ref") + cached, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, refs(complete), refs(cached)) +} + +func TestFailedFirstListingIsNeverCachedAsEmpty(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + failure := errors.New("registry unavailable") + e.registry.FailAll(failure) + result, err := e.list(artifact.ListRequest{}) + require.Error(t, err) + assert.Contains(t, err.Error(), failure.Error()) + assert.Empty(t, result.Artifacts) + assert.True(t, result.Incomplete) + assert.Empty(t, e.snapshots(), "a failed listing leaves no snapshot") + e.registry.FailAll(nil) + ok, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Len(t, ok.Artifacts, 1) +} + +func TestOfflineServesSnapshotsWithoutAnyRegistryTraffic(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + _, err = e.list(artifact.ListRequest{Name: "app"}) + require.NoError(t, err) + + e.offline.Store(true) + e.clock.advance(1000 * time.Hour) + e.registry.Reset() + opened := e.repos.opens.Load() + all, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0"}, refs(all)) + named, err := e.list(artifact.ListRequest{Name: "app"}) + require.NoError(t, err) + assert.Equal(t, refs(all), refs(named)) + assert.Zero(t, e.registry.Total(), "offline discovery makes no registry request and downloads no payload") + assert.Equal(t, opened, e.repos.opens.Load()) + + _, err = e.list(artifact.ListRequest{Refresh: true}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline, "an offline refresh is rejected") + assert.Zero(t, e.registry.Total()) +} + +func TestOfflineRejectsMissingAndCorruptSnapshots(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + e.registry.Reset() + e.offline.Store(true) + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline, "no snapshot") + assert.Zero(t, e.registry.Total()) + + e.offline.Store(false) + _, err = e.list(artifact.ListRequest{}) + require.NoError(t, err) + files := e.snapshots() + require.Len(t, files, 1) + original, err := os.ReadFile(files[0]) + require.NoError(t, err) + + e.offline.Store(true) + for name, corrupt := range map[string]string{ + "truncated": string(original[:len(original)/2]), + "empty": "", + "garbage": "not json at all", + "other kind": strings.Replace(string(original), `"kind":"collection"`, `"kind":"named"`, 1), + "bad entry": strings.Replace(string(original), `"name":"app"`, `"name":"UPPER"`, 2), + } { + t.Run(name, func(t *testing.T) { + require.NoError(t, os.WriteFile(files[0], []byte(corrupt), 0o600)) + e.registry.Reset() + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline) + assert.Zero(t, e.registry.Total()) + }) + } +} + +func TestOfflineDoesNotCrossAuthRealmsHostBindingsOrConfiguration(t *testing.T) { + e := newEnv(t, envOptions{scope: "realm-a", identity: "reader-a"}) + e.publish("app", "1.0.0", nil) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + e.offline.Store(true) + _, err = e.list(artifact.ListRequest{}) + require.NoError(t, err, "the same scope reuses its snapshot") + + t.Run("another auth realm", func(t *testing.T) { + other := e.scope + e.scope = "realm-b" + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline) + e.scope = other + }) + t.Run("another identity bound to the host", func(t *testing.T) { + e.info.Discovery.Identity = "reader-b" + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline) + e.info.Discovery.Identity = "reader-a" + }) + t.Run("another registry location", func(t *testing.T) { + e.info.Discovery.Location = "other.example.com/acme/atmos" + e.info.Discovery.Options = map[string]any{"registry": "other.example.com", "prefix": "acme/atmos"} + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactDiscoveryOffline) + }) +} + +func TestScopeDoesNotContainCredentialValues(t *testing.T) { + config := &schema.AtmosConfiguration{} + config.Artifacts.Repositories = map[string]schema.ArtifactRepositoryConfig{"shared": {Kind: "oci/registry", Identity: "reader"}} + first, err := discovery.ScopeOf(config) + require.NoError(t, err) + again, err := discovery.ScopeOf(config) + require.NoError(t, err) + assert.Equal(t, first, again) + assert.Len(t, first, 64) + + config.Auth.Realm = "other" + other, err := discovery.ScopeOf(config) + require.NoError(t, err) + assert.NotEqual(t, first, other, "the auth realm separates snapshots") + + config.Auth.Realm = "" + config.Artifacts.Repositories = map[string]schema.ArtifactRepositoryConfig{"shared": {Kind: "oci/registry", Identity: "writer"}} + rebound, err := discovery.ScopeOf(config) + require.NoError(t, err) + assert.NotEqual(t, first, rebound, "the host binding separates snapshots") +} + +// rejectCached is a policy that refuses every cached read. +type rejectCached struct{ discovery.LocationPolicy } + +var errPolicy = errors.New("source is no longer permitted") + +func (rejectCached) AuthorizeSource(_ context.Context, check *discovery.SourceCheck) error { + if check.Cached { + return errPolicy + } + return nil +} + +func TestPolicyIsReevaluatedOnCachedReads(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", nil) + _, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + + e.rebuild(envOptions{policy: rejectCached{}}) + e.registry.Reset() + _, err = e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errPolicy, "a fresh snapshot is not an authorization") + e.offline.Store(true) + _, err = e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errPolicy, "offline reads re-evaluate policy too") + assert.Zero(t, e.registry.Total()) +} + +func TestNamedListingWorksWithoutACatalogAndCollectionScanFailsLoudly(t *testing.T) { + e := newEnv(t, envOptions{noCatalog: true}) + e.publish("components/vpc", "1.0.0", nil) + e.publish("components/vpc", "1.1.0", nil) + e.publish("tools/helper", "2.0.0", nil) + + named, err := e.list(artifact.ListRequest{Name: "components/vpc"}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/vpc:1.0.0", "shared|components/vpc:1.1.0"}, refs(named)) + + result, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability, "an unsupported collection scan is an error, never an empty success") + assert.Empty(t, result.Artifacts) + assert.Contains(t, err.Error(), "catalog") + assert.Contains(t, err.Error(), "direct named operations") + assert.True(t, result.Incomplete) + require.Len(t, result.Repositories, 1) + assert.NotEmpty(t, result.Repositories[0].Diagnostics) +} + +func TestNamedListingIsCachedPerName(t *testing.T) { + e := newEnv(t, envOptions{noCatalog: true}) + e.publish("app", "1.0.0", nil) + e.publish("other", "1.0.0", nil) + _, err := e.list(artifact.ListRequest{Name: "app"}) + require.NoError(t, err) + e.registry.Reset() + _, err = e.list(artifact.ListRequest{Name: "app"}) + require.NoError(t, err) + assert.Zero(t, e.registry.Total()) + _, err = e.list(artifact.ListRequest{Name: "other"}) + require.NoError(t, err) + assert.Positive(t, e.registry.Total(), "another name has its own snapshot") +} + +func TestLabelsUseANDSemanticsOnPublishedVersionMetadata(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("app", "1.0.0", map[string]string{"team": "platform", "env": "dev"}) + e.publish("app", "1.1.0", map[string]string{"team": "platform", "env": "prod"}) + e.publish("other", "1.0.0", map[string]string{"team": "data", "env": "prod"}) + + both, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}, {Key: "env", Value: "prod"}}}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.1.0"}, refs(both), "every selector must match exactly") + one, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}}}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|app:1.0.0", "shared|app:1.1.0"}, refs(one)) + none, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "plat"}}}) + require.NoError(t, err) + assert.Empty(t, none.Artifacts, "label values match exactly, not by substring") + + // Repository defaults apply to later publications only; they never relabel older versions. + e.info.Defaults = artifact.Metadata{Labels: map[string]string{"retention": "long"}} + e.rebuild(envOptions{}) + retro, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "retention", Value: "long"}}}) + require.NoError(t, err) + assert.Empty(t, retro.Artifacts, "defaults are not retroactively applied to older artifacts") +} + +func TestSubstringSearchMatchesNameAndDescriptionCaseInsensitively(t *testing.T) { + e := newEnv(t, envOptions{}) + e.publish("components/networking", "1.0.0", nil) + e.publish("tools/helper", "1.0.0", nil) + byName, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "NETWORK"}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|components/networking:1.0.0"}, refs(byName)) + byDescription, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "package tools"}) + require.NoError(t, err) + assert.Equal(t, []string{"shared|tools/helper:1.0.0"}, refs(byDescription)) + none, err := e.svc.Search(context.Background(), artifact.SearchRequest{Query: "missing"}) + require.NoError(t, err) + assert.Empty(t, none.Artifacts) +} + +func newLocalRepo(t *testing.T) *local.Repository { + t.Helper() + repo, err := local.NewDir("backups", &schema.ArtifactRepositoryConfig{Kind: artifact.RepositoryKindLocalDir, Options: map[string]any{"path": testkit.LocalDir(t)}}) + require.NoError(t, err) + return repo +} + +func TestMultipleRepositoriesSearchTogetherDeterministically(t *testing.T) { + backups := newLocalRepo(t) + e := newEnv(t, envOptions{ + extraRepos: map[string]artifact.Repository{"backups": backups}, + extraInfo: map[string]service.RepositoryInfo{"backups": {Capabilities: backups.Capabilities(), Discovery: service.DiscoveryInfo{Kind: artifact.RepositoryKindLocalDir, Location: "x"}}}, + }) + e.publish("app", "1.0.0", map[string]string{"team": "platform"}) + e.publish("zeta", "1.0.0", nil) + testkit.Publish(t, backups, testkit.Spec{Name: "app", Version: "1.0.0", Content: "local", Packaging: artifact.PackagingNone, Labels: map[string]string{"team": "platform"}}) + testkit.Publish(t, backups, testkit.Spec{Name: "app", Version: "0.9.0", Content: "older", Packaging: artifact.PackagingNone}) + + result, err := e.list(artifact.ListRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"backups|app:0.9.0", "backups|app:1.0.0", "shared|app:1.0.0", "shared|zeta:1.0.0"}, refs(result), + "results sort by name, then version, then repository alias") + sources := map[string]string{} + for _, listing := range result.Repositories { + sources[listing.Repository] = listing.Source + } + assert.Equal(t, map[string]string{"backups": "live", "shared": "live"}, sources, "local repositories are always listed live") + + selected, err := e.list(artifact.ListRequest{Labels: []artifact.Label{{Key: "team", Value: "platform"}}}) + require.NoError(t, err) + assert.Equal(t, []string{"backups|app:1.0.0", "shared|app:1.0.0"}, refs(selected)) + for index := range selected.Artifacts { + assert.NotEmpty(t, selected.Artifacts[index].Pinned.ResolvedDigest, "each result keeps the repository alias and its pinned digest") + } + + // A local repository is never served stale after a publish; only remote ones are cached. + testkit.Publish(t, backups, testkit.Spec{Name: "fresh", Version: "1.0.0", Content: "new", Packaging: artifact.PackagingNone}) + again, err := e.list(artifact.ListRequest{Repositories: []string{"backups"}}) + require.NoError(t, err) + assert.Contains(t, refs(again), "backups|fresh:1.0.0") +} + +func TestUnauthorizedOrFailingRepositoryIsNeverPresentedAsComplete(t *testing.T) { + backups := newLocalRepo(t) + e := newEnv(t, envOptions{ + extraRepos: map[string]artifact.Repository{"backups": backups}, + extraInfo: map[string]service.RepositoryInfo{"backups": {Capabilities: backups.Capabilities(), Discovery: service.DiscoveryInfo{Kind: artifact.RepositoryKindLocalDir, Location: "x"}}}, + }) + testkit.Publish(t, backups, testkit.Spec{Name: "local-app", Version: "1.0.0", Content: "x", Packaging: artifact.PackagingNone}) + e.publish("app", "1.0.0", nil) + e.registry.FailAll(errors.New("401 unauthorized")) + + result, err := e.list(artifact.ListRequest{}) + require.NoError(t, err, "the readable repository is still listed") + assert.Equal(t, []string{"backups|local-app:1.0.0"}, refs(result)) + assert.True(t, result.Incomplete, "the result is flagged incomplete") + var failing artifact.RepositoryListing + for _, listing := range result.Repositories { + if listing.Repository == "shared" { + failing = listing + } + } + require.NotEmpty(t, failing.Diagnostics) + assert.Equal(t, artifact.SeverityError, failing.Diagnostics[0].Severity) + assert.Contains(t, failing.Diagnostics[0].Message, "401") + + _, err = e.list(artifact.ListRequest{Repositories: []string{"shared"}}) + require.Error(t, err, "an explicitly requested repository that fails fails the command") +} + +func TestPolicyDefaultRejectsUnreadableRepositories(t *testing.T) { + e := newEnv(t, envOptions{}) + e.info.Capabilities.Read = false + e.rebuild(envOptions{}) + _, err := e.list(artifact.ListRequest{}) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsupportedCapability) +} diff --git a/pkg/artifact/discovery/doc.go b/pkg/artifact/discovery/doc.go new file mode 100644 index 00000000000..6783e21e296 --- /dev/null +++ b/pkg/artifact/discovery/doc.go @@ -0,0 +1,25 @@ +// Package discovery serves artifact List and Search from a source-scoped discovery cache and +// optional catalogs, without ever downloading a payload. +// +// Directory, archive, and OCI layout repositories enumerate their managed records locally and are +// always listed live. Remote repositories (S3 and OCI registries) are listed once per fifteen +// minutes: the entries are stored as a snapshot under the XDG cache directory +// (atmos/artifacts/discovery), scoped by the effective authentication configuration, realm, and +// host bindings, never by credential values, and replaced atomically only after a complete refresh. +// A fresh snapshot answers a request with zero repository requests. An explicit refresh +// revalidates the source and reuses the metadata of references whose resolved digest is unchanged. +// Offline mode serves only an existing valid snapshot and fails on a missing or corrupt one. +// +// A registry cannot be portably enumerated, so a collection listing of an OCI registry uses the +// registry's repository catalog scoped to the repository prefix when it provides one, or the +// catalog configured on the repository. Listing the versions of one explicit name needs neither. +// A collection that cannot be enumerated fails with an unsupported-capability error; it is never +// an empty successful result. +// +// Catalogs and snapshots are discoverable data, never trust grants. The current policy is +// re-evaluated on every read, including a read of a cached snapshot, and every catalog entry must +// be bound to the destination repository's path. +// +// The shared snapshot mechanics live in the neutral cache subpackage, which plugin discovery also +// uses; the artifact-specific schema and policy live here. +package discovery diff --git a/pkg/artifact/discovery/internal/testkit/test_helpers.go b/pkg/artifact/discovery/internal/testkit/test_helpers.go new file mode 100644 index 00000000000..a9a05e73ab5 --- /dev/null +++ b/pkg/artifact/discovery/internal/testkit/test_helpers.go @@ -0,0 +1,225 @@ +// Package testkit provides in-process registries and publication helpers for discovery and +// catalog tests: an OCI registry that counts every request, repositories bound to it, and a +// deterministic way to publish small packages. +package testkit + +import ( + "context" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/oci" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/artifact/service" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/oci/ocitest" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Prefix is the collection prefix of every test repository. +const Prefix = "acme/atmos" + +// Owner is the lock owner of test repositories. +const Owner = "test-owner" + +const filePerm = 0o644 + +// PreparedAt is the fixed preparation time of published packages. +var PreparedAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + +// recorder counts the HTTP requests a registry client makes, by method and path. +type recorder struct { + mu sync.Mutex + requests []string + fail error + failAfter int +} + +func (r *recorder) RoundTrip(request *http.Request) (*http.Response, error) { + r.mu.Lock() + r.requests = append(r.requests, request.Method+" "+request.URL.Path) + fail := r.fail + if fail != nil && len(r.requests) <= r.failAfter { + fail = nil + } + r.mu.Unlock() + if fail != nil { + return nil, fail + } + return http.DefaultTransport.RoundTrip(request) +} + +// Registry is one in-process OCI registry with a request-counting client. +type Registry struct { + // Host is the registry address ("127.0.0.1:PORT"). + Host string + + // Transport is the recording registry client. + Transport atmosoci.Transport + + server *ocitest.Server + recorder *recorder +} + +// NewRegistry starts a registry. A non-nil credential makes the registry require it; the client +// then presents it, like a repository with a bound identity. +func NewRegistry(t *testing.T) *Registry { + t.Helper() + server := ocitest.NewServerWithOptions(t, nil, ocitest.WithTagsFollowManifestDelete()) + rec := &recorder{} + client := atmosoci.NewClient(nil, atmosoci.WithTransport(rec)) + return &Registry{Host: server.Host, Transport: client, server: server, recorder: rec} +} + +// TagExisting gives a second tag to an existing OCI manifest. +func (r *Registry) TagExisting(t *testing.T, artifactName, existing, alias string) { + t.Helper() + from, err := name.NewTag(r.Host + "/" + Prefix + "/" + artifactName + ":" + existing) + require.NoError(t, err) + to, err := name.NewTag(r.Host + "/" + Prefix + "/" + artifactName + ":" + alias) + require.NoError(t, err) + image, err := remote.Image(from) + require.NoError(t, err) + require.NoError(t, remote.Write(to, image)) +} + +// Reset forgets the recorded requests. +func (r *Registry) Reset() { + r.recorder.mu.Lock() + defer r.recorder.mu.Unlock() + r.recorder.requests = nil +} + +// Total returns how many requests were recorded since the last Reset. +func (r *Registry) Total() int { + r.recorder.mu.Lock() + defer r.recorder.mu.Unlock() + return len(r.recorder.requests) +} + +// Count returns how many recorded requests used the method and contain the path fragment. +func (r *Registry) Count(method, fragment string) int { + r.recorder.mu.Lock() + defer r.recorder.mu.Unlock() + total := 0 + for _, request := range r.recorder.requests { + if strings.HasPrefix(request, method+" ") && strings.Contains(request, fragment) { + total++ + } + } + return total +} + +// Requests returns a copy of the recorded requests. +func (r *Registry) Requests() []string { + r.recorder.mu.Lock() + defer r.recorder.mu.Unlock() + return append([]string(nil), r.recorder.requests...) +} + +// FailAll makes every later request fail, like a client with no network; nil restores it. +func (r *Registry) FailAll(err error) { + r.FailAfter(0, err) +} + +// FailAfter lets the next n requests through, then fails every later one; nil restores the client. +// It simulates a refresh that is interrupted part way through. +func (r *Registry) FailAfter(n int, err error) { + r.recorder.mu.Lock() + defer r.recorder.mu.Unlock() + r.recorder.fail, r.recorder.failAfter = err, len(r.recorder.requests)+n +} + +// Config returns the repository configuration of a registry-backed repository. +func (r *Registry) Config(catalog string) *schema.ArtifactRepositoryConfig { + return &schema.ArtifactRepositoryConfig{ + Kind: artifact.RepositoryKindOCIRegistry, + Options: map[string]any{"registry": r.Host, "prefix": Prefix}, + Catalog: catalog, + } +} + +// Location is the credential-free repository location ("registry/prefix"). +func (r *Registry) Location() string { return r.Host + "/" + Prefix } + +// CatalogURI returns the catalog reference of a tag below the repository path. +func (r *Registry) CatalogURI(name, tag string) string { + return "oci://" + r.Location() + "/" + name + ":" + tag +} + +// Repo builds a repository bound to the registry. Extra options follow the defaults. +func (r *Registry) Repo(t *testing.T, alias string, catalog string, extra ...oci.Option) *oci.Repository { + t.Helper() + options := append([]oci.Option{oci.WithTransport(r.Transport), oci.WithOwner(Owner), oci.WithLockDir(t.TempDir())}, extra...) + repo, err := oci.NewRepository(context.Background(), alias, r.Config(catalog), options...) + require.NoError(t, err) + return repo +} + +// Info returns the service description of a registry-backed repository. +func (r *Registry) Info(repo artifact.Repository, catalog string) service.RepositoryInfo { + return service.RepositoryInfo{ + Capabilities: repo.Capabilities(), + Discovery: service.DiscoveryInfo{ + Kind: artifact.RepositoryKindOCIRegistry, Location: r.Location(), Catalog: catalog, + Options: map[string]any{"registry": r.Host, "prefix": Prefix}, + }, + } +} + +// Spec describes one package to publish. +type Spec struct { + Name string + Version string + Content string + Labels map[string]string + NoDocs bool + + // Packaging defaults to tgz, which registries and archives store; local directories need none. + Packaging artifact.Packaging +} + +// Prepare builds a deterministic prepared artifact. +func Prepare(t *testing.T, spec *Spec) *artifact.PreparedArtifact { + t.Helper() + src := t.TempDir() + fixtures.WriteFile(t, src, "data.txt", spec.Content, filePerm) + if !spec.NoDocs { + fixtures.WriteFile(t, src, fixtures.ReservedDir+"/README.md", "readme for "+spec.Name+"\n", filePerm) + } + packaging := spec.Packaging + if packaging == "" { + packaging = artifact.PackagingTgz + } + prepared, err := produce.New(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: src, Name: spec.Name, Version: spec.Version, Packaging: packaging, + Metadata: artifact.Metadata{Description: "package " + spec.Name, Labels: spec.Labels}, + OperationID: "op-" + spec.Name + "-" + spec.Version, PreparedAt: PreparedAt, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +// Publish publishes a package to a repository and returns the result. +// +//nolint:gocritic // Test helper; the spec is a small literal. +func Publish(t *testing.T, repo artifact.Repository, spec Spec) *artifact.PublishResult { + t.Helper() + result, err := repo.Publish(context.Background(), Prepare(t, &spec), artifact.PublishOptions{}) + require.NoError(t, err) + return result +} + +// LocalDir returns a fresh directory for a local repository. +func LocalDir(t *testing.T) string { return filepath.Join(t.TempDir(), "repo") } diff --git a/pkg/artifact/discovery/options.go b/pkg/artifact/discovery/options.go new file mode 100644 index 00000000000..7b7543a311d --- /dev/null +++ b/pkg/artifact/discovery/options.go @@ -0,0 +1,59 @@ +package discovery + +import ( + "time" + + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Option customizes a [Discovery]. +type Option func(*Discovery) + +// WithDirectory sets the snapshot directory. The default is the XDG cache directory +// atmos/artifacts/discovery. +func WithDirectory(directory string) Option { + defer perf.Track(nil, "discovery.WithDirectory")() + + return func(d *Discovery) { d.directory = directory } +} + +// WithTTL sets how long a snapshot stays fresh. The default is fifteen minutes. +func WithTTL(ttl time.Duration) Option { + defer perf.Track(nil, "discovery.WithTTL")() + + return func(d *Discovery) { d.ttl = ttl } +} + +// WithClock injects the clock that stamps and ages snapshots. +func WithClock(now func() time.Time) Option { + defer perf.Track(nil, "discovery.WithClock")() + + return func(d *Discovery) { d.now = now } +} + +// WithOffline decides, per repository, whether discovery is offline. Offline discovery serves only +// an existing valid snapshot and never contacts a repository; a refresh is rejected. +func WithOffline(offline func(info *service.RepositoryInfo) bool) Option { + defer perf.Track(nil, "discovery.WithOffline")() + + return func(d *Discovery) { d.offline = offline } +} + +// WithPolicy replaces the default [LocationPolicy]. +func WithPolicy(policy Policy) Option { + defer perf.Track(nil, "discovery.WithPolicy")() + + return func(d *Discovery) { d.policy = policy } +} + +// WithMaxSize bounds one snapshot file in bytes. +func WithMaxSize(size int64) Option { + defer perf.Track(nil, "discovery.WithMaxSize")() + + return func(d *Discovery) { d.maxSize = size } +} + +// defaultTTL is the freshness window of a snapshot. +const defaultTTL = cache.DefaultTTL diff --git a/pkg/artifact/discovery/policy.go b/pkg/artifact/discovery/policy.go new file mode 100644 index 00000000000..1d5e05cde27 --- /dev/null +++ b/pkg/artifact/discovery/policy.go @@ -0,0 +1,75 @@ +package discovery + +import ( + "context" + "fmt" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// SourceCheck identifies one discovery source to a [Policy]. +type SourceCheck struct { + // Alias is the configured repository alias. + Alias string + + // Info describes the repository. + Info *service.RepositoryInfo + + // Kind is "collection", "named", or "catalog". + Kind string + + // Reference is the source reference: the repository location, or the catalog reference. + Reference string + + // Cached reports that the read is served from a snapshot rather than the source. + Cached bool +} + +// Policy decides whether a discovery source may be used now. It is evaluated on every read, so a +// policy that changed since a snapshot was written applies to the cached data too: a snapshot is +// never an authorization. +type Policy interface { + // AuthorizeSource runs before a snapshot is served from cache and before a source is read. + AuthorizeSource(ctx context.Context, check *SourceCheck) error + + // AuthorizeEntries runs on the entries about to be returned, cached or fresh. + AuthorizeEntries(ctx context.Context, check *SourceCheck, entries []artifact.ListEntry) error +} + +// LocationPolicy is the default policy. A repository must be readable, and every entry that came +// from a catalog must be bound to the destination repository path: a catalog is discoverable data +// and can never direct the repository's identity to another registry or path. +type LocationPolicy struct{} + +var _ Policy = LocationPolicy{} + +// AuthorizeSource implements [Policy]. +func (LocationPolicy) AuthorizeSource(_ context.Context, check *SourceCheck) error { + defer perf.Track(nil, "discovery.LocationPolicy.AuthorizeSource")() + + if !check.Info.Capabilities.Read { + return artifact.NewUnsupportedCapability(check.Alias, "list", "read", "the repository is write-only") + } + return nil +} + +// AuthorizeEntries implements [Policy]. +func (LocationPolicy) AuthorizeEntries(_ context.Context, check *SourceCheck, entries []artifact.ListEntry) error { + defer perf.Track(nil, "discovery.LocationPolicy.AuthorizeEntries")() + + if check.Kind != kindCatalog { + return nil + } + location := strings.ToLower(check.Info.Discovery.Location) + for index := range entries { + entry := &entries[index] + if want := location + "/" + entry.Name; entry.Pinned.Repository != want { + return fmt.Errorf("%w: catalog entry %s:%s is bound to %q, not to the repository path %q", errUtils.ErrArtifactCatalogInvalid, entry.Name, entry.Version, entry.Pinned.Repository, want) + } + } + return nil +} diff --git a/pkg/artifact/discovery/scope.go b/pkg/artifact/discovery/scope.go new file mode 100644 index 00000000000..aab57f5bd13 --- /dev/null +++ b/pkg/artifact/discovery/scope.go @@ -0,0 +1,42 @@ +package discovery + +import ( + "sort" + + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" + "github.com/cloudposse/atmos/pkg/auth/realm" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// binding is one host binding of an artifact repository: the registry host an identity is bound +// to. Only names appear here, never credential values. +type binding struct { + Repository string `json:"repository"` + Kind string `json:"kind"` + Identity string `json:"identity"` +} + +// ScopeOf returns the cache scope of a configuration: a hash of the authentication configuration, +// the effective authentication realm, and the identity bound to each repository. Credentials +// never enter it. A snapshot written under one scope is invisible under another, so a different +// realm, auth configuration, or identity binding can never reuse another's discovery data. +func ScopeOf(config *schema.AtmosConfiguration) (string, error) { + defer perf.Track(config, "discovery.ScopeOf")() + + info, err := realm.GetRealm(config.Auth.Realm, config.CliConfigPath) + if err != nil { + return "", err + } + aliases := make([]string, 0, len(config.Artifacts.Repositories)) + for alias := range config.Artifacts.Repositories { + aliases = append(aliases, alias) + } + sort.Strings(aliases) + bindings := make([]binding, 0, len(aliases)) + for _, alias := range aliases { + repo := config.Artifacts.Repositories[alias] + bindings = append(bindings, binding{Repository: alias, Kind: repo.Kind, Identity: repo.Identity}) + } + return cache.Scope(config.Auth, info.Value, bindings) +} diff --git a/pkg/artifact/discovery/snapshot.go b/pkg/artifact/discovery/snapshot.go new file mode 100644 index 00000000000..f07a1b5f504 --- /dev/null +++ b/pkg/artifact/discovery/snapshot.go @@ -0,0 +1,104 @@ +package discovery + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/opencontainers/go-digest" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" +) + +// snapshotVersion is the artifact snapshot schema version. +const snapshotVersion = 1 + +// Snapshot source kinds. They are part of the snapshot key and header. +const ( + kindCollection = "collection" + kindNamed = "named" + kindCatalog = "catalog" +) + +// snapshot is the artifact schema stored in a source's snapshot file: the shared header, the +// digest of the catalog it was read from (catalog snapshots only), and the unfiltered entries. +// Labels and text are filtered after the snapshot is read, so one snapshot answers every query. +type snapshot struct { + cache.Header + Digest string `json:"digest,omitempty"` + Entries []artifact.ListEntry `json:"entries"` +} + +// decodeSnapshot parses a snapshot file and rejects content that is not a coherent listing. A +// snapshot that fails here is treated as absent: replaced online, rejected offline. +func decodeSnapshot(raw []byte) (*snapshot, error) { + var value snapshot + if err := json.Unmarshal(raw, &value); err != nil { + return nil, err + } + seen := make(map[string]bool, len(value.Entries)) + for index := range value.Entries { + entry := &value.Entries[index] + if err := validEntry(entry); err != nil { + return nil, err + } + key := entry.Name + "\x00" + entry.Version + if seen[key] { + return nil, fmt.Errorf("%w: %s:%s is listed twice", errUtils.ErrArtifactRecordInvalid, entry.Name, entry.Version) + } + seen[key] = true + } + return &value, nil +} + +// validEntry checks that a snapshot entry names a version and carries a coherent pinned reference. +func validEntry(entry *artifact.ListEntry) error { + ref := artifact.Ref{Name: entry.Name, Version: entry.Version} + if err := ref.Validate(); err != nil { + return err + } + if err := artifact.ValidateKind(entry.Kind); err != nil { + return err + } + pinned := &entry.Pinned + if pinned.Ref != ref { + return fmt.Errorf("%w: %s is pinned as %s", errUtils.ErrArtifactRecordInvalid, ref, pinned.Ref) + } + if err := artifact.ValidateDigest(pinned.ResolvedDigest); err != nil { + return err + } + for _, value := range []digest.Digest{pinned.RecordDigest, pinned.ContentDigest} { + if value == "" { + continue + } + if err := artifact.ValidateDigest(value); err != nil { + return err + } + } + return nil +} + +// sortEntries orders entries deterministically: name, then version. +func sortEntries(entries []artifact.ListEntry) { + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].Name != entries[j].Name { + return entries[i].Name < entries[j].Name + } + return entries[i].Version < entries[j].Version + }) +} + +// knownByDigest indexes a previous snapshot's entries by resolved digest, so a refresh can reuse +// the metadata of references whose digest did not change. +func knownByDigest(previous *snapshot) map[digest.Digest]artifact.ListEntry { + if previous == nil { + return nil + } + known := make(map[digest.Digest]artifact.ListEntry, len(previous.Entries)) + for index := range previous.Entries { + known[previous.Entries[index].Pinned.ResolvedDigest] = previous.Entries[index] + } + return known +} diff --git a/pkg/artifact/execution.go b/pkg/artifact/execution.go new file mode 100644 index 00000000000..4050fd3350a --- /dev/null +++ b/pkg/artifact/execution.go @@ -0,0 +1,25 @@ +package artifact + +import "time" + +// Execution carries the per-call execution settings that a wrapper (a workflow runner, a custom +// command, or a hook) supplies when it owns the retry policy. The zero value is the behavior of a +// standalone CLI call: the service generates a fresh operation identity and applies its own +// transport retry budget. +// +// A wrapper that retries a whole operation captures one Execution before the first attempt and +// passes the same value on every attempt. The operation ID and the UTC preparation time then stay +// stable, so a publication keeps its creation time and a report can deduplicate attempts by the +// logical operation ID. The wrapper also sets [Execution.MaxAttempts] to 1, so the wrapper's retry +// budget and the service's transport retries never multiply. +type Execution struct { + // OperationID is the logical operation identifier. Empty means the service generates one. + OperationID string + + // PreparedAt is the single UTC preparation time. Zero means the service reads its clock. + PreparedAt time.Time + + // MaxAttempts overrides the number of transport attempts for this call. Zero keeps the + // service default. One disables transport retries. + MaxAttempts int +} diff --git a/pkg/artifact/invoke/doc.go b/pkg/artifact/invoke/doc.go new file mode 100644 index 00000000000..cd8f5c3071c --- /dev/null +++ b/pkg/artifact/invoke/doc.go @@ -0,0 +1,11 @@ +// Package invoke is the transport-neutral request and result mapping of the artifact operations. +// +// The CLI, workflow and custom-command steps, script step calls, and kind: step hooks all describe +// an operation as [Params], run it with [Run], and report the typed [Result] or [Collection]. One +// implementation builds the service requests, dispatches the action, populates the result, and +// names the outputs, so every entry point returns identical payloads, named outputs, and field +// names. Presentation (flags, text rendering, step output modes) stays with each entry point. +// +// The package depends only on the artifact domain. Production service assembly, which needs the +// plugin-aware registry preparation and the managed signer, lives in pkg/artifactwire. +package invoke diff --git a/pkg/artifact/invoke/invoke_test.go b/pkg/artifact/invoke/invoke_test.go new file mode 100644 index 00000000000..b10be47e558 --- /dev/null +++ b/pkg/artifact/invoke/invoke_test.go @@ -0,0 +1,135 @@ +package invoke_test + +import ( + "context" + "errors" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" +) + +const ( + recordDigest = digest.Digest("sha256:1111111111111111111111111111111111111111111111111111111111111111") + resolvedDigest = digest.Digest("sha256:2222222222222222222222222222222222222222222222222222222222222222") +) + +func operation(action artifact.Action) *artifact.OperationResult { + return &artifact.OperationResult{ + OperationID: "op-1", Action: action, Outcome: artifact.OutcomeCreated, Repository: "dir", + Ref: "team/app:1.0.0", Location: "/repo", ResolvedDigest: resolvedDigest, RecordDigest: recordDigest, Size: 7, + } +} + +func TestSpecCoversEveryCommandAndStepAction(t *testing.T) { + want := []artifact.Action{ + artifact.ActionCopy, artifact.ActionDelete, artifact.ActionInspect, artifact.ActionList, artifact.ActionPublish, + artifact.ActionPull, artifact.ActionSearch, artifact.ActionShow, artifact.ActionSign, artifact.ActionVerify, + } + assert.Equal(t, want, invoke.Actions()) + _, ok := invoke.SpecFor(artifact.ActionPin) + assert.False(t, ok, "pin is a service operation, not a command or step action") + + publish, ok := invoke.SpecFor(artifact.ActionPublish) + require.True(t, ok) + assert.True(t, publish.Accepts(invoke.FieldMetadata)) + assert.False(t, publish.Accepts(invoke.FieldLabels), "publish labels are metadata.labels; labels select in list and search") + assert.Equal(t, append([]invoke.Field{invoke.FieldSource, invoke.FieldRepository}, publish.Optional...), publish.Fields()) + + list, _ := invoke.SpecFor(artifact.ActionList) + assert.Empty(t, list.Required, "list and search default to every configured repository") + assert.Contains(t, invoke.AllFields(), invoke.FieldDocument) +} + +func TestOutputPairsOmitOptionalValues(t *testing.T) { + pairs := invoke.OutputPairs(&artifact.OperationResult{Outcome: artifact.OutcomeUnchanged, Ref: "team/app:1"}) + assert.Equal(t, []invoke.Pair{{Name: "outcome", Value: "unchanged"}, {Name: "ref", Value: "team/app:1"}}, pairs) + + full := invoke.NamedOutputs(operation(artifact.ActionPublish)) + assert.Equal(t, map[string]string{ + "outcome": "created", "ref": "team/app:1.0.0", "location": "/repo", "resolved_digest": string(resolvedDigest), + "record_digest": string(recordDigest), "size": "7", + }, full) + assert.Empty(t, invoke.NamedOutputs(&artifact.OperationResult{})) + assert.Equal(t, "0", invoke.NamedOutputs(&artifact.OperationResult{SizeMeasured: true})[invoke.OutputSize]) +} + +func TestRunMapsParamsOntoServiceRequests(t *testing.T) { + ctrl := gomock.NewController(t) + svc := artifact.NewMockService(ctrl) + execution := artifact.Execution{OperationID: "retained", MaxAttempts: 1} + + svc.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + assert.Equal(t, artifact.PublishRequest{ + Execution: execution, Repository: "dir", Source: "./dist", Name: "team/app", Version: "1", Kind: "files", + Metadata: artifact.Metadata{Labels: map[string]string{"a": "b"}}, Readme: "R.md", Changelog: "C.md", Force: true, DryRun: true, + }, r) + return operation(artifact.ActionPublish), nil + }) + outcome, err := invoke.Run(context.Background(), svc, &invoke.Params{ + Action: artifact.ActionPublish, Repository: "dir", Source: "./dist", Name: "team/app", Version: "1", Kind: "files", + Labels: map[string]string{"a": "b"}, Readme: "R.md", Changelog: "C.md", Force: true, DryRun: true, Execution: execution, + }) + require.NoError(t, err) + assert.Nil(t, outcome.Collection) + assert.Equal(t, outcome.Result, outcome.Payload()) + assert.Equal(t, "op-1", outcome.Result.OperationID) +} + +func TestRunReturnsAFailedResultWhenTheReferenceIsInvalid(t *testing.T) { + svc := artifact.NewMockService(gomock.NewController(t)) + outcome, err := invoke.Run(context.Background(), svc, &invoke.Params{Action: artifact.ActionDelete, Repository: "dir", Ref: "bad"}) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + require.NotNil(t, outcome.Result) + assert.Equal(t, artifact.OutcomeFailed, outcome.Result.Outcome) + assert.Equal(t, artifact.ActionDelete, outcome.Result.Action) + assert.Equal(t, "dir", outcome.Result.Repository) + assert.Equal(t, "bad", outcome.Result.Requested) + assert.Contains(t, outcome.Result.Error, "implicit latest") +} + +func TestRunRejectsUnknownActionsAndDocuments(t *testing.T) { + svc := artifact.NewMockService(gomock.NewController(t)) + _, err := invoke.Run(context.Background(), svc, &invoke.Params{Action: artifact.ActionPin, Repository: "dir", Ref: "a:1"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + + _, err = invoke.Run(context.Background(), svc, &invoke.Params{Action: artifact.ActionShow, Repository: "dir", Ref: "a:1", Document: "license"}) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) +} + +func TestRunCollectionEnvelopeSurvivesFailure(t *testing.T) { + svc := artifact.NewMockService(gomock.NewController(t)) + boom := errors.New("catalog unavailable") + svc.EXPECT().Search(gomock.Any(), gomock.Any()).Return(nil, boom) + outcome, err := invoke.Run(context.Background(), svc, &invoke.Params{Action: artifact.ActionSearch, Query: "x", Labels: map[string]string{"b": "2", "a": "1"}}) + require.ErrorIs(t, err, boom) + require.NotNil(t, outcome.Collection) + assert.NotNil(t, outcome.Collection.Artifacts) + assert.Empty(t, outcome.Collection.Artifacts) + assert.Equal(t, "catalog unavailable", outcome.Collection.Error) + assert.Equal(t, outcome.Collection, outcome.Payload()) +} + +func TestSelectorLabelsAreSorted(t *testing.T) { + assert.Nil(t, invoke.SelectorLabels(nil)) + assert.Equal(t, []artifact.Label{{Key: "a", Value: "1"}, {Key: "b", Value: "2"}}, invoke.SelectorLabels(map[string]string{"b": "2", "a": "1"})) +} + +func TestResultMarkFailedKeepsWhatTheServiceReported(t *testing.T) { + result := &invoke.Result{OperationResult: artifact.OperationResult{Action: artifact.ActionPull, Repository: "kept", Requested: "kept:1"}} + result.MarkFailed(nil, artifact.ActionPublish, "x", "y") + assert.Empty(t, result.Error, "no error, no failure markers") + + result.MarkFailed(errors.New("nope"), artifact.ActionPublish, "other", "other:2") + assert.Equal(t, artifact.OutcomeFailed, result.Outcome) + assert.Equal(t, artifact.ActionPull, result.Action) + assert.Equal(t, "kept", result.Repository) + assert.Equal(t, "kept:1", result.Requested) + assert.Equal(t, "nope", result.Error) +} diff --git a/pkg/artifact/invoke/params.go b/pkg/artifact/invoke/params.go new file mode 100644 index 00000000000..4a9233f3629 --- /dev/null +++ b/pkg/artifact/invoke/params.go @@ -0,0 +1,179 @@ +package invoke + +import ( + "slices" + "sort" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Field names an input of an artifact operation. The values are the artifact step field names, +// which are also the CLI flag names where one exists (`--name` is artifact_name, `--kind` is +// artifact_kind, `--output` is destination). +type Field string + +// Operation inputs. +const ( + FieldSource Field = "source" + FieldRepository Field = "repository" + FieldArtifactName Field = "artifact_name" + FieldVersion Field = "version" + FieldArtifactKind Field = "artifact_kind" + FieldForce Field = "force" + FieldDryRun Field = "dry_run" + FieldLabels Field = "labels" + FieldMetadata Field = "metadata.labels" + FieldReadme Field = "readme" + FieldChangelog Field = "changelog" + FieldRef Field = "ref" + FieldDestination Field = "destination" + FieldFrom Field = "from" + FieldTo Field = "to" + FieldDocument Field = "document" + FieldQuery Field = "query" + FieldRefresh Field = "refresh" +) + +// Spec lists the inputs an action requires and the optional inputs it also accepts. Any other +// input is invalid for the action. +type Spec struct { + // Required inputs must be present and non-empty. + Required []Field + + // Optional inputs may be present. + Optional []Field +} + +// specs is the contract table: "Public command and step mapping" in the artifacts PRD. +var specs = map[artifact.Action]Spec{ + artifact.ActionPublish: { + Required: []Field{FieldSource, FieldRepository}, + Optional: []Field{FieldArtifactName, FieldVersion, FieldArtifactKind, FieldForce, FieldDryRun, FieldMetadata, FieldReadme, FieldChangelog}, + }, + artifact.ActionPull: { + Required: []Field{FieldRef, FieldRepository, FieldDestination}, + Optional: []Field{FieldForce}, + }, + artifact.ActionCopy: { + Required: []Field{FieldRef, FieldFrom, FieldTo}, + Optional: []Field{FieldForce}, + }, + artifact.ActionInspect: {Required: []Field{FieldRef, FieldRepository}}, + artifact.ActionSign: {Required: []Field{FieldRef, FieldRepository}}, + artifact.ActionVerify: {Required: []Field{FieldRef, FieldRepository}}, + artifact.ActionDelete: {Required: []Field{FieldRef, FieldRepository}}, + artifact.ActionShow: {Required: []Field{FieldDocument, FieldRef, FieldRepository}}, + artifact.ActionList: {Optional: []Field{FieldArtifactName, FieldRepository, FieldLabels, FieldRefresh}}, + artifact.ActionSearch: {Optional: []Field{FieldQuery, FieldRepository, FieldLabels, FieldRefresh}}, +} + +// SpecFor returns the input contract of an action. The second result is false for an action that +// is not available as a CLI command or step action. +func SpecFor(action artifact.Action) (Spec, bool) { + defer perf.Track(nil, "invoke.SpecFor")() + + spec, ok := specs[action] + return spec, ok +} + +// Actions lists the actions in a stable order. +func Actions() []artifact.Action { + defer perf.Track(nil, "invoke.Actions")() + + actions := make([]artifact.Action, 0, len(specs)) + for action := range specs { + actions = append(actions, action) + } + sort.Slice(actions, func(i, j int) bool { return actions[i] < actions[j] }) + return actions +} + +// Accepts reports whether the field is required or optional for the action. +func (s Spec) Accepts(field Field) bool { + defer perf.Track(nil, "invoke.Spec.Accepts")() + + return slices.Contains(s.Required, field) || slices.Contains(s.Optional, field) +} + +// Fields lists every input the action accepts, required first. +func (s Spec) Fields() []Field { + defer perf.Track(nil, "invoke.Spec.Fields")() + + return append(slices.Clone(s.Required), s.Optional...) +} + +// AllFields lists every input of every action in a stable order. +func AllFields() []Field { + defer perf.Track(nil, "invoke.AllFields")() + + seen := make(map[Field]bool) + var all []Field + for _, action := range Actions() { + for _, field := range specs[action].Fields() { + if !seen[field] { + seen[field] = true + all = append(all, field) + } + } + } + return all +} + +// Params is the transport-neutral input of one artifact operation: the union of the CLI flags and +// the artifact step fields. Which fields an action reads is described by [SpecFor]. +type Params struct { + // Action selects the operation. + Action artifact.Action + + // Repository is the configured repository alias for single-repository actions. + Repository string + + // Repositories restricts list and search; empty means all configured repositories. + Repositories []string + + // Source is the publish source file or directory, already resolved to a path. + Source string + + // Name is the artifact name (--name; artifact_name). List uses it to restrict to one name. + Name string + + // Version is the explicit publish version. + Version string + + // Kind is the artifact kind (--kind; artifact_kind). Empty means the files kind. + Kind string + + // Force permits replacing conflicting state where the repository allows it. + Force bool + + // DryRun validates and plans a publication without writing. + DryRun bool + + // Refresh bypasses cached discovery snapshots for list and search. + Refresh bool + + // Labels are the publish metadata labels, or the exact list and search selectors. + Labels map[string]string + + // Readme and Changelog override the document source paths, relative to the source directory. + Readme, Changelog string + + // Ref is the artifact reference exactly as the caller typed it. + Ref string + + // Destination is the pull output directory (--output). + Destination string + + // From and To are the copy source and destination repository aliases. + From, To string + + // Document is readme or changelog for show. + Document string + + // Query is the search text. + Query string + + // Execution carries the wrapper-owned operation identity and retry budget. + Execution artifact.Execution +} diff --git a/pkg/artifact/invoke/result.go b/pkg/artifact/invoke/result.go new file mode 100644 index 00000000000..df4820793d3 --- /dev/null +++ b/pkg/artifact/invoke/result.go @@ -0,0 +1,206 @@ +package invoke + +import ( + "fmt" + + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Result is the structured result of one operation. The operation fields flatten into the top +// level with their named-output spelling (ref, location, resolved_digest, record_digest, +// content_digest, representation_digest, size, outcome), and the action-specific parts follow. +// JSON and YAML share these names; the display format never changes semantics. The CLI prints it +// and a step stores it under StepResult.Metadata["artifact"]. +type Result struct { + artifact.OperationResult + + // Error is set when the operation failed. The result is still reported, and the exit status is nonzero. + Error string `json:"error,omitempty"` + + // Record is the sealed Record of a publication (set for dry runs and publications). + Record *artifact.Record `json:"record,omitempty"` + + // Inspection is the metadata view of an inspected artifact. + Inspection *artifact.Inspection `json:"inspection,omitempty"` + + // Verification is the authenticated verification result; absent means unchecked. + Verification *artifact.VerificationResult `json:"verification,omitempty"` + + // Subject is the signed or verified subject. + Subject *artifact.SignatureSubject `json:"subject,omitempty"` + + // Evidence lists the evidence stored by sign. + Evidence []artifact.SignatureEvidence `json:"evidence,omitempty"` + + // Document is the descriptor of a shown document. + Document *artifact.Document `json:"document,omitempty"` + + // Content is the document text of show. + Content *string `json:"content,omitempty"` +} + +// MarkFailed fills the failure markers of a result. The service leaves the outcome empty on +// failure so a failure is never mistaken for success; the reported result states it explicitly. +// The requested values fill in what the service could not report, for example when the failure +// happened before the operation began. +func (r *Result) MarkFailed(err error, action artifact.Action, repository, requested string) { + defer perf.Track(nil, "invoke.Result.MarkFailed")() + + if err == nil { + return + } + r.Error = err.Error() + r.Outcome = artifact.OutcomeFailed + if r.Action == "" { + r.Action = action + } + if r.Repository == "" { + r.Repository = repository + } + if r.Requested == "" { + r.Requested = requested + } +} + +// Collection is the structured result of list and search: the `artifacts` array plus +// per-repository outcomes, and the error when the query failed. +type Collection struct { + // Artifacts are the entries across the selected repositories, sorted lexically. + Artifacts []Listed `json:"artifacts"` + + // Repositories reports per-repository outcomes. + Repositories []artifact.RepositoryListing `json:"repositories,omitempty"` + + // Incomplete is true when a selected repository could not be listed, so Artifacts is not the + // whole collection. The failing repository carries an error diagnostic. + Incomplete bool `json:"incomplete,omitempty"` + + // Error is set when the query failed. + Error string `json:"error,omitempty"` +} + +// Listed is one entry of the `artifacts` array. It carries the named outputs of an operation +// result (ref, resolved_digest, record_digest, content_digest) next to the listing fields, plus the +// pinned reference a consumer lockfile persists. +type Listed struct { + Repository string `json:"repository,omitempty"` + Ref string `json:"ref"` + Name string `json:"name"` + Version string `json:"version"` + Kind string `json:"kind"` + Description string `json:"description,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Documents []artifact.Document `json:"documents,omitempty"` + ResolvedDigest digest.Digest `json:"resolved_digest,omitempty"` + RecordDigest digest.Digest `json:"record_digest,omitempty"` + ContentDigest digest.Digest `json:"content_digest,omitempty"` + Pinned artifact.PinnedReference `json:"pinned"` +} + +// ListedFrom converts service entries into collection entries. The result is never nil, so an +// empty listing encodes as an empty array. +func ListedFrom(entries []artifact.ListEntry) []Listed { + defer perf.Track(nil, "invoke.ListedFrom")() + + out := make([]Listed, 0, len(entries)) + for index := range entries { + entry := &entries[index] + out = append(out, Listed{ + Repository: entry.Repository, + Ref: artifact.Ref{Name: entry.Name, Version: entry.Version}.String(), + Name: entry.Name, + Version: entry.Version, + Kind: entry.Kind, + Description: entry.Description, + Labels: entry.Labels, + Documents: entry.Documents, + ResolvedDigest: entry.Pinned.ResolvedDigest, + RecordDigest: entry.Pinned.RecordDigest, + ContentDigest: entry.Pinned.ContentDigest, + Pinned: entry.Pinned, + }) + } + return out +} + +// Pair is one named output. +type Pair struct { + // Name is the output name. + Name string + + // Value is the output text. + Value string +} + +// Named output names. These are the JSON field names of [artifact.OperationResult] too. +const ( + OutputOutcome = "outcome" + OutputRef = "ref" + OutputLocation = "location" + OutputResolvedDigest = "resolved_digest" + OutputRecordDigest = "record_digest" + OutputContentDigest = "content_digest" + OutputRepresentationDigest = "representation_digest" + OutputSize = "size" +) + +// OutputPairs returns the named outputs of an operation in a stable order. An optional value that +// the operation did not produce is omitted, never filled with a misleading empty string. +func OutputPairs(op *artifact.OperationResult) []Pair { + defer perf.Track(nil, "invoke.OutputPairs")() + + candidates := []Pair{ + {OutputOutcome, string(op.Outcome)}, + {OutputRef, op.Ref}, + {OutputLocation, op.Location}, + {OutputResolvedDigest, string(op.ResolvedDigest)}, + {OutputRecordDigest, string(op.RecordDigest)}, + {OutputContentDigest, string(op.ContentDigest)}, + {OutputRepresentationDigest, string(op.RepresentationDigest)}, + } + if op.SizeMeasured || op.Size > 0 { + candidates = append(candidates, Pair{OutputSize, fmt.Sprintf("%d", op.Size)}) + } + pairs := make([]Pair, 0, len(candidates)) + for _, pair := range candidates { + if pair.Value != "" { + pairs = append(pairs, pair) + } + } + return pairs +} + +// NamedOutputs is [OutputPairs] as a map, the shape of StepResult.Outputs. +func NamedOutputs(op *artifact.OperationResult) map[string]string { + defer perf.Track(nil, "invoke.NamedOutputs")() + + pairs := OutputPairs(op) + outputs := make(map[string]string, len(pairs)) + for _, pair := range pairs { + outputs[pair.Name] = pair.Value + } + return outputs +} + +// Outcome is the result of [Run]: a Result for single-artifact actions, or a Collection for list +// and search. Exactly one is set. +type Outcome struct { + // Result is set for every action except list and search. + Result *Result + + // Collection is set for list and search. + Collection *Collection +} + +// Payload returns the typed value that is reported as JSON or stored in step metadata. +func (o *Outcome) Payload() any { + defer perf.Track(nil, "invoke.Outcome.Payload")() + + if o.Collection != nil { + return o.Collection + } + return o.Result +} diff --git a/pkg/artifact/invoke/run.go b/pkg/artifact/invoke/run.go new file mode 100644 index 00000000000..8e15b4c8f38 --- /dev/null +++ b/pkg/artifact/invoke/run.go @@ -0,0 +1,222 @@ +package invoke + +import ( + "context" + "fmt" + "sort" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" +) + +// detailedPublisher is implemented by the real service. It returns the sealed Record next to the +// operation result, which a dry run needs in order to show the files it selected. +type detailedPublisher interface { + PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*service.PublishOutcome, error) +} + +// Run executes one operation against the service and returns its typed result. +// +// The returned Outcome is never nil, and the matching Result or Collection is never nil, even when +// the operation fails: a partial result (the identity, whatever the service resolved before it +// failed, and the failure markers) accompanies the error. Callers report it and then surface the +// error, so a failure is never reported as a success and never loses what was learned. +func Run(ctx context.Context, svc artifact.Service, params *Params) (*Outcome, error) { + defer perf.Track(nil, "invoke.Run")() + + if params.Action == artifact.ActionList || params.Action == artifact.ActionSearch { + collection, err := runCollection(ctx, svc, params) + return &Outcome{Collection: collection}, err + } + result, err := runOperation(ctx, svc, params) + if result == nil { + result = &Result{} + } + result.MarkFailed(err, params.Action, params.Repository, requestedOf(params)) + return &Outcome{Result: result}, err +} + +// requestedOf is the reference as the caller typed it, for failure results. Publish has no +// reference to type; the service reports the name and version it resolved. +func requestedOf(params *Params) string { + return params.Ref +} + +// runOperation dispatches the single-artifact actions. +func runOperation(ctx context.Context, svc artifact.Service, params *Params) (*Result, error) { + switch params.Action { + case artifact.ActionPublish: + return runPublish(ctx, svc, params) + case artifact.ActionPull: + return withRef(params, func(ref artifact.Ref) (*Result, error) { + op, err := svc.Pull(ctx, artifact.PullRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref, Destination: params.Destination, Force: params.Force}) + return fromOperation(op), err + }) + case artifact.ActionCopy: + return withRef(params, func(ref artifact.Ref) (*Result, error) { + op, err := svc.Copy(ctx, artifact.CopyRequest{Execution: params.Execution, Ref: ref, From: params.From, To: params.To, Force: params.Force}) + return fromOperation(op), err + }) + case artifact.ActionInspect: + return withRef(params, func(ref artifact.Ref) (*Result, error) { return runInspect(ctx, svc, params, ref) }) + case artifact.ActionDelete: + return withRef(params, func(ref artifact.Ref) (*Result, error) { + op, err := svc.Delete(ctx, artifact.DeleteRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref}) + return fromOperation(op), err + }) + case artifact.ActionShow: + return withRef(params, func(ref artifact.Ref) (*Result, error) { return runShow(ctx, svc, params, ref) }) + case artifact.ActionSign: + return withRef(params, func(ref artifact.Ref) (*Result, error) { return runSign(ctx, svc, params, ref) }) + case artifact.ActionVerify: + return withRef(params, func(ref artifact.Ref) (*Result, error) { return runVerify(ctx, svc, params, ref) }) + default: + return nil, fmt.Errorf("%w: unsupported artifact action %q", errUtils.ErrArtifactConfigInvalid, params.Action) + } +} + +// withRef parses the reference once, before any service call, and runs the action with it. +func withRef(params *Params, run func(ref artifact.Ref) (*Result, error)) (*Result, error) { + ref, err := artifact.ParseRef(params.Ref) + if err != nil { + return nil, err + } + return run(ref) +} + +// fromOperation builds a result from a plain operation result. +func fromOperation(op *artifact.OperationResult) *Result { + if op == nil { + return &Result{} + } + return &Result{OperationResult: *op} +} + +func runPublish(ctx context.Context, svc artifact.Service, params *Params) (*Result, error) { + request := artifact.PublishRequest{ + Execution: params.Execution, + Repository: params.Repository, + Source: params.Source, + Name: params.Name, + Version: params.Version, + Kind: params.Kind, + Metadata: artifact.Metadata{Labels: params.Labels}, + Readme: params.Readme, + Changelog: params.Changelog, + Force: params.Force, + DryRun: params.DryRun, + } + if detailed, ok := svc.(detailedPublisher); ok { + outcome, err := detailed.PublishDetailed(ctx, request) + res := fromOperation(&outcome.Operation) + if outcome.Record.Name != "" { + record := outcome.Record + res.Record = &record + } + return res, err + } + op, err := svc.Publish(ctx, request) + return fromOperation(op), err +} + +func runInspect(ctx context.Context, svc artifact.Service, params *Params, ref artifact.Ref) (*Result, error) { + inspected, err := svc.Inspect(ctx, artifact.InspectRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref}) + if inspected == nil { + return nil, err + } + res := fromOperation(&inspected.Operation) + res.Inspection = &inspected.Inspection + res.Verification = inspected.Verification + return res, err +} + +func runShow(ctx context.Context, svc artifact.Service, params *Params, ref artifact.Ref) (*Result, error) { + role := artifact.DocumentRole(params.Document) + if role != artifact.DocumentReadme && role != artifact.DocumentChangelog { + return nil, fmt.Errorf("%w: document must be %q or %q, got %q", errUtils.ErrArtifactConfigInvalid, artifact.DocumentReadme, artifact.DocumentChangelog, params.Document) + } + document, err := svc.ShowDocument(ctx, artifact.ShowDocumentRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref, Role: role}) + if document == nil { + return nil, err + } + res := fromOperation(&document.Operation) + if document.Document.Digest != "" { + descriptor := document.Document + res.Document = &descriptor + content := string(document.Content) + res.Content = &content + } + return res, err +} + +func runSign(ctx context.Context, svc artifact.Service, params *Params, ref artifact.Ref) (*Result, error) { + signed, err := svc.Sign(ctx, artifact.SignRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref}) + if signed == nil { + return nil, err + } + res := fromOperation(&signed.Operation) + if signed.Subject.Digest != "" { + res.Subject = &signed.Subject + } + res.Evidence = signed.Evidence + return res, err +} + +func runVerify(ctx context.Context, svc artifact.Service, params *Params, ref artifact.Ref) (*Result, error) { + verified, err := svc.Verify(ctx, artifact.VerifyRequest{Execution: params.Execution, Repository: params.Repository, Ref: ref}) + if verified == nil { + return nil, err + } + res := fromOperation(&verified.Operation) + verification := verified.Verification + res.Verification = &verification + if verification.Subject.Digest != "" { + res.Subject = &verification.Subject + } + return res, err +} + +// runCollection runs list or search and wraps the entries in the `artifacts` envelope. +func runCollection(ctx context.Context, svc artifact.Service, params *Params) (*Collection, error) { + selectors := SelectorLabels(params.Labels) + var ( + listing *artifact.ListResult + err error + ) + if params.Action == artifact.ActionSearch { + listing, err = svc.Search(ctx, artifact.SearchRequest{Repositories: params.Repositories, Query: params.Query, Labels: selectors, Refresh: params.Refresh}) + } else { + listing, err = svc.List(ctx, artifact.ListRequest{Repositories: params.Repositories, Name: params.Name, Labels: selectors, Refresh: params.Refresh}) + } + collection := &Collection{Artifacts: []Listed{}} + if listing != nil { + collection.Artifacts = ListedFrom(listing.Artifacts) + collection.Repositories = listing.Repositories + collection.Incomplete = listing.Incomplete + } + if err != nil { + collection.Error = err.Error() + } + return collection, err +} + +// SelectorLabels converts exact key/value selectors into the sorted label list the service takes. +func SelectorLabels(parsed map[string]string) []artifact.Label { + defer perf.Track(nil, "invoke.SelectorLabels")() + + if len(parsed) == 0 { + return nil + } + keys := make([]string, 0, len(parsed)) + for key := range parsed { + keys = append(keys, key) + } + sort.Strings(keys) + labels := make([]artifact.Label, 0, len(keys)) + for _, key := range keys { + labels = append(labels, artifact.Label{Key: key, Value: parsed[key]}) + } + return labels +} diff --git a/pkg/artifact/invoke/service.go b/pkg/artifact/invoke/service.go new file mode 100644 index 00000000000..3cdd08a5de0 --- /dev/null +++ b/pkg/artifact/invoke/service.go @@ -0,0 +1,17 @@ +package invoke + +import ( + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// ServiceOptions are the per-invocation settings that reach the composition root. +type ServiceOptions struct { + // AcceptTransparencyLog accepts Sigstore transparency log publication without prompting. + AcceptTransparencyLog bool +} + +// ServiceFactory builds the artifact service for the loaded configuration. Production composes the +// real repositories lazily, so building the service performs no network access, credential +// resolution, or directory creation. Tests inject a factory to run an entry point against a fake. +type ServiceFactory func(config *schema.AtmosConfiguration, options ServiceOptions) (artifact.Service, error) diff --git a/pkg/artifact/linkpath/linkpath.go b/pkg/artifact/linkpath/linkpath.go index e20aac6b8f3..75089edf116 100644 --- a/pkg/artifact/linkpath/linkpath.go +++ b/pkg/artifact/linkpath/linkpath.go @@ -70,6 +70,57 @@ func absolutePath(target string) (string, error) { func Escapes(links map[string]string, from string) bool { defer perf.Track(nil, "linkpath.Escapes")() + _, outcome := followRecorded(links, from, ignoreVisit) + return outcome == recordedEscaped +} + +// ResolveRecorded returns the slash-separated path, relative to the tree root, that from refers to +// once every recorded link on the way is followed. Like Escapes it reads only links, never the +// disk, so a caller can test where a captured link lands without looking at the source again. The +// root itself resolves to the empty path. ResolveRecorded reports false when the chain leaves the +// tree or loops, because neither names a path inside the tree. +func ResolveRecorded(links map[string]string, from string) (string, bool) { + defer perf.Track(nil, "linkpath.ResolveRecorded")() + + resolved, outcome := followRecorded(links, from, ignoreVisit) + if outcome != recordedInside { + return "", false + } + return strings.Join(resolved, "/"), true +} + +// TraversedRecorded returns, in the order the filesystem looks them up, the slash-separated path of +// every entry that resolving from through the recorded links passes through, relative to the tree +// root. A parent component is applied only after the directory before it was looked up, so +// "cache/p/../x" traverses "cache" and "cache/p" even though it reduces to "x". Paths reached +// through a recorded link are reported by the physical location they resolve to. It reports false +// when the chain leaves the tree or loops. +func TraversedRecorded(links map[string]string, from string) ([]string, bool) { + defer perf.Track(nil, "linkpath.TraversedRecorded")() + + var traversed []string + _, outcome := followRecorded(links, from, func(visited string) { traversed = append(traversed, visited) }) + return traversed, outcome == recordedInside +} + +// ignoreVisit is the visitor of callers that only need where a path ends up. +func ignoreVisit(string) {} + +// recordedOutcome is how the resolution of a path through recorded links ended. +type recordedOutcome int + +const ( + // The path resolved to a location inside the tree. + recordedInside recordedOutcome = iota + // The path climbed above the tree root. + recordedEscaped + // The link chain did not terminate. + recordedLooped +) + +// followRecorded resolves from through the recorded links and returns the components reached. It +// calls visit with the path of each entry it looks up that is not itself a recorded link. +func followRecorded(links map[string]string, from string, visit func(string)) ([]string, recordedOutcome) { pending := splitComponents(from) var resolved []string for hops := 0; len(pending) > 0; { @@ -79,7 +130,7 @@ func Escapes(links map[string]string, from string) bool { case ".": case parentComponent: if len(resolved) == 0 { - return true + return nil, recordedEscaped } resolved = resolved[:len(resolved)-1] default: @@ -87,16 +138,17 @@ func Escapes(links map[string]string, from string) bool { target, isLink := links[current] if !isLink { resolved = append(resolved, component) + visit(current) continue } hops++ if hops > maxLinkHops { - return false + return nil, recordedLooped } pending = append(splitComponents(target), pending...) } } - return false + return resolved, recordedInside } // walker resolves a path one component at a time. diff --git a/pkg/artifact/oci/assemble.go b/pkg/artifact/oci/assemble.go index 7b3326769de..9a1888c164c 100644 --- a/pkg/artifact/oci/assemble.go +++ b/pkg/artifact/oci/assemble.go @@ -99,6 +99,9 @@ func stagedLayers(prepared *artifact.PreparedArtifact) (stagedBlob, []stagedBlob if !ok { return stagedBlob{}, nil, fmt.Errorf("%w: OCI repositories store tar, tgz, or zip archives, not %q", errUtils.ErrArtifactPackagingInvalid, prepared.Packaging) } + if prepared.Packaging == artifact.PackagingCatalog && prepared.Record.Kind != artifact.KindCatalog { + return stagedBlob{}, nil, fmt.Errorf("%w: catalog packaging is reserved for Records of kind %q, not %q", errUtils.ErrArtifactPackagingInvalid, artifact.KindCatalog, prepared.Record.Kind) + } var ( payload *stagedBlob documents = map[artifact.DocumentRole]stagedBlob{} @@ -182,6 +185,8 @@ func payloadTitle(packaging artifact.Packaging) string { return "content.tar" case artifact.PackagingZip: return "content.zip" + case artifact.PackagingCatalog: + return "catalog.json" default: return "content.tar.gz" } diff --git a/pkg/artifact/oci/classify.go b/pkg/artifact/oci/classify.go index 7542a759ab6..4fcacd546c1 100644 --- a/pkg/artifact/oci/classify.go +++ b/pkg/artifact/oci/classify.go @@ -195,6 +195,8 @@ func packagingOf(mediaType string) (artifact.Packaging, bool) { return artifact.PackagingTgz, true case artifact.MediaTypeZip: return artifact.PackagingZip, true + case artifact.MediaTypeCatalogPayload: + return artifact.PackagingCatalog, true default: return "", false } @@ -209,6 +211,8 @@ func mediaTypeOf(packaging artifact.Packaging) (string, bool) { return artifact.MediaTypeTarGzip, true case artifact.PackagingZip: return artifact.MediaTypeZip, true + case artifact.PackagingCatalog: + return artifact.MediaTypeCatalogPayload, true default: return "", false } diff --git a/pkg/artifact/oci/list.go b/pkg/artifact/oci/list.go index b5fbde30018..93829d704ae 100644 --- a/pkg/artifact/oci/list.go +++ b/pkg/artifact/oci/list.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/opencontainers/go-digest" "golang.org/x/sync/errgroup" "github.com/cloudposse/atmos/pkg/artifact" @@ -21,6 +22,8 @@ const cursorSeparator = "|" // returns an unsupported-capability error that says direct named operations still work, never an // empty page. Only Atmos packages are listed: legacy plugin releases and foreign images that // share the prefix are never presented as generic artifacts. +// +//nolint:gocritic // The query is passed by value because artifact.Repository fixes the signature. func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { defer perf.Track(nil, "oci.Repository.List")() @@ -38,7 +41,7 @@ func (r *Repository) List(ctx context.Context, query artifact.ListQuery) (*artif if cursor.skipsName(name) { continue } - items, err := r.entriesOf(ctx, name, query.Labels, cursor) + items, err := r.entriesOf(ctx, name, query.Labels, query.Known, cursor) if err != nil { return nil, err } @@ -82,8 +85,9 @@ func (r *Repository) namesFor(ctx context.Context, name string) ([]string, error } // entriesOf lists the Atmos package versions of one name, sorted by version, that match labels. -// Each tag costs a manifest read and a Record read, bounded by the configured concurrency. -func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifact.Label, cursor listCursor) ([]artifact.ListEntry, error) { +// Each remaining tag costs a manifest read and a Record read, bounded by concurrency, except +// a tag whose resolved digest is in known: its entry is reused and only the tag is resolved. +func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifact.Label, known map[digest.Digest]artifact.ListEntry, cursor listCursor) ([]artifact.ListEntry, error) { versions, err := r.be.versions(ctx, name) if err != nil { return nil, err @@ -97,7 +101,7 @@ func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifa continue } group.Go(func() error { - entry, err := r.entryOf(groupCtx, artifact.Ref{Name: name, Version: version}) + entry, err := r.entryOf(groupCtx, artifact.Ref{Name: name, Version: version}, known) found[index] = entry return err }) @@ -114,15 +118,24 @@ func (r *Repository) entriesOf(ctx context.Context, name string, labels []artifa return entries, nil } -// entryOf builds the list entry of one tag, or nil when the tag is not an Atmos package. -func (r *Repository) entryOf(ctx context.Context, ref artifact.Ref) (*artifact.ListEntry, error) { - st, loaded, err := r.resolveRoot(ctx, ref) +// entryOf builds the list entry of one tag, or nil when the tag is not an Atmos package. A tag +// whose resolved digest names a known entry for the same name and version reuses that entry. +func (r *Repository) entryOf(ctx context.Context, ref artifact.Ref, known map[digest.Digest]artifact.ListEntry) (*artifact.ListEntry, error) { + st, desc, err := r.resolveDescriptor(ctx, ref) if err != nil { if errorsIsNotFound(err) { return nil, nil } return nil, err } + if entry, reused := reusable(known, desc.Digest, ref); reused { + entry.Repository = r.alias + return &entry, nil + } + loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) + if err != nil { + return nil, err + } if loaded.class != classAtmos { return nil, nil } @@ -158,3 +171,13 @@ func matchesLabels(labels map[string]string, selectors []artifact.Label) bool { func recordMatches(record *artifact.Record, ref artifact.Ref) bool { return requireSubjectOf(record, ref) == nil } + +// reusable returns the known entry of a resolved digest when it describes the same name and version +// as the reference and was bound to that same digest; anything else must be read again. +func reusable(known map[digest.Digest]artifact.ListEntry, resolved digest.Digest, ref artifact.Ref) (artifact.ListEntry, bool) { + entry, found := known[resolved] + if !found || entry.Name != ref.Name || entry.Version != ref.Version || entry.Pinned.ResolvedDigest != resolved { + return artifact.ListEntry{}, false + } + return entry, true +} diff --git a/pkg/artifact/oci/list_test.go b/pkg/artifact/oci/list_test.go index e55029e94a2..8d883256438 100644 --- a/pkg/artifact/oci/list_test.go +++ b/pkg/artifact/oci/list_test.go @@ -5,6 +5,7 @@ import ( "net/http" "testing" + "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "oras.land/oras-go/v2/registry/remote/errcode" @@ -25,6 +26,7 @@ type failingCatalog struct { func (f failingCatalog) Repositories(context.Context, string) ([]string, error) { return nil, f.err } +//nolint:gocritic // Test helper; the query is a small literal. func listAll(t *testing.T, repo *Repository, query artifact.ListQuery) []string { t.Helper() var seen []string @@ -56,6 +58,24 @@ func TestCollectionListingIsScopedSortedAndPaged(t *testing.T) { }) } +func TestPaginationRetainsKnownDigestMetadata(t *testing.T) { + backends(t, func(t *testing.T, repo *Repository) { + for _, version := range []string{"1", "2", "3"} { + publish(t, repo, publishSpec{name: "app", version: version, content: version}) + } + all, err := repo.List(context.Background(), artifact.ListQuery{Name: "app"}) + require.NoError(t, err) + require.Len(t, all.Entries, 3) + known := make(map[digest.Digest]artifact.ListEntry) + for _, entry := range all.Entries { + entry.Labels = map[string]string{"cached": "yes"} + known[entry.Pinned.ResolvedDigest] = entry + } + query := artifact.ListQuery{Name: "app", Cursor: "app|1", PageSize: 1, Known: known, Labels: []artifact.Label{{Key: "cached", Value: "yes"}}} + assert.Equal(t, []string{"app:2", "app:3"}, listAll(t, repo, query), "pagination skips preceding versions while reusing cached metadata") + }) +} + func TestRegistryListingStaysInsideThePrefix(t *testing.T) { reg := newRegistry(t, nil) inside := newRegistryRepo(t, reg) diff --git a/pkg/artifact/oci/resolve.go b/pkg/artifact/oci/resolve.go index 641932158f9..14ec0a3a510 100644 --- a/pkg/artifact/oci/resolve.go +++ b/pkg/artifact/oci/resolve.go @@ -82,15 +82,29 @@ func (r *Repository) pinnedFor(ref artifact.Ref, loaded *root) artifact.PinnedRe // resolveRoot resolves a reference once and classifies the root it names. func (r *Repository) resolveRoot(ctx context.Context, ref artifact.Ref) (store, *root, error) { - if err := ref.Validate(); err != nil { + st, desc, err := r.resolveDescriptor(ctx, ref) + if err != nil { return nil, nil, err } - if err := ctx.Err(); err != nil { + loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) + if err != nil { return nil, nil, err } + return st, loaded, nil +} + +// resolveDescriptor resolves a reference to the descriptor of the root it names without reading +// the root, so a caller that already knows the root by digest can skip the read. +func (r *Repository) resolveDescriptor(ctx context.Context, ref artifact.Ref) (store, ocispec.Descriptor, error) { + if err := ref.Validate(); err != nil { + return nil, ocispec.Descriptor{}, err + } + if err := ctx.Err(); err != nil { + return nil, ocispec.Descriptor{}, err + } st, err := r.be.open(ctx, ref.Name, false) if err != nil { - return nil, nil, err + return nil, ocispec.Descriptor{}, err } var desc ocispec.Descriptor if ref.IsDigest() { @@ -99,13 +113,9 @@ func (r *Repository) resolveRoot(ctx context.Context, ref artifact.Ref) (store, desc, err = st.Resolve(ctx, r.be.reference(ref.Name, ref.Version)) } if err != nil { - return nil, nil, resolveError(ref, err) + return nil, ocispec.Descriptor{}, resolveError(ref, err) } - loaded, err := r.loadCachedRoot(ctx, st, ref.Name, desc) - if err != nil { - return nil, nil, err - } - return st, loaded, nil + return st, desc, nil } // rootOf re-reads the root of an already pinned reference by its resolved digest. diff --git a/pkg/artifact/operation.go b/pkg/artifact/operation.go index 7573fc266c3..868b4b5eef8 100644 --- a/pkg/artifact/operation.go +++ b/pkg/artifact/operation.go @@ -117,6 +117,9 @@ type OperationResult struct { // Size is the transferred or described payload size in bytes. Size int64 `json:"size,omitempty"` + // SizeMeasured distinguishes a measured empty payload from an unavailable size. + SizeMeasured bool `json:"-"` + // Duration is the wall-clock duration of the operation. Duration time.Duration `json:"duration_ns,omitempty"` diff --git a/pkg/artifact/packaging.go b/pkg/artifact/packaging.go index fd06445137b..b1f6bf73ec5 100644 --- a/pkg/artifact/packaging.go +++ b/pkg/artifact/packaging.go @@ -17,6 +17,10 @@ const ( // PackagingZip stores one zip archive. PackagingZip Packaging = "zip" + // PackagingCatalog stores one catalog JSON document as the direct payload layer. Only Records + // of kind "catalog" use it. + PackagingCatalog Packaging = "catalog" + // PackagingOCIGraph identifies an existing native OCI graph carried unchanged. PackagingOCIGraph Packaging = "oci-graph" ) diff --git a/pkg/artifact/prepared.go b/pkg/artifact/prepared.go index 741d5e30417..02348419c6f 100644 --- a/pkg/artifact/prepared.go +++ b/pkg/artifact/prepared.go @@ -45,6 +45,18 @@ type StagedBlob struct { Open func(ctx context.Context) (io.ReadCloser, error) } +// Exclusion records one source entry that a kind-specific producer left out of the payload, and the +// visible rule that excluded it. Exclusions are reported with the plan; they are not part of the +// Record, so they never affect its digests. +type Exclusion struct { + // Path is the slash-separated source path relative to the source root. A directory is listed + // once, without its descendants. + Path string `json:"path"` + + // Rule is the default rule that excluded the entry, for example ".terraform/**". + Rule string `json:"rule"` +} + // PreparedArtifact is the output of a [Producer]: a sealed Record plus staged blobs. The preparer // captured one UTC preparation time and one logical operation ID before the first attempt. // @@ -69,6 +81,9 @@ type PreparedArtifact struct { // Blobs are the staged payload and document blobs. Blobs []StagedBlob + // Exclusions lists the source entries a kind-specific producer left out of the payload. + Exclusions []Exclusion + // Cleanup releases temporary resources owned by this operation; may be nil. Cleanup func() error diff --git a/pkg/artifact/produce/doc.go b/pkg/artifact/produce/doc.go index b53fb20a97b..1952518e270 100644 --- a/pkg/artifact/produce/doc.go +++ b/pkg/artifact/produce/doc.go @@ -7,4 +7,11 @@ // payload either as per-file blobs or as a deterministic tar, tgz, or zip archive. The user's // source tree is never modified; the generated manifest.json exists only inside the packaged // archive and never appears in the Record's inventory. +// +// Kind-specific producers reuse that pipeline and add their own validation, metadata, and visible +// default exclusions. [NewScaffold] validates scaffold configuration (through a [ScaffoldValidator] +// that the scaffold command registers) without generating output or running hooks. +// [NewTerraformComponent] leaves out Terraform working directories, state, and saved plans and +// reports each exclusion with its rule. [RegisterDefaults] registers all three. The generic files +// producer never applies another kind's exclusions. package produce diff --git a/pkg/artifact/produce/exclusions.go b/pkg/artifact/produce/exclusions.go new file mode 100644 index 00000000000..9de227974c6 --- /dev/null +++ b/pkg/artifact/produce/exclusions.go @@ -0,0 +1,58 @@ +package produce + +import ( + "path" + "strings" + + "github.com/cloudposse/atmos/pkg/artifact/snapshot" + "github.com/cloudposse/atmos/pkg/perf" +) + +// dirRuleSuffix marks a rule that excludes a whole directory subtree, for example ".terraform/**". +const dirRuleSuffix = "/**" + +// terraformExclusions are the visible default exclusions of the component/terraform producer: +// Terraform working directories, state files, and saved plans. `.terraform.lock.hcl` is a different +// name and stays in the payload, because the dependency lock file belongs to the component. +var terraformExclusions = []string{ + ".terraform" + dirRuleSuffix, + "*.tfstate", + "*.tfstate.*", + "*.tfplan", + "*.tfplan.*", +} + +// TerraformExclusions returns the default exclusion rules of the component/terraform producer, in the +// form dry-run output reports them. Each rule matches at any depth below the source root: a rule +// ending in "/**" excludes every directory with that name and its contents, and any other rule is a +// glob on the entry name. The generic files producer and backups do not apply them. +func TerraformExclusions() []string { + defer perf.Track(nil, "produce.TerraformExclusions")() + + return append([]string(nil), terraformExclusions...) +} + +// excluderFor builds the snapshot excluder for rules. The rule text returned for a match is the +// rule exactly as listed, so reports name the rule a user can read in the documentation. +func excluderFor(rules []string) snapshot.Excluder { + return func(entryPath string, isDir bool) (string, bool) { + name := path.Base(entryPath) + for _, rule := range rules { + if matchRule(rule, name, isDir) { + return rule, true + } + } + return "", false + } +} + +// matchRule reports whether rule excludes the entry named name. Directory rules (a "/**" suffix) match +// directories by name. Name globs match files, links, and directories alike, so the workspace state +// directory `terraform.tfstate.d` is excluded with the state it holds. +func matchRule(rule, name string, isDir bool) bool { + if dir, isDirRule := strings.CutSuffix(rule, dirRuleSuffix); isDirRule { + return isDir && name == dir + } + matched, err := path.Match(rule, name) + return err == nil && matched +} diff --git a/pkg/artifact/produce/files.go b/pkg/artifact/produce/files.go index 625670f472f..80fb35221aa 100644 --- a/pkg/artifact/produce/files.go +++ b/pkg/artifact/produce/files.go @@ -17,6 +17,13 @@ type Producer struct { limits snapshot.Limits metadata MetadataSource snapshotOps []snapshot.Option + + // profile layers kind-specific validation, metadata, and visible default exclusions over the + // shared pipeline. It is nil for the generic files producer, which has none. + profile *kindProfile + + // scaffoldValidator is the injected scaffold validator of a scaffold producer. + scaffoldValidator ScaffoldValidator } var _ artifact.Producer = (*Producer)(nil) @@ -72,10 +79,14 @@ func Register(registry *artifact.ProducerRegistry, opts ...Option) error { return registry.Register(New(opts...)) } -// Kind returns [artifact.KindFiles]. +// Kind returns [artifact.KindFiles] for the generic producer, or the kind of the producer built by +// [NewScaffold] or [NewComponent]. func (p *Producer) Kind() string { defer perf.Track(nil, "produce.Producer.Kind")() + if p.profile != nil { + return p.profile.kind + } return artifact.KindFiles } @@ -86,7 +97,7 @@ func (p *Producer) Kind() string { func (p *Producer) Prepare(ctx context.Context, request artifact.PrepareRequest) (*artifact.PreparedArtifact, error) { defer perf.Track(nil, "produce.Producer.Prepare")() - if err := validatePrepareRequest(&request); err != nil { + if err := validatePrepareRequest(&request, p.Kind()); err != nil { return nil, err } meta, err := p.producerMetadata(ctx, request.Source) @@ -109,23 +120,37 @@ func (p *Producer) Prepare(ctx context.Context, request artifact.PrepareRequest) _ = snap.Close() return nil, err } + prepared.Exclusions = snap.Exclusions return prepared, nil } func (p *Producer) captureOptions() []snapshot.Option { opts := []snapshot.Option{snapshot.WithStagingRoot(p.stagingRoot), snapshot.WithLimits(p.limits)} + if p.profile != nil && len(p.profile.rules) > 0 { + opts = append(opts, snapshot.WithExcluder(excluderFor(p.profile.rules))) + } return append(opts, p.snapshotOps...) } +// producerMetadata returns the kind's own validated metadata, overlaid by an injected metadata +// source (whose non-empty values win). The generic producer has only the injected source. func (p *Producer) producerMetadata(ctx context.Context, source string) (ProducerMetadata, error) { + var base ProducerMetadata + if p.profile != nil && p.profile.inspect != nil { + inspected, err := p.profile.inspect(ctx, source) + if err != nil { + return ProducerMetadata{}, err + } + base = inspected + } if p.metadata == nil { - return ProducerMetadata{}, nil + return base, nil } - meta, err := p.metadata(ctx, source) + injected, err := p.metadata(ctx, source) if err != nil { return ProducerMetadata{}, fmt.Errorf("%w: read producer metadata for %q: %w", errUtils.ErrArtifactPackagingInvalid, source, err) } - return meta, nil + return overlayMetadata(&base, &injected), nil } // identity is the resolved name, version, and producer metadata of one preparation. @@ -149,7 +174,7 @@ func (p *Producer) prepareFrom(ctx context.Context, snap *snapshot.Snapshot, req record := artifact.Record{ Name: id.name, Version: id.version, - Kind: artifact.KindFiles, + Kind: p.Kind(), CreatedAt: artifact.NewTimestamp(request.PreparedAt), Metadata: artifact.Metadata{ Description: firstNonEmpty(request.Metadata.Description, id.meta.Description), @@ -219,9 +244,9 @@ func documentDescriptors(docs []stagedDocument) []artifact.Document { } // validatePrepareRequest checks the fields the service is required to supply. -func validatePrepareRequest(request *artifact.PrepareRequest) error { - if request.Kind != "" && request.Kind != artifact.KindFiles { - return fmt.Errorf("%w: the files producer cannot prepare kind %q", errUtils.ErrArtifactConfigInvalid, request.Kind) +func validatePrepareRequest(request *artifact.PrepareRequest, kind string) error { + if request.Kind != "" && request.Kind != kind { + return fmt.Errorf("%w: the %s producer cannot prepare kind %q", errUtils.ErrArtifactConfigInvalid, kind, request.Kind) } if request.OperationID == "" || request.PreparedAt.IsZero() { return fmt.Errorf("%w: the operation ID and preparation time are captured once by the service before preparing", errUtils.ErrArtifactConfigInvalid) diff --git a/pkg/artifact/produce/kinds.go b/pkg/artifact/produce/kinds.go new file mode 100644 index 00000000000..25a758a7ff0 --- /dev/null +++ b/pkg/artifact/produce/kinds.go @@ -0,0 +1,195 @@ +package produce + +import ( + "context" + "fmt" + "os" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// Artifact kinds supplied by the kind-specific producers. +const ( + // KindScaffold is the kind of a scaffold template package. + KindScaffold = "scaffold" + + // ComponentTerraform is the component type of a Terraform root module. + ComponentTerraform = "terraform" + + // KindComponentTerraform is the kind of a Terraform component package. + KindComponentTerraform = "component/" + ComponentTerraform +) + +// kindProfile is the kind-specific behavior layered over the shared preparation pipeline: the +// kind name, visible default exclusion rules, and an inspection that validates the source and +// supplies producer metadata. The pipeline itself (snapshot, reserved distribution directory, +// documents, Record sealing, and packaging) is shared with the generic files producer. +type kindProfile struct { + kind string + rules []string + inspect func(ctx context.Context, source string) (ProducerMetadata, error) +} + +//go:generate mockgen -package produce -destination mock_scaffold_validator.go github.com/cloudposse/atmos/pkg/artifact/produce ScaffoldValidator + +// ScaffoldValidator validates a scaffold source directory and returns the metadata the scaffold +// declares. It validates configuration only: it never generates output and never runs scaffold hooks. +// +// The implementation lives above this package (it needs the scaffold manifest loader), so the +// scaffold command registers it with [RegisterScaffoldValidator] when the binary starts. +type ScaffoldValidator interface { + // ValidateScaffold checks the scaffold at dir and returns its declared name, version, and + // description. An invalid scaffold returns an error wrapping [errUtils.ErrArtifactScaffoldInvalid]. + ValidateScaffold(ctx context.Context, dir string) (ProducerMetadata, error) +} + +// ScaffoldValidatorFunc adapts a function to a [ScaffoldValidator]. +type ScaffoldValidatorFunc func(ctx context.Context, dir string) (ProducerMetadata, error) + +// ValidateScaffold calls the function. +func (f ScaffoldValidatorFunc) ValidateScaffold(ctx context.Context, dir string) (ProducerMetadata, error) { + defer perf.Track(nil, "produce.ScaffoldValidatorFunc.ValidateScaffold")() + + return f(ctx, dir) +} + +var ( + registeredValidatorMu sync.RWMutex + registeredValidator ScaffoldValidator +) + +// RegisterScaffoldValidator installs the process-wide scaffold validator used by scaffold +// producers that were not given one with [WithScaffoldValidator]. Registering nil removes it. +func RegisterScaffoldValidator(validator ScaffoldValidator) { + defer perf.Track(nil, "produce.RegisterScaffoldValidator")() + + registeredValidatorMu.Lock() + defer registeredValidatorMu.Unlock() + registeredValidator = validator +} + +func currentScaffoldValidator() ScaffoldValidator { + registeredValidatorMu.RLock() + defer registeredValidatorMu.RUnlock() + return registeredValidator +} + +// WithScaffoldValidator injects the scaffold validator, taking precedence over the registered one. +func WithScaffoldValidator(validator ScaffoldValidator) Option { + defer perf.Track(nil, "produce.WithScaffoldValidator")() + + return func(p *Producer) { p.scaffoldValidator = validator } +} + +// NewScaffold returns the scaffold producer (kind "scaffold"). It validates the scaffold +// configuration at the source root before it packages anything, takes the artifact name, version, +// and description from the scaffold manifest when they are not given explicitly, and treats the +// root README as ordinary payload while the reserved .atmos/artifact directory stays distribution +// metadata. It excludes nothing by default: a scaffold ships exactly what its author selected. +func NewScaffold(opts ...Option) *Producer { + defer perf.Track(nil, "produce.NewScaffold")() + + p := New(opts...) + p.profile = &kindProfile{kind: KindScaffold, inspect: p.inspectScaffold} + return p +} + +// NewTerraformComponent returns the component/terraform producer. It excludes Terraform working +// directories, state files, and saved plans by default (see [TerraformExclusions]) and reports each +// exclusion with the plan. Publishing never runs Terraform and needs no stack. Scripts keep their +// executable bits and `.terraform.lock.hcl` stays in the payload. +func NewTerraformComponent(opts ...Option) *Producer { + defer perf.Track(nil, "produce.NewTerraformComponent")() + + p := New(opts...) + p.profile = &kindProfile{ + kind: KindComponentTerraform, + rules: TerraformExclusions(), + inspect: inspectComponent(ComponentTerraform), + } + return p +} + +// RegisterDefaults registers every built-in producer: the generic files producer and the scaffold +// and component/terraform producers. Options apply to each of them. +func RegisterDefaults(registry *artifact.ProducerRegistry, opts ...Option) error { + defer perf.Track(nil, "produce.RegisterDefaults")() + + for _, producer := range []*Producer{New(opts...), NewScaffold(opts...), NewTerraformComponent(opts...)} { + if err := registry.Register(producer); err != nil { + return err + } + } + return nil +} + +// inspectScaffold validates the scaffold source and returns the metadata it declares. +func (p *Producer) inspectScaffold(ctx context.Context, source string) (ProducerMetadata, error) { + if err := requireDirectory(KindScaffold, source); err != nil { + return ProducerMetadata{}, err + } + validator := p.scaffoldValidator + if validator == nil { + validator = currentScaffoldValidator() + } + if validator == nil { + return ProducerMetadata{}, fmt.Errorf("%w: no scaffold validator is registered, so a scaffold cannot be published from this binary", errUtils.ErrArtifactProducerUnavailable) + } + meta, err := validator.ValidateScaffold(ctx, source) + if err != nil { + return ProducerMetadata{}, err + } + return meta, nil +} + +// componentKindData is the typed kind data of a component Record. It records the component subtype +// in portable metadata, next to the kind name. +type componentKindData struct { + ComponentType string `json:"component_type"` +} + +// inspectComponent checks that the source is a directory and records the component type. A +// component carries no name or version that is safe to reuse: the name in component.yaml names the +// vendoring manifest, and its source version is the upstream version, so both stay explicit. +func inspectComponent(componentType string) func(context.Context, string) (ProducerMetadata, error) { + return func(_ context.Context, source string) (ProducerMetadata, error) { + if err := requireDirectory("component/"+componentType, source); err != nil { + return ProducerMetadata{}, err + } + data, err := artifact.NewKindData(componentKindData{ComponentType: componentType}) + if err != nil { + return ProducerMetadata{}, err + } + return ProducerMetadata{KindData: data}, nil + } +} + +// requireDirectory fails unless source is a directory (a symlink to one is accepted, like every +// top-level source). +func requireDirectory(kind, source string) error { + info, err := os.Stat(source) + if err != nil { + return fmt.Errorf("%w: %s source %q: %w", errUtils.ErrArtifactPackagingInvalid, kind, source, err) + } + if !info.IsDir() { + return fmt.Errorf("%w: a %s source must be a directory, not the file %q", errUtils.ErrArtifactPackagingInvalid, kind, source) + } + return nil +} + +// overlayMetadata returns base with every non-empty value of over applied. Labels merge, and over +// wins on a shared key. +func overlayMetadata(base, over *ProducerMetadata) ProducerMetadata { + merged := *base + merged.Name = firstNonEmpty(over.Name, base.Name) + merged.Version = firstNonEmpty(over.Version, base.Version) + merged.Description = firstNonEmpty(over.Description, base.Description) + merged.Labels = mergeLabels(base.Labels, over.Labels) + if len(over.KindData) > 0 { + merged.KindData = over.KindData + } + return merged +} diff --git a/pkg/artifact/produce/kinds_test.go b/pkg/artifact/produce/kinds_test.go new file mode 100644 index 00000000000..5869eb4e898 --- /dev/null +++ b/pkg/artifact/produce/kinds_test.go @@ -0,0 +1,352 @@ +package produce_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" +) + +// Compile-time guard for the producer metadata fields these tests rely on. +var _ = produce.ProducerMetadata{Name: "", Version: "", Description: "", Labels: nil, KindData: nil} + +const lockFile = ".terraform.lock.hcl" + +// writeComponent writes a Terraform component with an executable helper, a dependency lock file, +// documentation, and every kind of transient Terraform data. +func writeComponent(t *testing.T, dir string) { + t.Helper() + fixtures.WriteFile(t, dir, "main.tf", "resource \"null_resource\" \"x\" {}\n", 0o644) + fixtures.WriteFile(t, dir, lockFile, "# lock\n", 0o644) + fixtures.WriteFile(t, dir, "README.md", fixtures.RootReadme, 0o644) + fixtures.WriteFile(t, dir, "scripts/helper.sh", fixtures.HelperScript, 0o755) + fixtures.WriteFile(t, dir, "modules/inner/main.tf", "# inner\n", 0o644) + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/README.md", fixtures.PackageReadme, 0o644) + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/CHANGELOG.md", fixtures.PackageChangelog, 0o644) + + fixtures.WriteFile(t, dir, ".terraform/providers/registry/provider.bin", "binary\n", 0o755) + fixtures.WriteFile(t, dir, "modules/inner/.terraform/modules.json", "{}\n", 0o644) + fixtures.WriteFile(t, dir, "terraform.tfstate", "{}\n", 0o644) + fixtures.WriteFile(t, dir, "terraform.tfstate.backup", "{}\n", 0o644) + fixtures.WriteFile(t, dir, "terraform.tfstate.d/dev/terraform.tfstate", "{}\n", 0o644) + fixtures.WriteFile(t, dir, "deploy.tfplan", "plan\n", 0o644) + fixtures.WriteFile(t, dir, "deploy.tfplan.json", "{}\n", 0o644) +} + +func prepareKind(t *testing.T, producer *produce.Producer, req *artifact.PrepareRequest) *artifact.PreparedArtifact { + t.Helper() + prepared, err := producer.Prepare(context.Background(), *req) + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + return prepared +} + +func TestTerraformComponent_ExcludesTransientDataAndKeepsLockFileAndScripts(t *testing.T) { + src := t.TempDir() + writeComponent(t, src) + req := request(src, artifact.PackagingTgz) + req.Kind = produce.KindComponentTerraform + + prepared := prepareKind(t, produce.NewTerraformComponent(produce.WithStagingRoot(t.TempDir())), req) + record := prepared.Record + + assert.Equal(t, "component/terraform", record.Kind) + assert.Equal(t, []string{ + lockFile, "README.md", "main.tf", "modules", "modules/inner", "modules/inner/main.tf", "scripts", "scripts/helper.sh", + }, inventoryPaths(&record), "transient data is excluded; the lock file, scripts, and docs stay") + + wantExclusions := []artifact.Exclusion{ + {Path: ".terraform", Rule: ".terraform/**"}, + {Path: "deploy.tfplan", Rule: "*.tfplan"}, + {Path: "deploy.tfplan.json", Rule: "*.tfplan.*"}, + {Path: "modules/inner/.terraform", Rule: ".terraform/**"}, + {Path: "terraform.tfstate", Rule: "*.tfstate"}, + {Path: "terraform.tfstate.backup", Rule: "*.tfstate.*"}, + {Path: "terraform.tfstate.d", Rule: "*.tfstate.*"}, + } + assert.Equal(t, wantExclusions, prepared.Exclusions, "each exclusion is reported once, with the rule that excluded it") + + require.Len(t, record.Documents, 2, "package README and CHANGELOG stay distinct from the root README payload") + var data struct { + ComponentType string `json:"component_type"` + } + require.NoError(t, record.KindData.Decode(&data)) + assert.Equal(t, "terraform", data.ComponentType, "the component subtype is recorded in portable metadata") + + if fixtures.POSIXModes() { + for _, entry := range record.Inventory { + if entry.Path == "scripts/helper.sh" { + assert.Equal(t, uint32(0o755), entry.Mode, "scripts keep their executable bit") + } + } + } +} + +func TestFilesProducer_DoesNotInheritComponentExclusions(t *testing.T) { + src := t.TempDir() + writeComponent(t, src) + + prepared, _ := prepare(t, request(src, artifact.PackagingNone)) + + paths := inventoryPaths(&prepared.Record) + for _, want := range []string{".terraform", ".terraform/providers/registry/provider.bin", "terraform.tfstate", "terraform.tfstate.backup", "deploy.tfplan", "deploy.tfplan.json", "terraform.tfstate.d/dev/terraform.tfstate"} { + assert.Contains(t, paths, want, "generic files and backups keep every requested file") + } + assert.Empty(t, prepared.Exclusions) +} + +func TestTerraformComponent_RuleMatrix(t *testing.T) { + tests := []struct { + name string + path string + excluded bool + }{ + {name: "working directory file", path: ".terraform/terraform.tfstate.d/x", excluded: true}, + {name: "lock file is kept", path: lockFile, excluded: false}, + {name: "tfstate", path: "a/b/prod.tfstate", excluded: true}, + {name: "tfstate backup", path: "terraform.tfstate.1700000000.backup", excluded: true}, + {name: "tfplan", path: "x.tfplan", excluded: true}, + {name: "tfplan json", path: "x.tfplan.json", excluded: true}, + {name: "tfvars is kept", path: "prod.tfvars", excluded: false}, + {name: "tf is kept", path: "state.tf", excluded: false}, + {name: "name that only contains the word", path: "tfstate.md", excluded: false}, + {name: "a file called .terraform is not a directory rule", path: "docs/.terraformrc", excluded: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "main.tf", "# keep\n", 0o644) + fixtures.WriteFile(t, src, tt.path, "content\n", 0o644) + req := request(src, artifact.PackagingNone) + req.Kind = produce.KindComponentTerraform + + prepared := prepareKind(t, produce.NewTerraformComponent(produce.WithStagingRoot(t.TempDir())), req) + + exclusionPaths := make([]string, 0, len(prepared.Exclusions)) + for _, exclusion := range prepared.Exclusions { + exclusionPaths = append(exclusionPaths, exclusion.Path) + } + if tt.excluded { + assert.NotEmpty(t, exclusionPaths) + assert.NotContains(t, inventoryPaths(&prepared.Record), tt.path) + return + } + assert.Empty(t, exclusionPaths) + assert.Contains(t, inventoryPaths(&prepared.Record), filepath.ToSlash(tt.path)) + }) + } +} + +func TestTerraformComponent_ExclusionsReturnsACopy(t *testing.T) { + rules := produce.TerraformExclusions() + require.Equal(t, []string{".terraform/**", "*.tfstate", "*.tfstate.*", "*.tfplan", "*.tfplan.*"}, rules) + rules[0] = "mutated" + assert.Equal(t, ".terraform/**", produce.TerraformExclusions()[0]) +} + +func TestTerraformComponent_IdentityPrecedence(t *testing.T) { + src := filepath.Join(t.TempDir(), "vpc") + require.NoError(t, os.MkdirAll(src, 0o755)) + fixtures.WriteFile(t, src, "main.tf", "# vpc\n", 0o644) + // component.yaml names the vendoring manifest and its upstream version, so neither is reused. + fixtures.WriteFile(t, src, "component.yaml", "kind: ComponentVendorConfig\nmetadata:\n name: vpc-vendor-config\nspec:\n source:\n version: 9.9.9\n", 0o644) + producer := produce.NewTerraformComponent(produce.WithStagingRoot(t.TempDir())) + + t.Run("basename is the default name and the version stays explicit", func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Name = "" + prepared := prepareKind(t, producer, req) + assert.Equal(t, "vpc", prepared.Record.Name) + assert.Equal(t, "1.0.0", prepared.Record.Version) + }) + t.Run("a missing version fails without a generated one", func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Version = "" + _, err := producer.Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + }) + t.Run("an injected metadata source supplies name and version", func(t *testing.T) { + injected := produce.NewTerraformComponent(produce.WithStagingRoot(t.TempDir()), produce.WithMetadataSource( + func(context.Context, string) (produce.ProducerMetadata, error) { + return produce.ProducerMetadata{Name: "network/vpc", Version: "2.3.4", Description: "VPC"}, nil + }, + )) + req := request(src, artifact.PackagingNone) + req.Name, req.Version = "", "" + prepared := prepareKind(t, injected, req) + assert.Equal(t, "network/vpc", prepared.Record.Name) + assert.Equal(t, "2.3.4", prepared.Record.Version) + assert.Equal(t, "VPC", prepared.Record.Metadata.Description) + assert.NotEmpty(t, prepared.Record.KindData, "the component subtype survives an injected metadata source") + }) +} + +func TestKindProducers_RejectMismatchedKindsAndFileSources(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "main.tf", "# x\n", 0o644) + file := filepath.Join(src, "main.tf") + + t.Run("component rejects another kind", func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Kind = artifact.KindFiles + _, err := produce.NewTerraformComponent().Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("files rejects a component kind", func(t *testing.T) { + req := request(src, artifact.PackagingNone) + req.Kind = produce.KindComponentTerraform + _, err := produce.New().Prepare(context.Background(), *req) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("component requires a directory", func(t *testing.T) { + _, err := produce.NewTerraformComponent().Prepare(context.Background(), *request(file, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) + t.Run("scaffold requires a directory", func(t *testing.T) { + validator := produce.ScaffoldValidatorFunc(func(context.Context, string) (produce.ProducerMetadata, error) { + t.Fatal("the validator must not run for a file source") + return produce.ProducerMetadata{}, nil + }) + _, err := produce.NewScaffold(produce.WithScaffoldValidator(validator)).Prepare(context.Background(), *request(file, artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) + t.Run("a missing component source fails", func(t *testing.T) { + _, err := produce.NewTerraformComponent().Prepare(context.Background(), *request(filepath.Join(src, "absent"), artifact.PackagingNone)) + require.ErrorIs(t, err, errUtils.ErrArtifactPackagingInvalid) + }) +} + +// scaffoldValidator records the directories it validated and returns canned results. +type scaffoldValidator struct { + meta produce.ProducerMetadata + err error + calls []string +} + +func (v *scaffoldValidator) ValidateScaffold(_ context.Context, dir string) (produce.ProducerMetadata, error) { + v.calls = append(v.calls, dir) + return v.meta, v.err +} + +// writeScaffold writes a scaffold with a rendered root README (payload) and distinct package documents. +func writeScaffold(t *testing.T, dir string) { + t.Helper() + fixtures.WriteFile(t, dir, "scaffold.yaml", "apiVersion: atmos/v1\nkind: AtmosScaffoldConfig\nmetadata:\n name: landing-zone\n", 0o644) + fixtures.WriteFile(t, dir, "README.md", "# {{ .project }}\n", 0o644) + fixtures.WriteFile(t, dir, "main.tf.tmpl", "# template\n", 0o644) + fixtures.WriteFile(t, dir, "bin/setup.sh", fixtures.HelperScript, 0o755) + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/README.md", fixtures.PackageReadme, 0o644) + fixtures.WriteFile(t, dir, fixtures.ReservedDir+"/CHANGELOG.md", fixtures.PackageChangelog, 0o644) +} + +func TestScaffoldProducer_ValidatesAndSuppliesIdentity(t *testing.T) { + src := t.TempDir() + writeScaffold(t, src) + validator := &scaffoldValidator{meta: produce.ProducerMetadata{Name: "scaffolds/landing-zone", Version: "2.0.0", Description: "A landing zone"}} + producer := produce.NewScaffold(produce.WithScaffoldValidator(validator), produce.WithStagingRoot(t.TempDir())) + + t.Run("name, version, and description default from the scaffold manifest", func(t *testing.T) { + req := request(src, artifact.PackagingTgz) + req.Name, req.Version, req.Kind = "", "", produce.KindScaffold + prepared := prepareKind(t, producer, req) + record := prepared.Record + + assert.Equal(t, "scaffold", record.Kind) + assert.Equal(t, "scaffolds/landing-zone", record.Name) + assert.Equal(t, "2.0.0", record.Version) + assert.Equal(t, "A landing zone", record.Metadata.Description) + assert.Equal(t, []string{src}, validator.calls[len(validator.calls)-1:], "validation reads the source, not a copy") + assert.Empty(t, prepared.Exclusions, "a scaffold ships exactly what its author selected") + }) + t.Run("explicit flags win over manifest metadata", func(t *testing.T) { + req := request(src, artifact.PackagingTgz) + req.Name, req.Version = "mine/zone", "3.0.0" + prepared := prepareKind(t, producer, req) + assert.Equal(t, "mine/zone", prepared.Record.Name) + assert.Equal(t, "3.0.0", prepared.Record.Version) + }) + t.Run("root README is payload and the package documents are distinct", func(t *testing.T) { + prepared := prepareKind(t, producer, request(src, artifact.PackagingNone)) + paths := inventoryPaths(&prepared.Record) + assert.Contains(t, paths, "README.md") + assert.Contains(t, paths, "scaffold.yaml") + for _, path := range paths { + assert.NotContains(t, path, ".atmos", "reserved distribution metadata is not scaffold payload: %s", path) + } + require.Len(t, prepared.Record.Documents, 2) + assert.Equal(t, ".atmos/artifact/CHANGELOG.md", prepared.Record.Documents[0].SourcePath) + assert.Equal(t, ".atmos/artifact/README.md", prepared.Record.Documents[1].SourcePath) + }) +} + +func TestScaffoldProducer_ValidationFailureStopsPublication(t *testing.T) { + src := t.TempDir() + writeScaffold(t, src) + cause := errors.New("metadata.name is required") + validator := produce.NewMockScaffoldValidator(gomock.NewController(t)) + validator.EXPECT().ValidateScaffold(gomock.Any(), src).Return(produce.ProducerMetadata{}, errors.Join(errUtils.ErrArtifactScaffoldInvalid, cause)) + staging := t.TempDir() + + _, err := produce.NewScaffold(produce.WithScaffoldValidator(validator), produce.WithStagingRoot(staging)). + Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + + require.ErrorIs(t, err, errUtils.ErrArtifactScaffoldInvalid) + require.ErrorIs(t, err, cause) + entries, readErr := os.ReadDir(staging) + require.NoError(t, readErr) + assert.Empty(t, entries, "a rejected scaffold leaves no staged data behind") +} + +func TestScaffoldProducer_ValidatorSelection(t *testing.T) { + src := t.TempDir() + writeScaffold(t, src) + + t.Run("a missing validator fails loudly", func(t *testing.T) { + produce.RegisterScaffoldValidator(nil) + _, err := produce.NewScaffold().Prepare(context.Background(), *request(src, artifact.PackagingTgz)) + require.ErrorIs(t, err, errUtils.ErrArtifactProducerUnavailable) + }) + t.Run("the registered validator is used and an injected one wins", func(t *testing.T) { + registered := &scaffoldValidator{meta: produce.ProducerMetadata{Version: "1.1.1"}} + produce.RegisterScaffoldValidator(registered) + t.Cleanup(func() { produce.RegisterScaffoldValidator(nil) }) + + prepared := prepareKind(t, produce.NewScaffold(produce.WithStagingRoot(t.TempDir())), requestWithoutVersion(src)) + assert.Equal(t, "1.1.1", prepared.Record.Version) + assert.Len(t, registered.calls, 1) + + injected := &scaffoldValidator{meta: produce.ProducerMetadata{Version: "2.2.2"}} + prepared = prepareKind(t, produce.NewScaffold(produce.WithScaffoldValidator(injected), produce.WithStagingRoot(t.TempDir())), requestWithoutVersion(src)) + assert.Equal(t, "2.2.2", prepared.Record.Version) + assert.Len(t, registered.calls, 1, "the registered validator is not consulted when one is injected") + }) +} + +func requestWithoutVersion(src string) *artifact.PrepareRequest { + req := request(src, artifact.PackagingTgz) + req.Version = "" + return req +} + +func TestRegisterDefaults(t *testing.T) { + registry := artifact.NewProducerRegistry() + require.NoError(t, produce.RegisterDefaults(registry)) + assert.Equal(t, []string{"component/terraform", "files", "scaffold"}, registry.Kinds()) + + for _, kind := range registry.Kinds() { + producer, ok := registry.Lookup(kind) + require.True(t, ok) + assert.Equal(t, kind, producer.Kind()) + } + require.ErrorIs(t, produce.RegisterDefaults(registry), errUtils.ErrArtifactConflict, "registering twice is an error") +} diff --git a/pkg/artifact/produce/mock_scaffold_validator.go b/pkg/artifact/produce/mock_scaffold_validator.go new file mode 100644 index 00000000000..ce6854735b6 --- /dev/null +++ b/pkg/artifact/produce/mock_scaffold_validator.go @@ -0,0 +1,56 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/produce (interfaces: ScaffoldValidator) +// +// Generated by this command: +// +// mockgen -package produce -destination mock_scaffold_validator.go github.com/cloudposse/atmos/pkg/artifact/produce ScaffoldValidator +// + +// Package produce is a generated GoMock package. +package produce + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockScaffoldValidator is a mock of ScaffoldValidator interface. +type MockScaffoldValidator struct { + ctrl *gomock.Controller + recorder *MockScaffoldValidatorMockRecorder + isgomock struct{} +} + +// MockScaffoldValidatorMockRecorder is the mock recorder for MockScaffoldValidator. +type MockScaffoldValidatorMockRecorder struct { + mock *MockScaffoldValidator +} + +// NewMockScaffoldValidator creates a new mock instance. +func NewMockScaffoldValidator(ctrl *gomock.Controller) *MockScaffoldValidator { + mock := &MockScaffoldValidator{ctrl: ctrl} + mock.recorder = &MockScaffoldValidatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockScaffoldValidator) EXPECT() *MockScaffoldValidatorMockRecorder { + return m.recorder +} + +// ValidateScaffold mocks base method. +func (m *MockScaffoldValidator) ValidateScaffold(ctx context.Context, dir string) (ProducerMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateScaffold", ctx, dir) + ret0, _ := ret[0].(ProducerMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ValidateScaffold indicates an expected call of ValidateScaffold. +func (mr *MockScaffoldValidatorMockRecorder) ValidateScaffold(ctx, dir any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateScaffold", reflect.TypeOf((*MockScaffoldValidator)(nil).ValidateScaffold), ctx, dir) +} diff --git a/pkg/artifact/report/collector.go b/pkg/artifact/report/collector.go new file mode 100644 index 00000000000..4067dcf4cc5 --- /dev/null +++ b/pkg/artifact/report/collector.go @@ -0,0 +1,46 @@ +package report + +import ( + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/perf" +) + +//go:generate mockgen -destination=mock_collector.go -package=report github.com/cloudposse/atmos/pkg/artifact/report Collector + +// Entry is one completed or failed operation attempt, recorded at an execution boundary. +type Entry struct { + // Step is the name of the step that ran the operation, or empty outside a step. + Step string + + // OperationID is the logical operation identifier. Every retry attempt of one operation + // carries the same ID, so a renderer deduplicates by it and keeps the final attempt. + OperationID string + + // Attempt counts the attempts of this logical operation, starting at one. + Attempt int + + // Outcome is the typed outcome of single-artifact actions. It is nil for list and search. + Outcome *invoke.Outcome + + // Err is the operation error, or nil on success. A failed attempt still carries the partial + // result in Outcome. + Err error +} + +// Collector receives the entries that artifact operations produce. Implementations must be safe +// for concurrent use: steps in a parallel group record concurrently. +type Collector interface { + // Record stores one entry. It must not block the operation and must not fail it: reporting + // problems are the collector's own to log. + Record(entry *Entry) +} + +// Discard is the default collector. It ignores every entry, so an unconfigured runner pays nothing. +type Discard struct{} + +// Record implements [Collector]. +func (Discard) Record(*Entry) { + defer perf.Track(nil, "report.Discard.Record")() +} + +var _ Collector = Discard{} diff --git a/pkg/artifact/report/collector_test.go b/pkg/artifact/report/collector_test.go new file mode 100644 index 00000000000..c06c0979e8d --- /dev/null +++ b/pkg/artifact/report/collector_test.go @@ -0,0 +1,28 @@ +package report_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifact/report" +) + +func TestDiscardIgnoresEveryEntry(t *testing.T) { + var collector report.Collector = report.Discard{} + assert.NotPanics(t, func() { + collector.Record(&report.Entry{Step: "publish", OperationID: "op-1", Attempt: 1}) + collector.Record(&report.Entry{Err: errors.New("failed"), Outcome: &invoke.Outcome{Result: &invoke.Result{}}}) + }) +} + +func TestCollectorMockRecordsEntries(t *testing.T) { + mock := report.NewMockCollector(gomock.NewController(t)) + entry := &report.Entry{Step: "publish", OperationID: "op-1", Attempt: 2} + mock.EXPECT().Record(entry) + var collector report.Collector = mock + collector.Record(entry) +} diff --git a/pkg/artifact/report/doc.go b/pkg/artifact/report/doc.go new file mode 100644 index 00000000000..acb3b1aacf5 --- /dev/null +++ b/pkg/artifact/report/doc.go @@ -0,0 +1,9 @@ +// Package report defines the seam through which artifact operations report their results to a +// summary renderer. +// +// This package holds only the [Collector] interface and a no-op default. Execution boundaries +// (the artifact step handler, and in the future the CLI) record each completed or failed +// operation into an injected collector; a renderer implements the interface and decides what to +// show. Collectors are passed explicitly through runner services and options, never through +// context values. +package report diff --git a/pkg/artifact/report/mock_collector.go b/pkg/artifact/report/mock_collector.go new file mode 100644 index 00000000000..751468ed24d --- /dev/null +++ b/pkg/artifact/report/mock_collector.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudposse/atmos/pkg/artifact/report (interfaces: Collector) +// +// Generated by this command: +// +// mockgen -destination=mock_collector.go -package=report github.com/cloudposse/atmos/pkg/artifact/report Collector +// + +// Package report is a generated GoMock package. +package report + +import ( + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockCollector is a mock of Collector interface. +type MockCollector struct { + ctrl *gomock.Controller + recorder *MockCollectorMockRecorder + isgomock struct{} +} + +// MockCollectorMockRecorder is the mock recorder for MockCollector. +type MockCollectorMockRecorder struct { + mock *MockCollector +} + +// NewMockCollector creates a new mock instance. +func NewMockCollector(ctrl *gomock.Controller) *MockCollector { + mock := &MockCollector{ctrl: ctrl} + mock.recorder = &MockCollectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCollector) EXPECT() *MockCollectorMockRecorder { + return m.recorder +} + +// Record mocks base method. +func (m *MockCollector) Record(entry *Entry) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Record", entry) +} + +// Record indicates an expected call of Record. +func (mr *MockCollectorMockRecorder) Record(entry any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Record", reflect.TypeOf((*MockCollector)(nil).Record), entry) +} diff --git a/pkg/artifact/repository_types.go b/pkg/artifact/repository_types.go index 3fa92a29840..feff0a5164a 100644 --- a/pkg/artifact/repository_types.go +++ b/pkg/artifact/repository_types.go @@ -156,6 +156,12 @@ type ListQuery struct { // Refresh bypasses cached discovery snapshots. Refresh bool + + // Known maps resolved digests to entries listed earlier (for example from a previous discovery + // snapshot). A repository may reuse a known entry instead of re-reading the metadata of a + // reference whose resolved digest is unchanged; it must still resolve every reference, so a + // disappeared or moved reference is never kept. Repositories that cannot reuse metadata ignore it. + Known map[digest.Digest]ListEntry } // ListEntry summarizes one artifact in a listing. diff --git a/pkg/artifact/requests.go b/pkg/artifact/requests.go index a2b19a4f13c..0c027464498 100644 --- a/pkg/artifact/requests.go +++ b/pkg/artifact/requests.go @@ -2,6 +2,9 @@ package artifact // PublishRequest publishes a local source to one repository. type PublishRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -35,6 +38,9 @@ type PublishRequest struct { // PullRequest materializes an artifact into a local directory. type PullRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -50,6 +56,9 @@ type PullRequest struct { // CopyRequest copies an artifact between two configured repositories. type CopyRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Ref selects the artifact in the source repository. Ref Ref @@ -65,6 +74,9 @@ type CopyRequest struct { // InspectRequest inspects an artifact without downloading its payload. type InspectRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -104,6 +116,9 @@ type SearchRequest struct { // DeleteRequest deletes an artifact from one repository. type DeleteRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -113,6 +128,9 @@ type DeleteRequest struct { // ShowDocumentRequest reads one package document. type ShowDocumentRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -125,6 +143,9 @@ type ShowDocumentRequest struct { // SignRequest adds signature evidence to an immutable subject. type SignRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -134,6 +155,9 @@ type SignRequest struct { // VerifyRequest verifies an artifact's signature evidence against the effective policy. type VerifyRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string @@ -143,6 +167,9 @@ type VerifyRequest struct { // PinRequest creates an immutable reference for consumer lockfiles. type PinRequest struct { + // Execution carries the wrapper-owned operation identity and retry budget. + Execution Execution + // Repository is the required configured repository alias. Repository string diff --git a/pkg/artifact/results.go b/pkg/artifact/results.go index ee7e34a3762..e3a6d576490 100644 --- a/pkg/artifact/results.go +++ b/pkg/artifact/results.go @@ -21,6 +21,13 @@ type RepositoryListing struct { // Count is the number of entries returned from the repository. Count int `json:"count"` + // Source says where the entries came from: "live" (read from the repository), "cache" (a + // discovery snapshot), or "catalog" (the repository's configured catalog). + Source string `json:"source,omitempty"` + + // FetchedAt is when a cached or catalog snapshot was fetched, in RFC 3339 UTC. + FetchedAt string `json:"fetched_at,omitempty"` + // Diagnostics explain capability or refresh problems. Diagnostics []Diagnostic `json:"diagnostics,omitempty"` } @@ -33,6 +40,10 @@ type ListResult struct { // Repositories reports per-repository outcomes. Repositories []RepositoryListing `json:"repositories,omitempty"` + + // Incomplete is true when at least one selected repository could not be listed, so Artifacts + // must not be read as the whole collection. Each failing repository carries an error diagnostic. + Incomplete bool `json:"incomplete,omitempty"` } // DocumentResult is the result of Service.ShowDocument. Content is bounded by [MaxDocumentSize]. diff --git a/pkg/artifact/roundtriptest/roundtriptest.go b/pkg/artifact/roundtriptest/roundtriptest.go new file mode 100644 index 00000000000..e6600781d38 --- /dev/null +++ b/pkg/artifact/roundtriptest/roundtriptest.go @@ -0,0 +1,127 @@ +// Package roundtriptest publishes artifacts to an in-process OCI registry so that tests of the +// existing consumers (component source provisioning, scaffold generation, and vendoring) can prove +// a published package works through them. It composes the real artifact service and records every +// registry request, so a test can also prove which blobs a consumer actually fetched. +// +// It is test support: it depends on the testing package and must not be imported by production code. +package roundtriptest + +import ( + "context" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/compose" + "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Alias is the repository alias of the registry published to. +const Alias = "registry" + +// Prefix is the repository path prefix below the registry host. +const Prefix = "acme/atmos" + +// Registry is an in-process registry with a composed artifact service publishing to it. +type Registry struct { + // Host is the "127.0.0.1:PORT" registry address. + Host string + + // Service is the real artifact service bound to the registry. + Service artifact.Service + + mu sync.Mutex + requests []string +} + +// anonymous resolves every registry anonymously, never reading the developer's Docker configuration. +type anonymous struct{} + +func (anonymous) Resolve(authn.Resource) (authn.Authenticator, error) { + defer perf.Track(nil, "roundtriptest.anonymous.Resolve")() + + return authn.Anonymous, nil +} + +// New starts the registry and composes the artifact service with an oci/registry repository named +// [Alias] bound to it. Extra compose options apply after the defaults. +func New(t *testing.T, opts ...compose.Option) *Registry { + defer perf.Track(nil, "roundtriptest.New")() + + t.Helper() + reg := &Registry{} + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0)), registry.WithReferrersSupport(true)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reg.mu.Lock() + reg.requests = append(reg.requests, r.Method+" "+r.URL.Path) + reg.mu.Unlock() + handler.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + reg.Host = strings.TrimPrefix(server.URL, "http://") + + config := &schema.AtmosConfiguration{Artifacts: schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + Alias: {Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": reg.Host, "prefix": Prefix}}, + }}} + factory := func(context.Context, string, *schema.ArtifactRepositoryConfig, artifact.IdentityResolver) (oci.Transport, error) { + return oci.NewClient(nil, oci.WithKeychain(anonymous{})), nil + } + composition, err := compose.New(config, append([]compose.Option{compose.WithTransportFactory(factory)}, opts...)...) + require.NoError(t, err) + reg.Service = composition.Service + return reg +} + +// Publish publishes request to the registry. The repository is always [Alias]. +func (r *Registry) Publish(t *testing.T, request *artifact.PublishRequest) *artifact.OperationResult { + defer perf.Track(nil, "roundtriptest.Registry.Publish")() + + t.Helper() + request.Repository = Alias + result, err := r.Service.Publish(context.Background(), *request) + require.NoError(t, err) + return result +} + +// Image returns the OCI image reference (without a scheme) of a published name and version. +func (r *Registry) Image(name, version string) string { + defer perf.Track(nil, "roundtriptest.Registry.Image")() + + return r.Host + "/" + Prefix + "/" + name + ":" + version +} + +// URI returns the oci:// source URI of a published name and version, as consumers declare it. +func (r *Registry) URI(name, version string) string { + defer perf.Track(nil, "roundtriptest.Registry.URI")() + + return "oci://" + r.Image(name, version) +} + +// Requests returns the "METHOD path" of every request the registry served so far. +func (r *Registry) Requests() []string { + defer perf.Track(nil, "roundtriptest.Registry.Requests")() + + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.requests...) +} + +// ResetRequests forgets the recorded requests, so a test can observe one consumer in isolation. +func (r *Registry) ResetRequests() { + defer perf.Track(nil, "roundtriptest.Registry.ResetRequests")() + + r.mu.Lock() + defer r.mu.Unlock() + r.requests = nil +} diff --git a/pkg/artifact/scaffold/scaffold.go b/pkg/artifact/scaffold/scaffold.go new file mode 100644 index 00000000000..a56d7c026ba --- /dev/null +++ b/pkg/artifact/scaffold/scaffold.go @@ -0,0 +1,64 @@ +// Package scaffold validates scaffold sources for publication. +// +// Validation loads the scaffold manifest (scaffold.yaml at the source root) through the same +// loader that generation and `atmos scaffold validate` use, so a published scaffold is exactly as +// valid as a generated one. It parses and validates configuration only: it never renders templates, +// writes generated output, or runs scaffold hooks. +// +// The package sits above pkg/artifact/produce in the dependency graph because the scaffold loader +// pulls in the generator, hook, and workflow packages. The scaffold command registers the validator +// with [Register] when the binary starts. +package scaffold + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/project/config" +) + +// Validator validates scaffold sources for the scaffold producer. +type Validator struct{} + +var _ produce.ScaffoldValidator = Validator{} + +// Register installs the validator as the process-wide scaffold validator of the producers. +func Register() { + defer perf.Track(nil, "scaffold.Register")() + + produce.RegisterScaffoldValidator(Validator{}) +} + +// ValidateScaffold validates the scaffold at dir and returns the name, version, and description its +// manifest declares. The manifest must be scaffold.yaml at the root, because generation looks only +// there. +func (Validator) ValidateScaffold(ctx context.Context, dir string) (produce.ProducerMetadata, error) { + defer perf.Track(nil, "scaffold.Validator.ValidateScaffold")() + + if err := ctx.Err(); err != nil { + return produce.ProducerMetadata{}, err + } + manifestPath := filepath.Join(dir, config.ScaffoldConfigFileName) + content, err := os.ReadFile(manifestPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return produce.ProducerMetadata{}, fmt.Errorf("%w: %s does not contain %s at its root", errUtils.ErrArtifactScaffoldInvalid, dir, config.ScaffoldConfigFileName) + } + return produce.ProducerMetadata{}, fmt.Errorf("%w: read %s: %w", errUtils.ErrArtifactScaffoldInvalid, manifestPath, err) + } + manifest, err := config.LoadScaffoldConfigFromContent(string(content), config.WithSourceDir(dir)) + if err != nil { + return produce.ProducerMetadata{}, fmt.Errorf("%w: %s: %w", errUtils.ErrArtifactScaffoldInvalid, manifestPath, err) + } + return produce.ProducerMetadata{ + Name: manifest.Metadata.Name, + Version: manifest.Metadata.Version, + Description: manifest.Metadata.Description, + }, nil +} diff --git a/pkg/artifact/scaffold/scaffold_test.go b/pkg/artifact/scaffold/scaffold_test.go new file mode 100644 index 00000000000..dc96542ad98 --- /dev/null +++ b/pkg/artifact/scaffold/scaffold_test.go @@ -0,0 +1,134 @@ +package scaffold_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/produce" + "github.com/cloudposse/atmos/pkg/artifact/scaffold" +) + +const validManifest = `apiVersion: atmos/v1 +kind: AtmosScaffoldConfig +metadata: + name: landing-zone + description: A landing zone + version: 2.0.0 +spec: + fields: + - name: project_name + type: input + default: demo +` + +func TestValidator_ReadsManifestMetadata(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "scaffold.yaml", validManifest, 0o644) + + meta, err := scaffold.Validator{}.ValidateScaffold(context.Background(), dir) + + require.NoError(t, err) + assert.Equal(t, produce.ProducerMetadata{Name: "landing-zone", Version: "2.0.0", Description: "A landing zone"}, meta) +} + +func TestValidator_RejectsInvalidScaffolds(t *testing.T) { + tests := []struct { + name string + manifest string + write bool + }{ + {name: "missing manifest", write: false}, + {name: "wrong kind", write: true, manifest: "apiVersion: atmos/v1\nkind: Other\nmetadata:\n name: x\n"}, + {name: "missing name", write: true, manifest: "apiVersion: atmos/v1\nkind: AtmosScaffoldConfig\nmetadata: {}\n"}, + {name: "unknown field type", write: true, manifest: "apiVersion: atmos/v1\nkind: AtmosScaffoldConfig\nmetadata:\n name: x\nspec:\n fields:\n - name: a\n type: nonsense\n"}, + {name: "not yaml", write: true, manifest: "{{{"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + if tt.write { + fixtures.WriteFile(t, dir, "scaffold.yaml", tt.manifest, 0o644) + } + + _, err := scaffold.Validator{}.ValidateScaffold(context.Background(), dir) + + require.ErrorIs(t, err, errUtils.ErrArtifactScaffoldInvalid) + }) + } +} + +func TestValidator_AcceptsOnlyTheRootManifest(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "nested/scaffold.yaml", validManifest, 0o644) + fixtures.WriteFile(t, dir, "scaffold.yml", validManifest, 0o644) + + _, err := scaffold.Validator{}.ValidateScaffold(context.Background(), dir) + + require.ErrorIs(t, err, errUtils.ErrArtifactScaffoldInvalid, "generation reads only scaffold.yaml at the root") +} + +func TestValidator_DoesNotGenerateOrRunHooks(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(t.TempDir(), "hook-ran") + manifest := validManifest + ` hooks: + marker: + events: [before.scaffold.generate, after.scaffold.generate] + kind: step + type: shell + with: + command: "touch ` + filepath.ToSlash(marker) + `" +` + fixtures.WriteFile(t, dir, "scaffold.yaml", manifest, 0o644) + fixtures.WriteFile(t, dir, "README.md", "# {{ .project_name }}\n", 0o644) + + _, err := scaffold.Validator{}.ValidateScaffold(context.Background(), dir) + + require.NoError(t, err) + _, statErr := os.Stat(marker) + require.ErrorIs(t, statErr, os.ErrNotExist, "validation must not execute scaffold hooks") + entries, readErr := os.ReadDir(dir) + require.NoError(t, readErr) + assert.Len(t, entries, 2, "validation writes no generated output into the source") +} + +func TestValidator_HonorsCancellation(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "scaffold.yaml", validManifest, 0o644) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := scaffold.Validator{}.ValidateScaffold(ctx, dir) + + require.ErrorIs(t, err, context.Canceled) +} + +func TestRegister_InstallsTheProcessWideValidator(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "scaffold.yaml", validManifest, 0o644) + produce.RegisterScaffoldValidator(nil) + t.Cleanup(func() { produce.RegisterScaffoldValidator(nil) }) + + scaffold.Register() + + prepared, err := produce.NewScaffold(produce.WithStagingRoot(t.TempDir())).Prepare(context.Background(), artifact.PrepareRequest{ + Source: dir, + Packaging: artifact.PackagingTgz, + OperationID: "op-1", + PreparedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC), + }) + + require.NoError(t, err) + t.Cleanup(func() { _ = prepared.Close() }) + assert.Equal(t, "landing-zone", prepared.Record.Name) + assert.Equal(t, "2.0.0", prepared.Record.Version) + assert.Equal(t, produce.KindScaffold, prepared.Record.Kind) +} diff --git a/pkg/artifact/service/config_repositories.go b/pkg/artifact/service/config_repositories.go index 70c6986f558..77388549d66 100644 --- a/pkg/artifact/service/config_repositories.go +++ b/pkg/artifact/service/config_repositories.go @@ -78,6 +78,7 @@ func (c *ConfigRepositories) Describe(alias string) (RepositoryInfo, error) { Packaging: desc.EffectivePackaging(repoCfg.Packaging), RequiredLockScope: plan.Required, Policy: policy, + Discovery: discoveryInfoOf(&repoCfg), } if repoCfg.Defaults != nil && repoCfg.Defaults.Metadata != nil { info.Defaults = artifact.Metadata{Description: repoCfg.Defaults.Metadata.Description, Labels: maps.Clone(repoCfg.Defaults.Metadata.Labels)} diff --git a/pkg/artifact/service/copy.go b/pkg/artifact/service/copy.go index db44ff588b5..f96bb8a82e0 100644 --- a/pkg/artifact/service/copy.go +++ b/pkg/artifact/service/copy.go @@ -40,7 +40,7 @@ type copyWrite struct { func (s *Service) Copy(ctx context.Context, request artifact.CopyRequest) (*artifact.OperationResult, error) { defer perf.Track(nil, "service.Service.Copy")() - op := s.begin(artifact.ActionCopy, request.To, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionCopy, request.To, request.Ref.String()) err := s.copyArtifact(ctx, op, &request) result := op.done(err) return &result, err diff --git a/pkg/artifact/service/discovery_info.go b/pkg/artifact/service/discovery_info.go new file mode 100644 index 00000000000..008a76043c2 --- /dev/null +++ b/pkg/artifact/service/discovery_info.go @@ -0,0 +1,48 @@ +package service + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/schema" +) + +// discoveryInfoOf describes where a configured repository's discoverable data lives. Decoding the +// options is best effort: configuration was validated when the composition was built, and a +// repository whose options cannot be decoded here simply has no location, which discovery treats +// as not cacheable. +func discoveryInfoOf(cfg *schema.ArtifactRepositoryConfig) DiscoveryInfo { + info := DiscoveryInfo{Kind: cfg.Kind, Catalog: cfg.Catalog, Identity: cfg.Identity, Options: cfg.Options} + switch cfg.Kind { + case artifact.RepositoryKindOCIRegistry: + var options artifact.OCIRegistryOptions + if artifact.DecodeOptions(cfg.Options, &options) == nil { + info.Location = joinLocation(strings.ToLower(options.Registry), options.Prefix) + } + case artifact.RepositoryKindS3: + var options artifact.S3Options + if artifact.DecodeOptions(cfg.Options, &options) == nil { + info.Location = "s3://" + joinLocation(options.Bucket, options.Prefix) + } + case artifact.RepositoryKindLocalDir, artifact.RepositoryKindLocalArchive: + var options artifact.LocalDirOptions + if artifact.DecodeOptions(cfg.Options, &options) == nil { + info.Location = options.Path + } + case artifact.RepositoryKindOCILayout: + var options artifact.OCILayoutOptions + if artifact.DecodeOptions(cfg.Options, &options) == nil { + info.Location = options.Path + } + } + return info +} + +// joinLocation joins a root and an optional slash-separated prefix. +func joinLocation(root, prefix string) string { + prefix = strings.Trim(prefix, "/") + if prefix == "" { + return root + } + return root + "/" + prefix +} diff --git a/pkg/artifact/service/enumerate.go b/pkg/artifact/service/enumerate.go index c62260cba9a..62a210265cb 100644 --- a/pkg/artifact/service/enumerate.go +++ b/pkg/artifact/service/enumerate.go @@ -5,6 +5,10 @@ import ( "fmt" "sort" "strings" + "sync" + "time" + + "github.com/opencontainers/go-digest" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/artifact" @@ -22,7 +26,8 @@ type listOutcome struct { // listRepository enumerates one repository, consuming pagination, and applies the label and text // filters. A repository without the needed enumeration capability yields a capability error that -// explains direct named operations still work. +// explains direct named operations still work. With an enumerator configured, the entries come from +// the discovery cache or catalog and the repository is opened only when the source needs it. func (s *Service) listRepository(ctx context.Context, alias string, request *enumeration) listOutcome { outcome := listOutcome{listing: artifact.RepositoryListing{Repository: alias}} fail := func(err error) listOutcome { @@ -34,14 +39,10 @@ func (s *Service) listRepository(ctx context.Context, alias string, request *enu if err != nil { return fail(err) } - if err := checkEnumeration(alias, &info.Capabilities, request.name); err != nil { - return fail(err) - } - repo, err := s.repos.Open(ctx, alias) - if err != nil { + if err := s.checkEnumeration(alias, &info, request.name); err != nil { return fail(err) } - entries, err := consumePages(ctx, repo, request) + entries, err := s.entriesOf(ctx, alias, &info, request, &outcome.listing) if err != nil { return fail(err) } @@ -55,20 +56,65 @@ func (s *Service) listRepository(ctx context.Context, alias string, request *enu return outcome } -func checkEnumeration(alias string, caps *artifact.Capabilities, name string) error { +// entriesOf returns the entries of one repository: through the enumerator when one is configured, +// otherwise directly from the repository with the label selectors pushed down. +func (s *Service) entriesOf(ctx context.Context, alias string, info *RepositoryInfo, request *enumeration, listing *artifact.RepositoryListing) ([]artifact.ListEntry, error) { + open := lazyOpen(s.repos, alias) + if s.enumerator == nil { + repo, err := open(ctx) + if err != nil { + return nil, err + } + return consumePages(ctx, repo, request, nil) + } + enumerated, err := s.enumerator.Enumerate(ctx, &EnumerateRequest{ + Alias: alias, Info: info, Name: request.name, Refresh: request.refresh, Open: open, + Live: func(ctx context.Context, known map[digest.Digest]artifact.ListEntry) ([]artifact.ListEntry, error) { + repo, err := open(ctx) + if err != nil { + return nil, err + } + return consumePages(ctx, repo, &enumeration{name: request.name, refresh: request.refresh}, known) + }, + }) + if err != nil { + return nil, err + } + listing.Source, listing.Diagnostics = enumerated.Source, append(listing.Diagnostics, enumerated.Diagnostics...) + if !enumerated.FetchedAt.IsZero() { + listing.FetchedAt = enumerated.FetchedAt.UTC().Format(time.RFC3339) + } + return enumerated.Entries, nil +} + +// lazyOpen opens a repository at most once, on first use, so a cache hit never resolves credentials. +func lazyOpen(repos Repositories, alias string) func(context.Context) (artifact.Repository, error) { + var ( + once sync.Once + repo artifact.Repository + err error + ) + return func(ctx context.Context) (artifact.Repository, error) { + once.Do(func() { repo, err = repos.Open(ctx, alias) }) + return repo, err + } +} + +func (s *Service) checkEnumeration(alias string, info *RepositoryInfo, name string) error { + caps := &info.Capabilities if !caps.Read { return artifact.NewUnsupportedCapability(alias, opList, "read", "the repository is write-only") } if name != "" && !caps.EnumerateNamed { return artifact.NewUnsupportedCapability(alias, opList, "enumerate/named", "this repository cannot list versions of a name; direct named operations such as inspect still work") } - if name == "" && !caps.EnumerateCollection { + if name == "" && !s.canEnumerateCollection(info) { return artifact.NewUnsupportedCapability(alias, opList, "enumerate/collection", "this repository has no native collection listing or catalog; direct named operations such as inspect still work") } return nil } -func consumePages(ctx context.Context, repo artifact.Repository, request *enumeration) ([]artifact.ListEntry, error) { +func consumePages(ctx context.Context, repo artifact.Repository, request *enumeration, known map[digest.Digest]artifact.ListEntry) ([]artifact.ListEntry, error) { var entries []artifact.ListEntry cursor := "" seen := map[string]bool{} @@ -76,7 +122,7 @@ func consumePages(ctx context.Context, repo artifact.Repository, request *enumer if err := ctx.Err(); err != nil { return entries, err } - result, err := repo.List(ctx, artifact.ListQuery{Name: request.name, Labels: request.labels, Cursor: cursor, Refresh: request.refresh}) + result, err := repo.List(ctx, artifact.ListQuery{Name: request.name, Labels: request.labels, Cursor: cursor, Refresh: request.refresh, Known: known}) if err != nil { return entries, err } diff --git a/pkg/artifact/service/enumerator.go b/pkg/artifact/service/enumerator.go new file mode 100644 index 00000000000..efdec7561f4 --- /dev/null +++ b/pkg/artifact/service/enumerator.go @@ -0,0 +1,99 @@ +package service + +import ( + "context" + "time" + + "github.com/opencontainers/go-digest" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/perf" +) + +// DiscoveryInfo is what discovery needs to know about a configured repository, without touching +// the network or resolving credentials. It never contains credential values. +type DiscoveryInfo struct { + // Kind is the repository kind, for example "oci/registry". + Kind string + + // Location is the credential-free collection root: "registry/prefix" for a registry, "s3://bucket/prefix" + // for S3, and the directory for local and layout repositories. + Location string + + // Catalog is the optional catalog reference ("oci://registry/path:tag"); empty when none is configured. + Catalog string + + // Identity is the name of the Atmos identity bound to the repository; never a credential. + Identity string + + // Options are the effective repository options. They hold no credentials. + Options map[string]any +} + +// Remote reports whether listing the repository crosses the network, which is what makes a +// discovery snapshot worthwhile. Local directories, archives, and layouts are always listed live. +func (d *DiscoveryInfo) Remote() bool { + defer perf.Track(nil, "service.DiscoveryInfo.Remote")() + + return d.Kind == artifact.RepositoryKindS3 || d.Kind == artifact.RepositoryKindOCIRegistry +} + +// Enumerator supplies the entries of one repository for List and Search. The service falls back to +// reading the repository directly when no enumerator is configured. An enumerator owns caching: +// when a fresh snapshot answers the request it must not call EnumerateRequest.Live or open the +// repository. +type Enumerator interface { + // Enumerate returns the unfiltered entries of the repository (or of one name). The service + // applies label and text filters afterward, so a snapshot holds the whole collection. + Enumerate(ctx context.Context, request *EnumerateRequest) (*Enumeration, error) +} + +// EnumerateRequest selects what an [Enumerator] lists. +type EnumerateRequest struct { + // Alias is the configured repository alias. + Alias string + + // Info is the repository description. + Info *RepositoryInfo + + // Name restricts the listing to the versions of one exact name; empty enumerates the collection. + Name string + + // Refresh bypasses a fresh snapshot. + Refresh bool + + // Open opens the repository, resolving credentials. It is lazy so a cache hit never authenticates. + Open func(ctx context.Context) (artifact.Repository, error) + + // Live lists the repository directly and consumes its pagination. The known entries, keyed by + // resolved digest, let the repository reuse metadata of unchanged references. + Live func(ctx context.Context, known map[digest.Digest]artifact.ListEntry) ([]artifact.ListEntry, error) +} + +// Enumeration is the result of [Enumerator.Enumerate]. +type Enumeration struct { + // Entries are the unfiltered entries. + Entries []artifact.ListEntry + + // Source is "live", "cache", or "catalog". + Source string + + // FetchedAt is when a cached or catalog snapshot was fetched; zero for a live listing. + FetchedAt time.Time + + // Diagnostics are non-fatal notes, for example that the entries come from a catalog. + Diagnostics []artifact.Diagnostic +} + +// WithEnumerator routes List and Search through an enumerator (the discovery cache and catalogs). +func WithEnumerator(enumerator Enumerator) Option { + defer perf.Track(nil, "service.WithEnumerator")() + + return func(s *Service) { s.enumerator = enumerator } +} + +// canEnumerateCollection reports whether a collection listing is possible: natively, or through a +// configured catalog read by the enumerator. +func (s *Service) canEnumerateCollection(info *RepositoryInfo) bool { + return info.Capabilities.EnumerateCollection || s.enumerator != nil && info.Discovery.Catalog != "" +} diff --git a/pkg/artifact/service/execute.go b/pkg/artifact/service/execute.go index 171f49fb47d..0896f0d86b5 100644 --- a/pkg/artifact/service/execute.go +++ b/pkg/artifact/service/execute.go @@ -65,7 +65,7 @@ func (s *Service) afterFailure(ctx context.Context, pub *publication, cur *curre if ctx.Err() != nil { return nil, false, errors.Join(writeErr, ctx.Err()) } - if !isTemporary(writeErr) || attempt >= s.attempts { + if !isTemporary(writeErr) || attempt >= pub.op.attempts { return nil, false, writeErr } if sleepErr := s.sleep(ctx, s.backoff<<(attempt-1)); sleepErr != nil { diff --git a/pkg/artifact/service/interrupt_test.go b/pkg/artifact/service/interrupt_test.go index 0e5f3f2bcf6..92b1bcd44f6 100644 --- a/pkg/artifact/service/interrupt_test.go +++ b/pkg/artifact/service/interrupt_test.go @@ -144,6 +144,35 @@ func TestPublish_TemporaryFailureExhaustsAttempts(t *testing.T) { assert.Equal(t, 3, e.repos["dst"].Calls("Publish")) } +func TestPublish_CancellationDuringUncommittedWriteIsPreserved(t *testing.T) { + caps := artifacttest.DefaultCapabilities() + m := newMockSetup(t, &caps) + m.expectLock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + gomock.InOrder( + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound), + m.repo.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *artifact.PreparedArtifact, artifact.PublishOptions) (*artifact.PublishResult, error) { + cancel() + return nil, errTransport + }), + m.repo.EXPECT().Resolve(gomock.Any(), exampleRef).Return(artifact.PinnedReference{}, errUtils.ErrArtifactNotFound), + ) + _, err := m.svc.Publish(ctx, m.e.publishRequest("dst", m.e.source("hello"))) + require.ErrorIs(t, err, errTransport) + require.ErrorIs(t, err, context.Canceled) +} + +func TestPublish_ExecutionAttemptLimitIsHonored(t *testing.T) { + e := newEnv(t, envConfig{}) + e.repos["dst"].InjectFailure(artifacttest.FailBeforePayload, &temporaryError{"throttled"}, 10) + request := e.publishRequest("dst", e.source("hello")) + request.Execution.MaxAttempts = 1 + _, err := e.svc.PublishDetailed(context.Background(), request) + require.Error(t, err) + assert.Equal(t, 1, e.repos["dst"].Calls("Publish")) +} + func TestPublish_TemporaryFailureAfterCommitIsReconciledWithoutRetry(t *testing.T) { e := newEnv(t, envConfig{}) e.repos["dst"].InjectFailure(artifacttest.FailAfterCommit, &temporaryError{"timeout"}, 5) @@ -323,3 +352,62 @@ func TestPublish_DelegatedLockIsTakenFromTheProvider(t *testing.T) { require.Error(t, err, "the second publish conflicts, but only after locking") assert.Equal(t, 1, e.repos["dst"].Calls("Lock"), "without delegation the target repository locks for itself") } + +// A listing over the default repositories may be a partial success, and is then marked incomplete. +// Cancellation is never that: the marker stays set, and the error says the listing did not finish. +func TestList_CancellationIsNeverASuccessfulPartialListing(t *testing.T) { + tests := []struct { + name string + beta func(ctx context.Context, cancel context.CancelFunc) error + wantErr error + wantIncomplete bool + }{ + {"canceled while one repository was still listing", func(ctx context.Context, cancel context.CancelFunc) error { + cancel() + return ctx.Err() + }, context.Canceled, true}, + {"canceled while one repository failed for another reason", func(_ context.Context, cancel context.CancelFunc) error { + cancel() + return errTransport + }, context.Canceled, true}, + {"one repository failed without cancellation", func(context.Context, context.CancelFunc) error { + return errTransport + }, nil, true}, + {"every repository listed", nil, nil, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := newEnv(t, envConfig{repos: map[string]artifacttest.Options{"alpha": {}, "beta": {}}}) + seedList(t, e, "alpha", map[string]string{"app/a:1": "a1"}, nil) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Alpha completes first, so the failure of beta always meets one repository that succeeded. + alphaDone := make(chan struct{}) + var once sync.Once + e.static.Repositories["alpha"] = delegatingRepository(t, e.repos["alpha"], repoHooks{list: func(ctx context.Context, query artifact.ListQuery) (*artifact.ListPage, error) { + page, err := e.repos["alpha"].List(ctx, query) + once.Do(func() { close(alphaDone) }) + return page, err + }}) + if tc.beta != nil { + e.static.Repositories["beta"] = delegatingRepository(t, e.repos["beta"], repoHooks{list: func(ctx context.Context, _ artifact.ListQuery) (*artifact.ListPage, error) { + <-alphaDone + return nil, tc.beta(ctx, cancel) + }}) + } + + result, err := e.svc.List(ctx, artifact.ListRequest{}) + + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr, "a canceled listing must not be reported as a success") + } else { + require.NoError(t, err) + } + require.NotNil(t, result) + assert.Equal(t, tc.wantIncomplete, result.Incomplete) + require.Len(t, result.Artifacts, 1, "what alpha returned stays visible") + assert.Equal(t, "app/a", result.Artifacts[0].Name) + }) + } +} diff --git a/pkg/artifact/service/mutate.go b/pkg/artifact/service/mutate.go index 9d3900d9c5a..640d948d8e2 100644 --- a/pkg/artifact/service/mutate.go +++ b/pkg/artifact/service/mutate.go @@ -15,7 +15,7 @@ import ( func (s *Service) Delete(ctx context.Context, request artifact.DeleteRequest) (*artifact.OperationResult, error) { defer perf.Track(nil, "service.Service.Delete")() - op := s.begin(artifact.ActionDelete, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionDelete, request.Repository, request.Ref.String()) err := s.deleteArtifact(ctx, op, &request) result := op.done(err) return &result, err @@ -68,6 +68,7 @@ func (s *Service) runDelete(ctx context.Context, op *operation, request *artifac op.result.Outcome = artifact.OutcomeDeleted op.result.LockScope = locks.scope() op.result.Size = payloadSize(inspection) + op.result.SizeMeasured = true if len(result.Deleted) > 0 { op.result.Location = result.Deleted[0] op.result.Aliases = result.Deleted[1:] @@ -109,7 +110,7 @@ func deleteLockTargets(ctx context.Context, request *artifact.DeleteRequest, inf func (s *Service) Pin(ctx context.Context, request artifact.PinRequest) (*artifact.PinResult, error) { defer perf.Track(nil, "service.Service.Pin")() - op := s.begin(artifact.ActionPin, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionPin, request.Repository, request.Ref.String()) result := &artifact.PinResult{} err := s.pin(ctx, op, &request, result) result.Operation = op.done(err) diff --git a/pkg/artifact/service/publish.go b/pkg/artifact/service/publish.go index 5140c00abd8..7dc58cb0908 100644 --- a/pkg/artifact/service/publish.go +++ b/pkg/artifact/service/publish.go @@ -41,7 +41,7 @@ func (s *Service) Publish(ctx context.Context, request artifact.PublishRequest) func (s *Service) PublishDetailed(ctx context.Context, request artifact.PublishRequest) (*PublishOutcome, error) { defer perf.Track(nil, "service.Service.PublishDetailed")() - op := s.begin(artifact.ActionPublish, request.Repository, requestedName(request.Name, request.Version)) + op := s.begin(&request.Execution, artifact.ActionPublish, request.Repository, requestedName(request.Name, request.Version)) state := &publishState{} err := s.publish(ctx, op, &request, state) return &PublishOutcome{Operation: op.done(err), Record: state.record, Existing: state.existing}, err @@ -92,6 +92,7 @@ func (s *Service) publish(ctx context.Context, op *operation, request *artifact. } state.record = prepared.Record fillPrepared(op, in) + reportExclusions(op, prepared.Exclusions) if request.DryRun { op.result.Outcome = artifact.OutcomeDryRun op.info("dry-run", fmt.Sprintf("would publish %s to repository %q as %s packaging; no repository was contacted", op.result.Ref, request.Repository, packaging)) @@ -190,6 +191,26 @@ func fillPrepared(op *operation, in *incoming) { op.result.RecordDigest = in.recordDigest op.result.ContentDigest = in.record.ContentDigest op.result.Size = in.size + op.result.SizeMeasured = true +} + +// diagnosticExcluded is the diagnostic code of a source entry that a kind-specific producer left out. +const ( + diagnosticExcluded = "excluded" + maxExclusionDiagnostics = 100 +) + +// reportExclusions makes the default exclusions of a kind-specific producer visible: every excluded +// entry becomes one info diagnostic, so dry runs and real publications, structured output, and +// step results all show what was left out and which rule excluded it. +func reportExclusions(op *operation, exclusions []artifact.Exclusion) { + limit := min(len(exclusions), maxExclusionDiagnostics) + for index := range limit { + op.info(diagnosticExcluded, fmt.Sprintf("excluded %s (rule %s)", exclusions[index].Path, exclusions[index].Rule)) + } + if remaining := len(exclusions) - limit; remaining > 0 { + op.info(diagnosticExcluded, fmt.Sprintf("%d additional exclusions omitted", remaining)) + } } // reportUnchanged fills the result from the EXISTING record; the current execution keeps its own @@ -207,6 +228,7 @@ func (s *Service) reportUnchanged(pub *publication, cur *current) { op.result.RecordDigest = cur.pinned.RecordDigest op.result.RepresentationDigest = cur.inspection.Representation.RepresentationDigest op.result.Size = payloadSize(cur.inspection) + op.result.SizeMeasured = true op.info("unchanged", fmt.Sprintf("an identical artifact already exists in repository %q; the existing record was returned unchanged", pub.tgt.alias)) } diff --git a/pkg/artifact/service/publish_test.go b/pkg/artifact/service/publish_test.go index 5a38342c700..95f46f6a9d6 100644 --- a/pkg/artifact/service/publish_test.go +++ b/pkg/artifact/service/publish_test.go @@ -2,6 +2,7 @@ package service_test import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -243,7 +244,7 @@ func TestPublish_NameAndVersionResolutionHappensBeforeAnyWrite(t *testing.T) { {"missing repository", func(r *artifact.PublishRequest) { r.Repository = "" }, errUtils.ErrArtifactConfigInvalid}, {"missing source", func(r *artifact.PublishRequest) { r.Source = "" }, errUtils.ErrArtifactConfigInvalid}, {"invalid kind", func(r *artifact.PublishRequest) { r.Kind = "Bad Kind" }, errUtils.ErrArtifactRecordInvalid}, - {"unregistered kind", func(r *artifact.PublishRequest) { r.Kind = "scaffold" }, errUtils.ErrArtifactConfigInvalid}, + {"unregistered kind", func(r *artifact.PublishRequest) { r.Kind = "toolchain/package" }, errUtils.ErrArtifactConfigInvalid}, {"bad labels", func(r *artifact.PublishRequest) { r.Metadata.Labels = map[string]string{"a=b": "c"} }, errUtils.ErrArtifactConfigInvalid}, {"unknown repository", func(r *artifact.PublishRequest) { r.Repository = "nope" }, errUtils.ErrArtifactConfigInvalid}, } @@ -409,3 +410,121 @@ func TestPublish_PackagingFollowsRepository(t *testing.T) { }) } } + +func TestPublish_ComponentKindReportsDefaultExclusionsAndGenericKindDoesNot(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + fixtures.WriteFile(t, src, "main.tf", "# component\n", 0o644) + fixtures.WriteFile(t, src, ".terraform/modules.json", "{}\n", 0o644) + fixtures.WriteFile(t, src, "terraform.tfstate", "{}\n", 0o644) + + excluded := func(outcome *service.PublishOutcome) []string { + var messages []string + for _, diagnostic := range outcome.Operation.Diagnostics { + if diagnostic.Code == "excluded" { + messages = append(messages, diagnostic.Message) + } + } + return messages + } + + request := e.publishRequest("dst", src) + request.Kind = "component/terraform" + request.DryRun = true + dryRun, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, artifact.OutcomeDryRun, dryRun.Operation.Outcome) + assert.Equal(t, "component/terraform", dryRun.Record.Kind) + assert.Equal(t, []string{"excluded .terraform (rule .terraform/**)", "excluded terraform.tfstate (rule *.tfstate)"}, excluded(dryRun), + "the dry run shows what the component producer leaves out and why") + for _, entry := range dryRun.Record.Inventory { + assert.NotContains(t, entry.Path, "terraform.tfstate") + assert.NotContains(t, entry.Path, ".terraform/") + } + + request.DryRun = false + published, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + assert.Len(t, excluded(published), 2, "a real publication reports the same exclusions") + + generic := e.publishRequest("dst", src) + generic.DryRun = true + plain, err := e.svc.PublishDetailed(context.Background(), generic) + require.NoError(t, err) + assert.Empty(t, excluded(plain), "generic files never inherit component exclusions") + assert.Contains(t, inventoryPathsOf(&plain.Record), "terraform.tfstate") +} + +func TestPublish_ExclusionDiagnosticsAreBounded(t *testing.T) { + e := newEnv(t, envConfig{}) + src := e.source("hello") + for i := 0; i < 103; i++ { + fixtures.WriteFile(t, src, fmt.Sprintf("state-%03d.tfstate", i), "{}", 0o644) + } + request := e.publishRequest("dst", src) + request.Kind, request.DryRun = "component/terraform", true + outcome, err := e.svc.PublishDetailed(context.Background(), request) + require.NoError(t, err) + var messages []string + for _, diagnostic := range outcome.Operation.Diagnostics { + if diagnostic.Code == "excluded" { + messages = append(messages, diagnostic.Message) + } + } + require.Len(t, messages, 101) + assert.Equal(t, "3 additional exclusions omitted", messages[100]) +} + +func inventoryPathsOf(record *artifact.Record) []string { + paths := make([]string, 0, len(record.Inventory)) + for index := range record.Inventory { + paths = append(paths, record.Inventory[index].Path) + } + return paths +} + +// Clone promises a copy that shares nothing mutable with the original, in both directions. +func TestRepositoryInfo_CloneIsolatesDiscoveryOptions(t *testing.T) { + newInfo := func() service.RepositoryInfo { + return service.RepositoryInfo{ + Alias: "repo", + Discovery: service.DiscoveryInfo{ + Kind: "s3", + Options: map[string]any{ + "prefix": "base", + "nested": map[string]any{"region": "us-east-1"}, + "list": []any{"a", map[string]any{"k": "v"}}, + }, + }, + } + } + + t.Run("mutating the clone leaves the original unchanged", func(t *testing.T) { + info := newInfo() + cloned := info.Clone() + + cloned.Discovery.Options["prefix"] = "other" + cloned.Discovery.Options["nested"].(map[string]any)["region"] = "eu-west-1" + cloned.Discovery.Options["list"].([]any)[0] = "changed" + cloned.Discovery.Options["list"].([]any)[1].(map[string]any)["k"] = "changed" + + assert.Equal(t, newInfo().Discovery.Options, info.Discovery.Options) + }) + + t.Run("mutating the original after cloning leaves the clone unchanged", func(t *testing.T) { + info := newInfo() + cloned := info.Clone() + + info.Discovery.Options["prefix"] = "other" + info.Discovery.Options["nested"].(map[string]any)["region"] = "eu-west-1" + info.Discovery.Options["list"].([]any)[0] = "changed" + info.Discovery.Options["list"].([]any)[1].(map[string]any)["k"] = "changed" + + assert.Equal(t, newInfo().Discovery.Options, cloned.Discovery.Options) + }) + + t.Run("a nil options map stays nil", func(t *testing.T) { + info := service.RepositoryInfo{Alias: "repo"} + assert.Nil(t, info.Clone().Discovery.Options) + }) +} diff --git a/pkg/artifact/service/pull.go b/pkg/artifact/service/pull.go index fe125659cc1..c62e2995124 100644 --- a/pkg/artifact/service/pull.go +++ b/pkg/artifact/service/pull.go @@ -19,7 +19,7 @@ import ( func (s *Service) Pull(ctx context.Context, request artifact.PullRequest) (*artifact.OperationResult, error) { defer perf.Track(nil, "service.Service.Pull")() - op := s.begin(artifact.ActionPull, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionPull, request.Repository, request.Ref.String()) err := s.pull(ctx, op, &request) result := op.done(err) return &result, err @@ -179,6 +179,7 @@ func fillFromInspection(op *operation, pinned *artifact.PinnedReference, inspect op.result.ContentDigest = pinned.ContentDigest op.result.RepresentationDigest = inspection.Representation.RepresentationDigest op.result.Size = payloadSize(inspection) + op.result.SizeMeasured = true if inspection.Record != nil { op.result.Ref = artifact.Ref{Name: inspection.Record.Name, Version: inspection.Record.Version}.String() op.result.ContentDigest = inspection.Record.ContentDigest diff --git a/pkg/artifact/service/query.go b/pkg/artifact/service/query.go index 172f0eac578..371be8b9c84 100644 --- a/pkg/artifact/service/query.go +++ b/pkg/artifact/service/query.go @@ -20,7 +20,7 @@ import ( func (s *Service) Inspect(ctx context.Context, request artifact.InspectRequest) (*artifact.InspectResult, error) { defer perf.Track(nil, "service.Service.Inspect")() - op := s.begin(artifact.ActionInspect, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionInspect, request.Repository, request.Ref.String()) result := &artifact.InspectResult{} err := s.inspect(ctx, op, &request, result) result.Operation = op.done(err) @@ -59,7 +59,7 @@ func (s *Service) inspect(ctx context.Context, op *operation, request *artifact. func (s *Service) ShowDocument(ctx context.Context, request artifact.ShowDocumentRequest) (*artifact.DocumentResult, error) { defer perf.Track(nil, "service.Service.ShowDocument")() - op := s.begin(artifact.ActionShow, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionShow, request.Repository, request.Ref.String()) result := &artifact.DocumentResult{} err := s.showDocument(ctx, op, &request, result) result.Operation = op.done(err) @@ -96,6 +96,7 @@ func (s *Service) showDocument(ctx context.Context, op *operation, request *arti result.Document, result.Content = stream.Document, content op.result.Outcome = artifact.OutcomeRead op.result.Size = int64(len(content)) + op.result.SizeMeasured = true return nil } @@ -215,6 +216,7 @@ func (s *Service) runEnumeration(ctx context.Context, request *enumeration, resu succeeded++ } sortEntries(result.Artifacts) + result.Incomplete = len(errs) > 0 return enumerationError(ctx, explicit, succeeded, errs) } diff --git a/pkg/artifact/service/repositories.go b/pkg/artifact/service/repositories.go index 8de41fea099..256f081ce9c 100644 --- a/pkg/artifact/service/repositories.go +++ b/pkg/artifact/service/repositories.go @@ -2,8 +2,10 @@ package service import ( "context" + "maps" "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/merge" "github.com/cloudposse/atmos/pkg/perf" ) @@ -35,6 +37,10 @@ type RepositoryInfo struct { // Policy is the effective verification policy: the global and repository trust sets, both of // which must be satisfied. Policy artifact.VerificationPolicy + + // Discovery describes where the repository's discoverable data lives, for the discovery cache + // and catalogs. It is empty for repositories described without configuration. + Discovery DiscoveryInfo } // Clone returns a RepositoryInfo that shares no map, slice, or pointer with i, so a caller editing a @@ -46,9 +52,21 @@ func (i *RepositoryInfo) Clone() RepositoryInfo { out.Capabilities = i.Capabilities.Clone() out.Defaults = i.Defaults.Clone() out.Policy = i.Policy.Clone() + out.Discovery.Options = cloneOptions(i.Discovery.Options) return out } +// cloneOptions deep-copies discovery options, including nested maps and slices. The copy can fail +// only for typed maps whose keys collide once normalized; the options then fall back to a copy of +// the top-level map, which still never shares that map with the original. +func cloneOptions(options map[string]any) map[string]any { + copied, err := merge.DeepCopyMap(options) + if err != nil { + return maps.Clone(options) + } + return copied +} + // Repositories resolves repository aliases. Describe must be free of network access and // credential resolution so dry runs never authenticate; Open may resolve identities. type Repositories interface { diff --git a/pkg/artifact/service/service.go b/pkg/artifact/service/service.go index 1aef0758221..6707168bc75 100644 --- a/pkg/artifact/service/service.go +++ b/pkg/artifact/service/service.go @@ -60,6 +60,7 @@ type Service struct { signer artifact.Signer journal Journal immutable ImmutabilityPolicy + enumerator Enumerator clock func() time.Time newID func() string sleep func(ctx context.Context, d time.Duration) error @@ -156,7 +157,8 @@ func WithRepacker(repacker Repacker) Option { return func(s *Service) { s.repacker = repacker } } -// New returns a Service. With a nil producer registry, only the generic files producer is available. +// New returns a Service. With a nil producer registry, the built-in producers are available: the +// generic files producer and the scaffold and component/terraform producers. func New(repos Repositories, producers *artifact.ProducerRegistry, opts ...Option) (*Service, error) { defer perf.Track(nil, "service.New")() @@ -186,7 +188,7 @@ func New(repos Repositories, producers *artifact.ProducerRegistry, opts ...Optio } if s.producers == nil { s.producers = artifact.NewProducerRegistry() - if err := produce.Register(s.producers, produce.WithStagingRoot(s.stagingRoot)); err != nil { + if err := produce.RegisterDefaults(s.producers, produce.WithStagingRoot(s.stagingRoot)); err != nil { return nil, err } } @@ -222,16 +224,32 @@ type operation struct { preparedAt time.Time started time.Time clock func() time.Time + attempts int result artifact.OperationResult } -func (s *Service) begin(action artifact.Action, repository, requested string) *operation { +// begin captures the operation identity. A wrapper that retries the whole operation supplies the +// identity it captured before its first attempt, so the ID and the preparation time survive every +// retry; otherwise the service generates both. The duration always measures the current call. +func (s *Service) begin(execution *artifact.Execution, action artifact.Action, repository, requested string) *operation { now := s.clock().UTC() + id, preparedAt := execution.OperationID, execution.PreparedAt.UTC() + if id == "" { + id = s.newID() + } + if execution.PreparedAt.IsZero() { + preparedAt = now + } + attempts := s.attempts + if execution.MaxAttempts > 0 { + attempts = execution.MaxAttempts + } return &operation{ - id: s.newID(), - preparedAt: now, + id: id, + preparedAt: preparedAt, started: now, clock: s.clock, + attempts: attempts, result: artifact.OperationResult{ OperationID: "", Action: action, diff --git a/pkg/artifact/service/sign.go b/pkg/artifact/service/sign.go index c21e6b87271..af57ca5ab7f 100644 --- a/pkg/artifact/service/sign.go +++ b/pkg/artifact/service/sign.go @@ -16,7 +16,7 @@ import ( func (s *Service) Sign(ctx context.Context, request artifact.SignRequest) (*artifact.SignResult, error) { defer perf.Track(nil, "service.Service.Sign")() - op := s.begin(artifact.ActionSign, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionSign, request.Repository, request.Ref.String()) result := &artifact.SignResult{} err := s.sign(ctx, op, &request, result) result.Operation = op.done(err) @@ -154,7 +154,7 @@ func checkSigningCapability(alias string, info *RepositoryInfo, signer artifact. func (s *Service) Verify(ctx context.Context, request artifact.VerifyRequest) (*artifact.VerifyResult, error) { defer perf.Track(nil, "service.Service.Verify")() - op := s.begin(artifact.ActionVerify, request.Repository, request.Ref.String()) + op := s.begin(&request.Execution, artifact.ActionVerify, request.Repository, request.Ref.String()) result := &artifact.VerifyResult{} err := s.verify(ctx, op, &request, result) result.Operation = op.done(err) diff --git a/pkg/artifact/snapshot/copy.go b/pkg/artifact/snapshot/copy.go index b35e1fe1bd7..b5525742f3c 100644 --- a/pkg/artifact/snapshot/copy.go +++ b/pkg/artifact/snapshot/copy.go @@ -114,6 +114,15 @@ func (c *capturer) recheck() error { return err } } + for _, record := range c.excludedSources { + now, err := os.Lstat(record.abs) + if err != nil { + return changedError(record.rel, err) + } + if now.Mode().Type() != record.info.Mode().Type() { + return fmt.Errorf("%w: %q changed type while it was being captured", errUtils.ErrArtifactSourceChanged, record.rel) + } + } for abs, names := range c.dirs { current, err := readDirNames(abs) if err != nil { diff --git a/pkg/artifact/snapshot/exclude_test.go b/pkg/artifact/snapshot/exclude_test.go new file mode 100644 index 00000000000..32dc4be9809 --- /dev/null +++ b/pkg/artifact/snapshot/exclude_test.go @@ -0,0 +1,210 @@ +package snapshot_test + +import ( + "context" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/fixtures" + "github.com/cloudposse/atmos/pkg/artifact/snapshot" +) + +// byName excludes directories called "cache" and any entry whose name ends in ".tmp". +func byName(entryPath string, isDir bool) (string, bool) { + name := path.Base(entryPath) + switch { + case isDir && name == "cache": + return "cache/**", true + case path.Ext(name) == ".tmp": + return "*.tmp", true + default: + return "", false + } +} + +func TestCapture_ExcluderLeavesEntriesOutAndReportsThem(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "keep.txt", "keep\n", 0o644) + fixtures.WriteFile(t, src, "scratch.tmp", "tmp\n", 0o644) + fixtures.WriteFile(t, src, "cache/a/b.txt", "deep\n", 0o644) + fixtures.WriteFile(t, src, "nested/cache/c.txt", "deep\n", 0o644) + fixtures.WriteFile(t, src, "nested/keep.txt", "keep\n", 0o644) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir()), snapshot.WithExcluder(byName)) + + require.NoError(t, err) + t.Cleanup(func() { _ = snap.Close() }) + assert.Equal(t, []string{"keep.txt", "nested", "nested/keep.txt"}, entryPaths(snap.Entries)) + assert.Equal(t, []artifact.Exclusion{ + {Path: "cache", Rule: "cache/**"}, + {Path: "nested/cache", Rule: "cache/**"}, + {Path: "scratch.tmp", Rule: "*.tmp"}, + }, snap.Exclusions, "an excluded directory is reported once, not once per descendant") + _, staged := snap.StagedPath("cache/a/b.txt") + assert.False(t, staged, "excluded content is never staged") +} + +func TestCapture_WithoutExcluderKeepsEverything(t *testing.T) { + src := t.TempDir() + fixtures.WriteFile(t, src, "scratch.tmp", "tmp\n", 0o644) + fixtures.WriteFile(t, src, "cache/a.txt", "a\n", 0o644) + + snap, err := snapshot.Capture(context.Background(), src, snapshot.WithStagingRoot(t.TempDir())) + + require.NoError(t, err) + t.Cleanup(func() { _ = snap.Close() }) + assert.Equal(t, []string{"cache", "cache/a.txt", "scratch.tmp"}, entryPaths(snap.Entries)) + assert.Empty(t, snap.Exclusions) +} + +func TestCapture_ExcludedFileAndLimit(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "a.tmp", "a", 0o644) + fixtures.WriteFile(t, dir, "b.tmp", "b", 0o644) + snap, err := snapshot.Capture(context.Background(), filepath.Join(dir, "a.tmp"), snapshot.WithExcluder(byName)) + require.NoError(t, err) + defer snap.Close() + assert.Empty(t, snap.Entries) + assert.Equal(t, []artifact.Exclusion{{Path: "a.tmp", Rule: "*.tmp"}}, snap.Exclusions) + _, err = snapshot.Capture(context.Background(), dir, snapshot.WithExcluder(byName), snapshot.WithLimits(snapshot.Limits{MaxEntries: 1})) + require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) +} + +func TestCapture_RejectsLinkToExcludedDirectory(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "cache/a", "a", 0o644) + if !fixtures.Symlink(t, dir, "link", "cache/a") { + t.Skip("symlinks are unavailable on this platform") + } + _, err := snapshot.Capture(context.Background(), dir, snapshot.WithExcluder(byName)) + require.ErrorIs(t, err, errUtils.ErrArtifactUnsafePath) +} + +func TestCapture_RejectsExcludedTypeChange(t *testing.T) { + dir := t.TempDir() + fixtures.WriteFile(t, dir, "a.tmp", "a", 0o644) + fixtures.WriteFile(t, dir, "z.txt", "z", 0o644) + _, err := snapshot.Capture(context.Background(), dir, snapshot.WithExcluder(byName), snapshot.WithCaptureHook(func(string) { + require.NoError(t, os.Remove(filepath.Join(dir, "a.tmp"))) + require.NoError(t, os.Mkdir(filepath.Join(dir, "a.tmp"), 0o755)) + })) + require.ErrorIs(t, err, errUtils.ErrArtifactSourceChanged) +} + +// Containment and exclusion are both decided from the links as they were captured. Each case builds +// a source, optionally changes it while the capture runs in a way the final recheck cannot see (below +// an excluded directory, which is never walked), and states the one outcome the capture may have. +func TestCapture_LinkTargetsAreJudgedFromTheCapturedLinks(t *testing.T) { + tests := []struct { + name string + files []string + links [][2]string + mutate func(t *testing.T, dir string) + wantErr error + wantMsg string + want []string + }{ + { + name: "a link to an excluded file is rejected", + files: []string{"keep.txt", "scratch.tmp"}, + links: [][2]string{{"link", "scratch.tmp"}}, + wantErr: errUtils.ErrArtifactUnsafePath, + wantMsg: `symlink "link" targets excluded path "scratch.tmp"`, + }, + { + name: "a chain of two links that ends below an excluded directory is rejected", + files: []string{"keep.txt", "cache/a"}, + links: [][2]string{{"first", "second"}, {"second", "cache/a"}}, + wantErr: errUtils.ErrArtifactUnsafePath, + wantMsg: `symlink "first" targets excluded path "cache"`, + }, + { + name: "a chain that climbs out of the source is rejected as an escape", + files: []string{"keep.txt", "sub/keep.txt"}, + links: [][2]string{{"sub/up", ".."}, {"out", "sub/up/../gone"}}, + wantErr: errUtils.ErrArtifactUnsafePath, + wantMsg: `symlink "out" resolves outside the source`, + }, + { + name: "a link to an allowed path is captured", + files: []string{"keep.txt", "cache/a"}, + links: [][2]string{{"link", "keep.txt"}}, + want: []string{"keep.txt", "link"}, + }, + { + // The filesystem looks up cache/p before it processes "..", so excluding cache turns this + // link into a broken one even though it reduces to keep.txt. A later change below the + // excluded directory (cache/p becoming a link to cache/q/r) cannot change the answer. + name: "a link that passes through an excluded directory is rejected even when it reduces to an allowed path", + files: []string{"keep.txt", "cache/p/a", "cache/q/r/a"}, + links: [][2]string{{"link", "cache/p/../../keep.txt"}}, + mutate: func(t *testing.T, dir string) { + require.NoError(t, os.RemoveAll(filepath.Join(dir, "cache", "p"))) + require.NoError(t, os.Symlink(filepath.Join("q", "r"), filepath.Join(dir, "cache", "p"))) + }, + wantErr: errUtils.ErrArtifactUnsafePath, + wantMsg: `symlink "link" targets excluded path "cache"`, + }, + { + name: "a link that only passes through allowed directories is captured", + files: []string{"keep.txt", "sub/a"}, + links: [][2]string{{"link", "sub/../keep.txt"}}, + want: []string{"keep.txt", "link", "sub", "sub/a"}, + }, + { + name: "removing the target below an excluded directory cannot turn a rejected link into an allowed one", + files: []string{"keep.txt", "cache/a"}, + links: [][2]string{{"link", "cache/a"}}, + mutate: func(t *testing.T, dir string) { + require.NoError(t, os.Remove(filepath.Join(dir, "cache", "a"))) + }, + wantErr: errUtils.ErrArtifactUnsafePath, + wantMsg: `symlink "link" targets excluded path "cache"`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + for _, file := range tc.files { + fixtures.WriteFile(t, dir, file, "x", 0o644) + } + for _, link := range tc.links { + if !fixtures.Symlink(t, dir, link[0], link[1]) { + t.Skip("symlinks are unavailable on this platform") + } + } + mutated := false + opts := []snapshot.Option{snapshot.WithStagingRoot(t.TempDir()), snapshot.WithExcluder(byName)} + if tc.mutate != nil { + // The hook runs after a regular file was copied. By then the walk has classified + // "cache", which sorts before every captured file in these sources. + opts = append(opts, snapshot.WithCaptureHook(func(string) { + if !mutated { + mutated = true + tc.mutate(t, dir) + } + })) + } + + snap, err := snapshot.Capture(context.Background(), dir, opts...) + + require.Equal(t, tc.mutate != nil, mutated, "the source is changed exactly when the case asks for it") + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + assert.Contains(t, err.Error(), tc.wantMsg) + return + } + require.NoError(t, err) + t.Cleanup(func() { _ = snap.Close() }) + require.NotEmpty(t, snap.Entries) + assert.Equal(t, tc.want, entryPaths(snap.Entries)) + }) + } +} diff --git a/pkg/artifact/snapshot/snapshot.go b/pkg/artifact/snapshot/snapshot.go index 413646e2b05..a509e4beb4f 100644 --- a/pkg/artifact/snapshot/snapshot.go +++ b/pkg/artifact/snapshot/snapshot.go @@ -50,6 +50,20 @@ type options struct { onCaptured func(path string) onRehashed func(path string) onResolved func(path string) + excluder Excluder +} + +// Excluder decides whether a source entry is left out of a capture. It receives the slash-separated +// path relative to the source root and whether the entry is a directory, and returns the visible +// rule that excludes it. An excluded directory is skipped with everything below it, and the exclusion +// is reported on the [Snapshot] so that dry-run output can show what was left out and why. +type Excluder func(entryPath string, isDir bool) (rule string, excluded bool) + +// WithExcluder leaves entries out of the capture. The generic files producer never sets one. +func WithExcluder(excluder Excluder) Option { + defer perf.Track(nil, "snapshot.WithExcluder")() + + return func(o *options) { o.excluder = excluder } } // WithStagingRoot sets the directory that holds the private staging directory. The default is the @@ -90,6 +104,10 @@ type Snapshot struct { // SourceIsFile reports that the source was a single file rather than a directory. SourceIsFile bool + // Exclusions lists the entries an [Excluder] left out, sorted by path. Excluded directories are + // listed once, not per descendant. + Exclusions []artifact.Exclusion + staged map[string]string } @@ -211,7 +229,16 @@ func captureInto(ctx context.Context, source, root string, opts *options) (*Snap if info.IsDir() { err = c.walkDir(resolved, "") } else { - err = c.captureFile(resolved, filepath.Base(resolved), info) + rel := filepath.Base(resolved) + if opts.excluder != nil { + if rule, excluded := opts.excluder(rel, false); excluded { + err = c.exclude(resolved, rel, rule, info) + } else { + err = c.captureFile(resolved, rel, info) + } + } else { + err = c.captureFile(resolved, rel, info) + } } if err != nil { return nil, err @@ -220,7 +247,8 @@ func captureInto(ctx context.Context, source, root string, opts *options) (*Snap return nil, err } sort.Slice(c.entries, func(i, j int) bool { return c.entries[i].Path < c.entries[j].Path }) - return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), staged: c.staged}, nil + sort.Slice(c.exclusions, func(i, j int) bool { return c.exclusions[i].Path < c.exclusions[j].Path }) + return &Snapshot{Root: root, Entries: c.entries, TotalBytes: c.total, SourceIsFile: !info.IsDir(), Exclusions: c.exclusions, staged: c.staged}, nil } // verify checks the finished capture: the source is unchanged, no link leaves it, the inventory is diff --git a/pkg/artifact/snapshot/walk.go b/pkg/artifact/snapshot/walk.go index ab17aa1e5be..b1aa856d7e0 100644 --- a/pkg/artifact/snapshot/walk.go +++ b/pkg/artifact/snapshot/walk.go @@ -34,11 +34,14 @@ type capturer struct { // rootInfo identifies the source root as it was resolved, so a replacement is detected. rootInfo os.FileInfo entries []artifact.Entry - staged map[string]string - dirs map[string][]string - sources []sourceRecord - total int64 - counter int + // exclusions are the entries the excluder left out. + exclusions []artifact.Exclusion + excludedSources []sourceRecord + staged map[string]string + dirs map[string][]string + sources []sourceRecord + total int64 + counter int // staging identifies the private staging directory, which is skipped when it lies inside the source. staging os.FileInfo @@ -78,6 +81,11 @@ func (c *capturer) captureChild(abs, rel string) error { return nil } mode := info.Mode() + if c.opts.excluder != nil { + if rule, excluded := c.opts.excluder(rel, mode.IsDir()); excluded { + return c.exclude(abs, rel, rule, info) + } + } switch { case mode.IsDir(): if err := c.addEntry(&artifact.Entry{Path: rel, Type: artifact.EntryDirectory, Mode: permBits(info)}); err != nil { @@ -94,6 +102,16 @@ func (c *capturer) captureChild(abs, rel string) error { } } +// exclude records a bounded exclusion and its source type for the final recheck. +func (c *capturer) exclude(abs, rel, rule string, info os.FileInfo) error { + if len(c.exclusions) >= c.opts.limits.MaxEntries { + return fmt.Errorf("%w: more than %d exclusions", errUtils.ErrArtifactTooLarge, c.opts.limits.MaxEntries) + } + c.exclusions = append(c.exclusions, artifact.Exclusion{Path: rel, Rule: rule}) + c.excludedSources = append(c.excludedSources, sourceRecord{abs: abs, rel: rel, info: info}) + return nil +} + // captureSymlink records a relative symlink without following it. Absolute or escaping targets // are rejected when the entry is validated. func (c *capturer) captureSymlink(abs, rel string, info os.FileInfo) error { @@ -149,6 +167,7 @@ func readDirNames(dir string) ([]string, error) { // can clean to a contained path yet resolve outside it. A dangling link is resolved through its // existing part plus the missing remainder, so it cannot escape once the missing path is created. // The check reads the links as captured and never the source, which may have changed since. +// A contained link is then checked against the excluder, so that it cannot name excluded content. func (c *capturer) checkLinkContainment() error { links := make(map[string]string) for index := range c.sources { @@ -156,11 +175,63 @@ func (c *capturer) checkLinkContainment() error { links[record.rel] = record.target } } + var captured map[string]bool + if c.opts != nil && c.opts.excluder != nil { + captured = c.capturedTypes() + } for index := range c.sources { record := &c.sources[index] - if record.target != "" && linkpath.Escapes(links, record.rel) { + if record.target == "" { + continue + } + if linkpath.Escapes(links, record.rel) { return fmt.Errorf("%w: symlink %q resolves outside the source", errUtils.ErrArtifactUnsafePath, record.rel) } + if captured == nil { + continue + } + if err := c.checkExcludedTarget(links, captured, record.rel); err != nil { + return err + } + } + return nil +} + +// capturedTypes maps the path of every entry the walk classified, captured or excluded, to whether +// it was a directory. It is the walk's own record of the source, so the excluder can be asked about +// a link target with the same arguments the walk used, without inspecting the source again. +func (c *capturer) capturedTypes() map[string]bool { + types := make(map[string]bool, len(c.sources)+len(c.excludedSources)) + for index := range c.sources { + types[c.sources[index].rel] = c.sources[index].info.IsDir() + } + for index := range c.excludedSources { + types[c.excludedSources[index].rel] = c.excludedSources[index].info.IsDir() + } + return types +} + +// checkExcludedTarget rejects a symlink whose target is an excluded entry, lies below one, or passes +// through one on the way. The filesystem looks up every directory before it processes a ".." that +// follows it, so a link such as "cache/p/../../keep.txt" breaks when "cache" is left out even +// though it reduces to "keep.txt". The target is therefore followed through the captured links the +// way the filesystem would, and each path it traverses is tested as the walk classified it, so the +// answer cannot change when the source changes after capture. A path the walk never classified is +// either missing (a dangling link) or below an excluded directory, which is traversed, and tested, +// first. +func (c *capturer) checkExcludedTarget(links map[string]string, captured map[string]bool, link string) error { + traversed, inside := linkpath.TraversedRecorded(links, link) + if !inside { + return nil + } + for _, candidate := range traversed { + isDir, classified := captured[candidate] + if !classified { + continue + } + if _, excluded := c.opts.excluder(candidate, isDir); excluded { + return fmt.Errorf("%w: symlink %q targets excluded path %q", errUtils.ErrArtifactUnsafePath, link, candidate) + } } return nil } diff --git a/pkg/artifactwire/catalog.go b/pkg/artifactwire/catalog.go new file mode 100644 index 00000000000..77a4c9818d2 --- /dev/null +++ b/pkg/artifactwire/catalog.go @@ -0,0 +1,29 @@ +package artifactwire + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/artifact/compose" + "github.com/cloudposse/atmos/pkg/artifact/discovery/catalog" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/plugin/bootstrap" + "github.com/cloudposse/atmos/pkg/schema" +) + +// NewCatalogs wires the catalog maintenance service of the configured repositories with the same +// plugin-aware registry authentication as [NewService]. Building it performs no network access, +// credential resolution, or directory creation. +func NewCatalogs(config *schema.AtmosConfiguration) (catalog.Maintainer, error) { + defer perf.Track(config, "artifactwire.NewCatalogs")() + + composition, err := compose.New( + config, + compose.WithRegistryPreparation(func(ctx context.Context) (func(), error) { + return bootstrap.PrepareAvailable(ctx, config) + }), + ) + if err != nil { + return nil, err + } + return composition.Catalogs, nil +} diff --git a/pkg/artifactwire/catalog_test.go b/pkg/artifactwire/catalog_test.go new file mode 100644 index 00000000000..47b8e07cbaa --- /dev/null +++ b/pkg/artifactwire/catalog_test.go @@ -0,0 +1,33 @@ +package artifactwire_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifactwire" + "github.com/cloudposse/atmos/pkg/schema" +) + +var _ = schema.ArtifactRepositoryConfig{Kind: "", Options: nil, Catalog: ""} + +func TestNewCatalogsComposesWithoutAccessingAnyRepository(t *testing.T) { + config := &schema.AtmosConfiguration{Artifacts: schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": { + Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "127.0.0.1:1", "prefix": "acme/atmos"}, + Catalog: "oci://127.0.0.1:1/acme/atmos/catalog:stable", + }, + }}} + maintainer, err := artifactwire.NewCatalogs(config) + require.NoError(t, err, "building the service performs no registry access") + require.NotNil(t, maintainer) +} + +func TestNewCatalogsRejectsInvalidConfiguration(t *testing.T) { + config := &schema.AtmosConfiguration{Artifacts: schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + "shared": {Kind: artifact.RepositoryKindOCIRegistry, Options: map[string]any{"registry": "127.0.0.1:1"}, Catalog: "https://example.com/catalog"}, + }}} + _, err := artifactwire.NewCatalogs(config) + require.Error(t, err) +} diff --git a/pkg/artifactwire/wire.go b/pkg/artifactwire/wire.go new file mode 100644 index 00000000000..709ae089452 --- /dev/null +++ b/pkg/artifactwire/wire.go @@ -0,0 +1,42 @@ +// Package artifactwire assembles the production artifact service that the CLI and the artifact +// workflow step share. +// +// It lives outside pkg/artifact because the assembly needs the plugin-aware registry preparation +// and the managed Cosign signer, and the artifact packages are forbidden from depending on the +// plugin subsystem. The artifact service itself, the request and result mapping, and every +// repository stay free of that dependency. +package artifactwire + +import ( + "context" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/compose" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/plugin/bootstrap" + pluginsign "github.com/cloudposse/atmos/pkg/plugin/sign" + "github.com/cloudposse/atmos/pkg/schema" +) + +// NewService wires the repositories, the shared verifier, the managed Cosign signer, and the +// plugin-aware registry authentication preparation into one artifact service. It is an +// [invoke.ServiceFactory]. Building the service performs no network access, credential +// resolution, or directory creation. +func NewService(config *schema.AtmosConfiguration, options invoke.ServiceOptions) (artifact.Service, error) { + defer perf.Track(config, "artifactwire.NewService")() + + composition, err := compose.New( + config, + compose.WithBlobSigner(pluginsign.Cosign{Config: config, Yes: options.AcceptTransparencyLog}), + compose.WithRegistryPreparation(func(ctx context.Context) (func(), error) { + return bootstrap.PrepareAvailable(ctx, config) + }), + ) + if err != nil { + return nil, err + } + return composition.Service, nil +} + +var _ invoke.ServiceFactory = NewService diff --git a/pkg/artifactwire/wiretest/wiretest.go b/pkg/artifactwire/wiretest/wiretest.go new file mode 100644 index 00000000000..24c0066f443 --- /dev/null +++ b/pkg/artifactwire/wiretest/wiretest.go @@ -0,0 +1,173 @@ +// Package wiretest is test support for the entry points that run artifact operations: the CLI, +// workflow and custom-command steps, script step calls, and kind: step hooks. +// +// A [Fixture] builds a real service over a real local directory repository in a temporary +// directory, pins the clock and the operation IDs, and installs the service through the artifact +// step's service seam. Every entry point then reaches the same wiring, and a test can compare what +// each one asked for and received against a baseline run of the shared implementation. +package wiretest + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/compose" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/perf" + runnerstep "github.com/cloudposse/atmos/pkg/runner/step" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Pinned values every fixture uses, so independent runs produce byte-identical records. +const ( + // OperationID is the operation ID every fixture service and artifact step generates. + OperationID = "op-fixed" + + // Alias is the repository alias of the fixture's local directory repository. + Alias = "dir" + + // Name and Version are the artifact the fixture source publishes. + Name = "team/app" + Version = "1.0.0" +) + +// File modes of the fixture source. +const ( + dirMode = 0o755 + fileMode = 0o600 +) + +// PreparedAt is the pinned UTC preparation time. +var PreparedAt = time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) + +// Fixture is a pinned artifact environment. +type Fixture struct { + // Root is the repository root of the entry point under test. + Root string + + // Source is a directory with the content to publish. + Source string + + // Config is the Atmos configuration that declares the repository. + Config *schema.AtmosConfiguration + + // Recorder records the publications that pass through the service seam. + Recorder *artifacttest.Recorder + + baselineRoot string +} + +// New creates the fixture and installs its service for artifact steps. The seams are restored when +// the test ends. +func New(t *testing.T) *Fixture { + defer perf.Track(nil, "wiretest.New")() + + t.Helper() + base := t.TempDir() + f := &Fixture{ + Root: filepath.Join(base, "repository"), + Source: filepath.Join(base, "dist"), + baselineRoot: filepath.Join(base, "baseline"), + } + require.NoError(t, os.MkdirAll(f.Source, dirMode)) + require.NoError(t, os.WriteFile(filepath.Join(f.Source, "app.txt"), []byte("hello"), fileMode)) + require.NoError(t, os.WriteFile(filepath.Join(f.Source, "README.md"), []byte("# App\n"), fileMode)) + + f.Config = Config(f.Root) + f.Recorder = artifacttest.NewRecorder(f.service(t, f.Config)) + restoreFactory := runnerstep.UseArtifactServiceFactory(func(*schema.AtmosConfiguration, invoke.ServiceOptions) (artifact.Service, error) { + return f.Recorder, nil + }) + t.Cleanup(restoreFactory) + restoreIdentity := runnerstep.UseArtifactIdentity(func() string { return OperationID }, func() time.Time { return PreparedAt }) + t.Cleanup(restoreIdentity) + return f +} + +// Config returns a configuration with one local directory repository rooted at root. +func Config(root string) *schema.AtmosConfiguration { + defer perf.Track(nil, "wiretest.Config")() + + return &schema.AtmosConfiguration{Artifacts: schema.ArtifactConfig{Repositories: map[string]schema.ArtifactRepositoryConfig{ + Alias: {Kind: "local/dir", Options: map[string]any{"path": root}}, + }}} +} + +// service builds a real service with the clock and the operation IDs pinned and no retry delay. +func (f *Fixture) service(t *testing.T, config *schema.AtmosConfiguration) artifact.Service { + t.Helper() + composition, err := compose.New(config, compose.WithServiceOptions( + service.WithClock(func() time.Time { return PreparedAt }), + service.WithIDGenerator(func() string { return OperationID }), + service.WithSleep(func(context.Context, time.Duration) error { return nil }), + )) + require.NoError(t, err) + return composition.Service +} + +// Params returns the parameters of the fixture publication. Every entry point is configured to +// publish exactly this. +func (f *Fixture) Params() *invoke.Params { + defer perf.Track(nil, "wiretest.Fixture.Params")() + + return &invoke.Params{ + Action: artifact.ActionPublish, Repository: Alias, Source: f.Source, Name: Name, Version: Version, + Labels: map[string]string{"team": "platform"}, + } +} + +// Baseline runs the fixture publication through the shared implementation, which is what the CLI +// runs, in a sibling repository. The result is normalized with [Fixture.Normalize] as if it came +// from the fixture's own repository. +func (f *Fixture) Baseline(t *testing.T) *invoke.Result { + defer perf.Track(nil, "wiretest.Fixture.Baseline")() + + t.Helper() + config := Config(f.baselineRoot) + params := f.Params() + outcome, err := invoke.Run(context.Background(), f.service(t, config), params) + require.NoError(t, err) + result := *outcome.Result + result.OperationResult = relocate(&result.OperationResult, f.baselineRoot) + return &result +} + +// Normalize rewrites the repository root of the fixture to a placeholder, so results from +// repositories rooted in different directories compare equal. +func (f *Fixture) Normalize(op *artifact.OperationResult) artifact.OperationResult { + defer perf.Track(nil, "wiretest.Fixture.Normalize")() + + return relocate(op, f.Root) +} + +func relocate(source *artifact.OperationResult, root string) artifact.OperationResult { + op := *source + op.Location = strings.ReplaceAll(op.Location, root, "") + if len(op.Aliases) > 0 { + aliases := make([]string, 0, len(op.Aliases)) + for _, alias := range op.Aliases { + aliases = append(aliases, strings.ReplaceAll(alias, root, "")) + } + op.Aliases = aliases + } + return op +} + +// StepFields returns the artifact step fields of the fixture publication, as authored in YAML. +func (f *Fixture) StepFields() map[string]any { + defer perf.Track(nil, "wiretest.Fixture.StepFields")() + + return map[string]any{ + "action": "publish", "source": f.Source, "repository": Alias, "artifact_name": Name, "version": Version, + "metadata": map[string]any{"labels": map[string]any{"team": "platform"}}, + } +} diff --git a/pkg/config/schema/overrides.go b/pkg/config/schema/overrides.go index 318ed68bbef..68c2f0eea70 100644 --- a/pkg/config/schema/overrides.go +++ b/pkg/config/schema/overrides.go @@ -22,6 +22,7 @@ const ( // The name of the shared `$defs` entry that models an Atmos YAML function // invocation in its authored (string) form. yamlFunctionDef = "yamlFunction" + yamlFunctionRef = "#/$defs/" + yamlFunctionDef // JSON Schema type names used when constructing and inspecting subschemas. typeString = "string" @@ -208,6 +209,12 @@ func applyDefinitionPropertyAlternatives(root *jsonschema.Schema) { // padding/margin: "1 2" string or bare integer shorthand. "padding": {{Type: typeInteger}}, "margin": {{Type: typeInteger}}, + // query: the http step maps query-string parameters; the artifact step takes search text. + "query": {{Type: typeString}}, + }, + "WorkflowStep": { + // query: preserve the mapping shape for http and scalar search text for artifact steps. + "query": {{Type: typeString}, {Ref: yamlFunctionRef}}, }, "Command": { // env accepts the map form ({KEY: value}) in addition to the list @@ -390,7 +397,7 @@ func wrapped(s *jsonschema.Schema) *jsonschema.Schema { s.AnyOf = append(s.AnyOf, &jsonschema.Schema{Type: typeNull}) } if !anyOfAcceptsYamlFunction(s.AnyOf) { - s.AnyOf = append(s.AnyOf, &jsonschema.Schema{Ref: "#/$defs/" + yamlFunctionDef}) + s.AnyOf = append(s.AnyOf, &jsonschema.Schema{Ref: yamlFunctionRef}) } return s case s.Ref == "" && s.Type == "" || s.Type == typeNull: @@ -410,7 +417,7 @@ func isAnyOfWrapper(s *jsonschema.Schema) bool { func authoredAlternatives(s *jsonschema.Schema) []*jsonschema.Schema { alternatives := []*jsonschema.Schema{{Type: typeNull}} if rejectsYamlFunctionString(s) { - alternatives = append(alternatives, &jsonschema.Schema{Ref: "#/$defs/" + yamlFunctionDef}) + alternatives = append(alternatives, &jsonschema.Schema{Ref: yamlFunctionRef}) } // String-list fields also accept a scalar string: the loader's // StringToSliceHookFunc coerces `import: "path"` into a one-element list. @@ -455,7 +462,7 @@ func anyOfAcceptsYamlFunction(branches []*jsonschema.Schema) bool { if branch == nil { continue } - if branch.Ref == "#/$defs/"+yamlFunctionDef { + if branch.Ref == yamlFunctionRef { return true } if branch.Type == typeString && !rejectsYamlFunctionString(branch) { @@ -470,7 +477,7 @@ func anyOfAcceptsYamlFunction(branches []*jsonschema.Schema) bool { // already accept it. func rejectsYamlFunctionString(s *jsonschema.Schema) bool { if s.Ref != "" { - return s.Ref != "#/$defs/"+yamlFunctionDef + return s.Ref != yamlFunctionRef } switch s.Type { case typeObject, typeArray, typeBoolean, typeInteger, typeNumber: diff --git a/pkg/config/schema/workflow_query_test.go b/pkg/config/schema/workflow_query_test.go new file mode 100644 index 00000000000..443131a5149 --- /dev/null +++ b/pkg/config/schema/workflow_query_test.go @@ -0,0 +1,38 @@ +package configschema + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + stjsonschema "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/stretchr/testify/require" +) + +func TestWorkflowQuerySchema(t *testing.T) { + root, err := RepoRoot() + require.NoError(t, err) + committed, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(EmbeddedPath))) + require.NoError(t, err) + for name, raw := range map[string][]byte{"generated": generatedSchema(t), "committed": committed} { + t.Run(name, func(t *testing.T) { + var document map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + defs := document["$defs"].(map[string]any) + step := defs["WorkflowStep"].(map[string]any) + props := step["properties"].(map[string]any) + query := props["query"].(map[string]any) + require.Contains(t, query["anyOf"], map[string]any{"$ref": "#/$defs/yamlFunction"}) + compiler := stjsonschema.NewCompiler() + require.NoError(t, compiler.AddResource("atmos-config.json", bytes.NewReader(raw))) + compiled, err := compiler.Compile("atmos-config.json#/$defs/WorkflowStep/properties/query") + require.NoError(t, err) + for _, query := range []any{"vpc", map[string]any{"page": "1"}, nil, "!env ARTIFACT_QUERY"} { + require.NoError(t, compiled.Validate(query), "query: %#v", query) + } + require.Error(t, compiled.Validate([]any{"vpc"})) + }) + } +} diff --git a/pkg/datafetcher/schema/atmos/config/1.0.json b/pkg/datafetcher/schema/atmos/config/1.0.json index a7dbe774c6e..d13ac315ee9 100644 --- a/pkg/datafetcher/schema/atmos/config/1.0.json +++ b/pkg/datafetcher/schema/atmos/config/1.0.json @@ -1856,6 +1856,36 @@ "type": "object", "description": "ArtifactRepositoryDefaults holds defaults applied to artifacts newly published to a repository." }, + "ArtifactStepMetadata": { + "properties": { + "labels": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Labels are the exact key/value labels attached to the published artifact version." + } + }, + "type": "object", + "description": "ArtifactStepMetadata is the `metadata:` block of an artifact publish step. Only labels are\naccepted: they are the explicit user metadata that participates in the publication's intent." + }, "ArtifactTrustedIdentity": { "properties": { "identity": { @@ -12301,10 +12331,10 @@ "type": "object" }, { - "type": "null" + "type": "string" }, { - "$ref": "#/$defs/yamlFunction" + "type": "null" } ], "description": "Query-string parameters (supports templates)." @@ -12682,6 +12712,186 @@ ], "description": "Mtime controls the modification-time metadata stamped into each archive entry\n(not the source files on disk, not the archive file's own OS-level mtime):\nfilesystem (default, same as omitting the field) | epoch (one shared timestamp\nfor the whole archive) | git (per-entry timestamps). epoch/git also normalize\npermission bits. See docs/prd/archive-step.md." }, + "repository": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configured artifact repository alias." + }, + "artifact_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact name (publish; list restricts to one name)." + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact version (publish)." + }, + "artifact_kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact kind (publish); defaults to files." + }, + "force": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Replace conflicting state (publish, pull, copy)." + }, + "dry_run": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Validate and plan a publication without writing. Distinct from DryRun, which executors set." + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactStepMetadata" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Publish metadata (labels)." + }, + "labels": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ], + "description": "List and search label selectors: a key=value string, a list of them, or a mapping." + }, + "readme": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "README path, relative to the publish source directory." + }, + "changelog": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "CHANGELOG path, relative to the publish source directory." + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact reference: name:version or name@sha256:\u003cdigest\u003e." + }, + "from": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Copy source repository alias." + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Copy destination repository alias." + }, + "document": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Document to show: readme or changelog." + }, + "refresh": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Bypass cached discovery snapshots (list, search)." + }, "tools": { "anyOf": [ { @@ -17789,10 +17999,13 @@ "type": "object" }, { - "type": "null" + "type": "string" }, { "$ref": "#/$defs/yamlFunction" + }, + { + "type": "null" } ], "description": "Query-string parameters (supports templates)." @@ -18220,6 +18433,186 @@ ], "description": "Mtime controls the modification-time metadata stamped into each archive entry\n(not the source files on disk, not the archive file's own OS-level mtime):\nfilesystem (default, same as omitting the field) | epoch (one shared timestamp\nfor the whole archive) | git (per-entry timestamps). epoch/git also normalize\npermission bits. See docs/prd/archive-step.md." }, + "repository": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Configured artifact repository alias." + }, + "artifact_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact name (publish; list restricts to one name)." + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact version (publish)." + }, + "artifact_kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact kind (publish); defaults to files." + }, + "force": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Replace conflicting state (publish, pull, copy)." + }, + "dry_run": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Validate and plan a publication without writing. Distinct from DryRun, which executors set." + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactStepMetadata" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Publish metadata (labels)." + }, + "labels": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ], + "description": "List and search label selectors: a key=value string, a list of them, or a mapping." + }, + "readme": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "README path, relative to the publish source directory." + }, + "changelog": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "CHANGELOG path, relative to the publish source directory." + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Artifact reference: name:version or name@sha256:\u003cdigest\u003e." + }, + "from": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Copy source repository alias." + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Copy destination repository alias." + }, + "document": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Document to show: readme or changelog." + }, + "refresh": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/yamlFunction" + } + ], + "description": "Bypass cached discovery snapshots (list, search)." + }, "files": { "anyOf": [ { diff --git a/pkg/datafetcher/schema/atmos/manifest/1.0.json b/pkg/datafetcher/schema/atmos/manifest/1.0.json index 5c108b606f5..6a87df27105 100644 --- a/pkg/datafetcher/schema/atmos/manifest/1.0.json +++ b/pkg/datafetcher/schema/atmos/manifest/1.0.json @@ -4325,11 +4325,18 @@ "description": "Request headers for the http step type (supports templates)." }, "query": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Query-string parameters for the http step type (supports templates)." + "description": "A mapping of query-string parameters for the http step type, or a string of search text for the artifact step type (action: search). Supports templates.", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "type": "string" + } + ] }, "body": { "type": "string", @@ -4410,7 +4417,7 @@ }, "action": { "type": "string", - "description": "Step-type-specific verb: build, push, run, or inspect for type: container; create, extract, update, or replace for type: archive; write for type: store." + "description": "Step-type-specific verb: build, push, run, or inspect for type: container; create, extract, update, or replace for type: archive; write for type: store; publish, pull, copy, inspect, list, search, delete, sign, verify, or show for type: artifact." }, "with": { "type": "object", @@ -4441,13 +4448,99 @@ "type": "boolean", "description": "Run the emulator without persistence for this step (emulator step type)." }, + "repository": { + "type": "string", + "description": "Configured artifact repository alias (artifact step type). Supports templates. List and search accept a comma-separated list and default to every configured repository." + }, + "artifact_name": { + "type": "string", + "description": "Artifact name (artifact step type): the name to publish, or the one name to list. The step's own `name` is unrelated." + }, + "version": { + "type": "string", + "description": "Artifact version to publish (artifact step type)." + }, + "artifact_kind": { + "type": "string", + "description": "Artifact kind to publish (artifact step type); defaults to files." + }, + "force": { + "type": "boolean", + "description": "Replace conflicting state for the artifact step type (publish, pull, copy). Force never bypasses verification." + }, + "dry_run": { + "type": "boolean", + "description": "Validate and plan an artifact publication without writing anything (artifact step type)." + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Exact key/value labels attached to the published artifact version." + } + }, + "description": "Artifact publish metadata (artifact step type)." + }, + "labels": { + "description": "Label selectors for artifact list and search: a comma-separated key=value string, a list of them, or a mapping. Every selector must match.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + ] + }, + "readme": { + "type": "string", + "description": "README path for artifact publish, relative to the source directory (artifact step type)." + }, + "changelog": { + "type": "string", + "description": "CHANGELOG path for artifact publish, relative to the source directory (artifact step type)." + }, + "ref": { + "type": "string", + "description": "Artifact reference, name:version or name@sha256: (artifact step type). There is no implicit latest." + }, + "from": { + "type": "string", + "description": "Source repository alias for artifact copy (artifact step type)." + }, + "to": { + "type": "string", + "description": "Destination repository alias for artifact copy (artifact step type)." + }, + "document": { + "type": "string", + "description": "Package document to read for artifact show: readme or changelog (artifact step type). Supports templates." + }, + "refresh": { + "type": "boolean", + "description": "Bypass cached discovery snapshots for artifact list and search (artifact step type)." + }, "format": { "type": "string", "description": "Archive format for the archive step type: zip, tar, tgz, tar.bz2, or tar.xz; inferred from destination/source extension when omitted." }, "destination": { "type": "string", - "description": "Archive output/extraction path, or publish destination relative to the target root." + "description": "Archive output/extraction path, publish destination relative to the target root, or the output directory of artifact pull." }, "subpath": { "type": "string", diff --git a/pkg/generator/directory/copy.go b/pkg/generator/directory/copy.go index 6e1a395b9fc..67dfdf6081a 100644 --- a/pkg/generator/directory/copy.go +++ b/pkg/generator/directory/copy.go @@ -10,6 +10,7 @@ import ( "path/filepath" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/filesystem" genfs "github.com/cloudposse/atmos/pkg/generator/filesystem" "github.com/cloudposse/atmos/pkg/perf" @@ -117,6 +118,12 @@ func inspectTree(ctx context.Context, source, target string) ([]entry, error) { if err != nil || rel == "." { return err } + if skip, err := isDistributionMetadata(name, rel, d); err != nil || skip { + if err == nil && d.IsDir() { + return filepath.SkipDir + } + return err + } if err := rejectDestinationSymlinks(target, rel); err != nil { return err } @@ -129,6 +136,27 @@ func inspectTree(ctx context.Context, source, target string) ([]entry, error) { return entries, err } +// isDistributionMetadata reports whether a source entry is the reserved distribution directory of a +// published scaffold (`.atmos/artifact`), which describes the distribution and never reaches the +// copy. The `.atmos` directory above it is skipped too when nothing else lives there. +func isDistributionMetadata(name, relative string, d fs.DirEntry) (bool, error) { + switch filepath.ToSlash(relative) { + case artifact.DistributionDir: + return true, nil + case artifact.DistributionParent: + if !d.IsDir() { + return false, nil + } + children, err := os.ReadDir(name) + if err != nil { + return false, err + } + return len(children) == 1 && children[0].Name() == filepath.Base(artifact.DistributionDir), nil + default: + return false, nil + } +} + func regularEntryInfo(name string, d fs.DirEntry) (fs.FileInfo, error) { info, err := d.Info() if err != nil { diff --git a/pkg/generator/directory/copy_distribution_test.go b/pkg/generator/directory/copy_distribution_test.go new file mode 100644 index 00000000000..cb9ee500b2b --- /dev/null +++ b/pkg/generator/directory/copy_distribution_test.go @@ -0,0 +1,73 @@ +package directory + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeSourceFile(t *testing.T, root, rel, content string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +func TestCopyWithholdsDistributionMetadata(t *testing.T) { + tests := []struct { + name string + files map[string]string + wantFiles []string + wantAbsent []string + }{ + { + name: "package documents only", + files: map[string]string{ + "README.md": "root readme\n", + ".atmos/artifact/README.md": "package readme\n", + ".atmos/artifact/CHANGELOG.md": "changelog\n", + }, + wantFiles: []string{"README.md"}, + wantAbsent: []string{".atmos"}, + }, + { + name: "other files under .atmos are ordinary content", + files: map[string]string{ + "README.md": "root readme\n", + ".atmos/scaffold.yaml": "kind: x\n", + ".atmos/artifact/README.md": "package readme\n", + }, + wantFiles: []string{"README.md", filepath.Join(".atmos", "scaffold.yaml")}, + wantAbsent: []string{filepath.Join(".atmos", "artifact")}, + }, + { + name: "similarly named directories are kept", + files: map[string]string{ + ".atmos/artifacts/x.txt": "x\n", + "docs/.atmos/artifact/y.md": "y\n", + }, + wantFiles: []string{filepath.Join(".atmos", "artifacts", "x.txt"), filepath.Join("docs", ".atmos", "artifact", "y.md")}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src, dst := t.TempDir(), filepath.Join(t.TempDir(), "target") + for rel, content := range tt.files { + writeSourceFile(t, src, rel, content) + } + + require.NoError(t, Copy(context.Background(), src, dst, false)) + + for _, kept := range tt.wantFiles { + assert.FileExists(t, filepath.Join(dst, kept)) + } + for _, absent := range tt.wantAbsent { + assert.NoDirExists(t, filepath.Join(dst, absent)) + } + }) + } +} diff --git a/pkg/generator/templates/distribution.go b/pkg/generator/templates/distribution.go new file mode 100644 index 00000000000..3a072df44de --- /dev/null +++ b/pkg/generator/templates/distribution.go @@ -0,0 +1,40 @@ +package templates + +import ( + "strings" + + "github.com/cloudposse/atmos/pkg/artifact" +) + +// withoutDistributionMetadata removes the reserved distribution directory (`.atmos/artifact`) from a +// template's files. That directory holds the package README and CHANGELOG of a published scaffold +// (and the manifest of a packaged one); it describes the distribution, not the generated project, so +// no template source strategy (local directory, git, OCI, S3, embedded, or a rendered base re-render) +// may generate it. The root README is ordinary template content and is kept. +// +// The `.atmos` directory entry is dropped only when nothing else lives below it, so scaffolds that +// really keep files under `.atmos` are unaffected. +func withoutDistributionMetadata(files []File) []File { + const parentPrefix = artifact.DistributionParent + "/" + hasOtherAtmos := false + for index := range files { + name := files[index].Path + if strings.HasPrefix(name, parentPrefix) && !isDistributionPath(name) { + hasOtherAtmos = true + break + } + } + kept := make([]File, 0, len(files)) + for index := range files { + name := files[index].Path + if isDistributionPath(name) || (name == artifact.DistributionParent && files[index].IsDirectory && !hasOtherAtmos) { + continue + } + kept = append(kept, files[index]) + } + return kept +} + +func isDistributionPath(name string) bool { + return name == artifact.DistributionDir || strings.HasPrefix(name, artifact.DistributionDir+"/") +} diff --git a/pkg/generator/templates/distribution_test.go b/pkg/generator/templates/distribution_test.go new file mode 100644 index 00000000000..20c773bc451 --- /dev/null +++ b/pkg/generator/templates/distribution_test.go @@ -0,0 +1,76 @@ +package templates + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func filePaths(files []File) []string { + paths := make([]string, 0, len(files)) + for index := range files { + paths = append(paths, files[index].Path) + } + return paths +} + +func TestWithoutDistributionMetadata(t *testing.T) { + dir := func(path string) File { return File{Path: path, IsDirectory: true} } + file := func(path string) File { return File{Path: path} } + tests := []struct { + name string + files []File + want []string + }{ + { + name: "package documents and their parent disappear", + files: []File{file("README.md"), dir(".atmos"), dir(".atmos/artifact"), file(".atmos/artifact/README.md"), file(".atmos/artifact/CHANGELOG.md"), file("scaffold.yaml")}, + want: []string{"README.md", "scaffold.yaml"}, + }, + { + name: "the parent stays when something else lives below it", + files: []File{dir(".atmos"), dir(".atmos/artifact"), file(".atmos/artifact/README.md"), file(".atmos/scaffold.yaml")}, + want: []string{".atmos", ".atmos/scaffold.yaml"}, + }, + { + name: "the root README is ordinary content", + files: []File{file("README.md"), file("docs/README.md")}, + want: []string{"README.md", "docs/README.md"}, + }, + { + name: "lookalike paths are kept", + files: []File{file(".atmos/artifacts/x"), file("docs/.atmos/artifact/y")}, + want: []string{".atmos/artifacts/x", "docs/.atmos/artifact/y"}, + }, + {name: "no files", files: nil, want: []string{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, filePaths(withoutDistributionMetadata(tt.files))) + }) + } +} + +func TestLoadConfigurationFromDir_NeverLoadsDistributionMetadata(t *testing.T) { + dir := t.TempDir() + writeTemplateFile(t, dir, "scaffold.yaml", "apiVersion: atmos/v1\nkind: AtmosScaffoldConfig\nmetadata:\n name: demo\n") + writeTemplateFile(t, dir, "README.md", "# root readme\n") + writeTemplateFile(t, dir, ".atmos/artifact/README.md", "# package readme\n") + writeTemplateFile(t, dir, ".atmos/artifact/CHANGELOG.md", "# changelog\n") + + configuration, err := LoadConfigurationFromDir("demo", dir) + + require.NoError(t, err) + assert.ElementsMatch(t, []string{"README.md", "scaffold.yaml"}, filePaths(configuration.Files)) + assert.Equal(t, "# root readme\n", configuration.README, "the root README keeps its behavior") +} + +func writeTemplateFile(t *testing.T, root, rel, content string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} diff --git a/pkg/generator/templates/embeds.go b/pkg/generator/templates/embeds.go index 17a57f8f3d1..c2edb331c53 100644 --- a/pkg/generator/templates/embeds.go +++ b/pkg/generator/templates/embeds.go @@ -279,6 +279,9 @@ func loadConfiguration(fsys fs.FS, templatePath, defaultName, source, excludeRoo Err() } + // The reserved distribution directory never reaches generated output. + files = withoutDistributionMetadata(files) + // Find README if it exists var readmeContent string // Use path.Join (forward slashes) not filepath.Join for fs.FS paths. diff --git a/pkg/oci/atmos_payload.go b/pkg/oci/atmos_payload.go new file mode 100644 index 00000000000..c51815efa1a --- /dev/null +++ b/pkg/oci/atmos_payload.go @@ -0,0 +1,91 @@ +package oci + +import ( + "fmt" + "io" + "path" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Atmos artifact packages (`atmos artifacts publish`, `atmos scaffold publish`) describe themselves +// with the shared Record config media type. Their layers are the archive payload followed by the +// README and CHANGELOG documents as separate Markdown blobs. Component and scaffold consumers that +// unpack an image into a directory must extract only the payload layer: the document layers are not +// archives, and the payload's generated `.atmos/artifact` distribution members are never part of +// what a consumer provisions or generates from. +// +// Legacy images (a single tar+gzip component layer, OpenTofu module zips, and generic images) do +// not carry that config media type, so they keep extracting every layer exactly as before. + +// isAtmosPackage reports whether the image is an Atmos artifact package. +func isAtmosPackage(img v1.Image) (bool, error) { + manifest, err := img.Manifest() + if err != nil { + return false, err + } + return string(manifest.Config.MediaType) == artifact.MediaTypeConfig, nil +} + +// atmosPayloadExtractor returns the extractor for an Atmos payload layer media type. Every extractor +// withholds the distribution metadata of the package. +func atmosPayloadExtractor(mediaType string) (func(io.Reader, string) error, bool) { + switch mediaType { + case artifact.MediaTypeTar, artifact.MediaTypeTarGzip: + return func(reader io.Reader, dest string) error { + return extractTarballSkipping(reader, dest, isDistributionMetadata) + }, true + case artifact.MediaTypeZip: + return func(reader io.Reader, dest string) error { + return extractZipSkipping(reader, dest, isDistributionMetadata) + }, true + default: + return nil, false + } +} + +// isDistributionMetadata reports whether an archive member belongs to the reserved distribution +// directory. The `.atmos` directory member itself is withheld too, because packaging adds it only +// to hold the distribution directory; files that a package really keeps under `.atmos` still create +// it when they are extracted. +func isDistributionMetadata(name string) bool { + clean := path.Clean(strings.TrimPrefix(filepath.ToSlash(name), "./")) + return clean == artifact.DistributionParent || clean == artifact.DistributionDir || strings.HasPrefix(clean, artifact.DistributionDir+"/") +} + +// selectLayers returns the layers a consumer extracts. A legacy image extracts every layer. An Atmos +// package extracts its single payload layer and leaves the document blobs untouched. +func selectLayers(img v1.Image, layers []v1.Layer) ([]v1.Layer, error) { + atmos, err := isAtmosPackage(img) + if err != nil { + return nil, err + } + if !atmos { + return layers, nil + } + manifest, err := img.Manifest() + if err != nil { + return nil, err + } + var payload []v1.Layer + for index := range manifest.Layers { + mediaType := string(manifest.Layers[index].MediaType) + if _, isPayload := atmosPayloadExtractor(mediaType); isPayload { + if index >= len(layers) { + return nil, fmt.Errorf("%w: missing payload layer", errUtils.ErrArtifactPayloadLayer) + } + payload = append(payload, layers[index]) + } else if mediaType != artifact.DocumentMediaType { + return nil, fmt.Errorf("%w: unexpected layer media type %q", errUtils.ErrArtifactPayloadLayer, mediaType) + } + } + if len(payload) != 1 { + return nil, fmt.Errorf("%w: found %d payload layers", errUtils.ErrArtifactPayloadLayer, len(payload)) + } + return payload, nil +} diff --git a/pkg/oci/atmos_payload_test.go b/pkg/oci/atmos_payload_test.go new file mode 100644 index 00000000000..202f1c95ba3 --- /dev/null +++ b/pkg/oci/atmos_payload_test.go @@ -0,0 +1,263 @@ +package oci + +import ( + "archive/tar" + "bytes" + "context" + "io" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/cloudposse/atmos/pkg/schema" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" +) + +// Compile-time guard for the artifact media types these tests build packages from. +var _ = []string{artifact.MediaTypeConfig, artifact.MediaTypeTar, artifact.MediaTypeTarGzip, artifact.MediaTypeZip, artifact.DocumentMediaType} + +// atmosFiles is a payload as packaging produces it: the user's files plus the generated +// distribution directory, with the `.atmos` directory member that holds it. +func atmosFiles() map[string]string { + return map[string]string{ + "main.tf": "# main\n", + ".terraform.lock.hcl": "# lock\n", + "scripts/helper.sh": "#!/bin/sh\n", + ".atmos/artifact/manifest.json": "{}\n", + ".atmos/artifact/README.md": "# package readme\n", + } +} + +// atmosLayer builds a payload layer. A gzip layer is compressed the way a registry serves it, so +// the layer's Uncompressed stream is the plain tar, like a layer read back from a registry. +func atmosLayer(t *testing.T, mediaType types.MediaType, files map[string]string) v1.Layer { + t.Helper() + switch string(mediaType) { + case artifact.MediaTypeZip: + return static.NewLayer(writeTestZip(t, files).Bytes(), mediaType) + case artifact.MediaTypeTarGzip: + data := writeTestTar(t, files).Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(data)), nil }, tarball.WithMediaType(mediaType)) + require.NoError(t, err) + return layer + default: + return static.NewLayer(writeTestTar(t, files).Bytes(), mediaType) + } +} + +func atmosPackage(t *testing.T, layers ...v1.Layer) v1.Image { + t.Helper() + img := mutate.ConfigMediaType(empty.Image, types.MediaType(artifact.MediaTypeConfig)) + img, err := mutate.AppendLayers(img, layers...) + require.NoError(t, err) + return img +} + +func documentLayer(content string) v1.Layer { + return static.NewLayer([]byte(content), types.MediaType(artifact.DocumentMediaType)) +} + +func TestSelectLayers_AtmosPackageExtractsOnlyThePayload(t *testing.T) { + payload := atmosLayer(t, types.MediaType(artifact.MediaTypeTarGzip), atmosFiles()) + img := atmosPackage(t, payload, documentLayer("# readme\n"), documentLayer("# changelog\n")) + layers, err := img.Layers() + require.NoError(t, err) + require.Len(t, layers, 3) + + selected, err := selectLayers(img, layers) + + require.NoError(t, err) + require.Len(t, selected, 1) + want, err := payload.Digest() + require.NoError(t, err) + got, err := selected[0].Digest() + require.NoError(t, err) + assert.Equal(t, want, got, "the archive payload is selected, not the Markdown documents") +} + +func TestSelectLayers_LegacyImageKeepsEveryLayer(t *testing.T) { + img, err := mutate.AppendLayers( + empty.Image, + static.NewLayer(writeTestTar(t, map[string]string{"a.tf": "a"}).Bytes(), types.OCILayer), + static.NewLayer(writeTestTar(t, map[string]string{"b.tf": "b"}).Bytes(), types.OCILayer), + ) + require.NoError(t, err) + layers, err := img.Layers() + require.NoError(t, err) + + selected, err := selectLayers(img, layers) + + require.NoError(t, err) + assert.Len(t, selected, 2, "legacy component and scaffold layer heuristics are untouched") +} + +func TestSelectLayers_RejectsPackagesWithoutExactlyOnePayload(t *testing.T) { + tarLayer := func(name string) v1.Layer { + return atmosLayer(t, types.MediaType(artifact.MediaTypeTar), map[string]string{name: "x"}) + } + tests := []struct { + name string + layers []v1.Layer + }{ + {name: "documents only", layers: []v1.Layer{documentLayer("# readme\n")}}, + {name: "two payloads", layers: []v1.Layer{tarLayer("a"), tarLayer("b")}}, + {name: "unexpected layer", layers: []v1.Layer{tarLayer("a"), static.NewLayer([]byte("x"), types.MediaType("application/octet-stream"))}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + img := atmosPackage(t, tt.layers...) + layers, err := img.Layers() + require.NoError(t, err) + + _, err = selectLayers(img, layers) + + require.ErrorIs(t, err, errUtils.ErrArtifactPayloadLayer) + }) + } +} + +func TestProcessLayer_AtmosPayloadWithholdsDistributionMetadata(t *testing.T) { + for _, mediaType := range []string{artifact.MediaTypeTar, artifact.MediaTypeTarGzip, artifact.MediaTypeZip} { + t.Run(mediaType, func(t *testing.T) { + dest := t.TempDir() + files := atmosFiles() + layer := atmosLayer(t, types.MediaType(mediaType), files) + + require.NoError(t, processLayer(layer, 0, dest)) + + for _, kept := range []string{"main.tf", ".terraform.lock.hcl", filepath.Join("scripts", "helper.sh")} { + assert.FileExists(t, filepath.Join(dest, kept)) + } + assert.NoDirExists(t, filepath.Join(dest, ".atmos"), "distribution metadata never reaches the consumer") + }) + } +} + +func TestProcessLayer_AtmosPayloadKeepsUserFilesUnderDotAtmos(t *testing.T) { + dest := t.TempDir() + layer := atmosLayer(t, types.MediaType(artifact.MediaTypeTar), map[string]string{ + "main.tf": "# main\n", + ".atmos/scaffold.yaml": "kind: AtmosScaffoldConfig\n", + ".atmos/artifact/README.md": "# package readme\n", + }) + + require.NoError(t, processLayer(layer, 0, dest)) + + assert.FileExists(t, filepath.Join(dest, ".atmos", "scaffold.yaml"), "only the reserved distribution directory is withheld") + assert.NoFileExists(t, filepath.Join(dest, ".atmos", "artifact", "README.md")) +} + +func TestProcessLayer_AtmosPayloadKeepsExecutableBits(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("this platform does not honor Unix permission bits") + } + var buf bytes.Buffer + writer := tar.NewWriter(&buf) + content := "#!/bin/sh\n" + require.NoError(t, writer.WriteHeader(&tar.Header{Name: "scripts/helper.sh", Mode: 0o755, Size: int64(len(content))})) + _, err := writer.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, writer.Close()) + dest := t.TempDir() + + require.NoError(t, processLayer(static.NewLayer(buf.Bytes(), types.MediaType(artifact.MediaTypeTar)), 0, dest)) + + info, err := os.Stat(filepath.Join(dest, "scripts", "helper.sh")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) +} + +func TestIsDistributionMetadata(t *testing.T) { + tests := []struct { + name string + want bool + }{ + {name: ".atmos", want: true}, + {name: ".atmos/", want: true}, + {name: "./.atmos/artifact", want: true}, + {name: ".atmos/artifact/", want: true}, + {name: ".atmos/artifact/manifest.json", want: true}, + {name: ".atmos/artifact/nested/x", want: true}, + {name: ".atmos/scaffold.yaml", want: false}, + {name: ".atmos/artifacts/x", want: false}, + {name: "docs/.atmos/artifact/x", want: false}, + {name: "main.tf", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isDistributionMetadata(tt.name)) + }) + } +} + +func TestSelectedLayers_AtmosPackageExtractsThroughTheRetryPath(t *testing.T) { + // A package shaped the way publishing shapes it extracts through the same per-layer retry path ProcessImage uses. + dest := t.TempDir() + img := atmosPackage(t, atmosLayer(t, types.MediaType(artifact.MediaTypeTarGzip), atmosFiles()), documentLayer("# readme\n")) + layers, err := img.Layers() + require.NoError(t, err) + selected, err := selectLayers(img, layers) + require.NoError(t, err) + + for index, layer := range selected { + require.NoError(t, processLayerWithRetry(context.Background(), layer, index, dest, defaultOCILayerRetryConfig())) + } + + assert.FileExists(t, filepath.Join(dest, "main.tf")) + assert.NoDirExists(t, filepath.Join(dest, ".atmos")) +} + +func TestProcessImageClassifiesAtmosPackages(t *testing.T) { + server := httptest.NewServer(registry.New()) + defer server.Close() + host := strings.TrimPrefix(server.URL, "http://") + payload := atmosLayer(t, types.MediaType(artifact.MediaTypeTar), atmosFiles()) + for _, tc := range []struct { + name string + image v1.Image + keepsMetadata bool + }{ + {name: "atmos", image: atmosPackage(t, payload, documentLayer("# readme"))}, + {name: "legacy", image: func() v1.Image { + img, err := mutate.AppendLayers(empty.Image, payload) + require.NoError(t, err) + return img + }(), keepsMetadata: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ref, err := name.NewTag(host + "/test/" + tc.name + ":latest") + require.NoError(t, err) + require.NoError(t, remote.Write(ref, tc.image)) + dest := t.TempDir() + require.NoError(t, ProcessImage(context.Background(), &schema.AtmosConfiguration{}, ref.String(), dest)) + assert.FileExists(t, filepath.Join(dest, "main.tf")) + if tc.keepsMetadata { + assert.FileExists(t, filepath.Join(dest, ".atmos", "artifact", "README.md")) + } else { + assert.NoDirExists(t, filepath.Join(dest, ".atmos")) + } + }) + } +} + +func TestExtractZipValidatesBeforeSkipping(t *testing.T) { + archive := writeTestZip(t, map[string]string{"../escape": "x"}) + err := extractZipSkipping(archive, t.TempDir(), func(string) bool { return true }) + require.ErrorIs(t, err, ErrInvalidFilePath) +} diff --git a/pkg/oci/pull.go b/pkg/oci/pull.go index 276311e01bf..23c7adf4f3a 100644 --- a/pkg/oci/pull.go +++ b/pkg/oci/pull.go @@ -22,6 +22,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" "github.com/cloudposse/atmos/pkg/filesystem" log "github.com/cloudposse/atmos/pkg/logger" // Charmbracelet structured logger "github.com/cloudposse/atmos/pkg/perf" @@ -197,8 +198,18 @@ func processImageWithFS(ctx context.Context, atmosConfig *schema.AtmosConfigurat return ErrNoLayers } + atmos, err := isAtmosPackage(img) + if err != nil { + return err + } + // An Atmos package extracts only its payload layer; legacy images extract every layer. + layers, err = selectLayers(img, layers) + if err != nil { + return err + } + for i, layer := range layers { - if err := processLayerWithRetry(ctx, layer, i, destDir, defaultOCILayerRetryConfig()); err != nil { + if err := processLayerWithRetryClassified(ctx, layer, i, destDir, layerRetryOptions{config: defaultOCILayerRetryConfig(), atmos: atmos}); err != nil { return errors.Join( errUtils.ErrProcessLayer, fmt.Errorf("layer %d: %w", i, err), @@ -302,6 +313,10 @@ func processLayer(layer v1.Layer, index int, destDir string) error { } func processLayerContext(ctx context.Context, layer v1.Layer, index int, destDir string) error { + return processLayerContextClassified(ctx, layer, index, destDir, true) +} + +func processLayerContextClassified(ctx context.Context, layer v1.Layer, index int, destDir string, atmos bool) error { layerDesc, err := layer.Digest() if err != nil { if !observed(ctx) { @@ -316,10 +331,7 @@ func processLayerContext(ctx context.Context, layer v1.Layer, index int, destDir } defer uncompressed.Close() - extract := extractTarball - if mediaType, mtErr := layer.MediaType(); mtErr == nil && mediaType == zipLayerMediaType { - extract = extractZip - } + extract := extractorForLayer(layer, atmos) if err := extract(uncompressed, destDir); err != nil { if !observed(ctx) { @@ -331,6 +343,20 @@ func processLayerContext(ctx context.Context, layer v1.Layer, index int, destDir return nil } +func extractorForLayer(layer v1.Layer, atmos bool) func(io.Reader, string) error { + mediaType, err := layer.MediaType() + if err != nil { + return extractTarball + } + if mediaType == zipLayerMediaType || (!atmos && string(mediaType) == artifact.MediaTypeZip) { + return extractZip + } + if extract, ok := atmosPayloadExtractor(string(mediaType)); atmos && ok { + return extract + } + return extractTarball +} + // remoteGetWithRetry calls remoteGet, retrying only transient connection-level // failures (see isRetryableOCIConnectError) on the same auth method. It does // not retry registry-level rejections (auth/status errors) -- those are @@ -392,14 +418,23 @@ func isRetryableOCIConnectError(err error) bool { // layer stream. Archive extraction failures are deliberately not retried: they // may reflect invalid image content and can leave partial files in destDir. func processLayerWithRetry(ctx context.Context, layer v1.Layer, index int, destDir string, retryConfig *schema.RetryConfig) error { + return processLayerWithRetryClassified(ctx, layer, index, destDir, layerRetryOptions{config: retryConfig, atmos: true}) +} + +type layerRetryOptions struct { + config *schema.RetryConfig + atmos bool +} + +func processLayerWithRetryClassified(ctx context.Context, layer v1.Layer, index int, destDir string, options layerRetryOptions) error { if err := ctx.Err(); err != nil { return err } attempts := 0 - err := retry.WithPredicate(ctx, retryConfig, func() error { + err := retry.WithPredicate(ctx, options.config, func() error { attempts++ - err := processLayerContext(ctx, layer, index, destDir) + err := processLayerContextClassified(ctx, layer, index, destDir, options.atmos) if err != nil && ctx.Err() == nil && isRetryableOCILayerError(err) && attempts < ociLayerRetryMaxAttempts { // Do not include the underlying error here: OCI blob errors can contain // signed URLs. The layer index is enough to correlate the retry with the @@ -456,7 +491,7 @@ func checkArtifactType(descriptor *remote.Descriptor, imageName string) { return } switch manifest.ArtifactType { - case targetArtifactType, opentofuModulePkgArtifactType: + case targetArtifactType, opentofuModulePkgArtifactType, artifact.MediaTypeComponent, artifact.MediaTypeScaffold, artifact.MediaTypeArtifact: // Recognized and supported artifact type; nothing to warn about. default: log.Warn("OCI image does not match a recognized artifact type", "image", imageName, "artifactType", manifest.ArtifactType) diff --git a/pkg/oci/tar.go b/pkg/oci/tar.go index 45718aeba44..d624c06ecf8 100644 --- a/pkg/oci/tar.go +++ b/pkg/oci/tar.go @@ -20,7 +20,13 @@ var ErrInvalidFilePath = errors.New("invalid file path") // extractTarball extracts the tarball file from an io.Reader into the destination directory. func extractTarball(reader io.Reader, extractPath string) error { // Call untar function to handle tar extraction. - return untar(reader, extractPath) + return untar(reader, extractPath, nil) +} + +// extractTarballSkipping extracts the tarball like extractTarball but leaves out every member for +// which skip reports true. Atmos package payloads use it to withhold distribution metadata. +func extractTarballSkipping(reader io.Reader, extractPath string, skip func(name string) bool) error { + return untar(reader, extractPath, skip) } // untar extracts a tar archive into the destination directory. @@ -30,8 +36,8 @@ func extractTarball(reader io.Reader, extractPath string) error { // without os.Root a malicious archive could plant a symlink via one entry // (e.g. "link" -> "/etc") and have a later entry (e.g. "link/passwd") follow // it out of extractPath -- os.Root's methods refuse to resolve a path that -// escapes the root, symlinks included. -func untar(reader io.Reader, extractPath string) error { +// escapes the root, symlinks included. A non-nil skip leaves out the members it selects. +func untar(reader io.Reader, extractPath string, skip func(name string) bool) error { if err := os.MkdirAll(extractPath, os.ModePerm); err != nil { return fmt.Errorf("failed to create extraction directory %s: %w", extractPath, err) } @@ -59,6 +65,9 @@ func untar(reader io.Reader, extractPath string) error { if err := layer.checkPath(header.Name); err != nil { return err } + if withheld(skip, header.Name) { + continue + } if err := processTarHeader(root, header, tarReader, extractPath); err != nil { return err } @@ -124,3 +133,8 @@ func createFileFromTar(root *os.Root, relPath string, tarReader *tar.Reader, hea } return nil } + +// withheld reports whether skip, when set, leaves the named archive member out of the extraction. +func withheld(skip func(name string) bool, name string) bool { + return skip != nil && skip(name) +} diff --git a/pkg/oci/zip.go b/pkg/oci/zip.go index 66e4d2a0583..b5b2169ff1e 100644 --- a/pkg/oci/zip.go +++ b/pkg/oci/zip.go @@ -37,6 +37,12 @@ var maxZipEntrySize uint64 = 512 * 1024 * 1024 // 512 MiB. // it out of extractPath -- os.Root's methods refuse to resolve a path that // escapes the root, symlinks included. func extractZip(reader io.Reader, extractPath string) error { + return extractZipSkipping(reader, extractPath, nil) +} + +// extractZipSkipping extracts the archive like extractZip but leaves out every entry for which skip +// reports true. Atmos package payloads use it to withhold distribution metadata. +func extractZipSkipping(reader io.Reader, extractPath string, skip func(name string) bool) error { data, err := io.ReadAll(io.LimitReader(reader, maxZipArchiveSize+1)) if err != nil { return fmt.Errorf("failed to read zip archive: %w", err) @@ -61,7 +67,7 @@ func extractZip(reader io.Reader, extractPath string) error { defer root.Close() for _, file := range zipReader.File { - if err := processZipFile(root, file, extractPath); err != nil { + if err := processZipEntry(root, file, extractPath, skip); err != nil { return err } } @@ -69,6 +75,17 @@ func extractZip(reader io.Reader, extractPath string) error { return nil } +// processZipEntry validates names before applying the optional skip rule. +func processZipEntry(root *os.Root, file *zip.File, extractPath string, skip func(string) bool) error { + if _, err := filesystem.SafeJoin(extractPath, file.Name); err != nil { + return fmt.Errorf("%w: %s", ErrInvalidFilePath, file.Name) + } + if skip != nil && skip(file.Name) { + return nil + } + return processZipFile(root, file, extractPath) +} + // processZipFile processes a zip.File entry and writes the corresponding file // to the destination directory via root. func processZipFile(root *os.Root, file *zip.File, extractPath string) error { diff --git a/pkg/plugin/discovery/cache.go b/pkg/plugin/discovery/cache.go index 94ecfcac67d..978fe0d05e7 100644 --- a/pkg/plugin/discovery/cache.go +++ b/pkg/plugin/discovery/cache.go @@ -2,36 +2,36 @@ package discovery import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" - "io" - "os" "path/filepath" "time" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/discovery/cache" "github.com/cloudposse/atmos/pkg/auth/realm" "github.com/cloudposse/atmos/pkg/auth/registryauth" - "github.com/cloudposse/atmos/pkg/filelock" - "github.com/cloudposse/atmos/pkg/filesystem" "github.com/cloudposse/atmos/pkg/plugin/catalog" pluginpolicy "github.com/cloudposse/atmos/pkg/plugin/policy" "github.com/cloudposse/atmos/pkg/xdg" ) const ( - maxCacheSize = 32 << 20 - privateDirectoryMode = 0o700 - privateFileMode = 0o600 + // The maximum size of one snapshot file is the shared cache bound. + maxCacheSize = cache.DefaultMaxSize + + // The snapshot version is the plugin snapshot schema version. + snapshotVersion = 1 ) +// indexCache is the plugin discovery cache: the shared store scoped to this configuration. type indexCache struct { - directory, scope string - ttl time.Duration + store *cache.Store } +// snapshot is the plugin schema stored in a source's snapshot file. It keeps the four shared header +// fields itself so the serialized keys stay version, source, kind, fetched_at, digest, entries, +// releases. type snapshot struct { Version int `json:"version"` Source string `json:"source"` @@ -42,6 +42,22 @@ type snapshot struct { Releases map[string]catalog.Entry `json:"releases,omitempty"` } +var _ cache.Snapshot = (*snapshot)(nil) + +// SnapshotHeader implements [cache.Snapshot]. +// +//nolint:lintroller // Trivial accessor. +func (s *snapshot) SnapshotHeader() cache.Header { + return cache.Header{Version: s.Version, Source: s.Source, Kind: s.Kind, FetchedAt: s.FetchedAt} +} + +// SetSnapshotHeader implements [cache.Snapshot]. +// +//nolint:lintroller // Trivial accessor. +func (s *snapshot) SetSnapshotHeader(header cache.Header) { + s.Version, s.Source, s.Kind, s.FetchedAt = header.Version, header.Source, header.Kind, header.FetchedAt +} + func (s *Service) newCache() (*indexCache, error) { directory := filepath.Join(s.Config.Plugins.CacheDir, "discovery") if s.Config.Plugins.CacheDir == "" { @@ -56,7 +72,7 @@ func (s *Service) newCache() (*indexCache, error) { if err != nil { return nil, err } - raw, err := json.Marshal([]any{s.Config.Auth, info.Value, bindings}) + scope, err := cache.Scope(s.Config.Auth, info.Value, bindings) if err != nil { return nil, err } @@ -67,26 +83,35 @@ func (s *Service) newCache() (*indexCache, error) { return nil, errors.Join(errUtils.ErrPluginInvalidConfig, err) } } - return &indexCache{directory: directory, scope: hash(raw), ttl: ttl}, nil + return &indexCache{store: cache.New(directory, scope, cache.WithTTL(ttl), cache.WithMaxSize(maxCacheSize))}, nil } -func hash(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) } - -func (s *Service) loadSource(ctx context.Context, cache *indexCache, item source, refresh bool) ([]catalog.Entry, error) { - key := hash([]byte(cache.scope + "\x00" + item.kind + "\x00" + item.reference)) - if err := os.MkdirAll(cache.directory, privateDirectoryMode); err != nil { +func (s *Service) loadSource(ctx context.Context, store *indexCache, item source, refresh bool) ([]catalog.Entry, error) { + spec := &cache.Source[snapshot, *snapshot]{ + Kind: item.kind, + Reference: item.reference, + Version: snapshotVersion, + Decode: decodeSnapshot, + Fetch: func(ctx context.Context, previous *snapshot) (*snapshot, error) { + return s.refresh(ctx, item, previous) + }, + Authorize: func(ctx context.Context, cached bool) error { + var options []pluginpolicy.Option + if cached { + options = append(options, pluginpolicy.WithCache()) + } + return pluginpolicy.Source(ctx, "search", item.reference, options...) + }, + ErrOffline: errUtils.ErrPluginOfflineArtifact, + ErrTooLarge: errUtils.ErrPluginPayloadTooLarge, + } + value, cached, err := cache.Load(ctx, store.store, spec, cache.Mode{Refresh: refresh, Offline: s.Config.Plugins.Policy.Offline}) + if err != nil { return nil, err } - path := filepath.Join(cache.directory, key+".json") var result []catalog.Entry - var cached bool - err := filelock.New(path+".lock").WithExclusive(ctx, func() error { - var err error - result, cached, err = s.updateSnapshot(ctx, cache, item, refresh, path) - return err - }) - if err != nil { - return nil, err + if value != nil { + result = value.Entries } if err := checkEntrySources(ctx, result, cached); err != nil { return nil, err @@ -107,65 +132,16 @@ func checkEntrySources(ctx context.Context, entries []catalog.Entry, cached bool return nil } -func (s *Service) updateSnapshot(ctx context.Context, cache *indexCache, item source, refresh bool, path string) ([]catalog.Entry, bool, error) { - old := readSnapshot(path, item) - if old != nil && (s.Config.Plugins.Policy.Offline || (!refresh && fresh(old.FetchedAt, cache.ttl))) { - if err := pluginpolicy.Source(ctx, "search", item.reference, pluginpolicy.WithCache()); err != nil { - return nil, true, err - } - return old.Entries, true, nil - } - if s.Config.Plugins.Policy.Offline { - return nil, false, errUtils.ErrPluginOfflineArtifact - } - if err := pluginpolicy.Source(ctx, "search", item.reference); err != nil { - return nil, false, err - } - return s.refreshSnapshot(ctx, item, old, path) -} - -func (s *Service) refreshSnapshot(ctx context.Context, item source, old *snapshot, path string) ([]catalog.Entry, bool, error) { - current, err := s.refresh(ctx, item, old) - if err != nil { - return nil, false, err - } - if current == nil { - // Do not cache an unavailable optional catalog as an empty result. - return nil, false, nil - } - current.Version, current.Source, current.Kind, current.FetchedAt = 1, item.reference, item.kind, time.Now().UTC() - if err := ctx.Err(); err != nil { - return nil, false, err - } - if err := writeSnapshot(path, current); err != nil { - return nil, false, err - } - return current.Entries, false, nil -} - -func fresh(fetched time.Time, ttl time.Duration) bool { - age := time.Since(fetched) - return age >= 0 && age < ttl -} - -func readSnapshot(path string, item source) *snapshot { - file, err := os.Open(path) - if err != nil { - return nil - } - defer file.Close() - raw, err := io.ReadAll(io.LimitReader(file, maxCacheSize+1)) - if err != nil || len(raw) > maxCacheSize { - return nil - } +// decodeSnapshot parses a plugin snapshot and rejects one whose entries or releases are invalid. +func decodeSnapshot(raw []byte) (*snapshot, error) { var value snapshot - if json.Unmarshal(raw, &value) != nil || value.Version != 1 || value.Source != item.reference || value.Kind != item.kind { - return nil + if err := json.Unmarshal(raw, &value); err != nil { + return nil, err } if !validSnapshot(&value) { - return nil + return nil, errUtils.ErrPluginInvalidConfig } - return &value + return &value, nil } func validSnapshot(value *snapshot) bool { @@ -192,13 +168,8 @@ func validEntries(entries []catalog.Entry) bool { return err == nil } +// writeSnapshot atomically replaces the snapshot file at path, leaving the previous contents in +// place when the new value cannot be serialized or exceeds the size bound. func writeSnapshot(path string, value *snapshot) error { - raw, err := json.Marshal(value) - if err != nil { - return err - } - if len(raw) > maxCacheSize { - return errUtils.ErrPluginPayloadTooLarge - } - return filesystem.NewOSFileSystem().WriteFileAtomic(path, raw, privateFileMode) + return cache.New("", "", cache.WithMaxSize(maxCacheSize)).WriteSnapshot(path, value, errUtils.ErrPluginPayloadTooLarge) } diff --git a/pkg/plugin/discovery/cache_characterization_test.go b/pkg/plugin/discovery/cache_characterization_test.go new file mode 100644 index 00000000000..565c1386496 --- /dev/null +++ b/pkg/plugin/discovery/cache_characterization_test.go @@ -0,0 +1,193 @@ +package discovery + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/auth/realm" + "github.com/cloudposse/atmos/pkg/auth/registryauth" + "github.com/cloudposse/atmos/pkg/plugin/catalog" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/xdg" +) + +// These tests characterize the plugin discovery cache's on-disk contract independently of the +// implementation: the directory, the file name derivation, the scope recipe, the serialized +// field order, the file modes, the default TTL, and the readability of a hand-written snapshot. +// They were written before the cache mechanics moved to the shared package so that the move can +// be proved byte-compatible. They deliberately re-derive every value from the documented recipe +// rather than calling the cache code. + +// characterizedScope re-derives the scope hash: sha256 over the JSON of the auth configuration, +// the realm value, and the host bindings, never any credential value. +func characterizedScope(t *testing.T, f *fixture) string { + t.Helper() + info, err := realm.GetRealm(f.service.Config.Auth.Realm, f.service.Config.CliConfigPath) + require.NoError(t, err) + bindings, err := registryauth.PluginBindings(f.service.Config) + require.NoError(t, err) + raw, err := json.Marshal([]any{f.service.Config.Auth, info.Value, bindings}) + require.NoError(t, err) + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// characterizedFile re-derives the snapshot path of a source. +func characterizedFile(t *testing.T, f *fixture, kind, reference string) string { + t.Helper() + sum := sha256.Sum256([]byte(characterizedScope(t, f) + "\x00" + kind + "\x00" + reference)) + return filepath.Join(f.service.Config.Plugins.CacheDir, "discovery", hex.EncodeToString(sum[:])+".json") +} + +func TestCacheCharacterizationFileNameAndSerialization(t *testing.T) { + f := newFixture(t) + release := f.publish(t, "1.0.0", "Original") + before := time.Now().UTC().Add(-time.Second) + _, err := f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + + path := characterizedFile(t, f, "repository", "oci://"+f.repository.Name()) + raw, err := os.ReadFile(path) + require.NoError(t, err, "the snapshot must live at /discovery/.json") + + // Top-level keys keep their historical order. + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + require.NoError(t, err) + require.Equal(t, json.Delim('{'), token) + var keys []string + for decoder.More() { + key, keyErr := decoder.Token() + require.NoError(t, keyErr) + keys = append(keys, key.(string)) + var skipped json.RawMessage + require.NoError(t, decoder.Decode(&skipped)) + } + require.Equal(t, []string{"version", "source", "kind", "fetched_at", "entries", "releases"}, keys) + + var value snapshot + require.NoError(t, json.Unmarshal(raw, &value)) + require.Equal(t, 1, value.Version) + require.Equal(t, "oci://"+f.repository.Name(), value.Source) + require.Equal(t, "repository", value.Kind) + require.False(t, value.FetchedAt.Before(before), "fetched_at records the refresh time") + require.Equal(t, time.UTC, value.FetchedAt.Location()) + require.Len(t, value.Entries, 1) + require.Equal(t, release.Digest, value.Entries[0].Versions[0].Digest) + require.Contains(t, value.Releases, release.Digest) + + if runtime.GOOS != "windows" { + info, statErr := os.Stat(path) + require.NoError(t, statErr) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + dir, statErr := os.Stat(filepath.Dir(path)) + require.NoError(t, statErr) + require.Equal(t, os.FileMode(0o700), dir.Mode().Perm()) + } + files, err := filepath.Glob(filepath.Join(filepath.Dir(path), "*.json")) + require.NoError(t, err) + require.Equal(t, []string{path}, files, "one snapshot per source and no stray temporary files") +} + +func TestCacheCharacterizationScopeSeparatesRealmsAndKeepsCredentialsOut(t *testing.T) { + f := newFixture(t) + f.publish(t, "1.0.0", "Original") + _, err := f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + first := characterizedFile(t, f, "repository", "oci://"+f.repository.Name()) + require.FileExists(t, first) + + f.service.Config.Auth.Realm = "other-realm" + other := characterizedFile(t, f, "repository", "oci://"+f.repository.Name()) + require.NotEqual(t, first, other, "a different auth realm changes the cache file") + _, err = f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + require.FileExists(t, other) + + for _, path := range []string{first, other} { + raw, readErr := os.ReadFile(path) + require.NoError(t, readErr) + require.NotContains(t, string(raw), "other-realm", "the snapshot never embeds the scope inputs") + } +} + +func TestCacheCharacterizationDefaultDirectoryTTLAndLegacySnapshot(t *testing.T) { + f := newFixture(t) + t.Setenv("XDG_CACHE_HOME", t.TempDir()) + t.Setenv("ATMOS_XDG_CACHE_HOME", "") + f.service.Config.Plugins.CacheDir = "" + release := f.publish(t, "1.0.0", "Original") + _, err := f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + + directory := xdg.LookupXDGCacheDir("plugins/discovery") + scope := characterizedScope(t, f) + sum := sha256.Sum256([]byte(scope + "\x00repository\x00oci://" + f.repository.Name())) + path := filepath.Join(directory, hex.EncodeToString(sum[:])+".json") + require.FileExists(t, path, "the default location is the XDG cache plugins/discovery directory") + raw, err := os.ReadFile(path) + require.NoError(t, err) + + var written snapshot + require.NoError(t, json.Unmarshal(raw, &written)) + // Rewrite the snapshot as a legacy writer would have: only the historical keys. + legacy := map[string]any{ + "version": 1, "source": written.Source, "kind": written.Kind, + "fetched_at": time.Now().UTC().Add(-14 * time.Minute).Format(time.RFC3339Nano), + "entries": written.Entries, "releases": written.Releases, + } + encoded, err := json.Marshal(legacy) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, encoded, 0o600)) + + // Fourteen minutes old is inside the fifteen minute default: no registry request. + calls := f.requests.Load() + results, err := f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + require.Equal(t, calls, f.requests.Load(), "a fresh legacy snapshot is served without contacting the registry") + require.Equal(t, release.Digest, results[0].Versions[0].Digest) + + // Sixteen minutes old is stale: the registry is contacted again and the file is replaced. + legacy["fetched_at"] = time.Now().UTC().Add(-16 * time.Minute).Format(time.RFC3339Nano) + encoded, err = json.Marshal(legacy) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, encoded, 0o600)) + _, err = f.service.Search(t.Context(), Request{}) + require.NoError(t, err) + require.Greater(t, f.requests.Load(), calls, "a stale snapshot is refreshed") + refreshed, err := os.ReadFile(path) + require.NoError(t, err) + var after snapshot + require.NoError(t, json.Unmarshal(refreshed, &after)) + require.WithinDuration(t, time.Now(), after.FetchedAt, time.Minute) +} + +func TestCacheCharacterizationCatalogSnapshotKeepsDigest(t *testing.T) { + f := newFixture(t) + release := f.publish(t, "1.0.0", "Original") + reference := "oci://" + f.server.Listener.Addr().String() + "/product/catalog:stable" + file := catalog.New("product") + require.NoError(t, catalog.Add(file, "oci://"+f.repository.Name(), release, f.service.Descriptors)) + require.NoError(t, catalog.Publish(t.Context(), f.service.Registry, reference, file, "")) + f.service.Config.Plugins.Registries["product"] = schema.PluginRegistry{Catalog: reference} + _, err := f.service.Search(t.Context(), Request{Registry: "product"}) + require.NoError(t, err) + + raw, err := os.ReadFile(characterizedFile(t, f, "catalog", reference)) + require.NoError(t, err) + var value snapshot + require.NoError(t, json.Unmarshal(raw, &value)) + require.Equal(t, "catalog", value.Kind) + require.Equal(t, reference, value.Source) + require.NotEmpty(t, value.Digest, "a catalog snapshot records the catalog manifest digest") + require.Empty(t, value.Releases) +} diff --git a/pkg/runner/step/artifact.go b/pkg/runner/step/artifact.go new file mode 100644 index 00000000000..c383bc49179 --- /dev/null +++ b/pkg/runner/step/artifact.go @@ -0,0 +1,212 @@ +package step + +import ( + "context" + "errors" + "strconv" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifact/report" + "github.com/cloudposse/atmos/pkg/artifactwire" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/ui" +) + +// Metadata keys of an artifact step result. +const ( + // ArtifactMetadataKey holds the typed operation result: an *invoke.Result for single-artifact + // actions, or an *invoke.Collection (with its `artifacts` array) for list and search. It is the + // same payload the CLI prints with --format=json. + ArtifactMetadataKey = "artifact" + + // ArtifactListMetadataKey holds the `artifacts` array of a list or search result. + ArtifactListMetadataKey = "artifacts" +) + +// ArtifactHandler runs the artifact operations (publish, pull, copy, inspect, list, search, +// delete, sign, verify, and show) through the same service and result mapping as `atmos +// artifacts`. It never shells out to the CLI. Available as a component lifecycle hook through the +// existing `kind: step` bridge (`kind: step` / `type: artifact` / flat fields under `with:`); no +// dedicated hook kind is registered for it. +// +// The handler does not retry: a wrapper that owns the step's `retry:` (the workflow runner, a +// custom command, or a hook) calls it again, and the retained operation identity makes every +// attempt the same logical operation. When `retry:` is set, the handler also turns off the +// service's own transport retries for the call, so the two budgets never multiply. +type ArtifactHandler struct { + BaseHandler +} + +func init() { + Register(&ArtifactHandler{ + BaseHandler: NewBaseHandler(schema.TaskTypeArtifact, CategoryCommand, false), + }) +} + +// artifactServiceFactory builds the artifact service. It is a variable so tests can inject a fake. +var artifactServiceFactory invoke.ServiceFactory = artifactwire.NewService + +// UseArtifactServiceFactory replaces the factory that builds the artifact service for artifact +// steps and returns a function that restores the previous one. It is the seam tests use to run a +// step against an injected service; production code never calls it. +func UseArtifactServiceFactory(factory invoke.ServiceFactory) (restore func()) { + defer perf.Track(nil, "step.UseArtifactServiceFactory")() + + previous := artifactServiceFactory + artifactServiceFactory = factory + return func() { artifactServiceFactory = previous } +} + +// Execute runs the configured artifact action. The returned result is non-nil whenever the +// operation produced one, even alongside an error, so a failed operation still reports the +// identity and whatever the service resolved before it failed. +func (h *ArtifactHandler) Execute(ctx context.Context, step *schema.WorkflowStep, vars *Variables) (*StepResult, error) { + defer perf.Track(nil, "step.ArtifactHandler.Execute")() + + session := vars.artifactSession() + stepKey, err := session.stepKey(step) + if err != nil { + return nil, err + } + params, retained, err := h.sessionParams(session, stepKey, step, vars) + if err != nil { + return nil, err + } + if err := checkResolvedArtifactParams(step, params); err != nil { + return nil, err + } + if h.applyDryRun(step, params) { + ui.Infof("Dry run: skipped artifact %s in step **%s**.", params.Action, step.Name) + return NewStepResult("").WithSkipped().WithMetadata("action", string(params.Action)), nil + } + if vars.AtmosConfig == nil { + return nil, errUtils.Build(errUtils.ErrArtifactConfigInvalid). + WithContext("step", step.Name). + WithExplanation("Atmos configuration is not loaded, so the configured artifact repositories are unknown"). + Err() + } + svc, err := artifactServiceFactory(vars.AtmosConfig, invoke.ServiceOptions{}) + if err != nil { + return nil, err + } + // The step's `timeout:` limits each attempt: the handler does not retry, so a wrapper's retry + // loop starts a fresh attempt, with a fresh limit, after a failure. + deadline, err := StartStepDeadline(ctx, step, vars) + if err != nil { + return nil, err + } + defer deadline.Stop() + + if !retained { + session.retainInput(stepKey, params) + } + result, runErr := h.runOperation(deadline.Context(), step, vars, svc, params) + if runErr == nil { + session.forgetInput(stepKey) + } + return result, deadline.Wrap(runErr) +} + +func (h *ArtifactHandler) sessionParams(session *artifactSession, key string, step *schema.WorkflowStep, vars *Variables) (*invoke.Params, bool, error) { + if params, retained := session.retainedInput(key); retained { + return params, true, nil + } + params, err := h.artifactParams(step, vars) + return params, false, err +} + +// runOperation runs one attempt of the operation under the retained logical identity, records the +// attempt into the report collector, and builds the step result. +func (h *ArtifactHandler) runOperation(ctx context.Context, step *schema.WorkflowStep, vars *Variables, svc artifact.Service, params *invoke.Params) (*StepResult, error) { + session := vars.artifactSession() + key := session.operationKey(step.Name, params) + operation, collector := session.begin(key) + params.Execution = artifact.Execution{OperationID: operation.id, PreparedAt: operation.preparedAt} + if step.Retry != nil { + // The wrapper owns the retry budget, so the service makes a single transport attempt. + params.Execution.MaxAttempts = 1 + } + + outcome, runErr := invoke.Run(ctx, svc, params) + if runErr != nil && ctx.Err() != nil && !errors.Is(runErr, ctx.Err()) { + // Keep the cancellation reachable so a retrying wrapper stops instead of starting another attempt. + runErr = errors.Join(ctx.Err(), runErr) + } + session.finish(key, runErr == nil) + collector.Record(&report.Entry{Step: step.Name, OperationID: operation.id, Attempt: operation.attempts, Outcome: outcome, Err: runErr}) + + result := artifactStepResult(params, outcome) + if runErr != nil { + return result, artifactStepError(step, runErr) + } + return result, nil +} + +// WrapperRetriesStep reports whether the step type leaves its `retry:` policy to the code that +// calls the handler. Such a caller must apply the policy around the step; the handler keeps the +// operation identity stable between the attempts. +func WrapperRetriesStep(stepType string) bool { + defer perf.Track(nil, "step.WrapperRetriesStep")() + + return strings.TrimSpace(stepType) == schema.TaskTypeArtifact +} + +// applyDryRun applies a workflow-level `--dry-run`. A publication switches to the service's own +// dry run, which validates and plans without writing; the actions that would mutate state without +// a dry run of their own are skipped; reads run normally. It reports whether to skip the step. +func (h *ArtifactHandler) applyDryRun(step *schema.WorkflowStep, params *invoke.Params) bool { + if !step.DryRun { + return false + } + switch params.Action { + case artifact.ActionPublish: + params.DryRun = true + case artifact.ActionPull, artifact.ActionCopy, artifact.ActionDelete, artifact.ActionSign: + return true + } + return false +} + +// artifactStepResult builds the step result from an operation outcome: the typed payload under the +// `artifact` metadata key, the resolved paths, and the named string outputs. Optional outputs the +// operation did not produce are omitted, never filled with empty strings. +func artifactStepResult(params *invoke.Params, outcome *invoke.Outcome) *StepResult { + result := NewStepResult(""). + WithMetadata("action", string(params.Action)). + WithMetadata(ArtifactMetadataKey, outcome.Payload()) + if params.Source != "" { + result.WithMetadata("source", params.Source) + } + if params.Destination != "" { + result.WithMetadata("destination", params.Destination) + } + if outcome.Collection != nil { + result.Value = strconv.Itoa(len(outcome.Collection.Artifacts)) + return result.WithMetadata(ArtifactListMetadataKey, outcome.Collection.Artifacts) + } + op := &outcome.Result.OperationResult + result.Value = op.Ref + for _, pair := range invoke.OutputPairs(op) { + result.WithOutput(pair.Name, pair.Value) + } + return result +} + +// artifactStepError adds the step context and the hints that fit a step to an operation failure, +// keeping the original error chain intact for errors.Is. +func artifactStepError(step *schema.WorkflowStep, err error) error { + builder := errUtils.Build(err).WithContext("step", step.Name).WithContext("action", step.Action) + switch { + case errors.Is(err, errUtils.ErrArtifactConflict): + builder = builder.WithHint("Set `force: true` on the step to replace the existing reference. Force never bypasses verification or immutable release formats.") + case errors.Is(err, errUtils.ErrArtifactNotFound): + builder = builder.WithHint("Add a step with `action: list` to see the published artifacts and versions.") + case errors.Is(err, errUtils.ErrArtifactVerificationFailed): + builder = builder.WithHint("Configure `artifacts.verification` with the signer identities or keys you trust.") + } + return builder.Err() +} diff --git a/pkg/runner/step/artifact_params.go b/pkg/runner/step/artifact_params.go new file mode 100644 index 00000000000..281a894c3d6 --- /dev/null +++ b/pkg/runner/step/artifact_params.go @@ -0,0 +1,223 @@ +package step + +import ( + "fmt" + "reflect" + "sort" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/tags" +) + +// artifactFieldResolver renders the templated string fields of one artifact step. It remembers the +// first failure so a long run of field renders reads as straight-line code. +type artifactFieldResolver struct { + handler *ArtifactHandler + step *schema.WorkflowStep + vars *Variables + err error +} + +// text renders a templated field. An empty value stays empty. +func (r *artifactFieldResolver) text(field, value string) string { + if r.err != nil || value == "" { + return "" + } + resolved, err := r.vars.ResolveStepField(r.step, field, value) + if err != nil { + r.err = r.vars.TemplateFieldError(r.step, field, err) + return "" + } + return resolved +} + +// path renders a templated path field and anchors a relative result to the step's working +// directory, using the same helper as the archive step, so a relative source or destination means +// the same thing in a workflow, a custom command, and a hook. +func (r *artifactFieldResolver) path(field, value string) string { + if r.err != nil || value == "" { + return "" + } + resolved, err := r.handler.ResolveInWorkingDirectory(r.step, r.vars, value, field) + if err != nil { + r.err = err + return "" + } + return resolved +} + +// artifactParams resolves the authored step into the transport-neutral operation parameters that +// the CLI also builds. Templates are rendered here, `source` and `destination` are anchored to the +// step's working directory, and the document paths stay relative to the publish source, exactly as +// `--readme` and `--changelog` do on the command line. +func (h *ArtifactHandler) artifactParams(step *schema.WorkflowStep, vars *Variables) (*invoke.Params, error) { + source, err := artifactSourceString(step) + if err != nil { + return nil, err + } + resolver := &artifactFieldResolver{handler: h, step: step, vars: vars} + params := &invoke.Params{ + Action: artifact.Action(step.Action), + Source: resolver.path("source", source), + Name: resolver.text("artifact_name", step.ArtifactName), + Version: resolver.text("version", step.Version), + Kind: resolver.text("artifact_kind", step.ArtifactKind), + Force: step.Force, + DryRun: step.ArtifactDryRun, + Refresh: step.Refresh, + Readme: resolver.text("readme", step.Readme), + Changelog: resolver.text("changelog", step.Changelog), + Ref: resolver.text("ref", step.Ref), + Destination: resolver.path("destination", step.Destination), + From: resolver.text("from", step.From), + To: resolver.text("to", step.To), + Document: resolver.text("document", step.Document), + Query: resolver.text("query", step.SearchQuery), + } + repository := resolver.text("repository", step.Repository) + if resolver.err != nil { + return nil, resolver.err + } + if params.Action == artifact.ActionList || params.Action == artifact.ActionSearch { + params.Repositories = splitRepositories(repository) + } else { + params.Repository = repository + } + if params.Labels, err = resolveArtifactLabels(step, resolver); err != nil { + return nil, err + } + return params, resolver.err +} + +// splitRepositories splits the comma-separated repository selector of list and search, the same +// way the repeatable `--repository` flag splits its values. +func splitRepositories(value string) []string { + var repositories []string + for _, alias := range strings.Split(value, ",") { + if alias = strings.TrimSpace(alias); alias != "" { + repositories = append(repositories, alias) + } + } + return repositories +} + +// resolveArtifactLabels returns the publish metadata labels, or the list and search selectors. The +// action's field matrix guarantees that at most one of the two forms is set. +func resolveArtifactLabels(step *schema.WorkflowStep, resolver *artifactFieldResolver) (map[string]string, error) { + if step.Metadata != nil && len(step.Metadata.Labels) > 0 { + labels := make(map[string]string, len(step.Metadata.Labels)) + for key, value := range step.Metadata.Labels { + labels[key] = resolver.text("metadata.labels."+key, value) + } + return labels, resolver.err + } + if step.Labels == nil { + return nil, nil + } + return parseArtifactSelectors(step, resolver) +} + +const artifactLabelsField = "labels" + +// parseArtifactSelectors accepts the three shapes of `labels`: a comma-separated key=value string, +// a list of them, or a mapping. The string forms use the CLI's exact key/value parser. +func parseArtifactSelectors(step *schema.WorkflowStep, resolver *artifactFieldResolver) (map[string]string, error) { + value := reflect.ValueOf(step.Labels) + if value.Kind() == reflect.Map { + labels, err := resolveLabelMapping(step, value, resolver) + if err != nil { + return nil, err + } + if resolver.err != nil { + return nil, resolver.err + } + return labels, artifact.ValidateLabels(labels) + } + var pairs []string + switch value.Kind() { + case reflect.String: + pairs = append(pairs, value.String()) + case reflect.Slice: + pairs = labelListItems(value) + default: + return nil, artifactInvalidField(step, artifactLabelsField, "use a key=value string, a list of them, or a mapping") + } + rendered := make([]string, 0, len(pairs)) + for _, pair := range pairs { + rendered = append(rendered, resolver.text(artifactLabelsField, pair)) + } + if resolver.err != nil { + return nil, resolver.err + } + return tags.ParseLabelsFlagFrom(strings.Join(rendered, ","), "the `labels` field of artifact step "+step.Name) +} + +// labelListItems returns the items of a label list as strings. +func labelListItems(list reflect.Value) []string { + items := make([]string, 0, list.Len()) + for index := range list.Len() { + items = append(items, unwrapInterface(list.Index(index)).String()) + } + return items +} + +// resolveLabelMapping renders the values of a label mapping as templates. +func resolveLabelMapping(step *schema.WorkflowStep, mapping reflect.Value, resolver *artifactFieldResolver) (map[string]string, error) { + labels := make(map[string]string, mapping.Len()) + for _, key := range mapping.MapKeys() { + entry := unwrapInterface(mapping.MapIndex(key)) + if !entry.IsValid() || entry.Kind() != reflect.String { + return nil, artifactInvalidField(step, artifactLabelsField, "mapping values must be strings") + } + labels[fmt.Sprint(key.Interface())] = resolver.text(artifactLabelsField, entry.String()) + } + return labels, nil +} + +// unwrapInterface returns the concrete value held by an interface-typed element. +func unwrapInterface(value reflect.Value) reflect.Value { + if value.Kind() == reflect.Interface { + return value.Elem() + } + return value +} + +// checkResolvedArtifactParams verifies, after templates are rendered, that every required input is +// still present. A template that renders to an empty string must fail here, with the field named, +// rather than deep inside the service. +func checkResolvedArtifactParams(step *schema.WorkflowStep, params *invoke.Params) error { + spec, err := artifactSpec(step) + if err != nil { + return err + } + present := map[invoke.Field]bool{ + invoke.FieldSource: params.Source != "", + invoke.FieldRepository: params.Repository != "", + invoke.FieldRef: params.Ref != "", + invoke.FieldDestination: params.Destination != "", + invoke.FieldFrom: params.From != "", + invoke.FieldTo: params.To != "", + invoke.FieldDocument: params.Document != "", + } + var missing []string + for _, field := range spec.Required { + if known, tracked := present[field]; tracked && !known { + missing = append(missing, string(field)) + } + } + if len(missing) == 0 { + return nil + } + sort.Strings(missing) + return errUtils.Build(fmt.Errorf("%w: %s", errUtils.ErrArtifactStepFieldRequired, strings.Join(missing, ", "))). + WithContext("step", step.Name). + WithContext("action", step.Action). + WithContext("fields", strings.Join(missing, ", ")). + WithExplanationf("Step `%s` (action `%s`) rendered an empty value for required field(s): %s", step.Name, step.Action, strings.Join(missing, ", ")). + WithHint("Check the templates in those fields; each must render to a non-empty value"). + Err() +} diff --git a/pkg/runner/step/artifact_session.go b/pkg/runner/step/artifact_session.go new file mode 100644 index 00000000000..60051a4fd81 --- /dev/null +++ b/pkg/runner/step/artifact_session.go @@ -0,0 +1,172 @@ +package step + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/google/uuid" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifact/report" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// artifactSession is the artifact state a [Variables] carries across the attempts and steps of one +// execution: the retained operation identities, and the report collector that execution +// boundaries record results into. +// +// A wrapper that retries a step (a workflow, a custom command, or a hook) calls the handler again +// with the same Variables, so the session is where an operation keeps its identity between +// attempts. Clones of a Variables share the session, so a parallel branch and its parent agree. +type artifactSession struct { + mu sync.Mutex + collector report.Collector + operations map[string]*artifactOperation + inputs map[string]invoke.Params + fingerprintKey [32]byte +} + +// artifactOperation is the identity of one logical operation: the ID and the UTC preparation time +// are captured once, before the first attempt, and reused by every retry. +type artifactOperation struct { + id string + preparedAt time.Time + attempts int +} + +func newArtifactSession() *artifactSession { + session := &artifactSession{collector: report.Discard{}, operations: make(map[string]*artifactOperation), inputs: make(map[string]invoke.Params)} + // Go 1.26+ guarantees Read fills the key or terminates the process on entropy failure. + if _, err := rand.Read(session.fingerprintKey[:]); err != nil { + panic("generate artifact session fingerprint key: " + err.Error()) + } + return session +} + +// Clocks and ID generation are variables so tests can pin them. +var ( + artifactNow = func() time.Time { return time.Now().UTC() } + artifactNewID = uuid.NewString +) + +// UseArtifactIdentity pins the clock and the operation ID generator that artifact steps use when they +// capture a new logical operation, and returns a function that restores them. It is the seam tests use +// to compare the results of different entry points byte for byte; production code never calls it. +func UseArtifactIdentity(newID func() string, now func() time.Time) (restore func()) { + defer perf.Track(nil, "step.UseArtifactIdentity")() + + previousNow, previousID := artifactNow, artifactNewID + artifactNow, artifactNewID = now, newID + return func() { artifactNow, artifactNewID = previousNow, previousID } +} + +// SetArtifactCollector injects the collector that artifact steps record their results into. A nil +// collector restores the no-op default. The collector is passed through the runner's services, +// never through a context value, and is shared by every Variables cloned from this one. +func (v *Variables) SetArtifactCollector(collector report.Collector) { + defer perf.Track(nil, "step.Variables.SetArtifactCollector")() + + session := v.artifactSession() + session.mu.Lock() + defer session.mu.Unlock() + if collector == nil { + collector = report.Discard{} + } + session.collector = collector +} + +// artifactSession returns the session, creating it for a Variables built without NewVariables. +func (v *Variables) artifactSession() *artifactSession { + if v.artifacts == nil { + v.artifacts = newArtifactSession() + } + return v.artifacts +} + +// begin returns the identity for the logical operation with the given key and counts the attempt. +// The first call captures the ID and preparation time; later calls, which are wrapper retries of +// the same step with the same resolved input, return them unchanged. +func (s *artifactSession) begin(key string) (operation artifactOperation, collector report.Collector) { + s.mu.Lock() + defer s.mu.Unlock() + + existing, ok := s.operations[key] + if !ok { + existing = &artifactOperation{id: artifactNewID(), preparedAt: artifactNow()} + s.operations[key] = existing + } + existing.attempts++ + return *existing, s.collector +} + +// finish ends the logical operation after a success, so the same step run again later is a new +// operation. A failed attempt keeps its identity: the next attempt is a retry of the same operation. +func (s *artifactSession) finish(key string, succeeded bool) { + if !succeeded { + return + } + s.mu.Lock() + defer s.mu.Unlock() + delete(s.operations, key) +} + +// stepKey identifies the authored call independently of resolved templates. The fingerprint is +// scoped to this session and keyed with a secret, since step environment values may be sensitive. +func (s *artifactSession) stepKey(step *schema.WorkflowStep) (string, error) { + encoded, err := json.Marshal(step) + if err != nil { + return "", fmt.Errorf("%w: cannot identify artifact step: %w", errUtils.ErrArtifactStepInvalidField, err) + } + return s.fingerprint("authored-step", encoded), nil +} + +func (s *artifactSession) retainedInput(key string) (*invoke.Params, bool) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.inputs[key] + return &value, ok +} + +func (s *artifactSession) retainInput(key string, params *invoke.Params) { + s.mu.Lock() + defer s.mu.Unlock() + s.inputs[key] = *params +} + +func (s *artifactSession) forgetInput(key string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.inputs, key) +} + +// fingerprint produces a session-local, domain-separated identifier without retaining a +// deterministic digest of potentially sensitive authored or resolved values. +func (s *artifactSession) fingerprint(domain string, encoded []byte) string { + mac := hmac.New(sha256.New, s.fingerprintKey[:]) + mac.Write([]byte(domain + "\x00")) + mac.Write(encoded) + return hex.EncodeToString(mac.Sum(nil)) +} + +// operationKey identifies a logical operation by the step and its fully resolved input. +// Explicit user metadata participates in the identity; the execution fields do not. +func (s *artifactSession) operationKey(stepName string, params *invoke.Params) string { + identity := *params + identity.Execution.OperationID = "" + identity.Execution.PreparedAt = time.Time{} + identity.Execution.MaxAttempts = 0 + encoded, err := json.Marshal(&identity) + if err != nil { + // Params holds only strings, booleans, and string maps, so encoding cannot fail. + return s.fingerprint("operation-fallback", []byte(stepName)) + } + return s.fingerprint("operation", append([]byte(stepName+"\x00"), encoded...)) +} diff --git a/pkg/runner/step/artifact_test.go b/pkg/runner/step/artifact_test.go new file mode 100644 index 00000000000..34ead5189fe --- /dev/null +++ b/pkg/runner/step/artifact_test.go @@ -0,0 +1,634 @@ +package step + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/api" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/artifacttest" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/artifact/report" + "github.com/cloudposse/atmos/pkg/artifact/service" + "github.com/cloudposse/atmos/pkg/schema" +) + +const ( + testRecordDigest = digest.Digest("sha256:1111111111111111111111111111111111111111111111111111111111111111") + testResolvedDigest = digest.Digest("sha256:2222222222222222222222222222222222222222222222222222222222222222") +) + +// artifactEnv is a step environment with an injected service and a fixed operation identity. +type artifactEnv struct { + t *testing.T + service *artifact.MockService + vars *Variables + collector *report.MockCollector + entries []report.Entry + dir string +} + +func newArtifactEnv(t *testing.T) *artifactEnv { + t.Helper() + ctrl := gomock.NewController(t) + env := &artifactEnv{t: t, service: artifact.NewMockService(ctrl), vars: NewVariables(), collector: report.NewMockCollector(ctrl), dir: t.TempDir()} + var mu sync.Mutex + env.collector.EXPECT().Record(gomock.Any()).Do(func(entry *report.Entry) { + mu.Lock() + defer mu.Unlock() + env.entries = append(env.entries, *entry) + }).AnyTimes() + env.vars.SetAtmosConfig(&schema.AtmosConfiguration{}) + env.vars.SetArtifactCollector(env.collector) + + restoreFactory := UseArtifactServiceFactory(func(*schema.AtmosConfiguration, invoke.ServiceOptions) (artifact.Service, error) { + return env.service, nil + }) + t.Cleanup(restoreFactory) + + previousNow, previousID := artifactNow, artifactNewID + ticks, ids := 0, 0 + artifactNow = func() time.Time { + ticks++ + return time.Date(2026, 10, 31, 12, 0, ticks, 0, time.UTC) + } + artifactNewID = func() string { + ids++ + return "op-" + string(rune('0'+ids)) + } + t.Cleanup(func() { artifactNow, artifactNewID = previousNow, previousID }) + return env +} + +func (e *artifactEnv) run(step *schema.WorkflowStep) (*StepResult, error) { + e.t.Helper() + step.WorkingDirectory = e.dir + handler := &ArtifactHandler{} + require.NoError(e.t, handler.Validate(step)) + return handler.Execute(context.Background(), step, e.vars) +} + +func completeOperation(action artifact.Action, outcome artifact.Outcome) *artifact.OperationResult { + return &artifact.OperationResult{ + OperationID: "op-1", Action: action, Outcome: outcome, Repository: "dir", Requested: "team/app:1.0.0", + Ref: "team/app:1.0.0", Location: "/repo/team/app/1.0.0", ResolvedDigest: testResolvedDigest, RecordDigest: testRecordDigest, + ContentDigest: testRecordDigest, Size: 42, + } +} + +func TestArtifactHandler_PublishMapsTheStepAndReportsNamedOutputs(t *testing.T) { + env := newArtifactEnv(t) + var request artifact.PublishRequest + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + request = r + return completeOperation(artifact.ActionPublish, artifact.OutcomeCreated), nil + }) + + step := artifactStepWith(artifact.ActionPublish, invoke.FieldRepository, invoke.FieldArtifactName, invoke.FieldVersion, invoke.FieldArtifactKind, + invoke.FieldForce, invoke.FieldMetadata, invoke.FieldReadme, invoke.FieldChangelog) + step.Source = "./dist" + step.Version = "{{ .env.APP_VERSION }}" + env.vars.SetEnv("APP_VERSION", "2.0.0") + + result, err := env.run(step) + require.NoError(t, err) + + assert.Equal(t, "dir", request.Repository) + assert.Equal(t, filepath.Join(env.dir, "dist"), request.Source, "source is anchored to the step working directory") + assert.Equal(t, "team/app", request.Name) + assert.Equal(t, "2.0.0", request.Version, "templates are rendered") + assert.Equal(t, "files", request.Kind) + assert.Equal(t, map[string]string{"team": "platform"}, request.Metadata.Labels) + assert.Equal(t, "README.md", request.Readme, "document paths stay relative to the source, as --readme does") + assert.Equal(t, "CHANGELOG.md", request.Changelog) + assert.True(t, request.Force) + assert.False(t, request.DryRun) + assert.Equal(t, "op-1", request.Execution.OperationID) + assert.False(t, request.Execution.PreparedAt.IsZero()) + assert.Zero(t, request.Execution.MaxAttempts, "without retry the service keeps its transport retries") + + assert.Equal(t, map[string]string{ + "outcome": "created", "ref": "team/app:1.0.0", "location": "/repo/team/app/1.0.0", + "resolved_digest": string(testResolvedDigest), "record_digest": string(testRecordDigest), + "content_digest": string(testRecordDigest), "size": "42", + }, result.Outputs, "optional outputs the operation did not produce (representation_digest) are omitted") + assert.Equal(t, "team/app:1.0.0", result.Value) + assert.Equal(t, filepath.Join(env.dir, "dist"), result.Metadata["source"]) + + payload, ok := result.Metadata[ArtifactMetadataKey].(*invoke.Result) + require.True(t, ok, "the typed result is stored under Metadata[artifact]") + assert.Equal(t, artifact.OutcomeCreated, payload.Outcome) + assert.Equal(t, "op-1", payload.OperationID) +} + +func TestArtifactHandler_ActionsMapToServiceCalls(t *testing.T) { + tests := []struct { + name string + step func() *schema.WorkflowStep + expect func(*artifactEnv) + check func(*testing.T, *StepResult) + }{ + { + name: "pull anchors the destination", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionPull, invoke.FieldRepository, invoke.FieldRef, invoke.FieldDestination, invoke.FieldForce) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Pull(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PullRequest) (*artifact.OperationResult, error) { + assert.Equal(t, filepath.Join(e.dir, "out"), r.Destination) + assert.Equal(t, artifact.Ref{Name: "team/app", Version: "1.0.0"}, r.Ref) + assert.True(t, r.Force) + return completeOperation(artifact.ActionPull, artifact.OutcomeRead), nil + }) + }, + check: func(t *testing.T, r *StepResult) { assert.Equal(t, "read", r.Outputs["outcome"]) }, + }, + { + name: "copy", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionCopy, invoke.FieldFrom, invoke.FieldTo, invoke.FieldRef) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Copy(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.CopyRequest) (*artifact.OperationResult, error) { + assert.Equal(t, "dir", r.From) + assert.Equal(t, "archive", r.To) + return completeOperation(artifact.ActionCopy, artifact.OutcomeCreated), nil + }) + }, + }, + { + name: "delete", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionDelete, invoke.FieldRepository, invoke.FieldRef) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(completeOperation(artifact.ActionDelete, artifact.OutcomeDeleted), nil) + }, + check: func(t *testing.T, r *StepResult) { assert.Equal(t, "deleted", r.Outputs["outcome"]) }, + }, + { + name: "inspect", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionInspect, invoke.FieldRepository, invoke.FieldRef) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.InspectResult{ + Operation: *completeOperation(artifact.ActionInspect, artifact.OutcomeRead), + }, nil) + }, + check: func(t *testing.T, r *StepResult) { + payload := r.Metadata[ArtifactMetadataKey].(*invoke.Result) + assert.NotNil(t, payload.Inspection) + }, + }, + { + name: "show", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionShow, invoke.FieldRepository, invoke.FieldRef, invoke.FieldDocument) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().ShowDocument(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.ShowDocumentRequest) (*artifact.DocumentResult, error) { + assert.Equal(t, artifact.DocumentReadme, r.Role) + return &artifact.DocumentResult{ + Operation: *completeOperation(artifact.ActionShow, artifact.OutcomeRead), + Document: artifact.Document{Role: artifact.DocumentReadme, Digest: testRecordDigest}, + Content: []byte("# App"), + }, nil + }) + }, + check: func(t *testing.T, r *StepResult) { + payload := r.Metadata[ArtifactMetadataKey].(*invoke.Result) + require.NotNil(t, payload.Content) + assert.Equal(t, "# App", *payload.Content) + }, + }, + { + name: "sign", + step: func() *schema.WorkflowStep { + step := artifactStepWith(artifact.ActionSign, invoke.FieldRepository) + step.Ref = "team/app@" + string(testRecordDigest) + return step + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Sign(gomock.Any(), gomock.Any()).Return(&artifact.SignResult{ + Operation: *completeOperation(artifact.ActionSign, artifact.OutcomeCreated), + Subject: artifact.SignatureSubject{Kind: artifact.SubjectRecord, Digest: testRecordDigest}, + }, nil) + }, + check: func(t *testing.T, r *StepResult) { + assert.NotNil(t, r.Metadata[ArtifactMetadataKey].(*invoke.Result).Subject) + }, + }, + { + name: "verify", + step: func() *schema.WorkflowStep { + return artifactStepWith(artifact.ActionVerify, invoke.FieldRepository, invoke.FieldRef) + }, + expect: func(e *artifactEnv) { + e.service.EXPECT().Verify(gomock.Any(), gomock.Any()).Return(&artifact.VerifyResult{ + Operation: *completeOperation(artifact.ActionVerify, artifact.OutcomeRead), + Verification: artifact.VerificationResult{Status: artifact.VerificationVerified}, + }, nil) + }, + check: func(t *testing.T, r *StepResult) { + assert.Equal(t, artifact.VerificationVerified, r.Metadata[ArtifactMetadataKey].(*invoke.Result).Verification.Status) + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + env := newArtifactEnv(t) + tc.expect(env) + result, err := env.run(tc.step()) + require.NoError(t, err) + require.NotNil(t, result.Metadata[ArtifactMetadataKey]) + if tc.check != nil { + tc.check(t, result) + } + }) + } +} + +func TestArtifactHandler_ListAndSearchReportAnArtifactsArray(t *testing.T) { + entries := []artifact.ListEntry{ + {Repository: "dir", Name: "team/app", Version: "1.0.0", Kind: "files", Pinned: artifact.PinnedReference{RecordDigest: testRecordDigest}}, + {Repository: "archive", Name: "team/app", Version: "2.0.0", Kind: "files"}, + } + t.Run("list parses the label selector forms and the repository list", func(t *testing.T) { + for _, labels := range []any{"team=platform,env:dev", []any{"team=platform", "env=dev"}, map[string]any{"team": "platform", "env": "dev"}} { + env := newArtifactEnv(t) + env.service.EXPECT().List(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.ListRequest) (*artifact.ListResult, error) { + assert.Equal(t, []string{"dir", "archive"}, r.Repositories) + assert.Equal(t, "team/app", r.Name) + assert.Equal(t, []artifact.Label{{Key: "env", Value: "dev"}, {Key: "team", Value: "platform"}}, r.Labels, "selectors are sorted") + assert.True(t, r.Refresh) + return &artifact.ListResult{Artifacts: entries}, nil + }) + step := artifactStepWith(artifact.ActionList, invoke.FieldArtifactName, invoke.FieldRefresh) + step.Repository = "dir, archive" + step.Labels = labels + result, err := env.run(step) + require.NoError(t, err) + + collection, ok := result.Metadata[ArtifactMetadataKey].(*invoke.Collection) + require.True(t, ok) + require.Len(t, collection.Artifacts, 2) + assert.Equal(t, "team/app:1.0.0", collection.Artifacts[0].Ref) + assert.Equal(t, "team/app:2.0.0", collection.Artifacts[1].Ref) + assert.Equal(t, collection.Artifacts, result.Metadata[ArtifactListMetadataKey]) + assert.Equal(t, "2", result.Value) + assert.Empty(t, result.Outputs, "a collection has no single-artifact named outputs") + } + }) + t.Run("search passes the text", func(t *testing.T) { + env := newArtifactEnv(t) + env.service.EXPECT().Search(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.SearchRequest) (*artifact.ListResult, error) { + assert.Equal(t, "app", r.Query) + assert.Empty(t, r.Repositories, "no repository means every configured repository") + return &artifact.ListResult{}, nil + }) + result, err := env.run(artifactStepWith(artifact.ActionSearch, invoke.FieldQuery)) + require.NoError(t, err) + collection := result.Metadata[ArtifactMetadataKey].(*invoke.Collection) + assert.NotNil(t, collection.Artifacts, "an empty listing encodes as an empty array, not null") + assert.Empty(t, collection.Artifacts) + }) +} + +func TestArtifactHandler_FailureReturnsThePartialResult(t *testing.T) { + env := newArtifactEnv(t) + partial := &artifact.OperationResult{OperationID: "op-1", Action: artifact.ActionPublish, Repository: "dir", Ref: "team/app:1.0.0", RecordDigest: testRecordDigest} + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(partial, errUtils.ErrArtifactConflict) + + result, err := env.run(artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository)) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NotNil(t, result, "a failed operation still returns what it learned") + + payload := result.Metadata[ArtifactMetadataKey].(*invoke.Result) + assert.Equal(t, artifact.OutcomeFailed, payload.Outcome, "a failure is never reported as a success") + assert.Contains(t, payload.Error, "conflict") + assert.Equal(t, testRecordDigest, payload.RecordDigest) + assert.Equal(t, "failed", result.Outputs["outcome"]) + assert.Equal(t, string(testRecordDigest), result.Outputs["record_digest"]) + assert.NotContains(t, result.Outputs, "location", "values the operation never resolved are omitted") + + require.Len(t, env.entries, 1) + assert.Error(t, env.entries[0].Err) + assert.Equal(t, "op-1", env.entries[0].OperationID) +} + +// TestArtifactHandler_RetryKeepsTheOperationIdentity covers the wrapper-level retry contract: the +// wrapper calls the handler again with the same Variables, the identity survives the failure, a +// success ends the operation, and the service's own transport retries are off while retry is set. +func TestArtifactHandler_RetryKeepsTheOperationIdentity(t *testing.T) { + env := newArtifactEnv(t) + var executions []artifact.Execution + temporary := errors.New("temporary registry failure") + calls := 0 + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + executions = append(executions, r.Execution) + calls++ + op := completeOperation(artifact.ActionPublish, artifact.OutcomeCreated) + op.OperationID = r.Execution.OperationID + if calls < 3 { + return &artifact.OperationResult{OperationID: r.Execution.OperationID, Action: artifact.ActionPublish}, temporary + } + return op, nil + }).Times(4) + + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository) + step.Retry = &schema.RetryConfig{} + + for attempt := 1; attempt <= 3; attempt++ { + _, err := env.run(step) + if attempt < 3 { + require.ErrorIs(t, err, temporary) + } else { + require.NoError(t, err) + } + } + require.Len(t, executions, 3) + for _, execution := range executions { + assert.Equal(t, executions[0].OperationID, execution.OperationID, "every attempt is the same logical operation") + assert.Equal(t, executions[0].PreparedAt, execution.PreparedAt, "the preparation time is captured once") + assert.Equal(t, 1, execution.MaxAttempts, "the wrapper owns the retry budget") + } + + // The operation ended with the success; running the step again is a new operation. + _, err := env.run(step) + require.NoError(t, err) + require.Len(t, executions, 4) + assert.NotEqual(t, executions[0].OperationID, executions[3].OperationID) + assert.NotEqual(t, executions[0].PreparedAt, executions[3].PreparedAt) + + require.Len(t, env.entries, 4) + for index, want := range []int{1, 2, 3, 1} { + assert.Equal(t, want, env.entries[index].Attempt) + } + assert.Equal(t, env.entries[0].OperationID, env.entries[2].OperationID, "a report deduplicates attempts by operation ID") +} + +func TestArtifactHandler_DifferentInputIsADifferentOperation(t *testing.T) { + env := newArtifactEnv(t) + var ids []string + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + ids = append(ids, r.Execution.OperationID) + return nil, errors.New("fail") + }).Times(2) + first := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository, invoke.FieldVersion) + second := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository, invoke.FieldVersion) + second.Version = "2.0.0" + _, _ = env.run(first) + _, _ = env.run(second) + require.Len(t, ids, 2) + assert.NotEqual(t, ids[0], ids[1]) +} + +func TestArtifactHandler_CancellationStaysReachable(t *testing.T) { + env := newArtifactEnv(t) + ctx, cancel := context.WithCancel(context.Background()) + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, artifact.PublishRequest) (*artifact.OperationResult, error) { + cancel() + return nil, errors.New("transfer aborted") + }) + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository) + step.WorkingDirectory = env.dir + result, err := (&ArtifactHandler{}).Execute(ctx, step, env.vars) + require.ErrorIs(t, err, context.Canceled, "a retrying wrapper must be able to see the cancellation and stop") + assert.NotNil(t, result) +} + +func TestArtifactHandler_WorkflowDryRun(t *testing.T) { + t.Run("publish switches to the service dry run", func(t *testing.T) { + env := newArtifactEnv(t) + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + assert.True(t, r.DryRun) + return completeOperation(artifact.ActionPublish, artifact.OutcomeDryRun), nil + }) + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository) + step.DryRun = true + _, err := env.run(step) + require.NoError(t, err) + }) + t.Run("mutating actions without a dry run are skipped", func(t *testing.T) { + for _, action := range []artifact.Action{artifact.ActionPull, artifact.ActionCopy, artifact.ActionDelete, artifact.ActionSign} { + env := newArtifactEnv(t) + spec, _ := invoke.SpecFor(action) + step := artifactStepWith(action, spec.Required...) + step.DryRun = true + result, err := env.run(step) + require.NoError(t, err, action) + assert.True(t, result.Skipped, action) + assert.Empty(t, env.entries, "a skipped step reports nothing") + } + }) + t.Run("reads run normally", func(t *testing.T) { + env := newArtifactEnv(t) + env.service.EXPECT().Delete(gomock.Any(), gomock.Any()).Times(0) + env.service.EXPECT().Inspect(gomock.Any(), gomock.Any()).Return(&artifact.InspectResult{Operation: *completeOperation(artifact.ActionInspect, artifact.OutcomeRead)}, nil) + step := artifactStepWith(artifact.ActionInspect, invoke.FieldRepository, invoke.FieldRef) + step.DryRun = true + result, err := env.run(step) + require.NoError(t, err) + assert.False(t, result.Skipped) + }) +} + +func TestArtifactHandler_ReportsConfigurationAndTemplateProblems(t *testing.T) { + t.Run("empty required value after rendering", func(t *testing.T) { + env := newArtifactEnv(t) + step := artifactStepWith(artifact.ActionInspect, invoke.FieldRepository) + step.Ref = `{{ "" }}` + _, err := env.run(step) + require.ErrorIs(t, err, errUtils.ErrArtifactStepFieldRequired) + assert.Contains(t, err.Error(), "ref") + }) + t.Run("template error names the field", func(t *testing.T) { + env := newArtifactEnv(t) + step := artifactStepWith(artifact.ActionInspect, invoke.FieldRepository) + step.Ref = "{{ .steps.nope.value | bogus }}" + _, err := env.run(step) + require.ErrorIs(t, err, errUtils.ErrTemplateEvaluation) + }) + t.Run("missing atmos configuration", func(t *testing.T) { + env := newArtifactEnv(t) + env.vars.AtmosConfig = nil + _, err := env.run(artifactStepWith(artifact.ActionInspect, invoke.FieldRepository, invoke.FieldRef)) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("service construction failure", func(t *testing.T) { + env := newArtifactEnv(t) + restore := UseArtifactServiceFactory(func(*schema.AtmosConfiguration, invoke.ServiceOptions) (artifact.Service, error) { + return nil, errUtils.ErrArtifactConfigInvalid + }) + defer restore() + _, err := env.run(artifactStepWith(artifact.ActionInspect, invoke.FieldRepository, invoke.FieldRef)) + require.ErrorIs(t, err, errUtils.ErrArtifactConfigInvalid) + }) + t.Run("invalid ref surfaces as a failed result", func(t *testing.T) { + env := newArtifactEnv(t) + step := artifactStepWith(artifact.ActionInspect, invoke.FieldRepository) + step.Ref = "no-version" + result, err := env.run(step) + require.ErrorIs(t, err, errUtils.ErrArtifactRefInvalid) + assert.Equal(t, "failed", result.Outputs["outcome"]) + assert.Equal(t, "no-version", result.Metadata[ArtifactMetadataKey].(*invoke.Result).Requested) + }) +} + +func TestArtifactHandler_TimeoutLimitsEachAttempt(t *testing.T) { + env := newArtifactEnv(t) + env.service.EXPECT().Delete(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, _ artifact.DeleteRequest) (*artifact.OperationResult, error) { + <-ctx.Done() + return nil, ctx.Err() + }) + step := artifactStepWith(artifact.ActionDelete, invoke.FieldRepository, invoke.FieldRef) + step.Timeout = "20ms" + _, err := env.run(step) + require.ErrorIs(t, err, errUtils.ErrStepTimeout) +} + +func TestArtifactHandler_UsesPriorStepOutputsInTemplates(t *testing.T) { + env := newArtifactEnv(t) + env.vars.Set("publish", NewStepResult("").WithOutput("ref", "team/app:1.0.0")) + env.service.EXPECT().Inspect(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.InspectRequest) (*artifact.InspectResult, error) { + assert.Equal(t, "team/app", r.Ref.Name) + return &artifact.InspectResult{Operation: *completeOperation(artifact.ActionInspect, artifact.OutcomeRead)}, nil + }) + step := artifactStepWith(artifact.ActionInspect, invoke.FieldRepository) + step.Ref = "{{ .steps.publish.outputs.ref }}" + _, err := env.run(step) + require.NoError(t, err) +} + +// TestArtifactHandler_ScriptDispatch runs the step through a direct step call, as an embedded +// script does with steps.artifact(...), and checks it reaches the same service with the same input. +func TestArtifactHandler_ScriptDispatch(t *testing.T) { + env := newArtifactEnv(t) + var request artifact.PublishRequest + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + request = r + return completeOperation(artifact.ActionPublish, artifact.OutcomeCreated), nil + }) + library := NewAutomationLibrary(env.vars, nil) + call := &api.StepCall{Type: "artifact", Configuration: map[string]any{ + "name": "publish", "action": "publish", "source": filepath.Join(env.dir, "dist"), "repository": "dir", + "artifact_name": "team/app", "version": "1.0.0", "metadata": map[string]any{"labels": map[string]any{"team": "platform"}}, + }} + require.NoError(t, library.Validate(call)) + result, err := library.Run(context.Background(), call) + require.NoError(t, err) + assert.Equal(t, "created", result.Outputs["outcome"]) + assert.Equal(t, filepath.Join(env.dir, "dist"), request.Source) + assert.Equal(t, map[string]string{"team": "platform"}, request.Metadata.Labels) + + t.Run("a field the artifact step does not read is rejected", func(t *testing.T) { + err := library.Validate(&api.StepCall{Type: "artifact", Configuration: map[string]any{"action": "inspect", "duration": "5s"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "duration") + }) + t.Run("the search text is a string query", func(t *testing.T) { + env.service.EXPECT().Search(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.SearchRequest) (*artifact.ListResult, error) { + assert.Equal(t, "vpc", r.Query) + return &artifact.ListResult{}, nil + }) + _, err := library.Run(context.Background(), &api.StepCall{Type: "artifact", Configuration: map[string]any{"action": "search", "query": "vpc"}}) + require.NoError(t, err) + }) +} + +func TestWrapperRetriesStep(t *testing.T) { + assert.True(t, WrapperRetriesStep("artifact")) + assert.True(t, WrapperRetriesStep(" artifact ")) + for _, other := range []string{"shell", "http", "archive", "script", ""} { + assert.False(t, WrapperRetriesStep(other), other) + } +} + +func TestArtifactHandler_RetryRetainsResolvedInputs(t *testing.T) { + env := newArtifactEnv(t) + var requests []artifact.PublishRequest + env.service.EXPECT().Publish(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, r artifact.PublishRequest) (*artifact.OperationResult, error) { + requests = append(requests, r) + if len(requests) == 1 { + return nil, errors.New("transient") + } + return completeOperation(artifact.ActionPublish, artifact.OutcomeCreated), nil + }).Times(2) + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository, invoke.FieldVersion, invoke.FieldMetadata) + step.Version = "{{ .env.VERSION }}" + step.Metadata.Labels["team"] = "{{ .env.TEAM }}" + env.vars.SetEnv("VERSION", "1") + env.vars.SetEnv("TEAM", "first") + _, err := env.run(step) + require.Error(t, err) + env.vars.SetEnv("VERSION", "2") + env.vars.SetEnv("TEAM", "second") + _, err = env.run(step) + require.NoError(t, err) + require.Len(t, requests, 2) + assert.Equal(t, requests[0].Version, requests[1].Version) + assert.Equal(t, requests[0].Metadata.Labels, requests[1].Metadata.Labels) + assert.Equal(t, requests[0].Execution.OperationID, requests[1].Execution.OperationID) +} + +type nilDetailedService struct { + artifact.Service + err error +} + +func (s nilDetailedService) PublishDetailed(context.Context, artifact.PublishRequest) (*service.PublishOutcome, error) { + return nil, s.err +} + +func TestRecorderNormalizesNilDetailedOutcome(t *testing.T) { + failure := errors.New("failed") + recorder := artifacttest.NewRecorder(nilDetailedService{err: failure}) + outcome, err := recorder.PublishDetailed(context.Background(), artifact.PublishRequest{}) + require.ErrorIs(t, err, failure) + require.NotNil(t, outcome) + require.Len(t, recorder.Results(), 1) + assert.Equal(t, outcome.Operation, recorder.Results()[0]) +} + +func TestArtifactSessionFingerprintIsScopedAndPreservesIdentity(t *testing.T) { + first := newArtifactSession() + second := newArtifactSession() + step := &schema.WorkflowStep{Name: "publish", Action: "publish", Env: map[string]string{"TOKEN": "sensitive-value"}} + key, err := first.stepKey(step) + require.NoError(t, err) + again, err := first.stepKey(step) + require.NoError(t, err) + otherSession, err := second.stepKey(step) + require.NoError(t, err) + assert.Equal(t, key, again, "retries in the same session retain their key") + assert.NotEqual(t, key, otherSession, "sessions do not expose a reusable fingerprint") + assert.NotContains(t, key, "sensitive-value") + step.Env["TOKEN"] = "different-value" + different, err := first.stepKey(step) + require.NoError(t, err) + assert.NotEqual(t, key, different, "authored inputs still participate in identity") + + params := &invoke.Params{Action: artifact.ActionPublish, Labels: map[string]string{"token": "sensitive-value"}} + operation := first.operationKey(step.Name, params) + assert.Equal(t, operation, first.operationKey(step.Name, params)) + assert.NotEqual(t, operation, second.operationKey(step.Name, params)) + params.Labels["token"] = "different-value" + assert.NotEqual(t, operation, first.operationKey(step.Name, params)) +} + +func TestArtifactSessionRejectsUnserializableStepIdentity(t *testing.T) { + step := &schema.WorkflowStep{Name: "publish", Action: "publish", With: map[string]any{"invalid": make(chan int)}} + _, err := newArtifactSession().stepKey(step) + require.ErrorIs(t, err, errUtils.ErrArtifactStepInvalidField) +} diff --git a/pkg/runner/step/artifact_validate.go b/pkg/runner/step/artifact_validate.go new file mode 100644 index 00000000000..82deff8f0bb --- /dev/null +++ b/pkg/runner/step/artifact_validate.go @@ -0,0 +1,217 @@ +package step + +import ( + "fmt" + "reflect" + "strings" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Validate checks the artifact step configuration before execution. It rejects an unknown action, +// a field the action does not accept, a missing required field, and a malformed `source` or +// `labels`, all from the authored step alone, so an invalid combination fails before any work. +func (h *ArtifactHandler) Validate(step *schema.WorkflowStep) error { + defer perf.Track(nil, "step.ArtifactHandler.Validate")() + + spec, err := artifactSpec(step) + if err != nil { + return err + } + if _, err := artifactSourceString(step); err != nil { + return err + } + if len(step.Query) > 0 { + return artifactInvalidField(step, "query", "the query mapping belongs to the http step; an artifact step takes the search text as a string") + } + if err := validateArtifactLabelShape(step); err != nil { + return err + } + if err := validateArtifactOtherFields(step); err != nil { + return err + } + return validateArtifactActionFields(step, spec) +} + +// validateArtifactActionFields checks the action-specific field contract. +func validateArtifactActionFields(step *schema.WorkflowStep, spec invoke.Spec) error { + set := artifactFieldsSet(step) + for _, field := range invoke.AllFields() { + if set[field] && !spec.Accepts(field) { + return errUtils.Build(fmt.Errorf("%w: `%s` for action `%s`", errUtils.ErrArtifactStepFieldForbidden, field, step.Action)). + WithContext("step", step.Name). + WithContext("action", step.Action). + WithContext("field", string(field)). + WithExplanationf("Field `%s` is not valid for artifact action `%s`", field, step.Action). + WithHintf("Fields for `%s`: %s", step.Action, artifactFieldList(spec)). + Err() + } + } + for _, field := range spec.Required { + if !set[field] { + return errUtils.Build(fmt.Errorf("%w: `%s` for action `%s`", errUtils.ErrArtifactStepFieldRequired, field, step.Action)). + WithContext("step", step.Name). + WithContext("action", step.Action). + WithContext("field", string(field)). + WithExplanationf("Step `%s` (action `%s`) is missing required field `%s`", step.Name, step.Action, field). + WithHintf("Fields for `%s`: %s", step.Action, artifactFieldList(spec)). + Err() + } + } + return nil +} + +// validateArtifactOtherFields rejects populated fields belonging to other step types. +func validateArtifactOtherFields(step *schema.WorkflowStep) error { + allowed := map[string]bool{} + for _, name := range []string{ + "Name", "Type", "Action", "Source", "Repository", "ArtifactName", "Version", "ArtifactKind", "Force", "ArtifactDryRun", "Metadata", "Labels", "Readme", "Changelog", "Ref", "Destination", "From", "To", "Document", "SearchQuery", "Refresh", + "WorkingDirectory", "Retry", "Identity", "Needs", "When", "Continue", "Output", "Timeout", "Env", "With", "Inputs", "Artifacts", "Outputs", "Show", "DryRun", "LiteralFields", "LoadError", + } { + allowed[name] = true + } + value := reflect.ValueOf(step).Elem() + kind := value.Type() + for index := 0; index < value.NumField(); index++ { + field := kind.Field(index) + if allowed[field.Name] || value.Field(index).IsZero() { + continue + } + key := strings.Split(field.Tag.Get("yaml"), ",")[0] + if key == "-" { + key = strings.Split(field.Tag.Get("json"), ",")[0] + } + if key == "" || key == "-" { + continue + } + return errUtils.Build(fmt.Errorf("%w: `%s` for action `%s`", errUtils.ErrArtifactStepFieldForbidden, key, step.Action)). + WithContext("step", step.Name).WithContext("action", step.Action).WithContext("field", key). + WithExplanationf("Field `%s` is not valid for artifact action `%s`", key, step.Action).Err() + } + return nil +} + +// artifactSpec returns the input contract of the step's action, or a typed error naming the valid +// actions when the action is missing or unknown. +func artifactSpec(step *schema.WorkflowStep) (invoke.Spec, error) { + action := artifact.Action(step.Action) + spec, ok := invoke.SpecFor(action) + if ok { + return spec, nil + } + valid := make([]string, 0, len(invoke.Actions())) + for _, candidate := range invoke.Actions() { + valid = append(valid, string(candidate)) + } + return invoke.Spec{}, errUtils.Build(fmt.Errorf("%w: %q", errUtils.ErrArtifactStepInvalidAction, step.Action)). + WithContext("step", step.Name). + WithContext("action", step.Action). + WithExplanationf("Artifact step `%s` has action %q, which is not an artifact operation", step.Name, step.Action). + WithHintf("Use one of: %s", strings.Join(valid, ", ")). + Err() +} + +// artifactFieldList renders the fields an action accepts, required fields first. +func artifactFieldList(spec invoke.Spec) string { + names := make([]string, 0, len(spec.Required)+len(spec.Optional)) + for _, field := range spec.Required { + names = append(names, string(field)+" (required)") + } + for _, field := range spec.Optional { + names = append(names, string(field)) + } + return strings.Join(names, ", ") +} + +// artifactSourceString reads step.Source as a plain string. The field is shared with the workdir +// step, which also accepts a source map; an artifact step requires a plain string path. An unset +// source returns the empty string and no error. +func artifactSourceString(step *schema.WorkflowStep) (string, error) { + if step.Source == nil { + return "", nil + } + source, ok := step.Source.(string) + if !ok { + return "", errUtils.Build(fmt.Errorf("%w, got %T", errUtils.ErrArtifactStepInvalidSource, step.Source)). + WithContext("step", step.Name). + WithContext("type", fmt.Sprintf("%T", step.Source)). + WithHint("Set `source` to a plain string path"). + Err() + } + return source, nil +} + +// artifactInvalidField builds the error for a field with the wrong shape or value. +func artifactInvalidField(step *schema.WorkflowStep, field, detail string) error { + return errUtils.Build(fmt.Errorf("%w: `%s`", errUtils.ErrArtifactStepInvalidField, field)). + WithContext("step", step.Name). + WithContext("field", field). + WithExplanationf("Artifact step `%s` field `%s` is invalid: %s", step.Name, field, detail). + Err() +} + +// validateArtifactLabelShape checks that `labels` is a string, a list of strings, or a mapping. +func validateArtifactLabelShape(step *schema.WorkflowStep) error { + if step.Labels == nil { + return nil + } + switch reflect.ValueOf(step.Labels).Kind() { + case reflect.String: + return nil + case reflect.Map: + return validateArtifactLabelMapping(step) + + case reflect.Slice: + for _, item := range reflect.ValueOf(step.Labels).Seq2() { + if item.Kind() == reflect.Interface { + item = item.Elem() + } + if item.Kind() != reflect.String { + return artifactInvalidField(step, "labels", "list items must be key=value strings") + } + } + return nil + default: + return artifactInvalidField(step, "labels", "use a key=value string, a list of them, or a mapping") + } +} + +func validateArtifactLabelMapping(step *schema.WorkflowStep) error { + mapping := reflect.ValueOf(step.Labels) + for _, key := range mapping.MapKeys() { + item := unwrapInterface(mapping.MapIndex(key)) + if !item.IsValid() || item.Kind() != reflect.String { + return artifactInvalidField(step, "labels", "mapping values must be strings") + } + } + return nil +} + +// artifactFieldsSet reports which operation inputs the authored step sets. A boolean counts as set +// only when true, because an absent boolean cannot be told from `false`. +func artifactFieldsSet(step *schema.WorkflowStep) map[invoke.Field]bool { + return map[invoke.Field]bool{ + invoke.FieldSource: step.Source != nil, + invoke.FieldRepository: step.Repository != "", + invoke.FieldArtifactName: step.ArtifactName != "", + invoke.FieldVersion: step.Version != "", + invoke.FieldArtifactKind: step.ArtifactKind != "", + invoke.FieldForce: step.Force, + invoke.FieldDryRun: step.ArtifactDryRun, + invoke.FieldMetadata: step.Metadata != nil, + invoke.FieldLabels: step.Labels != nil, + invoke.FieldReadme: step.Readme != "", + invoke.FieldChangelog: step.Changelog != "", + invoke.FieldRef: step.Ref != "", + invoke.FieldDestination: step.Destination != "", + invoke.FieldFrom: step.From != "", + invoke.FieldTo: step.To != "", + invoke.FieldDocument: step.Document != "", + invoke.FieldQuery: step.SearchQuery != "", + invoke.FieldRefresh: step.Refresh, + } +} diff --git a/pkg/runner/step/artifact_validate_test.go b/pkg/runner/step/artifact_validate_test.go new file mode 100644 index 00000000000..04e303ea6fa --- /dev/null +++ b/pkg/runner/step/artifact_validate_test.go @@ -0,0 +1,153 @@ +package step + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/artifact" + "github.com/cloudposse/atmos/pkg/artifact/invoke" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time guards for the schema fields these tests set. +var _ = schema.WorkflowStep{Repository: "", ArtifactName: "", Version: "", ArtifactKind: "", Force: false, ArtifactDryRun: false, Readme: "", Changelog: "", Ref: "", From: "", To: "", Document: "", SearchQuery: "", Refresh: false} + +// artifactFieldSetters sets one operation input on a step, keyed by the contract field name. +var artifactFieldSetters = map[invoke.Field]func(*schema.WorkflowStep){ + invoke.FieldSource: func(s *schema.WorkflowStep) { s.Source = "./dist" }, + invoke.FieldRepository: func(s *schema.WorkflowStep) { s.Repository = "dir" }, + invoke.FieldArtifactName: func(s *schema.WorkflowStep) { s.ArtifactName = "team/app" }, + invoke.FieldVersion: func(s *schema.WorkflowStep) { s.Version = "1.0.0" }, + invoke.FieldArtifactKind: func(s *schema.WorkflowStep) { s.ArtifactKind = "files" }, + invoke.FieldForce: func(s *schema.WorkflowStep) { s.Force = true }, + invoke.FieldDryRun: func(s *schema.WorkflowStep) { s.ArtifactDryRun = true }, + invoke.FieldMetadata: func(s *schema.WorkflowStep) { + s.Metadata = &schema.ArtifactStepMetadata{Labels: map[string]string{"team": "platform"}} + }, + invoke.FieldLabels: func(s *schema.WorkflowStep) { s.Labels = "team=platform" }, + invoke.FieldReadme: func(s *schema.WorkflowStep) { s.Readme = "README.md" }, + invoke.FieldChangelog: func(s *schema.WorkflowStep) { s.Changelog = "CHANGELOG.md" }, + invoke.FieldRef: func(s *schema.WorkflowStep) { s.Ref = "team/app:1.0.0" }, + invoke.FieldDestination: func(s *schema.WorkflowStep) { s.Destination = "./out" }, + invoke.FieldFrom: func(s *schema.WorkflowStep) { s.From = "dir" }, + invoke.FieldTo: func(s *schema.WorkflowStep) { s.To = "archive" }, + invoke.FieldDocument: func(s *schema.WorkflowStep) { s.Document = "readme" }, + invoke.FieldQuery: func(s *schema.WorkflowStep) { s.SearchQuery = "app" }, + invoke.FieldRefresh: func(s *schema.WorkflowStep) { s.Refresh = true }, +} + +// artifactStepWith builds a step for the action with exactly the given inputs set. +func artifactStepWith(action artifact.Action, fields ...invoke.Field) *schema.WorkflowStep { + step := &schema.WorkflowStep{Name: "artifact-step", Type: schema.TaskTypeArtifact, Action: string(action)} + for _, field := range fields { + artifactFieldSetters[field](step) + } + return step +} + +func TestArtifactHandler_Identity(t *testing.T) { + handler, ok := Get(schema.TaskTypeArtifact) + require.True(t, ok, "the artifact step type must be registered") + assert.Equal(t, "artifact", handler.GetName()) + assert.Equal(t, CategoryCommand, handler.GetCategory()) + assert.False(t, handler.RequiresTTY()) + + known, ok := handler.(knownFieldsHandler) + require.True(t, ok) + assert.ElementsMatch(t, []string{ + "action", "source", "repository", "artifact_name", "version", "artifact_kind", "force", "dry_run", "metadata", + "labels", "readme", "changelog", "ref", "destination", "from", "to", "document", "refresh", "query", + }, known.KnownFields()) +} + +// TestArtifactHandler_ValidateMatrix proves, for every action, that the required inputs are +// enforced and that every input outside the contract is rejected by name. +func TestArtifactHandler_ValidateMatrix(t *testing.T) { + handler := &ArtifactHandler{} + require.NotEmpty(t, invoke.Actions()) + require.NotEmpty(t, invoke.AllFields()) + + for _, action := range invoke.Actions() { + spec, ok := invoke.SpecFor(action) + require.True(t, ok) + + t.Run(string(action)+"/valid with required fields only", func(t *testing.T) { + require.NoError(t, handler.Validate(artifactStepWith(action, spec.Required...))) + }) + t.Run(string(action)+"/valid with every accepted field", func(t *testing.T) { + require.NoError(t, handler.Validate(artifactStepWith(action, spec.Fields()...))) + }) + for _, missing := range spec.Required { + t.Run(string(action)+"/missing "+string(missing), func(t *testing.T) { + var present []invoke.Field + for _, field := range spec.Required { + if field != missing { + present = append(present, field) + } + } + err := handler.Validate(artifactStepWith(action, present...)) + require.ErrorIs(t, err, errUtils.ErrArtifactStepFieldRequired) + assert.Contains(t, err.Error(), string(missing)) + }) + } + for _, field := range invoke.AllFields() { + if spec.Accepts(field) { + continue + } + t.Run(string(action)+"/forbids "+string(field), func(t *testing.T) { + err := handler.Validate(artifactStepWith(action, append(spec.Required, field)...)) + require.ErrorIs(t, err, errUtils.ErrArtifactStepFieldForbidden) + assert.Contains(t, err.Error(), string(field)) + }) + } + } +} + +func TestArtifactHandler_ValidateRejectsMalformedSteps(t *testing.T) { + handler := &ArtifactHandler{} + tests := []struct { + name string + mutate func(*schema.WorkflowStep) + want error + }{ + {"missing action", func(s *schema.WorkflowStep) { s.Action = "" }, errUtils.ErrArtifactStepInvalidAction}, + {"unknown action", func(s *schema.WorkflowStep) { s.Action = "build" }, errUtils.ErrArtifactStepInvalidAction}, + {"pin is not a step action", func(s *schema.WorkflowStep) { s.Action = "pin" }, errUtils.ErrArtifactStepInvalidAction}, + {"source map", func(s *schema.WorkflowStep) { s.Source = map[string]any{"uri": "git::https://example.com/repo.git"} }, errUtils.ErrArtifactStepInvalidSource}, + {"source number", func(s *schema.WorkflowStep) { s.Source = 3 }, errUtils.ErrArtifactStepInvalidSource}, + {"http URL", func(s *schema.WorkflowStep) { s.URL = "https://example.com" }, errUtils.ErrArtifactStepFieldForbidden}, + {"shell command", func(s *schema.WorkflowStep) { s.Command = "echo x" }, errUtils.ErrArtifactStepFieldForbidden}, + {"http query mapping", func(s *schema.WorkflowStep) { s.Query = map[string]string{"a": "b"} }, errUtils.ErrArtifactStepInvalidField}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository) + tc.mutate(step) + require.ErrorIs(t, handler.Validate(step), tc.want) + }) + } + + t.Run("label shapes", func(t *testing.T) { + for _, valid := range []any{"team=platform", []any{"a=b", "c=d"}, []string{"a=b"}, map[string]any{"a": "b"}, map[string]string{"a": "b"}} { + step := artifactStepWith(artifact.ActionList) + step.Labels = valid + require.NoError(t, handler.Validate(step), "%T", valid) + } + for _, invalid := range []any{3, true, []any{1, 2}, map[string]any{"a": 1}, map[string]any{"a": nil}} { + step := artifactStepWith(artifact.ActionList) + step.Labels = invalid + require.ErrorIs(t, handler.Validate(step), errUtils.ErrArtifactStepInvalidField, "%T", invalid) + } + }) +} + +func TestArtifactHandler_ValidateThroughTheRegistry(t *testing.T) { + step := artifactStepWith(artifact.ActionPublish, invoke.FieldSource, invoke.FieldRepository) + require.NoError(t, ValidateStep(step)) + + step.Action = "nope" + require.ErrorIs(t, ValidateStep(step), errUtils.ErrArtifactStepInvalidAction) +} diff --git a/pkg/runner/step/documentation_test.go b/pkg/runner/step/documentation_test.go index 9fa141d1b38..fc20ab2b7f3 100644 --- a/pkg/runner/step/documentation_test.go +++ b/pkg/runner/step/documentation_test.go @@ -48,7 +48,7 @@ func TestStepDocumentationCoversRegisteredTypes(t *testing.T) { // wait-all is documented with wait because both types operate on the same // background-service model. Every other canonical registry type owns a page. documents := map[string]string{ - "alert": "alert", "archive": "archive", "atmos": "atmos", "cancel": "cancel", + "alert": "alert", "archive": "archive", "artifact": "artifact", "atmos": "atmos", "cancel": "cancel", "cast": "cast", "choose": "choose", "clear": "clear", "confirm": "confirm", "container": "container", "emulator": "emulator", "env": "env", "exec": "exec", "exit": "exit", "file": "file", "filter": "filter", "format": "format", diff --git a/pkg/runner/step/known_fields.go b/pkg/runner/step/known_fields.go index ea4dc89f0e5..2631f9ee6f0 100644 --- a/pkg/runner/step/known_fields.go +++ b/pkg/runner/step/known_fields.go @@ -71,8 +71,14 @@ func knownFields(fields []string) []string { } var ( - alertFields = stepYAMLFields(contentField) - archiveFields = stepYAMLFields("Action", "Source", "Destination", "Format", "Subpath", "Include", "Exclude", "Mtime") + alertFields = stepYAMLFields(contentField) + archiveFields = stepYAMLFields("Action", "Source", "Destination", "Format", "Subpath", "Include", "Exclude", "Mtime") + // The artifact step reads these fields. The search text is the scalar form of the polymorphic + // `query` key, which has no YAML tag of its own, so it is listed explicitly. + artifactFields = append(stepYAMLFields( + "Action", "Source", "Repository", "ArtifactName", "Version", "ArtifactKind", "Force", "ArtifactDryRun", + "Metadata", "Labels", "Readme", "Changelog", "Ref", "Destination", "From", "To", "Document", "Refresh", + ), "query") atmosFields = stepYAMLFields("Command", "Stack", "Viewport") chooseFields = stepYAMLFields("Prompt", "Options", "Default") confirmFields = stepYAMLFields("Prompt", "Default") @@ -115,6 +121,13 @@ func (h *ArchiveHandler) KnownFields() []string { return knownFields(archiveFields) } +// KnownFields lists the fields the artifact step reads. +func (h *ArtifactHandler) KnownFields() []string { + defer perf.Track(nil, "step.ArtifactHandler.KnownFields")() + + return knownFields(artifactFields) +} + // KnownFields lists the fields the atmos step reads. func (h *AtmosHandler) KnownFields() []string { defer perf.Track(nil, "step.AtmosHandler.KnownFields")() diff --git a/pkg/runner/step/variables.go b/pkg/runner/step/variables.go index 0c9a2e4d575..c44a5efc5c9 100644 --- a/pkg/runner/step/variables.go +++ b/pkg/runner/step/variables.go @@ -83,6 +83,9 @@ type Variables struct { stageIndex int // totalStages tracks total number of stage steps in workflow. totalStages int + // artifacts holds the retained operation identities and the report collector of artifact steps. + // It is shared by every Variables cloned from this one. + artifacts *artifactSession } // OutputWriters routes step subprocess output through component-scoped streams. @@ -102,6 +105,7 @@ func NewVariables() *Variables { Flags: make(map[string]string), templateRoots: make(map[string]any), templatePasses: defaultTemplatePasses, + artifacts: newArtifactSession(), } // Pre-populate with OS environment variables. v.LoadOSEnv() diff --git a/pkg/schema/artifact_step.go b/pkg/schema/artifact_step.go new file mode 100644 index 00000000000..6b0474dbcf7 --- /dev/null +++ b/pkg/schema/artifact_step.go @@ -0,0 +1,87 @@ +package schema + +import ( + "fmt" + "reflect" + + "gopkg.in/yaml.v3" +) + +// TaskTypeArtifact is the step type of the artifact operations (publish, pull, copy, inspect, +// list, search, delete, sign, verify, and show). +const TaskTypeArtifact = "artifact" + +// ArtifactStepMetadata is the `metadata:` block of an artifact publish step. Only labels are +// accepted: they are the explicit user metadata that participates in the publication's intent. +type ArtifactStepMetadata struct { + // Labels are the exact key/value labels attached to the published artifact version. + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty" mapstructure:"labels"` +} + +// The `query` key is shape-dependent: a mapping holds the query-string parameters of an http step, +// and a scalar holds the search text of an artifact step. The mapstructure path sees the scalar +// form under the search_query key. +const ( + taskMapKeyQuery = "query" + taskMapKeySearchQuery = "search_query" +) + +// decodeStepQuery routes the polymorphic `query:` key. A mapping is the http step's query-string +// parameters; a scalar is the artifact step's search text. Both live in one YAML key because the +// artifact step contract names its search text `query`, while the http step already owns the +// mapping form. +func decodeStepQuery(node *yaml.Node, stepType string, params *map[string]string, search *string) error { + if node == nil { + return nil + } + if node.Kind == yaml.AliasNode { + node = node.Alias + } + if node.Kind == yaml.ScalarNode { + if node.Tag == "!!null" { + *params = nil + return nil + } + if stepType != TaskTypeArtifact || node.Tag != "!!str" { + return fmt.Errorf("%w: `query` must be a mapping (http) or a string (artifact search text)", ErrWorkflowControlStepInvalid) + } + *search = node.Value + return nil + } + if node.Kind != yaml.MappingNode { + return fmt.Errorf("%w: `query` must be a mapping (http) or a string (artifact search text)", ErrWorkflowControlStepInvalid) + } + // Decode into a fresh map so an explicit mapping replaces, rather than merges into, entries + // already inherited from merged defaults. + fresh := map[string]string{} + if err := node.Decode(&fresh); err != nil { + return err + } + *params = fresh + return nil +} + +// normalizeSearchQueryMap moves a scalar `query` into `search_query`, so mapstructure, which cannot +// decode a string into the http query mapping, sees each form under its own key. It returns the +// input map unchanged when `query` is absent or already a mapping, and never mutates the input. +func normalizeSearchQueryMap(m map[string]any) map[string]any { + value, ok := m[taskMapKeyQuery] + if !ok { + return m + } + if value == nil || reflect.ValueOf(value).Kind() == reflect.Map || m["type"] != TaskTypeArtifact { + return m + } + query, ok := value.(string) + if !ok { + return m + } + copied := make(map[string]any, len(m)) + for key, val := range m { + if key != taskMapKeyQuery { + copied[key] = val + } + } + copied[taskMapKeySearchQuery] = query + return copied +} diff --git a/pkg/schema/artifact_step_test.go b/pkg/schema/artifact_step_test.go new file mode 100644 index 00000000000..abc2c667707 --- /dev/null +++ b/pkg/schema/artifact_step_test.go @@ -0,0 +1,213 @@ +package schema + +import ( + "testing" + + "github.com/go-viper/mapstructure/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +var _ = WorkflowStep{ + Name: "", Type: "", Action: "", Source: nil, Repository: "", ArtifactName: "", ArtifactKind: "", Version: "", + Force: false, ArtifactDryRun: false, Readme: "", Changelog: "", Output: "", Metadata: nil, Artifacts: nil, + Ref: "", Destination: "", From: "", To: "", Document: "", SearchQuery: "", Query: nil, Refresh: false, Labels: nil, +} + +const publishStepYAML = ` +name: publish-vpc +type: artifact +action: publish +source: ./components/terraform/vpc +repository: shared +artifact_name: components/terraform/vpc +artifact_kind: component/terraform +version: "1.2.0" +force: true +dry_run: true +readme: README.md +changelog: CHANGELOG.md +output: log +metadata: + labels: + team: platform + env: dev +` + +// publishStep is the step the YAML above describes. +func publishStep() WorkflowStep { + return WorkflowStep{ + Name: "publish-vpc", Type: TaskTypeArtifact, Action: "publish", Source: "./components/terraform/vpc", + Repository: "shared", ArtifactName: "components/terraform/vpc", ArtifactKind: "component/terraform", Version: "1.2.0", + Force: true, ArtifactDryRun: true, Readme: "README.md", Changelog: "CHANGELOG.md", Output: "log", + Metadata: &ArtifactStepMetadata{Labels: map[string]string{"team": "platform", "env": "dev"}}, + } +} + +func TestWorkflowStep_UnmarshalYAML_ArtifactFields(t *testing.T) { + var step WorkflowStep + require.NoError(t, yaml.Unmarshal([]byte(publishStepYAML), &step)) + assert.Equal(t, publishStep(), step) + assert.Equal(t, "publish-vpc", step.Name, "the step name keeps its runner meaning") + assert.Equal(t, "components/terraform/vpc", step.ArtifactName, "the artifact name is a separate field") + assert.Equal(t, "log", step.Output, "output keeps its display-mode meaning") + assert.Nil(t, step.Artifacts, "the runner's expected-output declaration is untouched") +} + +func TestWorkflowStep_UnmarshalYAML_ArtifactStepOtherActions(t *testing.T) { + tests := []struct { + name string + input string + check func(*testing.T, WorkflowStep) + }{ + {"pull", "{type: artifact, action: pull, ref: 'team/app:1', repository: dir, destination: ./out}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, "team/app:1", s.Ref) + assert.Equal(t, "./out", s.Destination) + }}, + {"copy", "{type: artifact, action: copy, ref: 'team/app:1', from: dir, to: archive, force: true}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, "dir", s.From) + assert.Equal(t, "archive", s.To) + }}, + {"show", "{type: artifact, action: show, document: changelog, ref: 'team/app:1', repository: dir}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, "changelog", s.Document) + }}, + {"search scalar query", "{type: artifact, action: search, query: vpc, refresh: true}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, "vpc", s.SearchQuery) + assert.Nil(t, s.Query, "the http query mapping stays empty") + assert.True(t, s.Refresh) + }}, + {"list string labels", "{type: artifact, action: list, labels: 'team=platform,env=dev'}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, "team=platform,env=dev", s.Labels) + }}, + {"list sequence labels", "{type: artifact, action: list, labels: [team=platform, env=dev]}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, []any{"team=platform", "env=dev"}, s.Labels) + }}, + {"list mapping labels", "{type: artifact, action: list, labels: {team: platform}}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, map[string]any{"team": "platform"}, s.Labels) + }}, + {"http mapping query still decodes", "{type: http, url: 'https://example.com', query: {page: '2'}}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, map[string]string{"page": "2"}, s.Query) + assert.Empty(t, s.SearchQuery) + }}, + {"explicit null query clears merged defaults", "{type: http, <<: &defaults {query: {page: '1'}}, query: null}", func(t *testing.T, s WorkflowStep) { + assert.Empty(t, s.Query) + }}, + {"absent query preserves merged defaults", "{type: http, <<: &defaults {query: {page: '1'}}}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, map[string]string{"page": "1"}, s.Query) + }}, + {"explicit empty query replaces merged defaults", "{type: http, url: 'https://example.com', <<: {query: {page: '2'}}, query: {}}", func(t *testing.T, s WorkflowStep) { + assert.Empty(t, s.Query) + }}, + {"explicit query replaces merged default entries", "{type: http, url: 'https://example.com', <<: {query: {page: '2'}}, query: {limit: '5'}}", func(t *testing.T, s WorkflowStep) { + assert.Equal(t, map[string]string{"limit": "5"}, s.Query) + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var step WorkflowStep + require.NoError(t, yaml.Unmarshal([]byte(tc.input), &step)) + tc.check(t, step) + }) + } + + t.Run("a sequence query is rejected", func(t *testing.T) { + var step WorkflowStep + require.ErrorIs(t, yaml.Unmarshal([]byte("{type: artifact, query: [a, b]}"), &step), ErrWorkflowControlStepInvalid) + }) +} + +func TestTask_ConversionsCarryEveryArtifactField(t *testing.T) { + step := publishStep() + step.Labels = "a=b" + step.Ref, step.From, step.To, step.Document, step.SearchQuery, step.Refresh = "team/app:1", "dir", "archive", "readme", "vpc", true + step.Destination = "./out" + + task := TaskFromWorkflowStep(&step) + assert.Equal(t, step.Repository, task.Repository) + assert.Equal(t, step.ArtifactName, task.ArtifactName) + assert.Equal(t, step.ArtifactKind, task.ArtifactKind) + assert.Equal(t, step.Metadata, task.Metadata) + assert.Equal(t, step.SearchQuery, task.SearchQuery) + + assert.Equal(t, step, task.ToWorkflowStep(), "WorkflowStep -> Task -> WorkflowStep loses nothing") + roundTrip := TaskFromWorkflowStep(&step) + assert.Equal(t, task, roundTrip) +} + +func TestTasks_DecodeCustomCommandArtifactSteps(t *testing.T) { + type config struct { + Steps Tasks `mapstructure:"steps"` + } + decode := func(t *testing.T, steps ...any) Tasks { + t.Helper() + var result config + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &result, WeaklyTypedInput: true, DecodeHook: TasksDecodeHook()}) + require.NoError(t, err) + require.NoError(t, decoder.Decode(map[string]any{"steps": steps})) + return result.Steps + } + + t.Run("publish with metadata labels", func(t *testing.T) { + tasks := decode(t, map[string]any{ + "type": "artifact", "action": "publish", "source": "./dist", "repository": "shared", "artifact_name": "team/app", + "artifact_kind": "files", "version": "1", "force": true, "dry_run": true, "readme": "R.md", "changelog": "C.md", + "metadata": map[string]any{"labels": map[string]any{"team": "platform"}}, + }) + require.Len(t, tasks, 1) + require.NoError(t, tasks[0].LoadError) + step := tasks[0].ToWorkflowStep() + assert.Equal(t, "./dist", step.Source) + assert.Equal(t, "shared", step.Repository) + assert.Equal(t, "team/app", step.ArtifactName) + assert.True(t, step.ArtifactDryRun) + assert.Equal(t, map[string]string{"team": "platform"}, step.Metadata.Labels) + }) + t.Run("search text in a scalar query", func(t *testing.T) { + tasks := decode(t, map[string]any{"type": "artifact", "action": "search", "query": "vpc", "labels": []any{"a=b"}}) + require.NoError(t, tasks[0].LoadError) + assert.Equal(t, "vpc", tasks[0].SearchQuery) + assert.Nil(t, tasks[0].Query) + assert.Equal(t, []any{"a=b"}, tasks[0].Labels) + }) + t.Run("http query mapping is not disturbed", func(t *testing.T) { + tasks := decode(t, map[string]any{"type": "http", "url": "https://example.com", "query": map[string]any{"page": "2"}}) + require.NoError(t, tasks[0].LoadError) + assert.Equal(t, map[string]string{"page": "2"}, tasks[0].Query) + assert.Empty(t, tasks[0].SearchQuery) + }) + for _, input := range []map[string]any{ + {"type": "artifact", "query": 42}, + {"type": "http", "query": "page=2"}, + } { + tasks := decode(t, input) + require.Error(t, tasks[0].LoadError) + } + t.Run("an unknown artifact key still fails the step, not the load", func(t *testing.T) { + tasks := decode(t, map[string]any{"type": "artifact", "action": "pull", "repositry": "dir"}) + require.Error(t, tasks[0].LoadError) + }) +} + +func TestNormalizeSearchQueryMapDoesNotMutateItsInput(t *testing.T) { + input := map[string]any{"query": "vpc", "type": "artifact"} + normalized := normalizeSearchQueryMap(input) + assert.Equal(t, map[string]any{"search_query": "vpc", "type": "artifact"}, normalized) + assert.Equal(t, map[string]any{"query": "vpc", "type": "artifact"}, input) + + mapping := map[string]any{"query": map[string]any{"a": "b"}} + assert.Equal(t, mapping, normalizeSearchQueryMap(mapping)) + assert.Equal(t, map[string]any{"type": "x"}, normalizeSearchQueryMap(map[string]any{"type": "x"})) + assert.Equal(t, map[string]any{"query": nil}, normalizeSearchQueryMap(map[string]any{"query": nil})) +} + +func TestRejectInvalidScalarStepQueries(t *testing.T) { + for _, input := range []string{ + "{type: http, query: text}", + "{type: artifact, query: 42}", + "{type: artifact, query: true}", + } { + var step WorkflowStep + require.ErrorIs(t, yaml.Unmarshal([]byte(input), &step), ErrWorkflowControlStepInvalid) + } +} diff --git a/pkg/schema/task.go b/pkg/schema/task.go index 70e729b764f..1a6407f7951 100644 --- a/pkg/schema/task.go +++ b/pkg/schema/task.go @@ -290,6 +290,28 @@ type Task struct { // permission bits. See docs/prd/archive-step.md. Mtime string `yaml:"mtime,omitempty" json:"mtime,omitempty" mapstructure:"mtime"` + // Artifact step fields (type: artifact). Action, Source, and Destination reuse the container, + // workdir/archive, and archive fields; Source must be a plain string path for this step. The + // step's own Name, Output, and Artifacts keep their runner meaning: the artifact's name is + // ArtifactName, its kind ArtifactKind. A list or search step's scalar `query:` is SearchQuery. + + Repository string `yaml:"repository,omitempty" json:"repository,omitempty" mapstructure:"repository"` // Configured artifact repository alias. + ArtifactName string `yaml:"artifact_name,omitempty" json:"artifact_name,omitempty" mapstructure:"artifact_name"` // Artifact name (publish; list restricts to one name). + Version string `yaml:"version,omitempty" json:"version,omitempty" mapstructure:"version"` // Artifact version (publish). + ArtifactKind string `yaml:"artifact_kind,omitempty" json:"artifact_kind,omitempty" mapstructure:"artifact_kind"` // Artifact kind (publish); defaults to files. + Force bool `yaml:"force,omitempty" json:"force,omitempty" mapstructure:"force"` // Replace conflicting state (publish, pull, copy). + ArtifactDryRun bool `yaml:"dry_run,omitempty" json:"dry_run,omitempty" mapstructure:"dry_run"` // Validate and plan a publication without writing. Distinct from DryRun, which executors set. + Metadata *ArtifactStepMetadata `yaml:"metadata,omitempty" json:"metadata,omitempty" mapstructure:"metadata"` // Publish metadata (labels). + Labels any `yaml:"labels,omitempty" json:"labels,omitempty" mapstructure:"labels" jsonschema:"oneof_type=string;array;object"` // List and search label selectors: a key=value string, a list of them, or a mapping. + Readme string `yaml:"readme,omitempty" json:"readme,omitempty" mapstructure:"readme"` // README path, relative to the publish source directory. + Changelog string `yaml:"changelog,omitempty" json:"changelog,omitempty" mapstructure:"changelog"` // CHANGELOG path, relative to the publish source directory. + Ref string `yaml:"ref,omitempty" json:"ref,omitempty" mapstructure:"ref"` // Artifact reference: name:version or name@sha256:. + From string `yaml:"from,omitempty" json:"from,omitempty" mapstructure:"from"` // Copy source repository alias. + To string `yaml:"to,omitempty" json:"to,omitempty" mapstructure:"to"` // Copy destination repository alias. + Document string `yaml:"document,omitempty" json:"document,omitempty" mapstructure:"document"` // Document to show: readme or changelog. + SearchQuery string `yaml:"-" json:"search_query,omitempty" mapstructure:"search_query"` // Search text, decoded from a scalar `query:`. + Refresh bool `yaml:"refresh,omitempty" json:"refresh,omitempty" mapstructure:"refresh"` // Bypass cached discovery snapshots (list, search). + // Require step type fields (type: require; also accepts the alias type: assert). // The step is a read-only preconditions gate: it never mutates PATH or the environment. Tools []string `yaml:"tools,omitempty" json:"tools,omitempty" mapstructure:"tools"` // Executables that must be found on PATH (supports templates). @@ -368,6 +390,8 @@ func (task *Task) UnmarshalYAML(value *yaml.Node) error { color: &task.Background, forList: &task.For, steps: &task.Steps, + query: &task.Query, + search: &task.SearchQuery, generic: &task.With, container: containerActionTargets{Build: &task.Build, Run: &task.Run, Push: &task.Push, Inspect: &task.Inspect}, }) @@ -573,6 +597,24 @@ func (task *Task) ToWorkflowStep() WorkflowStep { Exclude: task.Exclude, Mtime: task.Mtime, + // Artifact step fields. + Repository: task.Repository, + ArtifactName: task.ArtifactName, + Version: task.Version, + ArtifactKind: task.ArtifactKind, + Force: task.Force, + ArtifactDryRun: task.ArtifactDryRun, + Metadata: task.Metadata, + Labels: task.Labels, + Readme: task.Readme, + Changelog: task.Changelog, + Ref: task.Ref, + From: task.From, + To: task.To, + Document: task.Document, + SearchQuery: task.SearchQuery, + Refresh: task.Refresh, + // S3 upload step fields. Target: task.Target, @@ -739,6 +781,24 @@ func TaskFromWorkflowStep(step *WorkflowStep) Task { Exclude: step.Exclude, Mtime: step.Mtime, + // Artifact step fields. + Repository: step.Repository, + ArtifactName: step.ArtifactName, + Version: step.Version, + ArtifactKind: step.ArtifactKind, + Force: step.Force, + ArtifactDryRun: step.ArtifactDryRun, + Metadata: step.Metadata, + Labels: step.Labels, + Readme: step.Readme, + Changelog: step.Changelog, + Ref: step.Ref, + From: step.From, + To: step.To, + Document: step.Document, + SearchQuery: step.SearchQuery, + Refresh: step.Refresh, + // S3 upload step fields. Target: step.Target, @@ -951,6 +1011,7 @@ func applyTaskPolymorphicMap(m map[string]any, task *Task, index int) error { // normalizeTaskMap prepares structured output, prompts, and nested steps. func normalizeTaskMap(m map[string]any, task *Task, index int) (map[string]any, error) { + m = normalizeSearchQueryMap(m) m, err := normalizeTaskOutputMap(m, task) if err != nil { return nil, fmt.Errorf("failed to decode task output at index %d: %w", index, err) @@ -1111,7 +1172,7 @@ func stringifyTaskMap(item any) (map[string]any, bool) { func normalizeWorkflowStepMap(m map[string]any) (map[string]any, error) { copied := make(map[string]any, len(m)) - for key, val := range m { + for key, val := range normalizeSearchQueryMap(m) { copied[key] = val } if mapHasKey(copied, "cursor") { diff --git a/pkg/schema/workflow.go b/pkg/schema/workflow.go index c595f5118eb..6efb659889a 100644 --- a/pkg/schema/workflow.go +++ b/pkg/schema/workflow.go @@ -275,6 +275,28 @@ type WorkflowStep struct { // permission bits. See docs/prd/archive-step.md. Mtime string `yaml:"mtime,omitempty" json:"mtime,omitempty" mapstructure:"mtime"` + // Artifact step fields (type: artifact). Action, Source, and Destination reuse the container, + // workdir/archive, and archive fields; Source must be a plain string path for this step. The + // step's own Name, Output, and Artifacts keep their runner meaning: the artifact's name is + // ArtifactName, its kind ArtifactKind. A list or search step's scalar `query:` is SearchQuery. + + Repository string `yaml:"repository,omitempty" json:"repository,omitempty" mapstructure:"repository"` // Configured artifact repository alias. + ArtifactName string `yaml:"artifact_name,omitempty" json:"artifact_name,omitempty" mapstructure:"artifact_name"` // Artifact name (publish; list restricts to one name). + Version string `yaml:"version,omitempty" json:"version,omitempty" mapstructure:"version"` // Artifact version (publish). + ArtifactKind string `yaml:"artifact_kind,omitempty" json:"artifact_kind,omitempty" mapstructure:"artifact_kind"` // Artifact kind (publish); defaults to files. + Force bool `yaml:"force,omitempty" json:"force,omitempty" mapstructure:"force"` // Replace conflicting state (publish, pull, copy). + ArtifactDryRun bool `yaml:"dry_run,omitempty" json:"dry_run,omitempty" mapstructure:"dry_run"` // Validate and plan a publication without writing. Distinct from DryRun, which executors set. + Metadata *ArtifactStepMetadata `yaml:"metadata,omitempty" json:"metadata,omitempty" mapstructure:"metadata"` // Publish metadata (labels). + Labels any `yaml:"labels,omitempty" json:"labels,omitempty" mapstructure:"labels" jsonschema:"oneof_type=string;array;object"` // List and search label selectors: a key=value string, a list of them, or a mapping. + Readme string `yaml:"readme,omitempty" json:"readme,omitempty" mapstructure:"readme"` // README path, relative to the publish source directory. + Changelog string `yaml:"changelog,omitempty" json:"changelog,omitempty" mapstructure:"changelog"` // CHANGELOG path, relative to the publish source directory. + Ref string `yaml:"ref,omitempty" json:"ref,omitempty" mapstructure:"ref"` // Artifact reference: name:version or name@sha256:. + From string `yaml:"from,omitempty" json:"from,omitempty" mapstructure:"from"` // Copy source repository alias. + To string `yaml:"to,omitempty" json:"to,omitempty" mapstructure:"to"` // Copy destination repository alias. + Document string `yaml:"document,omitempty" json:"document,omitempty" mapstructure:"document"` // Document to show: readme or changelog. + SearchQuery string `yaml:"-" json:"search_query,omitempty" mapstructure:"search_query"` // Search text, decoded from a scalar `query:`. + Refresh bool `yaml:"refresh,omitempty" json:"refresh,omitempty" mapstructure:"refresh"` // Bypass cached discovery snapshots (list, search). + // JUnit step fields. Files []string `yaml:"files,omitempty" json:"files,omitempty" mapstructure:"files"` // Glob(s) of JUnit XML files to summarize/annotate (junit step type). @@ -354,6 +376,8 @@ func (step *WorkflowStep) UnmarshalYAML(value *yaml.Node) error { color: &step.Background, forList: &step.For, steps: &step.Steps, + query: &step.Query, + search: &step.SearchQuery, generic: &step.With, container: containerActionTargets{Build: &step.Build, Run: &step.Run, Push: &step.Push, Inspect: &step.Inspect}, }) @@ -367,6 +391,7 @@ type stepPolyNodes struct { background *yaml.Node forNode *yaml.Node steps *yaml.Node + query *yaml.Node } // stepPolyTargets bundles the destinations a step exposes for its polymorphic keys. @@ -381,6 +406,8 @@ type stepPolyTargets struct { color *string forList *[]string steps *[]WorkflowStep + query *map[string]string + search *string generic *map[string]any container containerActionTargets } @@ -394,7 +421,8 @@ func splitStepPolymorphicNodes(value *yaml.Node) (stepPolyNodes, *yaml.Node) { backgroundNode, sanitized := splitMappingField(sanitized, "background") forNode, sanitized := splitMappingField(sanitized, "for") stepsNode, sanitized := splitMappingField(sanitized, "steps") - return stepPolyNodes{output: outputNode, prompt: promptNode, with: withNode, background: backgroundNode, forNode: forNode, steps: stepsNode}, sanitized + queryNode, sanitized := splitMappingField(sanitized, taskMapKeyQuery) + return stepPolyNodes{output: outputNode, prompt: promptNode, with: withNode, background: backgroundNode, forNode: forNode, steps: stepsNode, query: queryNode}, sanitized } // applyStepPolymorphicNodes decodes the extracted nodes into the step's targets. @@ -415,6 +443,9 @@ func applyStepPolymorphicNodes(nodes stepPolyNodes, stepType, action string, t * if err := decodeWorkflowStepList(nodes.steps, t.steps); err != nil { return err } + if err := decodeStepQuery(nodes.query, stepType, t.query, t.search); err != nil { + return err + } return decodeStepWith(nodes.with, stepType, action, t) } diff --git a/tests/snapshots/TestCLICommands_atmos_workflow_invalid_step_type.stderr.golden b/tests/snapshots/TestCLICommands_atmos_workflow_invalid_step_type.stderr.golden index 2bdf7f10b3b..e4df5ec1863 100644 --- a/tests/snapshots/TestCLICommands_atmos_workflow_invalid_step_type.stderr.golden +++ b/tests/snapshots/TestCLICommands_atmos_workflow_invalid_step_type.stderr.golden @@ -4,4 +4,4 @@ Workflow Error Error: invalid workflow step type -Step step1 of workflow invalid-step-type uses unsupported type invalid. Registered step types: alert, archive, atmos, cancel, cast, choose, clear, confirm, container, emulator, env, exec, exit, file, filter, format, hint, http, input, join, junit, linebreak, log, markdown, matrix, pager, parallel, publish, require, say, script, shell, sleep, spin, stage, store, style, table, test, tflint, title, toast, wait, wait-all, workdir, write. +Step step1 of workflow invalid-step-type uses unsupported type invalid. Registered step types: alert, archive, artifact, atmos, cancel, cast, choose, clear, confirm, container, emulator, env, exec, exit, file, filter, format, hint, http, input, join, junit, linebreak, log, markdown, matrix, pager, parallel, publish, require, say, script, shell, sleep, spin, stage, store, style, table, test, tflint, title, toast, wait, wait-all, workdir, write. diff --git a/website/docs/cli/commands/artifacts/catalog-add.mdx b/website/docs/cli/commands/artifacts/catalog-add.mdx new file mode 100644 index 00000000000..ab5ff715ab7 --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog-add.mdx @@ -0,0 +1,82 @@ +--- +title: atmos artifacts catalog add +sidebar_label: catalog add +sidebar_class_name: command +id: catalog-add +description: List releases in the repository's published catalog. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Resolve one or more releases and list them in the latest published catalog, creating the catalog when none exists. Only artifact metadata is read. + + + + +## Usage + +```shell +atmos artifacts catalog add ... --repository= [flags] +``` + +## Examples + +```shell +# List one release +atmos artifacts catalog add components/terraform/vpc:1.2.0 --repository=shared + +# List several releases in one write +atmos artifacts catalog add components/terraform/vpc:1.2.0 tools/helper:0.1.0 --repository=shared +``` + +## Behavior + +Add takes the repository's lock when it provides one, reads the latest catalog, resolves each ref, lists it, and publishes the result with a new revision. A version that is already listed is updated when its tag now resolves to a different digest, and reported under `updated`. + +## Arguments + +
+
`ref` (required, repeatable)
+
+ `name:version` or `name@sha256:`, relative to the repository. +
+
+ +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias. Its `catalog` setting selects the catalog. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Errors + +
+
Catalog not configured
+
The repository has no `catalog`. Nothing is written.
+ +
Reference not found
+
A ref does not resolve in the repository. Nothing is written.
+ +
Catalog changed
+
Another writer published between the read and the write. Run the command again, and run only one catalog writer at a time.
+
+ +Every failure exits with a nonzero status. + +## Related + +- [`atmos artifacts catalog remove`](/cli/commands/artifacts/catalog-remove) +- [`atmos artifacts catalog sync`](/cli/commands/artifacts/catalog-sync) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/artifacts/catalog-build.mdx b/website/docs/cli/commands/artifacts/catalog-build.mdx new file mode 100644 index 00000000000..7481871ffe0 --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog-build.mdx @@ -0,0 +1,97 @@ +--- +title: atmos artifacts catalog build +sidebar_label: catalog build +sidebar_class_name: command +id: catalog-build +description: Build a catalog snapshot from an inventory without publishing it. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Resolve the releases of an inventory in one repository and write them to a catalog snapshot file. Build reads artifact metadata only: it downloads no payload and publishes nothing. + + + + +## Usage + +```shell +atmos artifacts catalog build --repository= --input= --output= [flags] +``` + +## Examples + +```shell +# Build a JSON snapshot +atmos artifacts catalog build --repository=shared --input=inventory.yaml --output=catalog.json + +# Build a YAML snapshot with a custom catalog name +atmos artifacts catalog build --repository=shared --input=inventory.yaml --output=catalog.yaml --name=platform +``` + +## Behavior + +Build resolves each reference in the inventory and reads its Record, then writes an `AtmosArtifactCatalog` manifest. Entries and versions are sorted lexically. When the repository has a catalog that is already published, build records that catalog's revision in the snapshot and prints its digest as `base_digest`, so a later `publish` fails if someone published in between. Without a published catalog the base revision is `0`. + +Build works on a repository without a configured `catalog`. It does not need one until you publish. + +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias the inventory refs are relative to. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--input` (required)
+
+ An `AtmosArtifactInventory` manifest listing the releases to resolve. + Environment variable: `ATMOS_ARTIFACTS_INPUT` +
+ +
`--output` (required)
+
+ The snapshot file to write. The extension selects the format: `.yaml` and `.yml` write YAML, anything else writes JSON. The file is replaced atomically. + Environment variable: `ATMOS_ARTIFACTS_OUTPUT` +
+ +
`--name` (optional)
+
+ The catalog manifest name. Defaults to the name in the configured catalog reference, or `catalog`. + Environment variable: `ATMOS_ARTIFACTS_NAME` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Output + +Stdout carries `output`, `base_digest` (when a catalog is published), `revision`, and `listed`, the number of name and version pairs in the snapshot. + +## Errors + +
+
Reference not found
+
An inventory ref does not resolve in the repository. Nothing is written.
+ +
Not an Atmos artifact
+
The ref names a foreign image or plugin release, which a catalog cannot list.
+ +
Invalid inventory
+
The inventory is not an `AtmosArtifactInventory`, lists no refs, repeats a ref, or uses a ref without a version or digest.
+
+ +Every failure exits with a nonzero status. + +## Related + +- [`atmos artifacts catalog`](/cli/commands/artifacts/catalog) +- [`atmos artifacts catalog publish`](/cli/commands/artifacts/catalog-publish) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/artifacts/catalog-publish.mdx b/website/docs/cli/commands/artifacts/catalog-publish.mdx new file mode 100644 index 00000000000..968b9280e50 --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog-publish.mdx @@ -0,0 +1,94 @@ +--- +title: atmos artifacts catalog publish +sidebar_label: catalog publish +sidebar_class_name: command +id: catalog-publish +description: Publish a catalog snapshot to the repository's configured catalog. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Publish a catalog snapshot to the catalog configured on a repository. The command refuses to overwrite newer catalog metadata with a stale snapshot. + + + + +## Usage + +```shell +atmos artifacts catalog publish --repository= [flags] +``` + +## Examples + +```shell +# Publish a built snapshot +atmos artifacts catalog publish catalog.json --repository=shared + +# Publish only if the catalog is still the one the snapshot was built from +atmos artifacts catalog publish catalog.json --repository=shared --expected-digest=sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +``` + +## Behavior + +Publish replaces the catalog's entries with the snapshot. Before it writes, it checks that the repository has a `catalog` configured, that every entry is bound to the repository path, and that the snapshot's revision equals the published catalog's revision. With `--expected-digest` it also checks the catalog's manifest digest. After a successful write the revision increases by one. + +The write takes the repository's lock when the repository provides one, reads the latest catalog inside the lock, and checks the previous reference immediately before it moves the tag. Run only one catalog writer at a time. See [One writer at a time](/cli/commands/artifacts/catalog#one-writer-at-a-time). + +## Arguments + +
+
`file` (required)
+
+ A catalog snapshot written by `atmos artifacts catalog build`, in JSON or YAML. +
+
+ +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias. Its `catalog` setting selects where the catalog is published. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--expected-digest` (optional)
+
+ Require the published catalog to have this manifest digest. Use the `base_digest` that `build` printed. + Environment variable: `ATMOS_ARTIFACTS_EXPECTED_DIGEST` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Output + +Stdout carries `catalog`, the new `digest`, the `previous_digest`, `revision`, `listed`, `added`, and `lock_scope`. + +## Errors + +
+
Catalog not configured
+
The repository has no `catalog`. Nothing is written.
+ +
Catalog changed
+
The published catalog has a different revision or digest than the snapshot expects. Rebuild the snapshot from the latest catalog.
+ +
Location outside the repository
+
An entry names another registry, prefix, or path than the repository. Nothing is written.
+
+ +Every failure exits with a nonzero status. + +## Related + +- [`atmos artifacts catalog build`](/cli/commands/artifacts/catalog-build) +- [`atmos artifacts catalog sync`](/cli/commands/artifacts/catalog-sync) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/artifacts/catalog-remove.mdx b/website/docs/cli/commands/artifacts/catalog-remove.mdx new file mode 100644 index 00000000000..8255994b362 --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog-remove.mdx @@ -0,0 +1,76 @@ +--- +title: atmos artifacts catalog remove +sidebar_label: catalog remove +sidebar_class_name: command +id: catalog-remove +description: Unlist a release from the repository's published catalog. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Unlist a release from the latest published catalog. The artifact, its signatures, and every consumer lock file are untouched. + + + + +## Usage + +```shell +atmos artifacts catalog remove --repository= [flags] +``` + +## Examples + +```shell +# Unlist one release +atmos artifacts catalog remove components/terraform/vpc:1.2.0 --repository=shared +``` + +## Behavior + +Remove only unlists. It deletes no artifact content, no signature, and does not rewrite any lock file, so consumers that pinned the release keep working. When the removed version was the last one of its name, the name is dropped from the catalog. Removal publishes a new catalog revision under the repository's lock when one is available. + +## Arguments + +
+
`ref` (required)
+
+ `name:version` of a listed release. A digest reference cannot be unlisted. +
+
+ +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias. Its `catalog` setting selects the catalog. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Errors + +
+
Catalog not configured
+
The repository has no `catalog`. Nothing is written.
+ +
Not listed
+
The catalog does not exist or does not list that version.
+
+ +Every failure exits with a nonzero status. + +## Related + +- [`atmos artifacts catalog add`](/cli/commands/artifacts/catalog-add) +- [`atmos artifacts delete`](/cli/commands/artifacts/delete) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/artifacts/catalog-sync.mdx b/website/docs/cli/commands/artifacts/catalog-sync.mdx new file mode 100644 index 00000000000..38b3e164a63 --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog-sync.mdx @@ -0,0 +1,76 @@ +--- +title: atmos artifacts catalog sync +sidebar_label: catalog sync +sidebar_class_name: command +id: catalog-sync +description: Merge an inventory's releases into the repository's published catalog. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Resolve the releases of an inventory and merge them into the latest published catalog. Entries the inventory does not mention are kept, so sync is safe to run on a schedule. + + + + +## Usage + +```shell +atmos artifacts catalog sync --repository= --input= [flags] +``` + +## Examples + +```shell +# Merge an inventory into the published catalog +atmos artifacts catalog sync --repository=shared --input=inventory.yaml +``` + +## Behavior + +Sync takes the repository's lock when it provides one, reads the latest catalog, resolves each inventory ref, merges the releases, and publishes a new revision. A listed version whose tag now resolves to a different digest is updated and reported under `updated`. Nothing is removed: use [`remove`](/cli/commands/artifacts/catalog-remove) to unlist a release. + +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias the inventory refs are relative to. Its `catalog` setting selects the catalog. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--input` (required)
+
+ An `AtmosArtifactInventory` manifest listing the releases to merge. + Environment variable: `ATMOS_ARTIFACTS_INPUT` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Errors + +
+
Catalog not configured
+
The repository has no `catalog`. Nothing is written.
+ +
Reference not found
+
An inventory ref does not resolve in the repository. Nothing is written.
+ +
Catalog changed
+
Another writer published between the read and the write. Run the command again, and run only one catalog writer at a time.
+
+ +Every failure exits with a nonzero status. + +## Related + +- [`atmos artifacts catalog build`](/cli/commands/artifacts/catalog-build) +- [`atmos artifacts catalog add`](/cli/commands/artifacts/catalog-add) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/artifacts/catalog.mdx b/website/docs/cli/commands/artifacts/catalog.mdx new file mode 100644 index 00000000000..9ce6dbfed9a --- /dev/null +++ b/website/docs/cli/commands/artifacts/catalog.mdx @@ -0,0 +1,111 @@ +--- +title: atmos artifacts catalog +sidebar_label: catalog +sidebar_class_name: command +id: catalog +description: Build and maintain the optional discovery catalog of an artifact repository. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +A catalog is a maintained discovery snapshot of one repository. It lets you list and search a whole OCI registry collection when the registry cannot enumerate its repositories. Direct reads never need a catalog. + + + + +## Usage + +```shell +atmos artifacts catalog build --repository= --input= --output= +atmos artifacts catalog publish --repository= +atmos artifacts catalog add ... --repository= +atmos artifacts catalog remove --repository= +atmos artifacts catalog sync --repository= --input= +``` + +## Examples + +```shell +# Resolve an inventory into a snapshot, then publish it +atmos artifacts catalog build --repository=shared --input=inventory.yaml --output=catalog.json +atmos artifacts catalog publish catalog.json --repository=shared + +# List or unlist one release +atmos artifacts catalog add components/terraform/vpc:1.2.0 --repository=shared +atmos artifacts catalog remove components/terraform/vpc:1.2.0 --repository=shared + +# Merge an inventory into the published catalog and keep every other entry +atmos artifacts catalog sync --repository=shared --input=inventory.yaml +``` + +## Configure a catalog + +A catalog belongs to a repository and is configured on it. Only `oci/registry` repositories accept one, and the catalog must be on the repository's registry host and below its `prefix`. + +```yaml +artifacts: + repositories: + shared: + kind: oci/registry + identity: registry-reader + options: + registry: ghcr.io + prefix: acme/atmos + catalog: oci://ghcr.io/acme/atmos/catalog:stable +``` + +Commands that change the catalog fail before any write when the repository has no `catalog`. Directory, archive, S3, and OCI layout repositories list their own records and never need one. + +## Inventory + +An `AtmosArtifactInventory` lists the releases to resolve, relative to the selected repository. There is no implicit `latest`. + +```yaml +apiVersion: atmos/v1 +kind: AtmosArtifactInventory +metadata: + name: releases +spec: + refs: + - components/terraform/vpc:1.2.0 + - tools/helper@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +``` + +## Behavior + +The catalog is an `AtmosArtifactCatalog` manifest, separate from the plugin catalog. Each entry has an artifact name, kind, description, and the repository path it is bound to. Each version carries its resolved, record, and content digests, the labels published with it, and its README and CHANGELOG descriptors, never the Markdown. Version labels are opaque: names and versions sort lexically with no SemVer ordering. The manifest is stored in the repository as a regular artifact of kind `catalog` with artifact type `application/vnd.atmos.artifact.catalog.v1` and one JSON layer of type `application/vnd.atmos.artifact.catalog.v1+json`. + +A catalog is discoverable data, never a trust grant. Atmos checks every entry against the repository it is configured on and the current policy before using it, and it never follows a location a catalog names. Listing from a catalog does not prove that a payload is still available, and does not replace signature verification. + +Publishing, signing, or deleting an artifact never edits a catalog. Removing a catalog entry only unlists it: the artifact, its signatures, and every consumer lock file stay as they are. + +### One writer at a time + +Catalog changes need one externally serialized writer that spans reading the latest catalog, building, and publishing. Atmos reads the latest catalog after it takes the repository's write lock, rejects a snapshot whose revision or digest no longer matches the published catalog, and checks the previous reference immediately before it moves the tag. Registries have no atomic compare-and-swap, so these checks detect stale input but do not replace serializing your writers, for example with a single CI job. When the repository offers no write coordinator, serialize writers yourself. + +## Subcommands + +
+
[`build`](/cli/commands/artifacts/catalog-build)
+
Resolve an inventory into a catalog snapshot file. Reads metadata only and never publishes.
+ +
[`publish`](/cli/commands/artifacts/catalog-publish)
+
Publish a snapshot to the repository's configured catalog.
+ +
[`add`](/cli/commands/artifacts/catalog-add)
+
List releases in the latest published catalog.
+ +
[`remove`](/cli/commands/artifacts/catalog-remove)
+
Unlist a release without deleting the artifact.
+ +
[`sync`](/cli/commands/artifacts/catalog-sync)
+
Merge an inventory into the latest catalog, keeping every other entry.
+
+ +## Related + +- [`atmos artifacts list`](/cli/commands/artifacts/list) +- [`atmos artifacts search`](/cli/commands/artifacts/search) +- [Artifacts Configuration](/cli/configuration/artifacts) diff --git a/website/docs/cli/commands/scaffold/publish.mdx b/website/docs/cli/commands/scaffold/publish.mdx new file mode 100644 index 00000000000..0e419ed5767 --- /dev/null +++ b/website/docs/cli/commands/scaffold/publish.mdx @@ -0,0 +1,157 @@ +--- +title: atmos scaffold publish +sidebar_label: publish +sidebar_class_name: command +id: publish +description: Validate a scaffold and publish it as a versioned artifact in a configured repository. +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +Share a scaffold with your team as a pinned, versioned release. The command validates the scaffold configuration first, then publishes the directory to one configured artifact repository so that `atmos scaffold generate` can create and update projects from an exact version. + + + + +## Usage + +```shell +atmos scaffold publish --repository= [flags] +``` + +`atmos scaffold publish` delegates to the same publisher as [`atmos artifacts publish`](/cli/commands/artifacts/publish) with the `scaffold` kind. It takes the same flags except `--kind`, reads the same environment variables, and prints the same result. + +## Examples + +```shell +# Publish a scaffold, naming it explicitly +atmos scaffold publish ./scaffolds/landing-zone \ + --repository=shared --name=scaffolds/landing-zone --version=2.0.0 + +# Take the name and version from the scaffold manifest +atmos scaffold publish ./scaffolds/landing-zone --repository=shared + +# Validate the scaffold and plan the publication without writing anything +atmos scaffold publish ./scaffolds/landing-zone --repository=shared --dry-run + +# Attach labels and package documents +atmos scaffold publish ./scaffolds/landing-zone --repository=shared --version=2.0.0 \ + --labels=team=platform --readme=docs/PACKAGE.md --changelog=docs/CHANGES.md + +# Generate a project from the published release +atmos scaffold generate oci://registry.example.com/acme/atmos/scaffolds/landing-zone:2.0.0 ./my-project +``` + +## Behavior + +Publishing starts with validation. Atmos loads `scaffold.yaml` from the root of the source with the same loader that `atmos scaffold validate` and `atmos scaffold generate` use, so a scaffold that publishes is a scaffold that generates. Validation reads configuration only. It renders no templates, writes no generated output, and does not run scaffold hooks. A scaffold that fails validation is not packaged and nothing is written to the repository. + +The artifact name comes from `--name`, then from `metadata.name` in the manifest, then from the source basename. The version comes from `--version`, then from `metadata.version`. When neither exists, the command fails, because Atmos never generates a version. The manifest `metadata.description` becomes the artifact description. + +A scaffold carries two kinds of README, and they stay apart: + +
+
Root `README.md`
+
+ Ordinary scaffold content. It is part of the payload and keeps its generation behavior: it renders like any other template file and appears in the output and in the post-generation instructions. +
+ +
`.atmos/artifact/README.md` and `.atmos/artifact/CHANGELOG.md`
+
+ The package documents, stored as separate blobs next to the payload. Read them with [`atmos artifacts show`](/cli/commands/artifacts/show) without downloading the scaffold. The `.atmos/artifact/` directory is reserved for distribution metadata and never appears in generated output, whichever source you generate from: a published release, a local directory, a git repository, or an `atmos init` copy. +
+
+ +Files under `.atmos/` other than `.atmos/artifact/` are ordinary scaffold content. A scaffold publishes exactly the files its author put in the directory; the scaffold kind has no default exclusions. Use `--readme` and `--changelog` to pick package documents from other paths, given relative to the source directory. An explicit path does not remove the same file from the payload. + +Generating from a published scaffold produces the same output as generating from the original directory, including `--update`. Atmos reads only the payload when it generates, and legacy single-layer scaffold images keep working. + +## Arguments + +
+
`source` (required)
+
+ A scaffold directory with a `scaffold.yaml` at its root. +
+
+ +## Flags + +
+
`--repository`, `-r` (required)
+
+ Configured repository alias. + Environment variable: `ATMOS_ARTIFACTS_REPOSITORY` +
+ +
`--name` (optional)
+
+ Artifact name. Defaults to `metadata.name` in the manifest, then the source basename. + Environment variable: `ATMOS_ARTIFACTS_NAME` +
+ +
`--version` (required unless the manifest sets `metadata.version`)
+
+ Artifact version label. + Environment variable: `ATMOS_ARTIFACTS_VERSION` +
+ +
`--labels` (optional)
+
+ Exact `key=value` labels, comma-separated and/or repeated. + Environment variable: `ATMOS_ARTIFACTS_LABELS` +
+ +
`--readme` (optional)
+
+ Package README path, relative to the source directory. + Environment variable: `ATMOS_ARTIFACTS_README` +
+ +
`--changelog` (optional)
+
+ Package CHANGELOG path, relative to the source directory. + Environment variable: `ATMOS_ARTIFACTS_CHANGELOG` +
+ +
`--force`, `-f` (optional)
+
+ Replace an existing release that has different content. + Environment variable: `ATMOS_ARTIFACTS_FORCE` +
+ +
`--dry-run`, `-n` (optional)
+
+ Validate the scaffold and plan the publication without writing anything. + Environment variable: `ATMOS_ARTIFACTS_DRY_RUN` +
+ +
`--format` (optional)
+
+ Output format: `text` (default), `json`, or `yaml`. + Environment variable: `ATMOS_ARTIFACTS_FORMAT` +
+
+ +## Errors + +
+
Invalid scaffold
+
The manifest is missing from the source root or fails validation. The error names the file and the field. Nothing is packaged or written.
+ +
Source is a file
+
A scaffold source must be a directory.
+ +
Missing repository or version
+
Exit code 2 for a missing repository. A missing version fails before anything is staged, unless the manifest sets one.
+
+ +Conflicts, locks, and every other publication failure behave as described for [`atmos artifacts publish`](/cli/commands/artifacts/publish#errors). + +## Related + +- [`atmos scaffold validate`](/cli/commands/scaffold/validate) +- [`atmos scaffold generate`](/cli/commands/scaffold/generate) +- [`atmos artifacts publish`](/cli/commands/artifacts/publish) diff --git a/website/docs/functions/automation/steps.run.mdx b/website/docs/functions/automation/steps.run.mdx index 5df5bd5c756..749fb37f15e 100644 --- a/website/docs/functions/automation/steps.run.mdx +++ b/website/docs/functions/automation/steps.run.mdx @@ -205,6 +205,8 @@ prerequisites, and result details shared with YAML steps.
Ring the terminal bell and optionally print a message. See the alert step reference.
`steps.archive`
Create or extract an archive. See the archive step reference.
+
`steps.artifact`
+
Publish, pull, copy, inspect, list, search, delete, sign, verify, or show versioned artifacts. See the artifact step reference.
`steps.atmos`
Run an Atmos command through the step handler. See the atmos step reference.
`steps.cancel`
diff --git a/website/docs/steps/_partials/_type-catalog.mdx b/website/docs/steps/_partials/_type-catalog.mdx index b9940e12e27..5ba41d5d097 100644 --- a/website/docs/steps/_partials/_type-catalog.mdx +++ b/website/docs/steps/_partials/_type-catalog.mdx @@ -10,6 +10,7 @@ | [`emulator`](/steps/type/emulator) | Start, stop, or reset a local cloud emulator component. | `component`, `stack`, `action` | | [`http`](/steps/type/http) | Call an HTTP endpoint (alias: `webhook`). | `url`, `method`, `body`/`form`, `expect`, `retry` | | [`archive`](/steps/type/archive) | Pack (or update) a zip/tar/tgz archive. | `source`, `destination`, `action`, `format` | +| [`artifact`](/steps/type/artifact) | Publish, pull, copy, inspect, list, search, delete, sign, verify, or read documents of versioned artifacts. | `action`, `repository`, `source`, `ref`, `artifact_name`, `version` | | [`publish`](/steps/type/publish) | Publish files to S3 or Git targets. | `source`, `target`, `destination` | | [`require`](/steps/type/require) | Assert that required tools are on PATH and files/dirs exist (alias: `assert`). | `tools`, `files`, `dirs`, `hint` | | [`workdir`](/steps/type/workdir) | Provision a repeatable writable directory from a source. | `path`, `source`, `reset` | diff --git a/website/docs/steps/type/artifact.mdx b/website/docs/steps/type/artifact.mdx new file mode 100644 index 00000000000..1216d20356b --- /dev/null +++ b/website/docs/steps/type/artifact.mdx @@ -0,0 +1,184 @@ +--- +title: artifact +sidebar_label: artifact +slug: /steps/type/artifact +--- + +import Intro from '@site/src/components/Intro' +import Experimental from '@site/src/components/Experimental' + + +The `artifact` step type publishes, pulls, copies, inspects, lists, searches, deletes, signs, verifies, and reads documents of versioned [artifacts](/cli/commands/artifacts/usage) from a workflow, a custom command, an embedded script, or a lifecycle hook. It runs the same operations as `atmos artifacts`, in process, and returns the same typed result. No shelling out to the CLI. + + + + +```yaml +workflows: + publish-vpc: + steps: + - name: publish-vpc + type: artifact + action: publish + source: ./components/terraform/vpc + repository: shared + artifact_name: components/terraform/vpc + artifact_kind: component/terraform + version: "1.2.0" + metadata: + labels: + team: platform +``` + +Run the step as a component lifecycle hook through the generic [`kind: step` hook bridge](/stacks/hooks). The hook's `with:` block holds the same flat fields: + +```yaml +hooks: + backup-output: + events: [after.terraform.apply] + kind: step + type: artifact + on_failure: fail + with: + action: publish + source: ./dist + repository: backups + artifact_name: backups/app + version: build-123 +``` + +Repositories are defined under `artifacts.repositories` in `atmos.yaml`. See the [artifacts configuration reference](/cli/configuration/artifacts). + +## Fields + +The step's own `name`, `output`, and `artifacts` keep their usual meaning (the step name, the output display mode, and the expected-output declaration). The artifact's identity is `artifact_name` and `artifact_kind`. + +
+
`action`
+
Required. One of `publish`, `pull`, `copy`, `inspect`, `list`, `search`, `delete`, `sign`, `verify`, or `show`. Each action accepts only its own fields below. An unknown action, a field the action does not accept, or a missing required field fails validation before anything runs.
+ +
`repository`
+
The configured repository alias. Required by every action except `copy`, `list`, and `search`. A step never picks a default repository. `list` and `search` accept a comma-separated list and default to every configured repository. Supports Go templates.
+ +
`source`
+
`publish` only. The file or directory to publish. It must be a plain string path, even though other step types accept a map here. A relative path resolves against the step's `working_directory`. Supports Go templates.
+ +
`artifact_name`
+
`publish`: the artifact name. When omitted, the name comes from the producer metadata, then the source basename. `list`: restrict the listing to one name, which is the positional `[name]` argument of `atmos artifacts list`. For `publish`, this is the `--name` flag of the CLI. Supports Go templates.
+ +
`version`
+
`publish`. The explicit version. There is no generated version and no implicit `latest`; a publication fails when neither this field nor the producer supplies one. Supports Go templates.
+ +
`artifact_kind`
+
`publish`. The artifact kind. Defaults to `files`. This is the `--kind` flag of the CLI. Supports Go templates.
+ +
`force`
+
`publish`, `pull`, `copy`. Replace conflicting state. Force never bypasses signature verification or immutable release formats.
+ +
`dry_run`
+
`publish`. Validate and plan the publication without writing anything. A workflow-level `--dry-run` has the same effect on `publish`, and skips `pull`, `copy`, `delete`, and `sign`.
+ +
`metadata.labels`
+
`publish`. Exact key/value labels attached to the published version. Values support Go templates.
+ +
`readme`, `changelog`
+
`publish`. The document paths. A relative path resolves against the publish source directory, exactly like `--readme` and `--changelog`.
+ +
`ref`
+
`pull`, `copy`, `inspect`, `sign`, `verify`, `delete`, `show`. The artifact reference, `name:version` or `name@sha256:`. `sign` accepts a digest reference only. Supports Go templates.
+ +
`destination`
+
`pull`. The output directory (the `--output` flag of the CLI). A relative path resolves against the step's `working_directory`. Supports Go templates.
+ +
`from`, `to`
+
`copy`. The source and destination repository aliases.
+ +
`document`
+
`show`. Either `readme` or `changelog`.
+ +
`query`
+
`search`. The search text, a string. It matches names, kinds, descriptions, and labels.
+ +
`labels`
+
`list`, `search`. Exact key/value selectors, as a comma-separated `key=value` string, a list of them, or a mapping. Every selector must match.
+ +
`refresh`
+
`list`, `search`. Bypass the cached discovery snapshots.
+
+ +### Fields by action + +| Action | Required | Optional | +| --- | --- | --- | +| `publish` | `source`, `repository` | `artifact_name`, `version`, `artifact_kind`, `force`, `dry_run`, `metadata.labels`, `readme`, `changelog` | +| `pull` | `ref`, `repository`, `destination` | `force` | +| `copy` | `ref`, `from`, `to` | `force` | +| `inspect`, `sign`, `verify`, `delete` | `ref`, `repository` | none | +| `show` | `document`, `ref`, `repository` | none | +| `list` | none | `artifact_name`, `repository`, `labels`, `refresh` | +| `search` | none | `query`, `repository`, `labels`, `refresh` | + +## Result + +The typed result is stored under `metadata.artifact`. It is the payload that `atmos artifacts --format=json` prints, with the same field names. For `list` and `search` the payload is an object whose `artifacts` array holds the entries, also available as `metadata.artifacts`. + +Single-artifact actions also set these named string outputs. A value the operation did not produce is omitted, never filled with an empty string. + +
+
`{{ .steps..outputs.ref }}`
+
The resolved reference.
+ +
`{{ .steps..outputs.location }}`
+
The primary resolved storage location.
+ +
`{{ .steps..outputs.resolved_digest }}`, `record_digest`, `content_digest`, `representation_digest`
+
The digests the operation resolved. Each is labeled by what it covers: the manifest or record the repository resolved, the portable record, the content, and the transferred representation.
+ +
`{{ .steps..outputs.size }}`
+
The transferred or described payload size in bytes.
+ +
`{{ .steps..outputs.outcome }}`
+
`created`, `unchanged`, `replaced`, `deleted`, `read`, `dry-run`, or `failed`.
+ +
`{{ .steps..value }}`
+
The resolved reference, or the number of entries for `list` and `search`.
+
+ +A failed operation still returns its partial result: the operation identity, the outcome `failed`, the error text, and whatever the service resolved before it failed. + +```yaml +steps: + - name: publish + type: artifact + action: publish + source: ./dist + repository: backups + artifact_name: builds/app + version: build-123 + - name: verify + type: artifact + action: verify + repository: backups + ref: "{{ .steps.publish.outputs.ref }}" +``` + +## CI reports + +A workflow, custom command, embedded script, or hook that runs artifact steps writes one [CI report](/cli/commands/artifacts/usage) when it ends, even when it failed. The report has one section per operation, ordered by where each step was scheduled: sequential steps in order, and the branches of a parallel group in branch order, whatever order they finish in. Every attempt of a retried step is one operation, and the report shows the final attempt. + +The named outputs are also written as CI output variables when `ci.output.enabled` is on: `artifact_` carries the last operation in the report, and `artifact__` carries the last operation of the step with that name, with every character other than a letter or digit in the step name replaced by an underscore. A hook adds nothing to its own output, so the report is never rendered twice. + +## Retries, failure, and cancellation + +Set `retry:` to retry the whole operation. A publication keeps its operation identity and its captured preparation time across every attempt, so the creation time and the report of a retried publication do not change on every attempt, and the final attempt reports the outcome. + +The artifact service also retries a temporary transfer failure by itself, after it has proved that nothing was committed. When a step sets `retry:`, the service makes a single attempt per call, so the two budgets never multiply: `retry.max_attempts` is the total number of attempts. Without `retry:`, the service keeps its own small transport retry budget. + +`timeout:` limits each attempt. `continue:` and a hook's `on_failure` apply to the failed step as usual. Canceling the workflow cancels the operation in flight and stops further attempts. + +## See also + +- [`atmos artifacts`](/cli/commands/artifacts/usage) +- [Artifacts configuration](/cli/configuration/artifacts) +- [Manage lifecycle events with hooks](/stacks/hooks) +- [`retry`](/steps/retry)