Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f9ad720
feat(artifacts): add CI summaries, outputs, and entry-point tests (A10)
osterman Oct 31, 2026
ed1d552
docs(artifacts): document artifacts commands, catalogs, step, CI summ…
osterman Oct 31, 2026
e4e5f73
fix(artifacts): fix CI failures on Linux and Windows
osterman Oct 31, 2026
e73a0e4
test(artifacts): compare the workflow step source on slash form for W…
osterman Oct 31, 2026
e2f2fe3
fix(artifacts): address CodeRabbit review findings
osterman Oct 31, 2026
e07d8d1
ci: stop the floci go e2e job depending on apt
osterman Oct 31, 2026
313c41a
fix(artifacts): rerecord catalog and scaffold help casts
osterman Nov 11, 2026
8620dc2
fix(artifacts): claim qualified CI outputs per name and trim the comp…
osterman Nov 1, 2026
f4ab7cc
fix: isolate artifact steps and clean up shell sessions
osterman Nov 11, 2026
c1ba5b9
test(process): measure pipe drain after helper startup
osterman Nov 11, 2026
51292a5
fix(artifacts): confine validation and isolate task retries
osterman Nov 11, 2026
35c91b6
fix: preserve successful automation and compare artifact wire results
osterman Nov 12, 2026
56e5dee
test(process): verify drain deadline independently of startup
osterman Nov 12, 2026
9e964a7
test(step): adopt main's process-tree timeout test
osterman Nov 13, 2026
feaf0f9
test(auth): make the OIDC retry timeout case deterministic
osterman Nov 13, 2026
3805966
test(artifacts): use a generated ci.Reporter mock in report tests
osterman Nov 13, 2026
9bcc807
fix(manifest): wrap include-root confinement failures with a manifest…
osterman Nov 13, 2026
3d0da2d
refactor(manifest): move scaffold include confinement out of pkg/utils
osterman Nov 13, 2026
21170c9
docs(artifacts): exclude catalog commands from CI reporting and artif…
osterman Nov 13, 2026
e7d47f8
test(manifest): cover scaffold include publication-root boundary
osterman Nov 13, 2026
82b8ee4
test(step): use a generated mock for the detailed artifact service
osterman Nov 13, 2026
5b05b0c
test(artifacts): cover scaffold, confinement, parallel fork and repor…
osterman Nov 14, 2026
4174834
fix(artifacts): preserve historical L8 merge resolutions after rebase
osterman Nov 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 7 additions & 16 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -944,8 +944,7 @@ jobs:
# This is the same trap the govulncheck job documents (codeql.yml), but that
# job works around it by forcing CGO_ENABLED=0; that's not an option here since
# it would silently disable the race detector itself. Install libudev-dev
# instead, reusing the floci-go job's apt-get invocation below (same trap, hit
# there because that job also leaves CGO_ENABLED at the runner default).
# instead. The floci-go job hit the same trap and now disables CGO instead.
#
# GOFIPS140=latest (set inside the `race` command itself, .atmos.d/test.yaml)
# is unaffected by CGO_ENABLED: Go's FIPS 140-3 module is pure Go, no cgo.
Expand Down Expand Up @@ -1578,22 +1577,14 @@ jobs:
- name: Verify OpenTofu
run: tofu version

- name: Install Linux build dependencies
run: |
# The Azure mirror used by GitHub's Ubuntu runners can stall long
# enough to exhaust this job's timeout. Use Ubuntu's primary archive
# for this small dependency install. Current runner images use a
# mirror+file URI, while older ones contain the Azure URL directly.
# Pin both forms to the HTTP archive endpoint allowed by this job.
sudo sed -i \
-e 's|mirror+file:/etc/apt/apt-mirrors\.txt|http://archive.ubuntu.com/ubuntu|g' \
-e 's|http://azure\.archive\.ubuntu\.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' \
/etc/apt/sources.list.d/ubuntu.sources
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y --no-install-recommends libudev-dev pkg-config

# CGO stays off, as in the repository's other Go test paths. The only cgo
# code in the build is the USB HID library behind the SAML U2F support,
# which needs libudev-dev and therefore an apt install. Under this job's
# egress allowlist apt intermittently cannot reach archive.ubuntu.com
# (its address rotates), which failed the whole job before any test ran.
- name: Run Floci Go E2E tests
env:
CGO_ENABLED: "0"
ATMOS_TEST_FLOCI: "true"
FLOCI_ENDPOINT_URL: "http://localhost:4566"
FLOCI_GCP_ENDPOINT: "http://localhost:4588"
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ tools/gomodcheck/.gomodcheck
# Frozen compatibility fixtures must be committed even when they are archives.
!pkg/artifact/compat/testdata/**/*.tar
*.gz
# Byte-exact compatibility fixtures whose digests the tests pin.
!/pkg/artifact/compat/testdata/**/*.tar
/bin

# Nix
Expand Down
2 changes: 1 addition & 1 deletion agent-skills/skills/atmos-steps/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ type-specific subpages (e.g. `atmos.tools/steps/type/shell`) as the canonical
reference. Current canonical step types include:

- Command and integration: `atmos`, `shell`, `script`, `exec`, `container`,
`emulator`, `http`, `archive`, `require`, `workdir`, `cast`, `store`.
`emulator`, `http`, `archive`, `artifact`, `require`, `workdir`, `cast`, `store`.
- Orchestration: `test`, `parallel`, `matrix`, `wait`, `wait-all`, `cancel`.
- Interactive: `input`, `confirm`, `choose`, `filter`, `file`, `write`.
- UI and output: `toast`, `markdown`, `spin`, `table`, `pager`, `format`,
Expand Down
14 changes: 11 additions & 3 deletions cmd/artifacts/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
errUtils "github.com/cloudposse/atmos/errors"
"github.com/cloudposse/atmos/pkg/artifact"
"github.com/cloudposse/atmos/pkg/artifact/invoke"
"github.com/cloudposse/atmos/pkg/artifact/report"
"github.com/cloudposse/atmos/pkg/flags"
"github.com/cloudposse/atmos/pkg/list/column"
"github.com/cloudposse/atmos/pkg/list/format"
Expand Down Expand Up @@ -103,7 +104,11 @@ func (c *collection) execute(cmd *cobra.Command, args []string) error {
return err
}

envelope, err := c.run(commandContext(cmd), in)
// One report per command: the query is recorded after it ran and written when the command ends.
reporting := report.NewSession(atmosConfigPtr)
defer reporting.FlushAndLog()

envelope, err := c.run(commandContext(cmd), in, reporting)
if renderErr := renderCollection(outputFormat, envelope, err != nil); renderErr != nil && err == nil {
return renderErr
}
Expand All @@ -120,9 +125,11 @@ type listedArtifact = invoke.Listed

// run maps the flags onto operation parameters and runs the query through the shared
// implementation. The envelope is never nil: a failed query still yields an empty array and the
// error text, while the returned error keeps the exit code nonzero.
func (c *collection) run(ctx context.Context, in *input) (*listEnvelope, error) {
// error text, while the returned error keeps the exit code nonzero. A failure before the service
// runs is recorded into the report too, so CI shows a failed row instead of no report at all.
func (c *collection) run(ctx context.Context, in *input, reporting report.Collector) (*listEnvelope, error) {
failed := func(err error) (*listEnvelope, error) {
reporting.Record(report.EntryFor(c.action, nil, err))
return &listEnvelope{Artifacts: []listedArtifact{}, Error: err.Error()}, err
}
if atmosConfigPtr == nil {
Expand All @@ -149,6 +156,7 @@ func (c *collection) run(ctx context.Context, in *input) (*listEnvelope, error)
params.Name = firstArg(in)
}
outcome, err := invoke.Run(ctx, svc, params)
reporting.Record(report.EntryFor(c.action, outcome, err))
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return outcome.Collection, err
}

Expand Down
78 changes: 78 additions & 0 deletions cmd/artifacts/parity_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package artifacts

import (
"context"
"encoding/json"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/cloudposse/atmos/pkg/artifact"
"github.com/cloudposse/atmos/pkg/artifact/invoke"
"github.com/cloudposse/atmos/pkg/artifactwire/wiretest"
runnerstep "github.com/cloudposse/atmos/pkg/runner/step"
"github.com/cloudposse/atmos/pkg/schema"
)

// TestCLIAndArtifactStepReportTheSamePublication runs the same publication through the real
// command and through the artifact step against real local repositories, and compares the
// structured output of the command with the step's typed result and named outputs.
func TestCLIAndArtifactStepReportTheSamePublication(t *testing.T) {
f := wiretest.New(t)
h, _ := newHarness(t)
atmosConfigPtr = f.Config
newService = func(*schema.AtmosConfiguration, ServiceOptions) (artifact.Service, error) { return f.Recorder, nil }

cli := h.run("publish", f.Source, "--repository="+wiretest.Alias, "--name="+wiretest.Name, "--version="+wiretest.Version,
"--labels=team=platform", "--format=json")
require.NoError(t, cli.err)
var fromCLI invoke.Result
require.NoError(t, json.Unmarshal([]byte(cli.stdout), &fromCLI))

// The step publishes to a sibling repository, so it is a fresh creation with the same content.
sibling := wiretest.New(t)
vars := runnerstep.NewVariables()
vars.SetAtmosConfig(sibling.Config)
step := &schema.WorkflowStep{
Name: "publish", Type: schema.TaskTypeArtifact, Action: "publish", Source: sibling.Source, Repository: wiretest.Alias,
ArtifactName: wiretest.Name, Version: wiretest.Version, Metadata: &schema.ArtifactStepMetadata{Labels: map[string]string{"team": "platform"}},
}
result, err := (&runnerstep.ArtifactHandler{}).Execute(context.Background(), step, vars)
require.NoError(t, err)
fromStep, ok := result.Metadata[runnerstep.ArtifactMetadataKey].(*invoke.Result)
require.True(t, ok)

// SizeMeasured is internal state omitted from JSON; compare the public wire values while
// independently checking that the typed step remembers the measurement.
assert.True(t, fromStep.SizeMeasured)
cliOperation, err := json.Marshal(f.Normalize(&fromCLI.OperationResult))
require.NoError(t, err)
stepOperation, err := json.Marshal(sibling.Normalize(&fromStep.OperationResult))
require.NoError(t, err)
assert.JSONEq(t, string(cliOperation), string(stepOperation), "the command and the step report the same result")
assert.Equal(t, fromCLI.Record, fromStep.Record)

var named map[string]any
require.NoError(t, json.Unmarshal([]byte(cli.stdout), &named))
require.Contains(t, result.Outputs, "size")
require.Contains(t, named, "size")
for name, value := range result.Outputs {
if name == "location" {
continue // Rooted in each fixture's own repository directory.
}
if name == "size" {
assert.EqualValues(t, fromCLI.Size, mustSize(t, value), "size")
continue
}
assert.Equal(t, value, named[name], "named output %q has the same name and value in the command's structured output", name)
}
assert.Contains(t, named, "location")
}

func mustSize(t *testing.T, value string) int64 {
t.Helper()
var size int64
require.NoError(t, json.Unmarshal([]byte(value), &size))
return size
}
179 changes: 179 additions & 0 deletions cmd/artifacts/report_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package artifacts

import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"

errUtils "github.com/cloudposse/atmos/errors"
"github.com/cloudposse/atmos/pkg/artifact"
_ "github.com/cloudposse/atmos/pkg/ci/providers/generic" // Register the generic CI provider, as the binary does.
"github.com/cloudposse/atmos/pkg/schema"
)

// ciReportFiles enables CI reporting on the harness and points the generic CI provider at files.
func ciReportFiles(t *testing.T, h *harness) (summary, outputs string) {
t.Helper()
dir := t.TempDir()
summary, outputs = filepath.Join(dir, "summary.md"), filepath.Join(dir, "output.txt")
// A real CI runner advertises its own provider, which would receive the output instead of the files.
t.Setenv("GITHUB_ACTIONS", "")
t.Setenv("ATMOS_CI", "true")
t.Setenv("ATMOS_CI_SUMMARY", summary)
t.Setenv("ATMOS_CI_OUTPUT", outputs)
h.config.CI.Enabled = true
return summary, outputs
}

func contentOf(t *testing.T, path string) string {
t.Helper()
content, err := os.ReadFile(path)
if os.IsNotExist(err) {
return ""
}
require.NoError(t, err)
return string(content)
}

// The harness service is a strict mock: every report below is built from the result the operation
// returned, so an extra metadata or evidence request would fail the test as an unexpected call.
func TestCommandWritesOneReportAndKeepsStructuredStdoutValid(t *testing.T) {
h, svc := newHarness(t)
summary, outputs := ciReportFiles(t, h)
svc.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(operation(artifact.ActionPublish, artifact.OutcomeCreated), nil)

got := h.run("publish", "./dist", "--repository=dir", "--name=team/app", "--version=1.0.0", "--format=json")

require.NoError(t, got.err)
var envelope map[string]any
require.NoError(t, json.Unmarshal([]byte(got.stdout), &envelope), "stdout stays a valid JSON document")
assert.Equal(t, "created", envelope["outcome"])
assert.NotContains(t, got.stdout, "## Artifacts", "the report never goes to stdout")

assert.Equal(t, 1, strings.Count(contentOf(t, summary), "### ✅ 1. Publish `team/app:1.0.0`"))
assert.Contains(t, contentOf(t, outputs), "artifact_ref=team/app:1.0.0\n")
assert.Contains(t, contentOf(t, outputs), "artifact_resolved_digest="+string(manifestDigest)+"\n")
}

func TestFailedCommandReportsTheFailureAndStaysFailed(t *testing.T) {
h, svc := newHarness(t)
summary, _ := ciReportFiles(t, h)
failure := artifact.OperationResult{OperationID: "op-9", Action: artifact.ActionVerify, Repository: "dir", Requested: "team/app:1.0.0"}
svc.EXPECT().Verify(gomock.Any(), gomock.Any()).Return(
&artifact.VerifyResult{Operation: failure, Verification: artifact.VerificationResult{Status: artifact.VerificationUnchecked}},
errors.Join(errUtils.ErrArtifactVerificationFailed, errors.New("no permitted signature")),
)

got := h.run("verify", "team/app:1.0.0", "--repository=dir", "--format=json")

require.ErrorIs(t, got.err, errUtils.ErrArtifactVerificationFailed, "reporting never turns a verification failure into success")
content := contentOf(t, summary)
assert.Contains(t, content, "1 operation: 1 failed.")
assert.Contains(t, content, "no permitted signature")
assert.Contains(t, content, "No trust configuration was supplied", "an unchecked signature is never reported as verified")
}

func TestReportingFailurePreservesTheSuccessfulExitStatus(t *testing.T) {
cases := map[string]func(t *testing.T, h *harness){
"template failure": func(t *testing.T, h *harness) {
h.config.CI.Templates.BasePath = t.TempDir()
h.config.CI.Templates.Artifacts = map[string]string{"summary": "does-not-exist.md"}
},
"provider failure": func(t *testing.T, _ *harness) {
t.Setenv("ATMOS_CI_SUMMARY", filepath.Join(t.TempDir(), "missing-dir", "summary.md"))
t.Setenv("ATMOS_CI_OUTPUT", filepath.Join(t.TempDir(), "missing-dir", "output.txt"))
},
}
for name, breakReporting := range cases {
t.Run(name, func(t *testing.T) {
h, svc := newHarness(t)
ciReportFiles(t, h)
breakReporting(t, h)
svc.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(operation(artifact.ActionPublish, artifact.OutcomeCreated), nil)

got := h.run("publish", "./dist", "--repository=dir", "--name=team/app", "--version=1.0.0", "--format=json")

require.NoError(t, got.err, "a reporting error is logged and never changes the exit status")
assert.Contains(t, got.stdout, `"outcome": "created"`)
})
}
}

func TestListQueryIsReportedOnce(t *testing.T) {
h, svc := newHarness(t)
summary, _ := ciReportFiles(t, h)
svc.EXPECT().List(gomock.Any(), gomock.Any()).Return(&artifact.ListResult{Artifacts: []artifact.ListEntry{
{Name: "team/app", Version: "1.0.0", Kind: "files"},
}}, nil)

got := h.run("list", "--format=json")

require.NoError(t, got.err)
assert.Equal(t, 1, strings.Count(contentOf(t, summary), "## Artifacts"))
assert.Contains(t, contentOf(t, summary), "Listed 1 artifact.")
}

func TestDisabledReportingWritesNothing(t *testing.T) {
h, svc := newHarness(t)
summary, outputs := ciReportFiles(t, h)
h.config.CI.Enabled = false
svc.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(operation(artifact.ActionPublish, artifact.OutcomeCreated), nil)

got := h.run("publish", "./dist", "--repository=dir", "--name=team/app", "--version=1.0.0", "--format=json")

require.NoError(t, got.err)
assert.Empty(t, contentOf(t, summary))
assert.Empty(t, contentOf(t, outputs))
}

// A failure before the service runs still reaches the CI report, so the job summary shows a failed
// row instead of nothing.
func TestPreServiceFailuresAreReported(t *testing.T) {
failConstruction := func(*schema.AtmosConfiguration, ServiceOptions) (artifact.Service, error) {
return nil, errUtils.ErrArtifactConfigInvalid
}
tests := []struct {
name string
setup func(h *harness)
args []string
want error
}{
{"publish without a repository", func(*harness) {}, []string{"publish", "./dist", "--name=team/app", "--version=1.0.0"}, errUtils.ErrRequiredFlagNotProvided},
{"publish with a failing service construction", func(*harness) { newService = failConstruction }, []string{"publish", "./dist", "-r", "dir", "--name=team/app", "--version=1.0.0"}, errUtils.ErrArtifactConfigInvalid},
{"inspect with a failing service construction", func(*harness) { newService = failConstruction }, []string{"inspect", "team/app:1", "-r", "dir"}, errUtils.ErrArtifactConfigInvalid},
{"list with invalid labels", func(*harness) {}, []string{"list", "--labels=nope"}, errUtils.ErrInvalidFlag},
{"search with a failing service construction", func(*harness) { newService = failConstruction }, []string{"search", "vpc", "-r", "dir"}, errUtils.ErrArtifactConfigInvalid},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h, _ := newHarness(t)
summary, outputs := ciReportFiles(t, h)
tt.setup(h)

got := h.run(tt.args...)

require.ErrorIs(t, got.err, tt.want)
content := contentOf(t, summary)
assert.Contains(t, content, "1 operation: 1 failed.")
assert.Contains(t, content, "❌ Failed")
assert.Empty(t, contentOf(t, outputs), "a failure with no result writes no output variables")
})
}
}

func TestMissingConfigurationIsReported(t *testing.T) {
h, _ := newHarness(t)
summary, _ := ciReportFiles(t, h)
// The report session follows the loaded configuration, so a missing one has nothing to report to.
atmosConfigPtr = nil
got := h.run("inspect", "team/app:1", "-r", "dir", "--format=json")
require.ErrorIs(t, got.err, errUtils.ErrArtifactConfigInvalid)
assert.Empty(t, contentOf(t, summary))
}
Loading
Loading