diff --git a/.github/workflows/plugin-packages.yaml b/.github/workflows/plugin-packages.yaml index 300e705480a..9879ee6d464 100644 --- a/.github/workflows/plugin-packages.yaml +++ b/.github/workflows/plugin-packages.yaml @@ -32,11 +32,16 @@ jobs: # version bump in every layer that touches a package input. Compare such pull requests with the merge # base of the default branch instead, the same range the whole stack is released against. Pushes and # pull requests that target the default branch keep the provider's default base. - # The inventory --check also compares packed bytes with any published version in the registry, - # so a child cannot reuse its parent's published version for different package bytes. + # + # A manual run has no event base: the provider would fall back to refs/remotes/origin/HEAD, which + # actions/checkout never creates. Use the same default-branch merge base, so a manual run of a stack + # layer checks the same range as its pull request, and a manual run of the default branch compares + # its tip with itself. - name: Resolve the release comparison base id: base - if: github.event_name == 'pull_request' && github.event.pull_request.base.ref != github.event.repository.default_branch + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.base.ref != github.event.repository.default_branch) env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # Fail rather than fall back to the PR target when the branches share no history. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6df5844de5b..84935a9d6b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -220,6 +220,15 @@ jobs: run: | atmos build version --target=${{ matrix.target }} + - name: Verify Windows plan-binding ACL + if: matrix.target == 'windows' && ! github.event.pull_request.draft + shell: bash + # Match the CLI and acceptance harness; the hosted CGO runtime can + # crash during startup before this test reaches its ACL assertions. + env: + CGO_ENABLED: '0' + run: go test ./pkg/component/terraform -run '^TestPlanBindingPrivateACLUnderPermissiveParent$' -count=1 -v + # Mage keeps the cross-platform acceptance orchestration in typed, # unit-tested Go code while the workflow remains a one-command dispatch. # Reuse this job's warm Go cache to compile the large test binaries once. diff --git a/.golangci.yml b/.golangci.yml index a820f830cd8..9dd846e806a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -108,6 +108,11 @@ linters: # Only this cloud-specific leaf imports the SDK; the parent downloader stays generic. - "!**/pkg/downloader/s3/**" - "!**/pkg/github/**" + # The Terragrunt plugin bridge resolves the AWS identity of an invocation for Terragrunt's own + # AWS calls (get_aws_account_id, S3 dependency state). It is an AWS-specific leaf of a separate + # module that cannot import the Atmos auth packages. + - "!**/plugins/terragrunt/bridge/aws.go" + - "!**/plugins/terragrunt/bridge/aws_test.go" # The S3 deployment Mage target is an AWS-specific build tool and # intentionally uses the AWS SDK instead of shelling out to a CLI. - "!**/magefiles/s3/**" diff --git a/cmd/help_template.go b/cmd/help_template.go index 8e220fafc8c..8e88241f0ca 100644 --- a/cmd/help_template.go +++ b/cmd/help_template.go @@ -10,10 +10,12 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/muesli/reflow/wordwrap" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/cloudposse/atmos/cmd/internal" tuiUtils "github.com/cloudposse/atmos/internal/tui/utils" atmosansi "github.com/cloudposse/atmos/pkg/ansi" + "github.com/cloudposse/atmos/pkg/component/provideroptions" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/flags/compat" log "github.com/cloudposse/atmos/pkg/logger" @@ -495,10 +497,7 @@ func printFlags(w io.Writer, cmd *cobra.Command, atmosConfig *schema.AtmosConfig termWidth := getTerminalWidth() if cmd.HasAvailableLocalFlags() { - fmt.Fprintln(w, styles.heading.Render("FLAGS")) - fmt.Fprintln(w) - renderFlags(w, cmd.LocalFlags(), styles.commandName, styles.flagName, styles.flagDesc, termWidth, atmosConfig) - fmt.Fprintln(w) + renderLocalFlagSections(w, cmd.LocalFlags(), styles, termWidth, atmosConfig) } if cmd.HasAvailableInheritedFlags() { @@ -686,3 +685,19 @@ func printFooter(w io.Writer, cmd *cobra.Command, styles *helpStyles) { usageMsg = renderMarkdownDescription(usageMsg) fmt.Fprintf(w, "\n%s\n", styles.muted.Render(usageMsg)) } + +// renderLocalFlagSections prints the FLAGS section, followed by one section for each configuration +// provider whose declared options the command accepts. +func renderLocalFlagSections(w io.Writer, local *pflag.FlagSet, styles *helpStyles, termWidth int, atmosConfig *schema.AtmosConfiguration) { + rest, providerSections := provideroptions.SplitFlags(local) + fmt.Fprintln(w, styles.heading.Render("FLAGS")) + fmt.Fprintln(w) + renderFlags(w, rest, styles.commandName, styles.flagName, styles.flagDesc, termWidth, atmosConfig) + fmt.Fprintln(w) + for _, section := range providerSections { + fmt.Fprintln(w, styles.heading.Render(section.Title)) + fmt.Fprintln(w) + renderFlags(w, section.Flags, styles.commandName, styles.flagName, styles.flagDesc, termWidth, atmosConfig) + fmt.Fprintln(w) + } +} diff --git a/cmd/help_template_test.go b/cmd/help_template_test.go index 928ca54e058..0543df3dbf9 100644 --- a/cmd/help_template_test.go +++ b/cmd/help_template_test.go @@ -3,13 +3,17 @@ package cmd import ( "bytes" "os" + "slices" "strings" "testing" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" + "github.com/spf13/pflag" + terraformcmd "github.com/cloudposse/atmos/cmd/terraform" "github.com/cloudposse/atmos/pkg/ansi" + "github.com/cloudposse/atmos/pkg/component/provideroptions" "github.com/cloudposse/atmos/pkg/flags/compat" "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/ui/markdown" @@ -1422,3 +1426,59 @@ func TestGetExperimentalBadge(t *testing.T) { }) } } + +func TestPrintFlagsGroupsProviderFlagsUnderTheirProvider(t *testing.T) { + root := &cobra.Command{Use: "terraform"} + plan := &cobra.Command{Use: "plan", Run: func(*cobra.Command, []string) {}} + plan.Flags().Bool("affected", false, "Plan the affected components") + root.AddCommand(plan) + options, err := provideroptions.Parse("tool", []any{ + map[string]any{"name": "level", "type": "string", "description": "Provider log level.", "commands": []any{"plan"}}, + }) + if err != nil { + t.Fatal(err) + } + if err := provideroptions.Register(root, []provideroptions.Provider{{Section: "tool", Name: "tool", Mode: provideroptions.ModeNative, Options: options}}); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + styles := createHelpStyles(lipgloss.NewRenderer(&buf)) + printFlags(&buf, plan, &schema.AtmosConfiguration{}, &styles) + // Runners advertise color, and styling splits words with escape sequences; assert on the text. + output := ansi.Strip(buf.String()) + + flags, providerFlags, found := strings.Cut(output, "TOOL FLAGS") + if !found { + t.Fatalf("provider section missing: %q", output) + } + if !strings.Contains(flags, "--affected") || strings.Contains(flags, "--tool-level") { + t.Errorf("Atmos flags and provider flags are not separated: %q", flags) + } + if !strings.Contains(providerFlags, "--tool-level") || !strings.Contains(providerFlags, "Provider log level.") { + t.Errorf("provider section lacks its flag: %q", providerFlags) + } +} + +func TestRegisterProviderFlagsWithoutEnabledProvidersChangesNothing(t *testing.T) { + flagNames := func() []string { + var names []string + plan().Flags().VisitAll(func(flag *pflag.Flag) { names = append(names, flag.Name) }) + return names + } + before := flagNames() + for _, args := range [][]string{{"terraform", "plan"}, {"version"}, nil} { + if err := terraformcmd.RegisterProviderFlags(&schema.AtmosConfiguration{}, args); err != nil { + t.Errorf("args %v: %v", args, err) + } + } + if !slices.Equal(flagNames(), before) { + t.Error("registration without a provider changed the flags") + } +} + +// plan finds the terraform plan command in the real command tree. +func plan() *cobra.Command { + command, _, _ := RootCmd.Find([]string{"terraform", "plan"}) + return command +} diff --git a/cmd/help_topics_render.go b/cmd/help_topics_render.go index 562fc08ef46..3884108c53b 100644 --- a/cmd/help_topics_render.go +++ b/cmd/help_topics_render.go @@ -21,10 +21,7 @@ func printLocalFlagsOnly(w io.Writer, cmd *cobra.Command, atmosConfig *schema.At return } - fmt.Fprintln(w, styles.heading.Render("FLAGS")) - fmt.Fprintln(w) - renderFlags(w, localFlags, styles.commandName, styles.flagName, styles.flagDesc, getTerminalWidth(), atmosConfig) - fmt.Fprintln(w) + renderLocalFlagSections(w, localFlags, styles, getTerminalWidth(), atmosConfig) } func commandSpecificFlagSet(cmd *cobra.Command) *pflag.FlagSet { diff --git a/cmd/plugin/author.go b/cmd/plugin/author.go index 87e5d2c315b..f53b24bf439 100644 --- a/cmd/plugin/author.go +++ b/cmd/plugin/author.go @@ -46,7 +46,7 @@ func validateCommand() *cobra.Command { } func packCommand() *cobra.Command { - parser := newOutputParser(flags.WithStringFlag(packOutputFlag, "o", "", "Destination OCI layout directory")) + parser := newOutputParser(append(sourceFlags(), flags.WithStringFlag(packOutputFlag, "o", "", "Destination OCI layout directory"))...) command := &cobra.Command{Use: "pack [directory] --output ", Short: "Pack precompiled plugins into a multi-platform OCI layout", Args: cobra.MaximumNArgs(1)} registerOutputFlags(command, parser) _ = command.MarkFlagRequired(packOutputFlag) @@ -68,7 +68,11 @@ func packCommand() *cobra.Command { return err } output := values.GetString(packOutputFlag) - if err := pluginoci.Pack(directory, output, registry, pluginoci.WithExecutableFormatCheck()); err != nil { + opts, err := packOptions(values) + if err != nil { + return err + } + if err := pluginoci.Pack(directory, output, registry, opts...); err != nil { return err } files, err := pluginoci.LayoutFiles(output) diff --git a/cmd/plugin/publish_public_test.go b/cmd/plugin/publish_public_test.go index cde15420b76..3b00d15f700 100644 --- a/cmd/plugin/publish_public_test.go +++ b/cmd/plugin/publish_public_test.go @@ -82,7 +82,7 @@ func TestPublishPublicAccessAfterUpload(t *testing.T) { require.NoError(t, err, "ordinary publication must support private registries") } -func publicReleaseLayout(t *testing.T, descriptors *extension.Registry) string { +func publicReleaseLayout(t *testing.T, descriptors *extension.Registry, opts ...pluginoci.Option) string { t.Helper() directory := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(directory, "bin"), 0o755)) @@ -107,6 +107,6 @@ spec: example/fixture: {} `), 0o644)) layout := filepath.Join(t.TempDir(), "layout") - require.NoError(t, pluginoci.Pack(directory, layout, descriptors)) + require.NoError(t, pluginoci.Pack(directory, layout, descriptors, opts...)) return layout } diff --git a/cmd/plugin/publish_safety_test.go b/cmd/plugin/publish_safety_test.go index f3476dab906..df1e9c26b8b 100644 --- a/cmd/plugin/publish_safety_test.go +++ b/cmd/plugin/publish_safety_test.go @@ -12,6 +12,8 @@ import ( "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" + atmosoci "github.com/cloudposse/atmos/pkg/oci" + "github.com/cloudposse/atmos/pkg/plugin/extension" ) //go:generate go run go.uber.org/mock/mockgen -destination=mock_registry_test.go -package=plugin github.com/cloudposse/atmos/pkg/oci Registry @@ -121,3 +123,42 @@ func TestDeleteReleaseConfirmation(t *testing.T) { require.ErrorIs(t, deleteRelease(t.Context(), client, reference, true, deletePrompts{interactive: neverInteractive, confirm: noConfirm}), denied) }) } + +func TestPackOptionsSourceIsOptInAndNeverImplicit(t *testing.T) { + t.Setenv("GITHUB_REPOSITORY", "") + t.Setenv("GITHUB_SERVER_URL", "") + values := func(source string) *viper.Viper { + result := viper.New() + result.Set(sourceFlag, source) + return result + } + + opts, err := packOptions(values("")) + require.NoError(t, err) + require.Len(t, opts, 1, "without --source only the executable format check applies") + + opts, err = packOptions(values("https://github.com/example/repo")) + require.NoError(t, err) + require.Len(t, opts, 2) + + _, err = packOptions(values(sourceGitHubActions)) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig, "github-actions must fail loudly outside GitHub Actions") + + t.Setenv("GITHUB_REPOSITORY", "example/repo") + opts, err = packOptions(values(sourceGitHubActions)) + require.NoError(t, err) + require.Len(t, opts, 2) + descriptors := extension.New() + require.NoError(t, descriptors.Register(extension.Descriptor{Capability: "fixture", Path: []string{"fixtures"}, Version: 1, Validate: func(extension.Binding) error { return nil }})) + layout := publicReleaseLayout(t, descriptors, opts...) + index, err := atmosoci.LoadIndex(layout, "") + require.NoError(t, err) + manifest, err := index.IndexManifest() + require.NoError(t, err) + require.Equal(t, "https://github.com/example/repo", manifest.Annotations["org.opencontainers.image.source"]) + + t.Setenv("GITHUB_REPOSITORY", "example/repo") + opts, err = packOptions(values("")) + require.NoError(t, err) + require.Len(t, opts, 1, "the CI environment alone never links a package") +} diff --git a/cmd/plugin/release.go b/cmd/plugin/release.go index fb89a94a35e..2a87bee22dc 100644 --- a/cmd/plugin/release.go +++ b/cmd/plugin/release.go @@ -3,6 +3,8 @@ package plugin import ( "context" "fmt" + "os" + "slices" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -23,7 +25,7 @@ import ( ) func releaseCommand() *cobra.Command { - parser := newOutputParser(append(publicationFlags(), inventoryReleaseFlags()...)...) + parser := newOutputParser(slices.Concat(publicationFlags(), inventoryReleaseFlags(), sourceFlags())...) command := &cobra.Command{ Use: "release [directory] [--to | --inventory ]", Short: "Build all targets, test the host target, and publish a plugin release", @@ -122,11 +124,13 @@ func (backend *releaseBackend) Test(ctx context.Context, directory string) error } func (backend *releaseBackend) Pack(ctx context.Context, directory, layout string) error { - if err := pluginoci.Pack(directory, layout, backend.manager.Installer.Descriptors, pluginoci.WithExecutableFormatCheck()); err != nil { + opts, err := packOptions(backend.values) + if err != nil { + return err + } + if err := pluginoci.Pack(directory, layout, backend.manager.Installer.Descriptors, opts...); err != nil { return err } - // Inventory entries always supply a destination. Directory-only checks can - // remain offline unless the caller supplies --to. destination := backend.values.GetString(publishDestinationFlag) if backend.values.GetBool(releaseCheckFlag) && destination != "" { return pluginoci.CheckPublication(ctx, backend.manager.Installer.Registry, layout, destination) @@ -134,6 +138,41 @@ func (backend *releaseBackend) Pack(ctx context.Context, directory, layout strin return nil } +const ( + // The source flag selects the repository recorded as the package source. + sourceFlag = "source" + // The github-actions value asks packing to derive the repository link from the GitHub Actions environment. + sourceGitHubActions = "github-actions" +) + +// sourceFlags declares the opt-in repository link recorded as org.opencontainers.image.source. +func sourceFlags() []flags.Option { + return []flags.Option{ + flags.WithStringFlag(sourceFlag, "", "", "Repository URL recorded as the package source so registries such as GHCR link the package to it (https URL, or "+sourceGitHubActions+" to derive it from GITHUB_REPOSITORY); defaults to metadata.repository in plugin.yaml"), + flags.WithEnvVars(sourceFlag, "ATMOS_PLUGIN_SOURCE"), + } +} + +// packOptions translates the shared pack flags into packing options. Nothing is +// derived from the environment unless --source asks for it. +func packOptions(values *viper.Viper) ([]pluginoci.Option, error) { + opts := []pluginoci.Option{pluginoci.WithExecutableFormatCheck()} + switch source := values.GetString(sourceFlag); source { + case "": + case sourceGitHubActions: + derived := pluginoci.GitHubActionsSource(os.Getenv) //nolint:forbidigo // GITHUB_* is the runner's own context, not Atmos configuration, and is read only when --source asks for it. + if derived == "" { + return nil, errUtils.Build(fmt.Errorf("%w: --%s=%s requires GITHUB_REPOSITORY (and optionally an https GITHUB_SERVER_URL)", errUtils.ErrPluginInvalidConfig, sourceFlag, sourceGitHubActions)). + WithHintf("Run inside GitHub Actions, or pass the repository explicitly with `--%s https://github.com//`.", sourceFlag). + Err() + } + opts = append(opts, pluginoci.WithSource(derived)) + default: + opts = append(opts, pluginoci.WithSource(source)) + } + return opts, nil +} + func (backend *releaseBackend) Publish(ctx context.Context, layout string) (*pluginoci.Release, error) { return publishLayout(ctx, backend.manager, layout, backend.values) } diff --git a/cmd/root.go b/cmd/root.go index edd523315b3..e21fde9d18d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,7 +59,7 @@ import ( _ "github.com/cloudposse/atmos/cmd/secret" stackcmd "github.com/cloudposse/atmos/cmd/stack" _ "github.com/cloudposse/atmos/cmd/store" - _ "github.com/cloudposse/atmos/cmd/terraform" + terraformcmd "github.com/cloudposse/atmos/cmd/terraform" "github.com/cloudposse/atmos/cmd/terraform/backend" terraformcache "github.com/cloudposse/atmos/cmd/terraform/cache" "github.com/cloudposse/atmos/cmd/terraform/workdir" @@ -1915,6 +1915,11 @@ func Execute() error { if err != nil { return err } + + // Provider options come from static package metadata and must exist before Cobra parses flags. + if err = terraformcmd.RegisterProviderFlags(&atmosConfig, os.Args[1:]); err != nil { + return err + } } // Custom commands and configured aliases can now use inherited discovery metadata. diff --git a/cmd/terraform/planfile/download.go b/cmd/terraform/planfile/download.go index 5209afe7c1e..a6e466bcaeb 100644 --- a/cmd/terraform/planfile/download.go +++ b/cmd/terraform/planfile/download.go @@ -179,8 +179,7 @@ func resolveDownloadPlanfilePath(cmd *cobra.Command, opts *DownloadOptions, atmo return "", fmt.Errorf("%w: failed to resolve component path: %w", errUtils.ErrPlanfileDownloadFailed, err) } - planfilePath := exec.ConstructTerraformComponentPlanfilePath(atmosConfig, &info) - return planfilePath, nil + return canonicalPlanfilePath(atmosConfig, &info) } // printDownloadSuccess prints the success message for a download. diff --git a/cmd/terraform/planfile/resolve.go b/cmd/terraform/planfile/resolve.go index b6ecd634f9e..bc1e5223f48 100644 --- a/cmd/terraform/planfile/resolve.go +++ b/cmd/terraform/planfile/resolve.go @@ -5,12 +5,33 @@ import ( "os" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/internal/exec" "github.com/cloudposse/atmos/pkg/ci/plugins/terraform/planfile" + "github.com/cloudposse/atmos/pkg/component" + cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/git" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" + provWorkdir "github.com/cloudposse/atmos/pkg/provisioner/workdir" + "github.com/cloudposse/atmos/pkg/schema" ) +// canonicalPlanfilePath returns where `atmos terraform plan` keeps a component's plan file. A +// workdir-enabled component (every Terragrunt unit) keeps it in its workdir, which is derived from +// the configuration here because nothing has been provisioned in a new process or on a fresh runner. +func canonicalPlanfilePath(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (string, error) { + defer perf.Track(atmosConfig, "planfile.canonicalPlanfilePath")() + + if provWorkdir.IsWorkdirEnabled(info.ComponentSection) { + workdir, _, err := component.BuildAndResolveWorkdirPath(atmosConfig, info, cfg.TerraformComponentType) + if err != nil { + return "", err + } + info.ComponentSection[provWorkdir.WorkdirPathKey] = workdir + } + return exec.ConstructTerraformComponentPlanfilePath(atmosConfig, info), nil +} + // resolvedContext holds SHA and branch resolved from environment or git. type resolvedContext struct { SHA string diff --git a/cmd/terraform/planfile/resolve_test.go b/cmd/terraform/planfile/resolve_test.go index 505546e0e1f..4c82a84ffb9 100644 --- a/cmd/terraform/planfile/resolve_test.go +++ b/cmd/terraform/planfile/resolve_test.go @@ -2,12 +2,15 @@ package planfile import ( "errors" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" ) func TestResolveContext(t *testing.T) { @@ -149,3 +152,29 @@ func TestGetFirstEnvValue(t *testing.T) { assert.Empty(t, result) }) } + +func TestCanonicalPlanfilePathFollowsTheComponentWorkdir(t *testing.T) { + base := t.TempDir() + config := &schema.AtmosConfiguration{BasePath: base, TerraformDirAbsolutePath: filepath.Join(base, "components", "terraform")} + config.Components.Terraform.BasePath = filepath.Join("components", "terraform") + + t.Run("workdir component keeps its plan in the workdir, which need not exist yet", func(t *testing.T) { + info := &schema.ConfigAndStacksInfo{ + Stack: "live/dev", ComponentFromArg: "app", FinalComponent: "app", + ComponentSettingsSection: map[string]any{"configuration": true}, + ComponentSection: map[string]any{"provision": map[string]any{"workdir": map[string]any{"enabled": true}}}, + } + path, err := canonicalPlanfilePath(config, info) + require.NoError(t, err) + assert.Contains(t, filepath.ToSlash(path), ".workdir/terraform/live/dev-app-") + assert.True(t, strings.HasSuffix(path, ".planfile"), path) + assert.NoDirExists(t, filepath.Dir(path), "resolving the path must not provision anything") + }) + + t.Run("ordinary component keeps its plan beside its sources", func(t *testing.T) { + info := &schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "vpc", FinalComponent: "vpc", ComponentSection: map[string]any{}} + path, err := canonicalPlanfilePath(config, info) + require.NoError(t, err) + assert.Equal(t, filepath.Join(base, "components", "terraform", "vpc"), filepath.Dir(path)) + }) +} diff --git a/cmd/terraform/planfile/upload.go b/cmd/terraform/planfile/upload.go index 4b1a4d36ccd..8a8ae19e592 100644 --- a/cmd/terraform/planfile/upload.go +++ b/cmd/terraform/planfile/upload.go @@ -156,6 +156,16 @@ func runUpload(cmd *cobra.Command, args []string) error { } } + // Include the plan record when the component keeps one, so another runner can verify and apply the plan. + entry, closer, err := planfile.OpenBinding(planfilePath) + if err != nil { + return err + } + if closer != nil { + defer closer.Close() + files = append(files, entry) + } + // Resolve SHA: explicit flag > context resolution. sha := opts.SHA if sha == "" { @@ -264,8 +274,7 @@ func resolveUploadPlanfilePath(opts *UploadOptions, atmosConfig *schema.AtmosCon return "", fmt.Errorf("%w: failed to resolve component path: %w", errUtils.ErrPlanfileUploadFailed, err) } - planfilePath := exec.ConstructTerraformComponentPlanfilePath(atmosConfig, &info) - return planfilePath, nil + return canonicalPlanfilePath(atmosConfig, &info) } // resolveLockfilePath returns the lock file path from the explicit flag or auto-detected diff --git a/cmd/terraform/provider_options_prerun_test.go b/cmd/terraform/provider_options_prerun_test.go new file mode 100644 index 00000000000..09797c52b55 --- /dev/null +++ b/cmd/terraform/provider_options_prerun_test.go @@ -0,0 +1,31 @@ +package terraform + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/component/provideroptions" +) + +// Before-hooks describe the component through discovery, so the hook context carries the provider +// options the user set, exactly as RunE resolves them. +func TestPrepareHookContextResolvesProviderOptions(t *testing.T) { + t.Chdir("../../tests/fixtures/scenarios/terraform-apply-all-dependencies") + root := &cobra.Command{Use: "terraform"} + cmd := newHookTestCmd() + root.AddCommand(cmd) + options, err := provideroptions.Parse("prerun", []any{ + map[string]any{"name": "feature", "type": "map", "description": "Features.", "commands": []any{"plan"}}, + }) + require.NoError(t, err) + require.NoError(t, provideroptions.Register(root, []provideroptions.Provider{{Section: "prerun", Name: "prerun", Mode: provideroptions.ModeNative, Options: options}})) + require.NoError(t, cmd.ParseFlags([]string{"--stack", "reported-order-alias", "--prerun-feature", "tier=premium"})) + + ctx, err := prepareHookContext(cmd, []string{"platform-kms"}) + + require.NoError(t, err) + assert.Equal(t, map[string]any{"tier": "premium"}, ctx.info.ComponentProviderOptions["prerun"]["feature"].Value) +} diff --git a/cmd/terraform/shared/execution.go b/cmd/terraform/shared/execution.go index cfb22c45775..9133f56d2dd 100644 --- a/cmd/terraform/shared/execution.go +++ b/cmd/terraform/shared/execution.go @@ -166,7 +166,7 @@ func HandleInteractiveComponentStackSelection(info *schema.ConfigAndStacksInfo, } if info.Stack != "" && info.ComponentFromArg == "" { - if err := ValidateStackExists(cmd, info.Stack); err != nil { + if err := ValidateStackExists(cmd, info.Stack, info); err != nil { return err } } @@ -189,7 +189,7 @@ func promptMissingComponent(info *schema.ConfigAndStacksInfo, cmd *cobra.Command if info.ComponentFromArg != "" { return nil } - component, err := PromptForComponent(cmd, info.Stack) + component, err := PromptForComponent(cmd, info.Stack, info) if err = HandlePromptError(err, "component"); err != nil { return err } @@ -202,7 +202,7 @@ func promptMissingStack(info *schema.ConfigAndStacksInfo, cmd *cobra.Command) er if info.Stack != "" { return nil } - stack, err := PromptForStack(cmd, info.ComponentFromArg) + stack, err := PromptForStack(cmd, info.ComponentFromArg, info) if err = HandlePromptError(err, "stack"); err != nil { return err } diff --git a/cmd/terraform/shared/execution_coverage_test.go b/cmd/terraform/shared/execution_coverage_test.go index bbb3e92ee87..3a7d6d7ac70 100644 --- a/cmd/terraform/shared/execution_coverage_test.go +++ b/cmd/terraform/shared/execution_coverage_test.go @@ -637,6 +637,40 @@ func TestPromptMissingComponentAndStack_SetsPromptedFlags(t *testing.T) { assert.False(t, preSupplied.StackPrompted, "command-line-supplied stack must not be flagged as prompted") } +func TestInteractiveSelectionPassesProviderOptionsToDiscovery(t *testing.T) { + origInteractive, origSelect := isInteractiveFn, selectFromOptions + origDescribe, origInit := executeDescribeStacks, initCliConfig + t.Cleanup(func() { + isInteractiveFn, selectFromOptions = origInteractive, origSelect + executeDescribeStacks, initCliConfig = origDescribe, origInit + }) + + options := map[string]map[string]schema.ComponentProviderOption{ + "terragrunt": {"feature": {}}, + } + var seen int + initCliConfig = func(info schema.ConfigAndStacksInfo, _ bool) (schema.AtmosConfiguration, error) { + assert.Equal(t, options, info.ComponentProviderOptions) + seen++ + return schema.AtmosConfiguration{}, nil + } + isInteractiveFn = func() bool { return true } + executeDescribeStacks = func(_ *schema.AtmosConfiguration, _ string, _, _, _ []string, _, _, _, _ bool, _ []string, _ auth.AuthManager) (map[string]any, error) { + return map[string]any{"dev": map[string]any{ + "components": map[string]any{"terraform": map[string]any{"vpc": map[string]any{}}}, + }}, nil + } + selectFromOptions = func(_ string, _ string, values []string) (string, error) { return values[0], nil } + + cmd := &cobra.Command{Use: "plan"} + cmd.Flags().String("stack", "", "Stack") + info := &schema.ConfigAndStacksInfo{ComponentProviderOptions: options} + require.NoError(t, HandleInteractiveComponentStackSelection(info, cmd)) + assert.Equal(t, "vpc", info.ComponentFromArg) + assert.Equal(t, "dev", info.Stack) + assert.Equal(t, 2, seen, "component and stack discovery must both receive the options") +} + func TestHandleInteractiveComponentStackSelectionPropagatesComponentPromptError(t *testing.T) { // Force interactive mode and make the component picker's underlying stack // enumeration fail, so PromptForComponent surfaces ErrLoadSelectionOptions diff --git a/cmd/terraform/shared/prompt.go b/cmd/terraform/shared/prompt.go index 1e751fc9345..742515c4c29 100644 --- a/cmd/terraform/shared/prompt.go +++ b/cmd/terraform/shared/prompt.go @@ -51,18 +51,22 @@ var ( // buildConfigAndStacksInfo creates a ConfigAndStacksInfo populated with global CLI flags. // This ensures --base-path, --config, --config-path, and --profile flags are respected. -func buildConfigAndStacksInfo(cmd *cobra.Command) schema.ConfigAndStacksInfo { +func buildConfigAndStacksInfo(cmd *cobra.Command, source ...*schema.ConfigAndStacksInfo) schema.ConfigAndStacksInfo { if cmd == nil { return schema.ConfigAndStacksInfo{} } v := viper.GetViper() globalFlags := flags.ParseGlobalFlags(cmd, v) - return schema.ConfigAndStacksInfo{ + info := schema.ConfigAndStacksInfo{ AtmosBasePath: globalFlags.BasePath, AtmosConfigFilesFromArg: globalFlags.Config, AtmosConfigDirsFromArg: globalFlags.ConfigPath, ProfilesFromArg: globalFlags.Profile, } + if len(source) > 0 && source[0] != nil { + info.ComponentProviderOptions = source[0].ComponentProviderOptions + } + return info } // PromptForComponent shows an interactive selector for component selection. @@ -72,14 +76,14 @@ func buildConfigAndStacksInfo(cmd *cobra.Command) schema.ConfigAndStacksInfo { // handling so that a failure to enumerate components (e.g. `describe stacks` errored) // surfaces the real cause, and an empty list yields a clear "no components" message — // instead of silently falling through to a misleading "component is required" error. -func PromptForComponent(cmd *cobra.Command, stack string) (string, error) { +func PromptForComponent(cmd *cobra.Command, stack string, source ...*schema.ConfigAndStacksInfo) (string, error) { defer perf.Track(nil, "shared.PromptForComponent")() if !isInteractiveFn() { return "", nil // Non-interactive: let the caller's required-arg validation handle it. } - components, err := listTerraformComponentsForStack(cmd, stack) + components, err := listTerraformComponentsForStack(cmd, stack, source...) if err != nil { return "", errUtils.Build(errUtils.ErrLoadSelectionOptions). WithCause(err). @@ -112,14 +116,14 @@ func noComponentsExplanation(stack string) string { // // Like PromptForComponent, it surfaces load errors and an explicit "no stacks" // message instead of silently returning empty. -func PromptForStack(cmd *cobra.Command, component string) (string, error) { +func PromptForStack(cmd *cobra.Command, component string, source ...*schema.ConfigAndStacksInfo) (string, error) { defer perf.Track(nil, "shared.PromptForStack")() if !isInteractiveFn() { return "", nil // Non-interactive: let the caller's required-arg validation handle it. } - stacks, err := stacksForSelection(cmd, component) + stacks, err := stacksForSelection(cmd, component, source...) if err != nil { return "", errUtils.Build(errUtils.ErrLoadSelectionOptions). WithCause(err). @@ -153,11 +157,11 @@ func PromptForStack(cmd *cobra.Command, component string) (string, error) { // stacksForSelection returns the candidate stacks for the picker, filtered to the // given component when one was already chosen. -func stacksForSelection(cmd *cobra.Command, component string) ([]string, error) { +func stacksForSelection(cmd *cobra.Command, component string, source ...*schema.ConfigAndStacksInfo) ([]string, error) { if component != "" { - return listStacksForComponent(cmd, component) + return listStacksForComponent(cmd, component, source...) } - return listAllStacks(cmd) + return listAllStacks(cmd, source...) } // noStacksExplanation tailors the empty-stack message to whether a component filter applied. @@ -295,8 +299,8 @@ func FilterDeployableComponents(terraformComponents map[string]any) []string { // listTerraformComponents lists all deployable terraform components across all stacks. // Filters out abstract and disabled components. // The cmd parameter is used to respect global CLI flags (--base-path, --config, --config-path, --profile). -func listTerraformComponents(cmd *cobra.Command) ([]string, error) { - configAndStacksInfo := buildConfigAndStacksInfo(cmd) +func listTerraformComponents(cmd *cobra.Command, source ...*schema.ConfigAndStacksInfo) ([]string, error) { + configAndStacksInfo := buildConfigAndStacksInfo(cmd, source...) atmosConfig, err := initCliConfig(configAndStacksInfo, true) if err != nil { return nil, err @@ -335,12 +339,12 @@ func listTerraformComponents(cmd *cobra.Command) ([]string, error) { // Filters out abstract and disabled components. // If stack is empty, returns components from all stacks. // The cmd parameter is used to respect global CLI flags (--base-path, --config, --config-path, --profile). -func listTerraformComponentsForStack(cmd *cobra.Command, stack string) ([]string, error) { +func listTerraformComponentsForStack(cmd *cobra.Command, stack string, source ...*schema.ConfigAndStacksInfo) ([]string, error) { if stack == "" { - return listTerraformComponents(cmd) + return listTerraformComponents(cmd, source...) } - configAndStacksInfo := buildConfigAndStacksInfo(cmd) + configAndStacksInfo := buildConfigAndStacksInfo(cmd, source...) atmosConfig, err := initCliConfig(configAndStacksInfo, true) if err != nil { return nil, err @@ -378,8 +382,8 @@ func listTerraformComponentsForStack(cmd *cobra.Command, stack string) ([]string // listStacksForComponent returns stacks that contain the specified component. // The cmd parameter is used to respect global CLI flags (--base-path, --config, --config-path, --profile). -func listStacksForComponent(cmd *cobra.Command, component string) ([]string, error) { - configAndStacksInfo := buildConfigAndStacksInfo(cmd) +func listStacksForComponent(cmd *cobra.Command, component string, source ...*schema.ConfigAndStacksInfo) ([]string, error) { + configAndStacksInfo := buildConfigAndStacksInfo(cmd, source...) atmosConfig, err := initCliConfig(configAndStacksInfo, true) if err != nil { return nil, err @@ -432,8 +436,8 @@ func stackContainsComponent(stackData any, component string) bool { // listAllStacks returns all stacks. // The cmd parameter is used to respect global CLI flags (--base-path, --config, --config-path, --profile). -func listAllStacks(cmd *cobra.Command) ([]string, error) { - configAndStacksInfo := buildConfigAndStacksInfo(cmd) +func listAllStacks(cmd *cobra.Command, source ...*schema.ConfigAndStacksInfo) ([]string, error) { + configAndStacksInfo := buildConfigAndStacksInfo(cmd, source...) atmosConfig, err := initCliConfig(configAndStacksInfo, true) if err != nil { return nil, err @@ -455,8 +459,8 @@ func listAllStacks(cmd *cobra.Command) ([]string, error) { // ValidateStackExists checks if the provided stack name exists and returns // an error with suggestions if it doesn't. // The cmd parameter is used to respect global CLI flags (--base-path, --config, --config-path, --profile). -func ValidateStackExists(cmd *cobra.Command, stack string) error { - stacks, err := listAllStacks(cmd) +func ValidateStackExists(cmd *cobra.Command, stack string, source ...*schema.ConfigAndStacksInfo) error { + stacks, err := listAllStacks(cmd, source...) if err != nil { return err } diff --git a/cmd/terraform/terraform.go b/cmd/terraform/terraform.go index 3092a51474e..0e957a4d165 100644 --- a/cmd/terraform/terraform.go +++ b/cmd/terraform/terraform.go @@ -1,6 +1,7 @@ package terraform import ( + "context" "strings" "github.com/spf13/cobra" @@ -19,6 +20,9 @@ import ( cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/flags" "github.com/cloudposse/atmos/pkg/flags/compat" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/perf" + pluginbootstrap "github.com/cloudposse/atmos/pkg/plugin/bootstrap" "github.com/cloudposse/atmos/pkg/schema" ) @@ -106,6 +110,32 @@ func init() { ) } +// RegisterProviderFlags adds the options that enabled configuration providers declare in their +// package metadata to the terraform subcommands they apply to. It reads static metadata only, so no +// plugin starts, and it runs before Cobra parses the command line. +func RegisterProviderFlags(config *schema.AtmosConfiguration, args []string) error { + defer perf.Track(config, "terraform.RegisterProviderFlags")() + + err := pluginbootstrap.RegisterComponentFlags(context.Background(), config, terraformCmd) + if err != nil && !targetsTerraform(args) { + // A package that declares a bad option must not break commands that never run a unit. + log.Debug("Provider flag registration failed", "error", err) + return nil + } + return err +} + +// targetsTerraform reports whether the command line selects `atmos terraform` or one of its subcommands. +func targetsTerraform(args []string) bool { + target, _, _ := terraformCmd.Root().Find(args) + for command := target; command != nil; command = command.Parent() { + if command == terraformCmd { + return true + } + } + return false +} + // TerraformCommandProvider implements the CommandProvider interface. type TerraformCommandProvider struct{} diff --git a/cmd/terraform/utils.go b/cmd/terraform/utils.go index 1e8a6163eba..be8d6c7b44c 100644 --- a/cmd/terraform/utils.go +++ b/cmd/terraform/utils.go @@ -31,6 +31,7 @@ import ( citerraform "github.com/cloudposse/atmos/pkg/ci/plugins/terraform" "github.com/cloudposse/atmos/pkg/ci/plugins/terraform/planfile" "github.com/cloudposse/atmos/pkg/component" + "github.com/cloudposse/atmos/pkg/component/provideroptions" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/flags" "github.com/cloudposse/atmos/pkg/flags/compat" @@ -40,6 +41,7 @@ import ( "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/proexec" "github.com/cloudposse/atmos/pkg/provisioner" + provWorkdir "github.com/cloudposse/atmos/pkg/provisioner/workdir" "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/store/authbridge" u "github.com/cloudposse/atmos/pkg/utils" @@ -180,6 +182,11 @@ func preResolveInteractiveSelection(cmd_ *cobra.Command, args []string) error { if err != nil { return err } + // The prompt discovers components, so it must see the same provider options RunE uses: a feature + // override can decide which dependencies an imported unit has. + if info.ComponentProviderOptions, err = provideroptions.Resolve(cmd_, viper.GetViper()); err != nil { + return err + } // resolveAndPromptForArgs is a no-op when not interactive or when both values are // already provided; otherwise it shows the component/stack pickers. @@ -359,6 +366,10 @@ func prepareHookContext(cmd_ *cobra.Command, args []string) (hookContext, error) // Honor a component chosen interactively in PreRunE so before-hooks resolve // against the selected component (the stack comes from the persisted --stack flag). applyPreResolvedComponent(&info) + // Hooks describe the component through discovery, which must see the provider options RunE uses. + if info.ComponentProviderOptions, err = provideroptions.Resolve(cmd_, viper.GetViper()); err != nil { + return hookContext{info: info}, err + } if authCtx, authMgr := e.GetLastAuthContext(); authCtx != nil { info.AuthContext = authCtx @@ -1823,6 +1834,10 @@ func terraformRunWithOptions(parentCmd, actualCmd *cobra.Command, args []string, // Apply parsed options to info BEFORE prompting, so hasMultiComponentFlags() works correctly. // This fixes issue #1945: --all flag must be set before resolveAndPromptForArgs checks it. applyOptionsToInfo(&info, opts) + // Provider options are validated here, before any discovery or preparation can start a plugin. + if info.ComponentProviderOptions, err = provideroptions.Resolve(actualCmd, viper.GetViper()); err != nil { + return err + } applyFromPlanSelection(&info, actualCmd) // Honor a component already chosen interactively in PreRunE (runBeforeHooks) so the @@ -1999,6 +2014,9 @@ func verifyStoredPlanForDeploy(subCommand string, info *schema.ConfigAndStacksIn } canonicalPlanPath := e.ConstructTerraformComponentPlanfilePath(&verifyAtmosConfig, info) + if workdirPath, ok := retainedWorkdirPlanfilePath(&verifyAtmosConfig, info); ok { + canonicalPlanPath = workdirPath + } storedPlanPath := filepath.Join(filepath.Dir(canonicalPlanPath), planfile.StoredPlanPrefix+planfile.PlanFilename) if _, statErr := os.Stat(storedPlanPath); statErr != nil { // No stored planfile was downloaded. Whether that blocks the deploy is @@ -2017,6 +2035,21 @@ func verifyStoredPlanForDeploy(subCommand string, info *schema.ConfigAndStacksIn return verifyPlanfile(info, storedPlanPath, mode) } +// retainedWorkdirPlanfilePath returns the plan file path inside the workdir of a workdir-enabled +// component (every Terragrunt unit), where `plan` keeps its plan and the CI hooks store the +// downloaded one. It derives the directory from configuration, so it works before anything is +// provisioned on a fresh runner. It reports false for components without a workdir. +func retainedWorkdirPlanfilePath(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (string, bool) { + if !provWorkdir.IsWorkdirEnabled(info.ComponentSection) { + return "", false + } + workdir, _, err := component.BuildAndResolveWorkdirPath(atmosConfig, info, cfg.TerraformComponentType) + if err != nil { + return "", false + } + return filepath.Join(workdir, filepath.Base(e.ConstructTerraformComponentPlanfilePath(atmosConfig, info))), true +} + // handleUnconfiguredPlanfileStorage resolves a verification request that cannot be // honored because no planfile storage is configured. An explicit --verify-plan / // ATMOS_TERRAFORM_VERIFY_PLAN=true errors: verification depends on storage settings @@ -2145,11 +2178,15 @@ func handlePromptError(err error, name string) error { // promptForComponent delegates to shared.PromptForComponent. // If stack is provided, filters components to only those in that stack. // Declared as a var so tests can stub the interactive prompt. -var promptForComponent = shared.PromptForComponent +var promptForComponent = func(cmd *cobra.Command, stack string) (string, error) { + return shared.PromptForComponent(cmd, stack) +} // promptForStack delegates to shared.PromptForStack. // Declared as a var so tests can stub the interactive prompt. -var promptForStack = shared.PromptForStack +var promptForStack = func(cmd *cobra.Command, component string) (string, error) { + return shared.PromptForStack(cmd, component) +} // enableHeatmapIfRequested checks os.Args for --heatmap flag and enables performance tracking. // This is needed because --heatmap must be detected before flag parsing occurs. diff --git a/cmd/terraform/utils_test.go b/cmd/terraform/utils_test.go index 6300d9b85d9..a229098482a 100644 --- a/cmd/terraform/utils_test.go +++ b/cmd/terraform/utils_test.go @@ -3,6 +3,7 @@ package terraform import ( "bytes" "errors" + "path/filepath" "strings" "testing" @@ -1325,3 +1326,24 @@ func normalizeUIOutput(s string) string { } return strings.Join(strings.Fields(b.String()), " ") } + +func TestRetainedWorkdirPlanfilePathLocatesTheWorkdirOfAFreshRunner(t *testing.T) { + base := t.TempDir() + config := &schema.AtmosConfiguration{BasePath: base, TerraformDirAbsolutePath: filepath.Join(base, "components", "terraform")} + config.Components.Terraform.BasePath = filepath.Join("components", "terraform") + + unit := &schema.ConfigAndStacksInfo{ + Stack: "live/dev", ComponentFromArg: "app", FinalComponent: "app", + ComponentSettingsSection: map[string]any{"configuration": true}, + ComponentSection: map[string]any{"provision": map[string]any{"workdir": map[string]any{"enabled": true}}}, + } + path, ok := retainedWorkdirPlanfilePath(config, unit) + require.True(t, ok) + require.Contains(t, filepath.ToSlash(path), ".workdir/terraform/live/dev-app-") + require.NotContains(t, path, filepath.Join("components", "terraform")) + require.NoDirExists(t, filepath.Dir(path), "locating the workdir must not provision it") + + ordinary := &schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "vpc", FinalComponent: "vpc", ComponentSection: map[string]any{}} + _, ok = retainedWorkdirPlanfilePath(config, ordinary) + require.False(t, ok, "components without a workdir keep the canonical path") +} diff --git a/errors/plugin.go b/errors/plugin.go index abeb0df834e..f8e610a519c 100644 --- a/errors/plugin.go +++ b/errors/plugin.go @@ -97,7 +97,17 @@ var ( ErrComponentRunIgnored = errors.New("component run ended by an ignored error") // ErrComponentExcluded ends a component run successfully because the provider excluded the operation. ErrComponentExcluded = errors.New("component excluded from this operation") - // ErrComponentOutputCapture identifies command output that could not be recorded for error hook matching. + // ErrProviderOption identifies a configuration provider option that is declared or supplied incorrectly. + ErrProviderOption = errors.New("invalid configuration provider option") + // ErrProviderOptionCollision identifies a provider option whose flag name an Atmos or another provider flag already uses. + ErrProviderOptionCollision = errors.New("configuration provider option collides with an existing flag") + // ErrExecutableModeUnsupported identifies a command or stack override that executable mode cannot honor. + ErrExecutableModeUnsupported = errors.New("not supported in executable mode") + // ErrExecutableNotFound identifies a missing or unusable terragrunt executable in executable mode. + ErrExecutableNotFound = errors.New("terragrunt executable not found") + // ErrExecutableVersion identifies a terragrunt executable that is older than the supported minimum. + ErrExecutableVersion = errors.New("terragrunt version is not supported") + // ErrComponentOutputCapture identifies output that could not be recorded for error hook matching. ErrComponentOutputCapture = errors.New("failed to capture command output for error hooks") ErrInvalidCredentialCodec = errors.New("invalid credential codec") diff --git a/examples/terragrunt-native/README.md b/examples/terragrunt-native/README.md index 754f4d92e62..8b63a9953c7 100644 --- a/examples/terragrunt-native/README.md +++ b/examples/terragrunt-native/README.md @@ -18,36 +18,42 @@ run the Terragrunt executable. The module uses the built-in `terraform_data` resource and local state. No cloud credentials or provider downloads are needed. -## Build the development package +## Install the plugin -This example requires the experimental Atmos branch and its local Terragrunt -plugin. See [plugin usage](https://atmos.tools/cli/commands/plugin/usage) for package configuration. The plugin is not yet published. Install Go and OpenTofu, then run these -commands from the root of the Atmos checkout containing this example: +The Terragrunt configuration provider is a plugin package that Atmos installs from the registry. +`atmos.yaml` declares it with a source and a version constraint: -```shell -go build -o build/atmos . -export PATH="$PWD/build:$PATH" -atmos plugin build plugins/terragrunt -export ATMOS_TERRAGRUNT_PLUGIN_PATH="$PWD/plugins/terragrunt" +```yaml +plugins: + packages: + terragrunt: + source: oci://ghcr.io/cloudposse/atmos/plugins/terragrunt + version: "^0.1.0" ``` -Copy the example into a new directory and select its `dev` profile, which points the -plugin package at your build: +Install Atmos and OpenTofu, copy the example into a new directory, and install the plugin: ```shell atmos init ./examples/terragrunt-native ./terragrunt-native \ --copy --no-git cd terragrunt-native -export ATMOS_PROFILE=dev +atmos plugin install ``` -The example already enables native discovery, and `profiles/dev/atmos.yaml` reads the -plugin path from `ATMOS_TERRAGRUNT_PLUGIN_PATH`. To enable the same behavior in an -existing project, run these commands from that project's root: +`atmos plugin install` downloads the release, verifies its digest, and writes it to +`.atmos.plugins.lock.yaml`. Commit that file: the lock pins the exact release, so every machine +and CI run restores the same bytes, and CI refuses to create a new lock entry on its own. +Without this step the first command that needs the plugin installs it and creates the lock entry; +`atmos plugin install --frozen` restores exactly what the lock records and never changes it. +If the package cannot be fetched (for example because the registry is unreachable or the release is +not yet published), Atmos reports which source and version constraint it could not prepare and +changes nothing. + +To enable the same behavior in an existing project, run these commands from that project's root: ```shell -atmos config set plugins.packages.terragrunt.path \ - "$ATMOS_TERRAGRUNT_PLUGIN_PATH" +atmos plugin install oci://ghcr.io/cloudposse/atmos/plugins/terragrunt \ + --as=terragrunt atmos config set components.terraform.base_path \ components/terraform atmos config set components.terraform.command tofu @@ -60,14 +66,30 @@ atmos config set \ components.terraform.terragrunt.discovery.enabled true ``` -In an existing project the plugin path is stored in `atmos.yaml`, so Atmos does not -need the environment variable on subsequent runs. Adjust `included_paths` to match your Terragrunt directory layout; +Adjust `included_paths` to match your Terragrunt directory layout; see [discovery paths](https://atmos.tools/migration/terragrunt#discovery-paths) for the pattern rules. `components.terraform.base_path` is required by Atmos even though Terragrunt units do not live there; the directory does not need to exist. If you use Terraform instead, set `components.terraform.command` to `terraform` and install Terraform 1.4 or newer. +## Develop against a local build + +Only plugin developers need this. The `dev` profile replaces the registry package with a local +build, which is never fetched, locked, or signature-checked. From the root of the Atmos checkout +containing this example: + +```shell +go build -o build/atmos . +export PATH="$PWD/build:$PATH" +atmos plugin build plugins/terragrunt +export ATMOS_TERRAGRUNT_PLUGIN_PATH="$PWD/plugins/terragrunt" +export ATMOS_PROFILE=dev +``` + +Run the commands below from the copied example directory with `ATMOS_PROFILE=dev` set. +`profiles/dev/atmos.yaml` reads the plugin path from `ATMOS_TERRAGRUNT_PLUGIN_PATH`. + ## Discover and plan ```shell diff --git a/examples/terragrunt-native/atmos.yaml b/examples/terragrunt-native/atmos.yaml index cc43645367c..c3248752bdf 100644 --- a/examples/terragrunt-native/atmos.yaml +++ b/examples/terragrunt-native/atmos.yaml @@ -6,10 +6,22 @@ components: base_path: components/terraform command: tofu # Settings for the Terragrunt configuration provider. The provider itself comes from - # plugins.packages.terragrunt; the `dev` profile supplies a local build (see profiles/dev). + # plugins.packages.terragrunt below. terragrunt: enabled: true mode: native discovery: enabled: true included_paths: [live/**] + +# The Terragrunt configuration provider is installed from the registry. The first command that needs it +# creates the lock entry (`.atmos.plugins.lock.yaml`); `atmos plugin install` does the same up front, +# and `atmos plugin install --frozen` restores exactly what the lock records, as CI does. +# +# To run a local build while developing the plugin itself, activate the `dev` profile instead +# (ATMOS_PROFILE=dev, see profiles/dev/atmos.yaml): its `path` takes the place of this registry package. +plugins: + packages: + terragrunt: + source: oci://ghcr.io/cloudposse/atmos/plugins/terragrunt + version: "^0.1.0" diff --git a/examples/terragrunt-native/profiles/dev/atmos.yaml b/examples/terragrunt-native/profiles/dev/atmos.yaml index cb44173e002..53b119121ff 100644 --- a/examples/terragrunt-native/profiles/dev/atmos.yaml +++ b/examples/terragrunt-native/profiles/dev/atmos.yaml @@ -4,8 +4,8 @@ # export ATMOS_PROFILE=dev # # A package with `path` is not fetched, locked, or signature-checked, and Atmos warns once per run. -# Once the plugin is published, declare `source` and `version` under plugins.packages.terragrunt in -# atmos.yaml and drop this profile. +# This profile is for developing the plugin itself. Everyone else uses the registry package declared +# by `source` and `version` in atmos.yaml; a `path` takes precedence over them while this profile is active. plugins: packages: terragrunt: diff --git a/pkg/ci/plugins/terraform/handlers.go b/pkg/ci/plugins/terraform/handlers.go index 65848980cca..ca7f97a8c2b 100644 --- a/pkg/ci/plugins/terraform/handlers.go +++ b/pkg/ci/plugins/terraform/handlers.go @@ -569,6 +569,16 @@ func (p *Plugin) uploadPlanfile(ctx *plugin.HookContext) error { files = append(files, planfile.FileEntry{Name: planfile.LockFilename, Data: lf, Size: -1}) } + // Include the plan record when the component keeps one, so another runner can verify and apply the plan. + entry, closer, err := planfile.OpenBinding(planfilePath) + if err != nil { + return err + } + if closer != nil { + defer closer.Close() + files = append(files, entry) + } + metadata := p.buildPlanfileMetadata(ctx) if err := metadata.Validate(); err != nil { return errUtils.Build(errUtils.ErrPlanfileUploadFailed).WithCause(err). diff --git a/pkg/ci/plugins/terraform/handlers_test.go b/pkg/ci/plugins/terraform/handlers_test.go index 6cb807ffe40..66670ea3a25 100644 --- a/pkg/ci/plugins/terraform/handlers_test.go +++ b/pkg/ci/plugins/terraform/handlers_test.go @@ -12,8 +12,12 @@ import ( "github.com/stretchr/testify/require" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/ci/artifact" + _ "github.com/cloudposse/atmos/pkg/ci/artifact/local" // Register the local artifact store. "github.com/cloudposse/atmos/pkg/ci/internal/plugin" "github.com/cloudposse/atmos/pkg/ci/internal/provider" + "github.com/cloudposse/atmos/pkg/ci/plugins/terraform/planfile" + "github.com/cloudposse/atmos/pkg/ci/plugins/terraform/planfile/adapter" "github.com/cloudposse/atmos/pkg/ci/templates" cfg "github.com/cloudposse/atmos/pkg/config" provWorkdir "github.com/cloudposse/atmos/pkg/provisioner/workdir" @@ -2622,3 +2626,75 @@ func TestDestroyBinding_WarningsStayInsideCallouts(t *testing.T) { } assert.NotContains(t, summary, "> > Warning:") } + +// TestUploadPlanfile_CarriesThePlanRecordToAnotherRunner uploads a plan with its record through a real +// local planfile store and restores it the way `planfile download` does, so a different runner gets +// both files and can verify and apply the plan. +func TestUploadPlanfile_CarriesThePlanRecordToAnotherRunner(t *testing.T) { + storeDir := filepath.Join(t.TempDir(), "store") + backend, err := artifact.NewStore(artifact.StoreOptions{Type: "local/dir", Options: map[string]any{"path": storeDir}}) + require.NoError(t, err) + store := adapter.NewStore(backend) + + runnerA := t.TempDir() + planPath := filepath.Join(runnerA, "atmos-key.planfile") + require.NoError(t, os.WriteFile(planPath, []byte("plan bytes"), 0o600)) + require.NoError(t, os.WriteFile(planfile.BindingPath(planPath), []byte(`{"Version":3}`), 0o600)) + + ctx := &plugin.HookContext{ + Config: &schema.AtmosConfiguration{CI: schema.CIConfig{Enabled: true}}, + Command: "plan", + CICtx: &provider.Context{SHA: "abc123"}, + Info: &schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "app", PlanFile: planPath}, + CreatePlanfileStore: func() (any, error) { + return store, nil + }, + } + require.NoError(t, (&Plugin{}).uploadPlanfile(ctx)) + + key, err := (&Plugin{}).getArtifactKey(ctx.Info, ctx.CICtx) + require.NoError(t, err) + results, _, err := store.Download(t.Context(), key) + require.NoError(t, err) + defer func() { + for _, r := range results { + r.Data.Close() + } + }() + runnerB := filepath.Join(t.TempDir(), "atmos-key.planfile") + require.NoError(t, planfile.WritePlanfileResults(results, runnerB)) + + restoredPlan, err := os.ReadFile(runnerB) + require.NoError(t, err) + assert.Equal(t, "plan bytes", string(restoredPlan)) + restoredRecord, err := os.ReadFile(planfile.BindingPath(runnerB)) + require.NoError(t, err) + assert.Equal(t, `{"Version":3}`, string(restoredRecord)) +} + +// A plan without a record (every ordinary component) uploads exactly what it did before. +func TestUploadPlanfile_OrdinaryPlanUploadsNoRecord(t *testing.T) { + backend, err := artifact.NewStore(artifact.StoreOptions{Type: "local/dir", Options: map[string]any{"path": filepath.Join(t.TempDir(), "store")}}) + require.NoError(t, err) + store := adapter.NewStore(backend) + planPath := filepath.Join(t.TempDir(), "plan.tfplan") + require.NoError(t, os.WriteFile(planPath, []byte("plan bytes"), 0o600)) + ctx := &plugin.HookContext{ + Config: &schema.AtmosConfiguration{CI: schema.CIConfig{Enabled: true}}, + Command: "plan", + CICtx: &provider.Context{SHA: "abc123"}, + Info: &schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "vpc", PlanFile: planPath}, + CreatePlanfileStore: func() (any, error) { return store, nil }, + } + require.NoError(t, (&Plugin{}).uploadPlanfile(ctx)) + key, err := (&Plugin{}).getArtifactKey(ctx.Info, ctx.CICtx) + require.NoError(t, err) + results, _, err := store.Download(t.Context(), key) + require.NoError(t, err) + names := []string{} + for _, r := range results { + names = append(names, r.Name) + r.Data.Close() + } + assert.Equal(t, []string{planfile.PlanFilename}, names) +} diff --git a/pkg/ci/plugins/terraform/planfile/interface.go b/pkg/ci/plugins/terraform/planfile/interface.go index be7db7b442a..65c74ca3ebe 100644 --- a/pkg/ci/plugins/terraform/planfile/interface.go +++ b/pkg/ci/plugins/terraform/planfile/interface.go @@ -27,6 +27,15 @@ const ( // LockFilename is the well-known name for the lock file within a bundle. LockFilename = ".terraform.lock.hcl" + // BindingFilename is the well-known name for the plan record within a bundle. Components + // whose configuration provider owns their saved plans (for example Terragrunt units) keep a + // record beside the plan file that proves what was reviewed; without it another runner cannot + // apply the plan. Components that have no record upload nothing under this name. + BindingFilename = PlanFilename + ".atmos.json" + + // BindingSuffix is the suffix of the record stored beside a plan file on disk. + bindingSuffix = ".atmos.json" + // StoredPlanPrefix is the prefix used for stored planfiles during plan verification. // When --verify-plan is set in CI mode, the downloaded planfile gets this prefix // so terraform can generate a fresh plan at the canonical path for comparison. diff --git a/pkg/ci/plugins/terraform/planfile/write.go b/pkg/ci/plugins/terraform/planfile/write.go index 77e65195a97..1ffb26d127a 100644 --- a/pkg/ci/plugins/terraform/planfile/write.go +++ b/pkg/ci/plugins/terraform/planfile/write.go @@ -1,8 +1,10 @@ package planfile import ( + "errors" "fmt" "io" + "io/fs" "os" "path/filepath" @@ -10,8 +12,41 @@ import ( "github.com/cloudposse/atmos/pkg/perf" ) +const ( + // Mode of a downloaded plan or lock file. + downloadedFileMode os.FileMode = 0o644 + // Owner-only mode of a downloaded plan record. + privateFileMode os.FileMode = 0o600 +) + +// BindingPath returns where the plan record of a plan file is kept on disk. +func BindingPath(planfilePath string) string { + defer perf.Track(nil, "planfile.BindingPath")() + + return planfilePath + bindingSuffix +} + +// OpenBinding opens the plan record beside a plan file for upload. It returns a nil closer when there +// is none, which is the normal case for components that do not keep one. A record that exists but cannot +// be opened is an error: uploading the plan without it would leave a fresh runner unable to apply it. +// The caller closes the returned file. +func OpenBinding(planfilePath string) (FileEntry, io.Closer, error) { + defer perf.Track(nil, "planfile.OpenBinding")() + + path := BindingPath(planfilePath) + file, err := os.Open(path) + switch { + case errors.Is(err, fs.ErrNotExist): + return FileEntry{}, nil, nil + case err != nil: + return FileEntry{}, nil, fmt.Errorf("%w: failed to open plan record %s: %w", errUtils.ErrPlanfileUploadFailed, path, err) + } + return FileEntry{Name: BindingFilename, Data: file, Size: -1}, file, nil +} + // WritePlanfileResults writes downloaded planfile results to disk. -// It maps PlanFilename to planfilePath and LockFilename to the same directory. +// It maps PlanFilename to planfilePath, BindingFilename to the plan record beside it, and +// LockFilename to the same directory. // Parent directories are created as needed. Unknown filenames are skipped. func WritePlanfileResults(results []FileResult, planfilePath string) error { defer perf.Track(nil, "planfile.WritePlanfileResults")() @@ -23,13 +58,15 @@ func WritePlanfileResults(results []FileResult, planfilePath string) error { switch r.Name { case PlanFilename: destPath = planfilePath + case BindingFilename: + destPath = BindingPath(planfilePath) case LockFilename: destPath = filepath.Join(componentDir, LockFilename) default: continue } - if err := writeFileResult(r, destPath); err != nil { + if err := writeFileResult(r, destPath, fileMode(r.Name)); err != nil { return err } } @@ -37,6 +74,15 @@ func WritePlanfileResults(results []FileResult, planfilePath string) error { return nil } +// fileMode keeps the plan record owner-only: it can carry generated credentials or extra-argument +// environment values, which other users of a shared runner must not read. +func fileMode(name string) os.FileMode { + if name == BindingFilename { + return privateFileMode + } + return downloadedFileMode +} + // WritePlanfileResultsForVerification writes downloaded planfile results for plan verification. // The plan file is written to storedPlanPath (with stored prefix), while the lock file // is written relative to canonicalPlanPath (the canonical location where terraform expects it). @@ -64,7 +110,7 @@ func WritePlanfileResultsForVerification(results []FileResult, storedPlanPath, c continue } - if err := writeFileResult(r, destPath); err != nil { + if err := writeFileResult(r, destPath, fileMode(r.Name)); err != nil { return err } } @@ -72,8 +118,8 @@ func WritePlanfileResultsForVerification(results []FileResult, storedPlanPath, c return nil } -// writeFileResult writes a single file result to the given destination path. -func writeFileResult(r FileResult, destPath string) error { +// writeFileResult writes a single file result to the given destination path with mode. +func writeFileResult(r FileResult, destPath string, mode os.FileMode) error { // Ensure parent directory exists. if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { return fmt.Errorf("%w: failed to create directory for %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, err) @@ -83,7 +129,29 @@ func writeFileResult(r FileResult, destPath string) error { if err != nil { return fmt.Errorf("%w: failed to read %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, err) } - if err := os.WriteFile(destPath, fileData, 0o644); err != nil { + flags := os.O_WRONLY | os.O_CREATE + if r.Name != BindingFilename { + flags |= os.O_TRUNC + } + file, err := os.OpenFile(destPath, flags, mode) + if err != nil { + return fmt.Errorf("%w: failed to write %s to %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, destPath, err) + } + // Restrict an existing record before replacing its contents. Other files retain + // their existing permissions, or the permissions selected by the umask. + if r.Name == BindingFilename { + if err := file.Chmod(mode); err != nil { + _ = file.Close() + return fmt.Errorf("%w: failed to write %s to %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, destPath, err) + } + if err := file.Truncate(0); err != nil { + _ = file.Close() + return fmt.Errorf("%w: failed to write %s to %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, destPath, err) + } + } + _, writeErr := file.Write(fileData) + closeErr := file.Close() + if err := errors.Join(writeErr, closeErr); err != nil { return fmt.Errorf("%w: failed to write %s to %s: %w", errUtils.ErrPlanfileDownloadFailed, r.Name, destPath, err) } diff --git a/pkg/ci/plugins/terraform/planfile/write_test.go b/pkg/ci/plugins/terraform/planfile/write_test.go index a21657de7a4..8325f7afb27 100644 --- a/pkg/ci/plugins/terraform/planfile/write_test.go +++ b/pkg/ci/plugins/terraform/planfile/write_test.go @@ -1,14 +1,18 @@ package planfile import ( + "fmt" "io" "os" "path/filepath" + "runtime" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" ) func TestWritePlanfileResults(t *testing.T) { @@ -202,3 +206,110 @@ func TestVerificationStopsOnUnreadableLockfile(t *testing.T) { require.NoError(t, err) assert.Equal(t, LockFilename, target, "a stat error must not overwrite the current lockfile") } + +func TestWritePlanfileResultsRestoresThePlanRecordBesideThePlan(t *testing.T) { + tmpDir := t.TempDir() + planPath := filepath.Join(tmpDir, "atmos-key.planfile") + results := []FileResult{ + {Name: PlanFilename, Data: io.NopCloser(strings.NewReader("plan data"))}, + {Name: BindingFilename, Data: io.NopCloser(strings.NewReader(`{"Version":3}`))}, + } + require.NoError(t, WritePlanfileResults(results, planPath)) + + record, err := os.ReadFile(BindingPath(planPath)) + require.NoError(t, err) + assert.JSONEq(t, `{"Version":3}`, string(record)) + assert.Equal(t, planPath+".atmos.json", BindingPath(planPath)) +} + +func TestWritePlanfileResultsForVerificationDoesNotRestoreARecord(t *testing.T) { + tmpDir := t.TempDir() + stored := filepath.Join(tmpDir, StoredPlanPrefix+PlanFilename) + canonical := filepath.Join(tmpDir, "atmos-key.planfile") + results := []FileResult{ + {Name: PlanFilename, Data: io.NopCloser(strings.NewReader("plan data"))}, + {Name: BindingFilename, Data: io.NopCloser(strings.NewReader("{}"))}, + } + require.NoError(t, WritePlanfileResultsForVerification(results, stored, canonical)) + + assert.FileExists(t, stored) + assert.NoFileExists(t, BindingPath(stored), "a stored plan is only compared against; the fresh plan gets its own record") + assert.NoFileExists(t, BindingPath(canonical)) +} + +func TestOpenBindingFindsOnlyAnExistingRecord(t *testing.T) { + planPath := filepath.Join(t.TempDir(), "atmos-key.planfile") + _, closer, err := OpenBinding(planPath) + require.NoError(t, err) + assert.Nil(t, closer, "components without a record upload nothing under the record name") + + require.NoError(t, os.WriteFile(BindingPath(planPath), []byte("record"), 0o600)) + entry, closer, err := OpenBinding(planPath) + require.NoError(t, err) + require.NotNil(t, closer) + defer closer.Close() + assert.Equal(t, BindingFilename, entry.Name) + data, err := io.ReadAll(entry.Data) + require.NoError(t, err) + assert.Equal(t, "record", string(data)) +} + +func TestOpenBindingReportsARecordThatCannotBeOpened(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("needs POSIX permission bits enforced for the current user") + } + planPath := filepath.Join(t.TempDir(), "atmos-key.planfile") + require.NoError(t, os.WriteFile(BindingPath(planPath), []byte("record"), 0o600)) + require.NoError(t, os.Chmod(BindingPath(planPath), 0o000)) + t.Cleanup(func() { _ = os.Chmod(BindingPath(planPath), 0o600) }) + + _, closer, err := OpenBinding(planPath) + require.ErrorIs(t, err, errUtils.ErrPlanfileUploadFailed, "a record that exists is never silently left out") + assert.Nil(t, closer) +} + +func TestWritePlanfileResultsKeepsThePlanRecordOwnerOnly(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not use POSIX permission bits") + } + planPath := filepath.Join(t.TempDir(), "atmos-key.planfile") + // An earlier, looser copy must not keep the replaced record readable by others. + require.NoError(t, os.WriteFile(BindingPath(planPath), []byte("old"), 0o644)) + require.NoError(t, os.Chmod(BindingPath(planPath), 0o644)) + results := []FileResult{ + {Name: PlanFilename, Data: io.NopCloser(strings.NewReader("plan data"))}, + {Name: BindingFilename, Data: io.NopCloser(strings.NewReader("{\"secret\":true}"))}, + } + require.NoError(t, WritePlanfileResults(results, planPath)) + + info, err := os.Stat(BindingPath(planPath)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + data, err := os.ReadFile(BindingPath(planPath)) + require.NoError(t, err) + assert.Equal(t, "{\"secret\":true}", string(data)) + plan, err := os.Stat(planPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0), plan.Mode().Perm()&^os.FileMode(0o644), "the plan file respects the umask") +} + +func TestWritePlanfileResultsPreservesExistingPlanPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not use POSIX permission bits") + } + for _, verification := range []bool{false, true} { + t.Run(fmt.Sprintf("verification=%t", verification), func(t *testing.T) { + planPath := filepath.Join(t.TempDir(), "plan") + require.NoError(t, os.WriteFile(planPath, []byte("old"), 0o600)) + results := []FileResult{{Name: PlanFilename, Data: io.NopCloser(strings.NewReader("new"))}} + if verification { + require.NoError(t, WritePlanfileResultsForVerification(results, planPath, planPath+"-canonical")) + } else { + require.NoError(t, WritePlanfileResults(results, planPath)) + } + info, err := os.Stat(planPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + }) + } +} diff --git a/pkg/component/configuration/files.go b/pkg/component/configuration/files.go index 4d16ef31aa3..a8e4c4419e8 100644 --- a/pkg/component/configuration/files.go +++ b/pkg/component/configuration/files.go @@ -15,9 +15,10 @@ import ( ) const ( - fileMode = 0o644 - secretMode = 0o600 - directoryMode = 0o755 + fileMode = 0o644 + secretMode = 0o600 + directoryMode = 0o755 + sensitiveTempAttempts = 10 // Requests carry at most the permission, setuid, setgid, and sticky bits. maxRequestedMode = 0o7777 // Granted modes keep owner rwx and group/other read-execute bits only. @@ -25,6 +26,29 @@ const ( ownerReadWrite = 0o600 ) +// CreateSensitiveTemp creates a private file in directory for data that will be +// renamed into place. The file is restricted before the caller writes to it. +func CreateSensitiveTemp(directory string) (*os.File, string, error) { + defer perf.Track(nil, "configuration.CreateSensitiveTemp")() + root, err := os.OpenRoot(directory) + if err != nil { + return nil, "", fmt.Errorf("%w: open temporary file directory %s: %w", errUtils.ErrFileOperation, directory, err) + } + defer root.Close() + for range sensitiveTempAttempts { + name := ".atmos-plan-" + rand.Text() + file, err := createSensitiveTempFile(root, name) + if errors.Is(err, os.ErrExist) { + continue + } + if err != nil { + return nil, "", fmt.Errorf("%w: create temporary file in %s: %w", errUtils.ErrFileOperation, directory, err) + } + return file, filepath.Join(directory, name), nil + } + return nil, "", fmt.Errorf("%w: create temporary file in %s after %d attempts: %w", errUtils.ErrFileOperation, directory, sensitiveTempAttempts, os.ErrExist) +} + // WriteFiles materializes evaluated bytes under the provisioned module, without templating. // // It also keeps a manifest of what it generated. A file generated by an earlier run that the @@ -107,10 +131,10 @@ func validateFile(file schema.ComponentConfigurationFile) error { func writeFile(root *os.Root, file schema.ComponentConfigurationFile, recorded string) (string, error) { remove, err := checkFileOwnership(root, file, recorded) if err != nil { - return "", err + return "", generatedFileError(file.Path, err) } if remove { - return "", removeFile(root, file.Path) + return "", removeGeneratedFile(root, file.Path) } // Register sensitive bytes even when the file is kept, so a hook that prints it is masked. if file.Sensitive { @@ -129,6 +153,23 @@ func writeFile(root *os.Root, file schema.ComponentConfigurationFile, recorded s return digest, replaceFile(root, file, exists) } +// removeFile deletes a provider-owned file, treating an already absent file as removed. +func removeFile(root *os.Root, path string) error { + err := root.Remove(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +// removeGeneratedFile removes a generated file and classifies a failure as invalid configuration. +func removeGeneratedFile(root *os.Root, path string) error { + if err := removeFile(root, path); err != nil { + return generatedFileError(path, err) + } + return nil +} + // unchangedFileOutcome retains ownership only when an identical file belongs to this host. func unchangedFileOutcome(root *os.Root, file schema.ComponentConfigurationFile, recorded, digest string) (string, error) { if (file.IfExists == "skip" || file.IfExists == "error") && recorded != digest { @@ -136,6 +177,7 @@ func unchangedFileOutcome(root *os.Root, file schema.ComponentConfigurationFile, // removing the block keeps it. Its bytes already match, so if_exists = "error" has nothing to reject. return "", nil } + // Already what would be written, so even if_exists = "error" has nothing to protect. return digest, ensureMode(root, file) } @@ -169,12 +211,12 @@ func recordedIfOwned(recorded, existing string) string { // ensureMode applies the file's permission bits to a file written by an earlier run, which may // predate them. Like a fresh write, it leaves the mode alone unless the file is sensitive or -// requests one. Windows keeps its ACLs. +// requests one. Windows keeps its ACLs, except that sensitive files are re-restricted. func ensureMode(root *os.Root, file schema.ComponentConfigurationFile) error { - if (!file.Sensitive && file.Mode == 0) || runtime.GOOS == "windows" { + if !file.Sensitive && file.Mode == 0 { return nil } - if err := root.Chmod(file.Path, writtenMode(file)); err != nil { + if err := restrictExistingGeneratedFile(root, file); err != nil { return generatedFileError(file.Path, err) } return nil @@ -189,12 +231,47 @@ func replaceFile(root *os.Root, file schema.ComponentConfigurationFile, exists b if err := root.MkdirAll(filepath.Dir(file.Path), directoryMode); err != nil { return generatedFileError(file.Path, err) } - // Policy has already approved replacing an existing file; creating one must not race another writer. + // Sensitive files on Windows need the secured handle to retain their ACL and + // reject concurrent readers while replacing existing bytes. + if runtime.GOOS == "windows" && file.Sensitive { + return replaceSensitiveWindowsFile(root, file, exists) + } if !exists { return createFile(root, file.Path, file) } temporary := filepath.Join(filepath.Dir(file.Path), "."+filepath.Base(file.Path)+"."+rand.Text()+".tmp") if err := createFile(root, temporary, file); err != nil { + return generatedFileError(file.Path, err) + } + if err := root.Rename(temporary, file.Path); err != nil { + _ = root.Remove(temporary) + return generatedFileError(file.Path, err) + } + return nil +} + +// replaceSensitiveWindowsFile writes the bytes to a secured temporary file and renames it over +// an existing destination only after the write and close succeed, so a failed write leaves the +// previous file unchanged. +func replaceSensitiveWindowsFile(root *os.Root, file schema.ComponentConfigurationFile, exists bool) error { + if !exists { + output, err := openGeneratedFile(root, file, false) + if err != nil { + return generatedFileError(file.Path, err) + } + if err := writeSensitiveGeneratedFile(output, file); err != nil { + _ = root.Remove(file.Path) + return err + } + return nil + } + temporary := filepath.Join(filepath.Dir(file.Path), "."+filepath.Base(file.Path)+"."+rand.Text()+".tmp") + output, err := createSensitiveTempFile(root, temporary) + if err != nil { + return generatedFileError(file.Path, err) + } + if err := writeSensitiveGeneratedFile(output, file); err != nil { + _ = root.Remove(temporary) return err } if err := root.Rename(temporary, file.Path); err != nil { @@ -204,6 +281,15 @@ func replaceFile(root *os.Root, file schema.ComponentConfigurationFile, exists b return nil } +// writeSensitiveGeneratedFile retains both operation and close failures on the secured handle. +func writeSensitiveGeneratedFile(output *os.File, file schema.ComponentConfigurationFile) error { + _, writeErr := output.Write(file.Content) + if err := errors.Join(writeErr, output.Close()); err != nil { + return generatedFileError(file.Path, err) + } + return nil +} + // createFile writes a new file with the permission bits the file carries and removes it again if // the write does not complete, so a partial file is never left behind. func createFile(root *os.Root, path string, file schema.ComponentConfigurationFile) error { @@ -231,15 +317,6 @@ func generatedFileError(path string, err error) error { return fmt.Errorf("%w: generated file %s: %w", errUtils.ErrPluginInvalidConfig, path, err) } -// removeFile deletes a provider-owned file, treating an already absent file as removed. -func removeFile(root *os.Root, path string) error { - err := root.Remove(path) - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err -} - // applyMode sets the permission bits explicitly because open applies the umask // and leaves an existing file's mode alone. Windows keeps its ACLs. func applyMode(output *os.File, file schema.ComponentConfigurationFile, mode os.FileMode) error { diff --git a/pkg/component/configuration/files_other.go b/pkg/component/configuration/files_other.go new file mode 100644 index 00000000000..a16af52221b --- /dev/null +++ b/pkg/component/configuration/files_other.go @@ -0,0 +1,41 @@ +//go:build !windows + +package configuration + +import ( + "errors" + "os" + + "github.com/cloudposse/atmos/pkg/schema" +) + +func createSensitiveTempFile(root *os.Root, name string) (*os.File, error) { + file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, secretMode) + if err != nil { + return nil, err + } + if err := file.Chmod(secretMode); err != nil { + return nil, errors.Join(err, file.Close()) + } + return file, nil +} + +func openGeneratedFile(root *os.Root, file schema.ComponentConfigurationFile, exists bool) (*os.File, error) { + flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL + if exists { + flags = os.O_WRONLY + } + output, err := root.OpenFile(file.Path, flags, secretMode) + if err != nil { + return nil, err + } + if err := output.Chmod(secretMode); err != nil { + return nil, errors.Join(err, output.Close()) + } + return output, nil +} + +// restrictExistingGeneratedFile applies the file's clamped permission bits to an existing file. +func restrictExistingGeneratedFile(root *os.Root, file schema.ComponentConfigurationFile) error { + return root.Chmod(file.Path, writtenMode(file)) +} diff --git a/pkg/component/configuration/files_test.go b/pkg/component/configuration/files_test.go index 0b355167e7d..43267de4d4f 100644 --- a/pkg/component/configuration/files_test.go +++ b/pkg/component/configuration/files_test.go @@ -78,6 +78,49 @@ func TestWriteFilesRegistersSensitiveContentItKeeps(t *testing.T) { require.Equal(t, "key=", atmosio.MaskString("key="+secret)) } +func TestSensitiveGeneratedFileRestrictsBeforeTruncating(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + previous := []byte("preserve until restricted") + require.NoError(t, os.WriteFile(path, previous, 0o644)) + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer root.Close() + output, err := openGeneratedFile(root, schema.ComponentConfigurationFile{ + Path: "secret.tf", IfExists: "overwrite", Sensitive: true, + }, true) + require.NoError(t, err) + defer output.Close() + info, err := output.Stat() + require.NoError(t, err) + require.EqualValues(t, len(previous), info.Size()) + if runtime.GOOS != "windows" { + require.Equal(t, os.FileMode(secretMode), info.Mode().Perm()) + } +} + +func TestSensitiveGeneratedFileExistingPolicies(t *testing.T) { + for _, policy := range []string{"skip", "error"} { + t.Run(policy, func(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + previous := []byte("preserve existing content") + require.NoError(t, os.WriteFile(path, previous, 0o644)) + err := WriteFiles(directory, []schema.ComponentConfigurationFile{{ + Path: "secret.tf", IfExists: policy, Sensitive: true, Content: []byte("replacement"), + }}) + if policy == "skip" { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + } + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, previous, content) + }) + } +} + // A module file that already matches a skip block stays the module's: removing the block later // must not delete it as a stale generated file. func TestSkipKeepsOwnershipOfAnIdenticalModuleFile(t *testing.T) { @@ -156,6 +199,105 @@ func TestErrorPolicyDoesNotClaimAnIdenticalModuleFile(t *testing.T) { require.Equal(t, content, got, "removing the block keeps the module's file") } +func TestWriteFilesClassifiesOwnershipFailure(t *testing.T) { + directory := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(directory, "parent"), []byte("file"), 0o600)) + err := WriteFiles(directory, []schema.ComponentConfigurationFile{{Path: "parent/child.tf", IfExists: "overwrite"}}) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + require.ErrorContains(t, err, "generated file parent/child.tf") + var cause *os.PathError + require.ErrorAs(t, err, &cause) + require.ErrorIs(t, err, cause.Err) +} + +func TestWriteFilesClassifiesRemovalFailure(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permissions do not prevent deletion on Windows") + } + directory := t.TempDir() + parent := filepath.Join(directory, "parent") + require.NoError(t, os.Mkdir(parent, 0o700)) + path := filepath.Join(parent, "generated.tf") + require.NoError(t, os.WriteFile(path, []byte("original"), 0o600)) + require.NoError(t, os.Chmod(parent, 0o500)) + t.Cleanup(func() { require.NoError(t, os.Chmod(parent, 0o700)) }) + // Privileged processes can bypass directory permissions. + probe := filepath.Join(parent, "probe") + if err := os.WriteFile(probe, nil, 0o600); err == nil { + t.Skip("process can bypass directory permissions") + } + err := WriteFiles(directory, []schema.ComponentConfigurationFile{{Path: "parent/generated.tf", IfExists: "remove"}}) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + require.ErrorIs(t, err, os.ErrPermission) + require.ErrorContains(t, err, "generated file parent/generated.tf") + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "original", string(content)) +} + +func TestSensitiveGeneratedFileClassifiesHandleFailures(t *testing.T) { + path := filepath.Join(t.TempDir(), "secret.tf") + require.NoError(t, os.WriteFile(path, []byte("original"), 0o600)) + output, err := os.Open(path) + require.NoError(t, err) + require.NoError(t, output.Close()) + err = writeSensitiveGeneratedFile(output, schema.ComponentConfigurationFile{Path: "secret.tf", Content: []byte("replacement")}) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + require.ErrorIs(t, err, os.ErrClosed) + require.ErrorContains(t, err, "generated file secret.tf") + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "original", string(content)) +} + +// A failed replacement of a sensitive file must leave the existing destination untouched and +// remove the temporary file it staged. +func TestReplaceSensitiveFilePreservesDestinationOnFailure(t *testing.T) { + directory := t.TempDir() + // The destination is a non-empty directory, so the final rename cannot succeed. + blocked := filepath.Join(directory, "secret.tf") + require.NoError(t, os.MkdirAll(blocked, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(blocked, "keep"), []byte("original"), 0o600)) + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer root.Close() + err = replaceSensitiveWindowsFile(root, schema.ComponentConfigurationFile{Path: "secret.tf", Content: []byte("replacement"), Sensitive: true}, true) + require.ErrorIs(t, err, errUtils.ErrPluginInvalidConfig) + content, err := os.ReadFile(filepath.Join(blocked, "keep")) + require.NoError(t, err) + require.Equal(t, "original", string(content)) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +func TestReplaceSensitiveFileReplacesExistingBytes(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + require.NoError(t, os.WriteFile(path, []byte("original"), 0o600)) + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer root.Close() + require.NoError(t, replaceSensitiveWindowsFile(root, schema.ComponentConfigurationFile{Path: "secret.tf", Content: []byte("replacement"), Sensitive: true}, true)) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "replacement", string(content)) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +func TestCreateSensitiveTempClassifiesFailures(t *testing.T) { + _, _, err := CreateSensitiveTemp(filepath.Join(t.TempDir(), "missing")) + require.ErrorIs(t, err, errUtils.ErrFileOperation) + require.ErrorIs(t, err, os.ErrNotExist) + + file, name, err := CreateSensitiveTemp(t.TempDir()) + require.NoError(t, err) + require.NoError(t, file.Close()) + require.FileExists(t, name) +} + func TestWriteFilesAppliesClampedMode(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows ignores POSIX permission bits.") diff --git a/pkg/component/configuration/files_windows.go b/pkg/component/configuration/files_windows.go new file mode 100644 index 00000000000..9beaf0dd74e --- /dev/null +++ b/pkg/component/configuration/files_windows.go @@ -0,0 +1,177 @@ +package configuration + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "unsafe" + + "golang.org/x/sys/windows" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +func createSensitiveTempFile(root *os.Root, name string) (*os.File, error) { + return openSensitiveGeneratedFile(root, name, false) +} + +func openGeneratedFile(root *os.Root, file schema.ComponentConfigurationFile, exists bool) (*os.File, error) { + if !file.Sensitive { + flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL + if exists { + flags = os.O_WRONLY + } + output, err := root.OpenFile(file.Path, flags, secretMode) + if err != nil { + return nil, err + } + if err := output.Chmod(secretMode); err != nil { + return nil, errors.Join(err, output.Close()) + } + return output, nil + } + + return openSensitiveGeneratedFile(root, file.Path, exists) +} + +// restrictExistingGeneratedFile re-secures an existing sensitive file. Other files keep their ACLs. +func restrictExistingGeneratedFile(root *os.Root, file schema.ComponentConfigurationFile) error { + if !file.Sensitive { + return nil + } + output, err := openSensitiveGeneratedFile(root, file.Path, true) + if err != nil { + return err + } + return output.Close() +} + +func openSensitiveGeneratedFile(root *os.Root, path string, exists bool) (*os.File, error) { + // Resolve the parent through os.Root, then open only a basename relative to + // its handle. Never turn the contained path back into an absolute pathname. + parent, err := root.Open(filepath.Dir(path)) + if err != nil { + return nil, err + } + defer parent.Close() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return nil, err + } + security, err := windows.SecurityDescriptorFromString("O:" + user.User.Sid.String() + "D:P(A;;FA;;;" + user.User.Sid.String() + ")") + if err != nil { + return nil, err + } + name, err := windows.NewNTUnicodeString(filepath.Base(path)) + if err != nil { + return nil, err + } + attributes := windows.OBJECT_ATTRIBUTES{ + RootDirectory: windows.Handle(parent.Fd()), ObjectName: name, + Attributes: windows.OBJ_CASE_INSENSITIVE | windows.OBJ_DONT_REPARSE, + SecurityDescriptor: security, + } + attributes.Length = uint32(unsafe.Sizeof(attributes)) + disposition := uint32(windows.FILE_CREATE) + if exists { + disposition = windows.FILE_OPEN + } + var handle windows.Handle + // Exclusive sharing rejects existing readers as well as new readers until + // close: replacing a DACL alone cannot revoke an already-open read handle. + err = windows.NtCreateFile(&handle, windows.FILE_GENERIC_WRITE|windows.FILE_READ_ATTRIBUTES|windows.WRITE_DAC, + &attributes, &windows.IO_STATUS_BLOCK{}, nil, windows.FILE_ATTRIBUTE_NORMAL, 0, disposition, + windows.FILE_NON_DIRECTORY_FILE|windows.FILE_SYNCHRONOUS_IO_NONALERT, 0, 0) + runtime.KeepAlive(parent) + if err != nil { + var status windows.NTStatus + if errors.As(err, &status) { + return nil, status.Errno() + } + return nil, err + } + output := os.NewFile(uintptr(handle), path) + if err := restrictGeneratedFile(handle, user.User.Sid, security); err != nil { + return nil, errors.Join(err, output.Close()) + } + return output, nil +} + +func validateSensitiveGeneratedFile(handle windows.Handle, user *windows.SID) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return err + } + if info.NumberOfLinks != 1 || info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return fmt.Errorf("%w: sensitive generated file must be a regular file with one link", errUtils.ErrPluginInvalidConfig) + } + current, err := windows.GetSecurityInfo(handle, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION) + if err != nil { + return err + } + owner, _, err := current.Owner() + if err != nil { + return err + } + // A different owner could grant themselves read access again after close. + if owner == nil || !owner.Equals(user) { + return fmt.Errorf("%w: sensitive generated file must be owned by the current user", errUtils.ErrPluginInvalidConfig) + } + return nil +} + +func restrictGeneratedFile(handle windows.Handle, user *windows.SID, security *windows.SECURITY_DESCRIPTOR) error { + if err := validateSensitiveGeneratedFile(handle, user); err != nil { + return err + } + dacl, _, err := security.DACL() + if err != nil { + return err + } + if err := windows.SetSecurityInfo(handle, windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, dacl, nil); err != nil { + return err + } + // Fail closed on filesystems that cannot persist the requested ACL. + actual, err := windows.GetSecurityInfo(handle, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if err != nil { + return err + } + return verifyGeneratedFileACL(actual, user, dacl) +} + +func verifyGeneratedFileACL(actual *windows.SECURITY_DESCRIPTOR, user *windows.SID, expected *windows.ACL) error { + invalid := fmt.Errorf("%w: sensitive generated file ACL was not retained", errUtils.ErrPluginInvalidConfig) + control, _, err := actual.Control() + if err != nil { + return err + } + if control&windows.SE_DACL_PROTECTED == 0 { + return invalid + } + dacl, _, err := actual.DACL() + if err != nil { + return err + } + if dacl == nil || dacl.AceCount != 1 { + return invalid + } + var ace, wanted *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(dacl, 0, &ace); err != nil { + return err + } + if err := windows.GetAce(expected, 0, &wanted); err != nil { + return err + } + if *ace != *wanted { + return invalid + } + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if !sid.Equals(user) { + return invalid + } + return nil +} diff --git a/pkg/component/configuration/files_windows_test.go b/pkg/component/configuration/files_windows_test.go new file mode 100644 index 00000000000..f304ea41e0d --- /dev/null +++ b/pkg/component/configuration/files_windows_test.go @@ -0,0 +1,189 @@ +package configuration + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" + + "github.com/cloudposse/atmos/pkg/schema" +) + +func assertPrivateGeneratedACL(t *testing.T, handle windows.Handle) { + t.Helper() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + actual, err := windows.GetSecurityInfo(handle, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + require.NoError(t, err) + expected, err := windows.SecurityDescriptorFromString("D:P(A;;FA;;;" + user.User.Sid.String() + ")") + require.NoError(t, err) + dacl, _, err := expected.DACL() + require.NoError(t, err) + require.NoError(t, verifyGeneratedFileACL(actual, user.User.Sid, dacl)) +} + +func TestSensitiveGeneratedFileWindowsACLBeforeWrite(t *testing.T) { + for _, existing := range []bool{false, true} { + t.Run(map[bool]string{false: "create", true: "overwrite"}[existing], func(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + previous := []byte("old content remains until access is restricted") + if existing { + require.NoError(t, os.WriteFile(path, previous, 0o644)) + // Explicit Everyone read access must be removed on overwrite. + security, err := windows.SecurityDescriptorFromString("D:(A;;FA;;;OW)(A;;GR;;;WD)") + require.NoError(t, err) + dacl, _, err := security.DACL() + require.NoError(t, err) + require.NoError(t, windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)) + } + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer root.Close() + file := schema.ComponentConfigurationFile{Path: "secret.tf", IfExists: "overwrite", Sensitive: true, Content: []byte("new")} + output, err := openGeneratedFile(root, file, existing) + require.NoError(t, err) + t.Cleanup(func() { _ = output.Close() }) + assertPrivateGeneratedACL(t, windows.Handle(output.Fd())) + info, err := output.Stat() + require.NoError(t, err) + if existing { + require.EqualValues(t, len(previous), info.Size(), "opening must not truncate before ACL restriction") + } else { + require.Zero(t, info.Size()) + } + reader, err := os.Open(path) + if reader != nil { + _ = reader.Close() + } + require.ErrorIs(t, err, windows.ERROR_SHARING_VIOLATION) + require.NoError(t, output.Close()) + require.NoError(t, WriteFiles(directory, []schema.ComponentConfigurationFile{file})) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, file.Content, content) + reader, err = os.Open(path) + require.NoError(t, err) + defer reader.Close() + assertPrivateGeneratedACL(t, windows.Handle(reader.Fd())) + }) + } +} + +func TestSensitiveGeneratedFileWindowsExistingReader(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + previous := []byte("original") + require.NoError(t, os.WriteFile(path, previous, 0o644)) + reader, err := os.Open(path) + require.NoError(t, err) + defer reader.Close() + err = WriteFiles(directory, []schema.ComponentConfigurationFile{{Path: "secret.tf", IfExists: "overwrite", Sensitive: true, Content: []byte("secret")}}) + // Replacing a file another process holds open fails: the temporary file cannot be renamed over + // it. Windows reports that as a sharing violation or as access denied depending on the handle. + require.Error(t, err) + require.True(t, errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_ACCESS_DENIED), "unexpected error: %v", err) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, previous, content) +} + +// A sensitive file is replaced through a temporary file renamed over it, which replaces the +// directory entry and never writes through a hard link, so the other name keeps its bytes. +func TestSensitiveGeneratedFileWindowsReplacementLeavesHardLinkUntouched(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "original.tf") + previous := []byte("original") + require.NoError(t, os.WriteFile(path, previous, 0o644)) + require.NoError(t, os.Link(path, filepath.Join(directory, "secret.tf"))) + require.NoError(t, WriteFiles(directory, []schema.ComponentConfigurationFile{{Path: "secret.tf", IfExists: "overwrite", Sensitive: true, Content: []byte("secret")}})) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, previous, content, "the other hard link must not be written through") + replaced, err := os.ReadFile(filepath.Join(directory, "secret.tf")) + require.NoError(t, err) + require.Equal(t, []byte("secret"), replaced) +} + +func TestSensitiveGeneratedFileWindowsACLFailurePreservesContent(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + previous := []byte("original") + require.NoError(t, os.WriteFile(path, previous, 0o644)) + // A read-only handle cannot set the DACL. Exercise the real Windows failure. + reader, err := os.Open(path) + require.NoError(t, err) + defer reader.Close() + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + security, err := windows.SecurityDescriptorFromString("D:P(A;;FA;;;" + user.User.Sid.String() + ")") + require.NoError(t, err) + require.Error(t, restrictGeneratedFile(windows.Handle(reader.Fd()), user.User.Sid, security)) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, previous, content) +} + +func TestSensitiveGeneratedFileWindowsRejectsDifferentOwner(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "secret.tf") + require.NoError(t, os.WriteFile(path, []byte("original"), 0o644)) + reader, err := os.Open(path) + require.NoError(t, err) + defer reader.Close() + other, err := windows.CreateWellKnownSid(windows.WinWorldSid) + require.NoError(t, err) + require.ErrorContains(t, validateSensitiveGeneratedFile(windows.Handle(reader.Fd()), other), "owned by the current user") +} + +func TestSensitiveGeneratedFileWindowsRejectsReparsePoint(t *testing.T) { + directory := t.TempDir() + outside := filepath.Join(t.TempDir(), "original.tf") + previous := []byte("original") + require.NoError(t, os.WriteFile(outside, previous, 0o644)) + if err := os.Symlink(outside, filepath.Join(directory, "secret.tf")); err != nil { + t.Skipf("symlink creation requires Windows developer mode or privilege: %v", err) + } + require.Error(t, WriteFiles(directory, []schema.ComponentConfigurationFile{{Path: "secret.tf", IfExists: "overwrite", Sensitive: true, Content: []byte("secret")}})) + content, err := os.ReadFile(outside) + require.NoError(t, err) + require.Equal(t, previous, content) +} + +func TestSensitiveGeneratedFileWindowsACLVerification(t *testing.T) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + require.NoError(t, err) + entry := "(A;;FA;;;" + user.User.Sid.String() + ")" + expected, err := windows.SecurityDescriptorFromString("D:P" + entry) + require.NoError(t, err) + dacl, _, err := expected.DACL() + require.NoError(t, err) + for _, tc := range []struct { + name string + sddl string + valid bool + }{ + {"protected", "D:P" + entry, true}, + {"auto-inheritance-history", "D:PAI" + entry, true}, + {"unprotected", "D:" + entry, false}, + {"everyone", "D:P(A;;FA;;;WD)", false}, + {"extra-reader", "D:P" + entry + "(A;;GR;;;WD)", false}, + {"empty", "D:P", false}, + {"null", "D:NO_ACCESS_CONTROL", false}, + } { + t.Run(tc.name, func(t *testing.T) { + actual, err := windows.SecurityDescriptorFromString(tc.sddl) + require.NoError(t, err) + err = verifyGeneratedFileACL(actual, user.User.Sid, dacl) + if tc.valid { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/pkg/component/helm/aggregate_ci_test.go b/pkg/component/helm/aggregate_ci_test.go index 366b58bcfce..0212de11b1c 100644 --- a/pkg/component/helm/aggregate_ci_test.go +++ b/pkg/component/helm/aggregate_ci_test.go @@ -81,8 +81,10 @@ func (p *helmBulkGraphTestProvider) Execute(ctx *component.ExecutionContext) err } return nil } + func (p *helmBulkGraphTestProvider) GenerateArtifacts(*component.ExecutionContext) error { return nil } -func (p *helmBulkGraphTestProvider) GetAvailableCommands() []string { return nil } + +func (p *helmBulkGraphTestProvider) GetAvailableCommands() []string { return nil } func TestHelmBulkCICollectorRecordsDependencyBlockedComponents(t *testing.T) { collector := newHelmBulkCICollector("apply") diff --git a/pkg/component/provideroptions/flags.go b/pkg/component/provideroptions/flags.go new file mode 100644 index 00000000000..7693729e0f2 --- /dev/null +++ b/pkg/component/provideroptions/flags.go @@ -0,0 +1,403 @@ +package provideroptions + +import ( + "fmt" + "maps" + "os" + "slices" + "strconv" + "strings" + "sync" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/flags" + atmosio "github.com/cloudposse/atmos/pkg/io" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// sectionAnnotation marks a flag as belonging to a provider, so help can group it. +const sectionAnnotation = "atmos_provider_section" + +// registration records the flags one provider added to one command. +type registration struct { + provider Provider + options []Option + parser *flags.StandardParser +} + +var registered = struct { + sync.Mutex + byCommand map[*cobra.Command][]*registration +}{byCommand: make(map[*cobra.Command][]*registration)} + +// Register adds the flags of every option to the subcommands that accept it. Registration needs no +// running plugin: it uses only the static declarations. A flag name that an Atmos flag or another +// provider already uses is rejected, so a provider can never shadow selection, scheduling, or UI flags. +func Register(root *cobra.Command, providers []Provider) error { + defer perf.Track(nil, "provideroptions.Register")() + + for index := range providers { + provider := &providers[index] + commands := make(map[string][]Option) + for i := range provider.Options { + option := &provider.Options[i] + if !option.AppliesTo(provider.Mode) { + continue + } + for _, name := range option.Commands { + commands[name] = append(commands[name], *option) + } + } + for _, name := range slices.Sorted(maps.Keys(commands)) { + sub := findSubcommand(root, name) + if sub == nil { + continue + } + if err := registerCommand(sub, provider, commands[name]); err != nil { + return err + } + } + } + return nil +} + +func findSubcommand(root *cobra.Command, name string) *cobra.Command { + for _, sub := range root.Commands() { + if sub.Name() == name { + return sub + } + } + return nil +} + +func registerCommand(sub *cobra.Command, provider *Provider, options []Option) error { + if alreadyRegistered(sub, provider) { + return nil + } + var parserOptions []flags.Option + for index := range options { + option := &options[index] + primary := option.FlagName(provider.Name) + for _, name := range append([]string{primary}, option.Aliases...) { + if err := checkCollision(sub, provider, option, name); err != nil { + return err + } + } + parserOptions = append(parserOptions, flagOptions(provider.Name, option, primary)...) + } + parser := flags.NewStandardParser(parserOptions...) + // A command with Cobra children, such as `state` (`state rm`) or `workspace` (`workspace select`), + // is run through its leaf, which only parses flags it inherits. Persistent flags reach the leaf, so + // the provider flags are honored there instead of being discarded as unknown. + if sub.HasSubCommands() { + parser.RegisterPersistentFlags(sub) + } else { + parser.RegisterFlags(sub) + } + if err := parser.BindToViper(viper.GetViper()); err != nil { + return fmt.Errorf("%w: %w", errUtils.ErrProviderOption, err) + } + for index := range options { + decorate(sub, provider.Name, &options[index]) + } + registered.Lock() + registered.byCommand[sub] = append(registered.byCommand[sub], ®istration{provider: *provider, options: options, parser: parser}) + registered.Unlock() + return nil +} + +func alreadyRegistered(sub *cobra.Command, provider *Provider) bool { + registered.Lock() + defer registered.Unlock() + for _, existing := range registered.byCommand[sub] { + if existing.provider.Name == provider.Name && existing.provider.Section == provider.Section { + return true + } + } + return false +} + +func checkCollision(sub *cobra.Command, provider *Provider, option *Option, name string) error { + if sub.Flags().Lookup(name) == nil && sub.InheritedFlags().Lookup(name) == nil && sub.PersistentFlags().Lookup(name) == nil { + return nil + } + return errUtils.Build(errUtils.ErrProviderOptionCollision). + WithExplanationf("The `%s` configuration provider declares option `%s` with flag `--%s`, but `atmos terraform %s` already has a flag with that name.", provider.Name, option.Name, name, sub.Name()). + WithContext("provider", provider.Name). + WithContext("option", option.Name). + WithContext("command", sub.Name()). + WithHint("Atmos owns stack selection, `--affected`, `--all`, `--ui`, and scheduling flags; a provider option cannot reuse them."). + WithHint("Ask the package author to set a different `flag` for the option in `plugin.yaml`, or remove the conflicting package."). + Err() +} + +func flagOptions(provider string, option *Option, name string) []flags.Option { + description := option.Description + if option.Default != nil { + description += fmt.Sprintf(" (default: %v)", option.Default) + } + var result []flags.Option + switch option.Type { + case TypeBool: + value, _ := option.Default.(bool) + result = append(result, flags.WithBoolFlag(name, "", value, description)) + case TypeInt: + if option.Sensitive { + // pflag includes invalid integer input in its parse errors. Accept the raw + // value here and validate it in convert, which never echoes secrets. + result = append(result, flags.WithStringFlag(name, "", "", description)) + break + } + value, _ := option.Default.(int) + result = append(result, flags.WithIntFlag(name, "", value, description)) + case TypeList: + value, _ := option.Default.([]string) + result = append(result, flags.WithStringArrayFlag(name, "", value, description)) + case TypeMap: + result = append(result, flags.WithStringMapFlag(name, "", nil, description)) + default: + value, _ := option.Default.(string) + result = append(result, flags.WithStringFlag(name, "", value, description)) + } + return append(result, flags.WithEnvVars(name, option.EnvNames(provider)...)) +} + +// decorate groups the flags under their provider in help and adds aliases and deprecation. +func decorate(sub *cobra.Command, provider string, option *Option) { + set := sub.Flags() + if sub.HasSubCommands() { + set = sub.PersistentFlags() + } + primary := set.Lookup(option.FlagName(provider)) + if primary == nil { + return + } + mark(primary, provider, option.Sensitive) + for _, alias := range option.Aliases { + shared := &pflag.Flag{Name: alias, Usage: "Alias of --" + primary.Name, Value: primary.Value, DefValue: primary.DefValue, NoOptDefVal: primary.NoOptDefVal, Hidden: true} + mark(shared, provider, option.Sensitive) + set.AddFlag(shared) + } + if option.Deprecated != "" { + _ = set.MarkDeprecated(primary.Name, option.Deprecated) + } +} + +// lookupFlag finds a provider flag whether it was registered as a local or a persistent flag. +func lookupFlag(cmd *cobra.Command, name string) *pflag.Flag { + if flag := cmd.Flags().Lookup(name); flag != nil { + return flag + } + return cmd.PersistentFlags().Lookup(name) +} + +func mark(flag *pflag.Flag, provider string, sensitive bool) { + if flag.Annotations == nil { + flag.Annotations = make(map[string][]string) + } + flag.Annotations[sectionAnnotation] = []string{provider} + if sensitive { + flag.Annotations[flags.SensitiveMetadataAnnotation] = []string{"true"} + } +} + +// Resolve reads the options the user set for the command, from the command line or the +// environment, and validates them against their declared types. Options the user did not set are +// absent from the result. Values of sensitive options are registered with the output masker and +// never appear in an error. +func Resolve(cmd *cobra.Command, v *viper.Viper) (map[string]map[string]schema.ComponentProviderOption, error) { + defer perf.Track(nil, "provideroptions.Resolve")() + + registered.Lock() + entries := slices.Clone(registered.byCommand[cmd]) + registered.Unlock() + result := make(map[string]map[string]schema.ComponentProviderOption) + for _, entry := range entries { + if err := entry.parser.BindFlagsToViper(cmd, v); err != nil { + return nil, fmt.Errorf("%w: %w", errUtils.ErrProviderOption, err) + } + for index := range entry.options { + option := &entry.options[index] + value, set, err := entry.read(cmd, v, option) + if err != nil { + return nil, err + } + if !set { + continue + } + if result[entry.provider.Section] == nil { + result[entry.provider.Section] = make(map[string]schema.ComponentProviderOption) + } + result[entry.provider.Section][option.Name] = schema.ComponentProviderOption{Value: value, Sensitive: option.Sensitive, FromCLI: entry.fromCLI(cmd, option)} + if option.Sensitive { + atmosio.RegisterSecretValue(value) + } + } + } + return result, nil +} + +func (r *registration) read(cmd *cobra.Command, v *viper.Viper, option *Option) (any, bool, error) { + primary := option.FlagName(r.provider.Name) + if !r.explicit(cmd, option, primary) { + return nil, false, nil + } + value, err := convert(option, primary, r.raw(cmd, v, option, primary)) + if err != nil { + return nil, false, err + } + return value, true, nil +} + +func (r *registration) explicit(cmd *cobra.Command, option *Option, primary string) bool { + for _, name := range append([]string{primary}, option.Aliases...) { + if flag := lookupFlag(cmd, name); flag != nil && flag.Changed { + return true + } + } + return r.parser.IsFlagExplicitlySet(cmd, primary) +} + +func (r *registration) fromCLI(cmd *cobra.Command, option *Option) bool { + for _, name := range append([]string{option.FlagName(r.provider.Name)}, option.Aliases...) { + if flag := lookupFlag(cmd, name); flag != nil && flag.Changed { + if _, fromEnv := flags.FlagValueFromEnv(flag); !fromEnv { + return true + } + } + } + return false +} + +// convert turns a flag or environment value into the declared type. +func convert(option *Option, flag string, raw any) (any, error) { + switch option.Type { + case TypeBool: + parsed, err := strconv.ParseBool(strings.TrimSpace(fmt.Sprint(raw))) + if err != nil { + return nil, invalidValue(option, flag, "expected true or false") + } + return parsed, nil + case TypeInt: + parsed, err := strconv.Atoi(strings.TrimSpace(fmt.Sprint(raw))) + if err != nil { + return nil, invalidValue(option, flag, "expected a whole number") + } + return parsed, nil + case TypeList: + return listValues(raw), nil + case TypeMap: + return mapValues(option, flag, raw) + } + return fmt.Sprint(raw), nil +} + +// listValues reads a repeatable flag or a comma-separated environment value. An explicit empty +// value is an empty list, not an absent one. +func listValues(raw any) []string { + switch typed := raw.(type) { + case []string: + if len(typed) == 1 && typed[0] == "" { + return []string{} + } + return append([]string{}, typed...) + case []any: + items := make([]string, 0, len(typed)) + for _, item := range typed { + items = append(items, fmt.Sprint(item)) + } + return listValues(items) + case string: + items := []string{} + for _, item := range strings.Split(typed, ",") { + if item = strings.TrimSpace(item); item != "" { + items = append(items, item) + } + } + return items + } + return []string{} +} + +// mapValues parses name=value entries and rejects malformed ones instead of dropping them. +func mapValues(option *Option, flag string, raw any) (map[string]any, error) { + result := make(map[string]any) + for _, entry := range listValues(raw) { + key, value, found := strings.Cut(entry, "=") + if !found || strings.TrimSpace(key) == "" { + return nil, invalidValue(option, flag, "each entry must look like name=value") + } + result[strings.TrimSpace(key)] = value + } + return result, nil +} + +// invalidValue never echoes the value, which may be a secret. +func invalidValue(option *Option, flag, problem string) error { + return errUtils.Build(errUtils.ErrProviderOption). + WithExplanationf("The value for `--%s` (option `%s`) is invalid: %s.", flag, option.Name, problem). + WithContext("option", option.Name). + WithHintf("Check `--%s` or the environment variable that supplies it. Run the command with `--help` to see the accepted values.", flag). + Err() +} + +// Section is a titled group of provider flags, for help output. +type Section struct { + Title string + Flags *pflag.FlagSet +} + +// SplitFlags separates provider flags from the rest, so help can show them under a provider heading. +func SplitFlags(set *pflag.FlagSet) (*pflag.FlagSet, []Section) { + defer perf.Track(nil, "provideroptions.SplitFlags")() + + rest := pflag.NewFlagSet("flags", pflag.ContinueOnError) + groups := make(map[string]*pflag.FlagSet) + set.VisitAll(func(flag *pflag.Flag) { + provider := flag.Annotations[sectionAnnotation] + if len(provider) == 0 { + rest.AddFlag(flag) + return + } + if groups[provider[0]] == nil { + groups[provider[0]] = pflag.NewFlagSet(provider[0], pflag.ContinueOnError) + } + groups[provider[0]].AddFlag(flag) + }) + sections := make([]Section, 0, len(groups)) + for _, name := range slices.Sorted(maps.Keys(groups)) { + sections = append(sections, Section{Title: strings.ToUpper(name) + " FLAGS", Flags: groups[name]}) + } + return rest, sections +} + +// raw returns the value of an alias the user typed, which Viper does not see because only the +// primary flag is bound, and otherwise the layered value of the primary flag. +func (r *registration) raw(cmd *cobra.Command, v *viper.Viper, option *Option, primary string) any { + for _, name := range option.Aliases { + flag := lookupFlag(cmd, name) + if flag == nil || !flag.Changed { + continue + } + if slice, ok := flag.Value.(pflag.SliceValue); ok { + return slice.GetSlice() + } + return flag.Value.String() + } + if flag := lookupFlag(cmd, primary); flag == nil || !flag.Changed { + // Viper treats an empty environment variable as unset and falls through to the next name, so an + // explicit `ATMOS__