diff --git a/cmd/git/env.go b/cmd/git/env.go index 4f5283d51ec..faf870c0a47 100644 --- a/cmd/git/env.go +++ b/cmd/git/env.go @@ -4,18 +4,22 @@ import ( "context" "os" - atmosgit "github.com/cloudposse/atmos/pkg/git" + "github.com/spf13/viper" + + "github.com/cloudposse/atmos/pkg/auth/gitenv" + cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/perf" ) -// composeEnv builds the subprocess environment for a Git operation. -// When identity is non-empty, the identity environment (GIT_CONFIG_* from -// GitHub STS etc.) is merged over the current process environment via -// atmosgit.ComposeEnvironment. When identity is empty, the process environment -// is returned unchanged (ambient credentials from the developer's Git config -// or GITHUB_TOKEN apply). Auth manager injection is a planned future enhancement. +// envComposer builds Git subprocess environments from Atmos Auth identities. Tests replace it. +var envComposer = gitenv.NewComposer() + +// composeEnv builds the subprocess environment for a Git operation: the process environment plus +// the environment of the effective identity. The global --identity flag (or ATMOS_IDENTITY) wins +// over the repository's configured identity; with neither, the process environment is returned +// unchanged (ambient credentials from the developer's Git config or GITHUB_TOKEN apply). func composeEnv(ctx context.Context, identity string) ([]string, error) { defer perf.Track(nil, "git.composeEnv")() - return atmosgit.ComposeEnvironment(ctx, os.Environ(), identity, nil) + return envComposer.Compose(ctx, atmosConfigPtr, os.Environ(), viper.GetString(cfg.IdentityFlagName), identity) } diff --git a/cmd/git/orchestration_test.go b/cmd/git/orchestration_test.go index 77f4cf74ffc..ccd88ac038c 100644 --- a/cmd/git/orchestration_test.go +++ b/cmd/git/orchestration_test.go @@ -7,14 +7,17 @@ import ( "context" "errors" "os" + "strings" "testing" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/auth/gitenv" "github.com/cloudposse/atmos/pkg/ci" atmosgit "github.com/cloudposse/atmos/pkg/git" "github.com/cloudposse/atmos/pkg/schema" @@ -576,6 +579,48 @@ func TestComposeEnv_EmptyIdentity(t *testing.T) { assert.NotEmpty(t, env) } +// ---- composeEnv identity precedence ---- + +func TestComposeEnv_IdentityReachesSubprocessEnvironment(t *testing.T) { + original := envComposer + t.Cleanup(func() { envComposer = original }) + manager := gitenv.NewMockIdentityManager(gomock.NewController(t)) + // Each identity is authenticated only by the case that selects it: the mock fails the test if + // the repository identity is resolved when the flag overrides it, or if any is with neither. + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "repo-id").Return(map[string]string{"ATMOS_TEST_GIT_SOCK": "repo"}, nil).Times(1) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "flag-id").Return(map[string]string{"ATMOS_TEST_GIT_SOCK": "flag"}, nil).Times(1) + envComposer = gitenv.NewComposer(gitenv.WithManagerFactory(func(*schema.AtmosConfiguration) (gitenv.IdentityManager, error) { + return manager, nil + })) + setAtmosConfigPtr(t, &schema.AtmosConfiguration{}) + t.Setenv("ATMOS_TEST_GIT_SOCK", "ambient") + + tests := []struct { + name, flag, repo string + want string + }{ + {"no identity keeps the process environment", "", "", "ambient"}, + {"repository identity", "", "repo-id", "repo"}, + {"flag overrides repository identity", "flag-id", "repo-id", "flag"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + viper.Set("identity", tt.flag) + t.Cleanup(func() { viper.Set("identity", "") }) + + env, err := composeEnv(context.Background(), tt.repo) + require.NoError(t, err) + var got []string + for _, entry := range env { + if strings.HasPrefix(entry, "ATMOS_TEST_GIT_SOCK=") { + got = append(got, strings.TrimPrefix(entry, "ATMOS_TEST_GIT_SOCK=")) + } + } + assert.Equal(t, []string{tt.want}, got) + }) + } +} + func TestCloneRefExcludesBranchAndOverridesConfiguredBranch(t *testing.T) { opts := &cloneOptions{Ref: "revision", Branch: "main"} require.ErrorIs(t, runClone(context.Background(), opts, []string{"https://example.com/repo.git"}), errUtils.ErrInvalidConfig) diff --git a/cmd/root.go b/cmd/root.go index 430fd66c3c8..a11b77d31ef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -77,6 +77,9 @@ import ( // Import the Atmos Pro credential broker so it registers itself (init) and is consulted // before the first remote read in CI (lazily provisions the github/sts integration). _ "github.com/cloudposse/atmos/pkg/auth/providers/atmospro/broker" + // Import the remote-read identity resolver so `identity:` on vendored sources, component sources, + // and remote imports resolves through Atmos Auth (registers itself with the downloader in init). + _ "github.com/cloudposse/atmos/pkg/auth/remoteread" "github.com/cloudposse/atmos/pkg/ci" cistartup "github.com/cloudposse/atmos/pkg/ci/startup" // Import component providers to register them with the component registry. diff --git a/docs/fixes/2026-11-12-remote-read-identity-environment.md b/docs/fixes/2026-11-12-remote-read-identity-environment.md new file mode 100644 index 00000000000..6fd722a37e5 --- /dev/null +++ b/docs/fixes/2026-11-12-remote-read-identity-environment.md @@ -0,0 +1,47 @@ +# Fix: identity environments now reach `atmos git`, `atmos ansible playbook`, and stored plugin credentials + +**Date:** 2026-11-12 + +## Summary + +Three places that should have used an Atmos Auth identity did not. `atmos git` ignored +`git.repositories..auth.identity` and the `--identity` flag. `atmos ansible playbook --identity` +authenticated but never gave the identity's environment to `ansible-playbook`. A plugin credential of +kind `environment/v1` that was stored by an earlier process failed in a fresh process. + +## Context + +These surfaced while adding `identity:` to vendored sources, component sources, and remote stack +imports, which all depend on an identity's environment reaching the subprocess that does the work. + +- `atmos git` built each Git subprocess environment from the process environment only, so a configured + repository identity (for example a `github/sts`-linked identity) and `--identity` had no effect. +- The Ansible executor processed the stack with the identity but then built the playbook environment from + the component environment alone. Helmfile and Packer already layered the identity's environment on top. +- After `atmos auth login`, a later process finds the stored credential still valid and skips + authentication. The `environment/v1` materializer was only populated by authentication, so in that fresh + process (`atmos auth exec`, `atmos auth shell`, `atmos git`) it held no environment and the command failed. + +## Changes + +- `atmos git` composes the Git subprocess environment from the process environment plus the effective + identity. The global `--identity` flag (or `ATMOS_IDENTITY`) wins over the repository's configured + identity. With neither, the process environment is used unchanged. +- `atmos ansible playbook` layers the identity's prepared environment over the component environment before + starting `ansible-playbook`, the same way Helmfile and Packer do. Its debug log lists the names of the + environment variables the playbook runs with, not their values, because that environment now carries the + identity's credentials. +- The `environment/v1` materializer can now be restored from a credential read back from the credential + store. It applies the same validation and secret masking as a fresh authentication, and refuses an + expired or malformed credential. + +## Validation + +- Unit tests cover the identity environment reaching the `atmos git` subprocess, the Ansible playbook + environment with an auth manager, without one, and when preparation fails, and restoring a stored + `environment/v1` credential, including refusal of an expired one. +- The website build covers the documentation for these behaviors. + +## Follow-ups + +None. diff --git a/docs/prd/ssh-agent-plugin.md b/docs/prd/ssh-agent-plugin.md new file mode 100644 index 00000000000..3c53b0a3871 --- /dev/null +++ b/docs/prd/ssh-agent-plugin.md @@ -0,0 +1,470 @@ +# PRD: SSH identities (`cloudposse/ssh` plugin, and identities on remote reads) + +**Status:** Proposed. Backed by a working proof of concept (`~/se-agent-poc`, 2026-11-06); no plugin or core code exists yet. Builds on the [plugin platform](plugins/README.md), its [authentication](plugins/extensions/authentication.md) and [command](plugins/extensions/commands.md) contracts, and [Git operations](git-ops.md). + +## Problem and outcome + +SSH is how Ansible reaches hosts, how Git signs commits, and how private Git sources are fetched. In each case the key is who you are. Atmos has identities for clouds but none for SSH, so SSH keys are managed outside it: exportable files on disk, keys in environment variables or `sshkey=` URL parameters, or a desktop agent (1Password, Secretive) that refuses to sign while the screen is locked. That last point blocks unattended work, such as a coding agent committing or a playbook running while nobody is at the keyboard. + +Outcome, in two parts: + +1. **A plugin makes an SSH key an Atmos identity.** Its private key is generated inside the machine's security hardware and cannot be exported. It works while the screen is locked, edits no global configuration, and puts no secret in the environment. +2. **Core lets every remote read name an identity.** Vendored sources, component sources, and remote imports accept `identity:`, as Git repositories, stores, and plugin registries already do. + +```shell +atmos auth login --identity ops # once per machine: creates and enrolls the key +atmos ansible playbook web -s prod --identity ops # Ansible logs in to hosts with it +atmos git commit infra -m "..." # commits are signed with it +atmos vendor pull # sources that name the identity are fetched with it +atmos auth exec --identity ops -- claude # a coding agent's own git and ssh use it +``` + +## Evidence from the proof of concept + +| Finding | Result | +|---|---| +| Secure Enclave key with `AfterFirstUnlockThisDeviceOnly` and `.privateKeyUsage` only | Signed 43 of 43 commits with the screen locked, all verified by `git verify-commit` | +| Control key with `WhenUnlockedThisDeviceOnly` | Refused 42 of 43 while locked (`errSecInteractionNotAllowed`, -25308); the one success was inside the grace period right after locking | +| Entitlements, keychain, Developer Program | None needed. An ad-hoc-signed binary run as a per-user LaunchAgent is enough | +| Same key through the C `Security.framework` API (no CryptoKit, no Swift) | Create, reload from blob, and sign all work; blobs created by CryptoKit load unchanged | +| Key blob used by an unrelated binary | **Works.** Any process running as the user that can read the blob can sign | +| 1Password `op-ssh-sign` in global Git config | Intercepts signing unless `gpg.ssh.program` is overridden per process | +| `gh` token on the test machine | Lacks the `ssh_signing_key` scope, so it cannot register a signing key as is | + +The fourth row is what makes a plugin possible: official plugins are cross-built on Linux with `CGO_ENABLED=0` (`.github/workflows/plugin-packages.yaml`), so the macOS code must reach the Secure Enclave without cgo or Swift. + +## What exists today + +Read from the code on this branch. + +**Plugin platform** + +- **Exports:** auth providers, auth identities, and commands (`pkg/plugin/bootstrap/descriptors.go`). One package can export several. Integrations and stores are follow-ons. +- **Plugin identities need an upstream** (`via` a provider or another identity). +- **Credential types are fixed.** Between a provider and an identity only `secret-values/v1` (a string map), `oidc/v1`, and `aws/v1` can be decoded; an identity can output only `environment/v1` or `aws/v1`. A plugin cannot define its own type. +- **Every use authenticates.** A chain rooted at a plugin provider is never served from a cache: the provider's and the identity's authentication both run on each use, each in a fresh plugin process. Only `atmos auth exec`, `shell`, `env`, and `atmos git` first try the stored identity credential. +- **Credentials must have an expiry.** A missing one is rejected; a far-future one is accepted. +- **A plugin cannot show the user text during authentication.** The host discards progress events, interaction events, and error messages from auth calls. A plugin can only open a browser URL and listen on a loopback port. A device code cannot be displayed. +- **Environment credentials** are a fixed map. Every value is masked in Atmos output, and the plugin cannot see or append to the caller's environment. +- **Private state** is separate for each provider and identity, is not available to commands, and is wiped by the host on logout. +- **Commands** run through `atmos plugin run -- ` and receive the raw arguments. They get HTTP but no stdin, TTY, or state. There is no top-level shortcut yet; the `aliases:` setting provides one. +- **Empty environment.** Plugin processes start with no `HOME` or `PATH`, in the install directory. The home directory is still available from the user database (verified with cgo off). +- **The process group is killed after every call,** so a daemon cannot be a child of a plugin call. +- **Only one of `spec` and `principal` is validated** against the identity's schema. +- **Project scope only.** The release is pinned per project in `.atmos.plugins.lock.yaml`. +- **Release contents:** `plugin.yaml`, `schemas/**`, and one native executable per platform, whose install path changes with every release. No app bundle, helper binary, or provisioning profile. + +**Remote reads** + +- Everything Atmos fetches itself goes through go-getter in `pkg/downloader`: `atmos vendor pull`, component `source:` provisioning, remote stack imports, remote `atmos.yaml` imports. Its Git getter runs the system `git` with the process environment. +- **None of these accept an identity.** `AtmosVendorSource`, `VendorComponentSource`, `StackImport`, and `ConfigImport` have no such field. Private reads rely on whatever is in the process environment, plus one hard-wired broker (`pkg/auth/broker`) that provisions Atmos Pro's `github/sts` in CI. +- `git.repositories..auth.identity` is specified to work this way for `atmos git` ([Git operations](git-ops.md)), but `cmd/git/env.go` passes no identity provider, so the setting has no effect today. Git hooks and provisioner targets do pass one. +- `atmos vendor update` and `vendor diff` list remote versions in-process with go-git and no credentials. +- Vendoring fetches sources concurrently, so credentials exported into the process (as the broker does) cannot differ per source. + +## Design + +### Model: the upstream is who vouches for the key + +A hardware key on its own proves nothing. It becomes an identity when a party others already trust says "this key speaks for X". Registering a key on GitHub, an `authorized_keys` entry, an EC2 Instance Connect push, and an SSH certificate are the same act with different authorities and lifetimes. Atmos already has a slot for "who vouches for this identity": the `via` link. + +| Upstream (`via`) | What `atmos auth login --identity ` does | Trust lasts | Phase | +|---|---|---|---| +| `ssh/self` provider | Creates the key if missing. You distribute the public key yourself | Until removed | 1 | +| A provider that yields a GitHub token (`secrets/store` from `cloudposse/auth-secrets`) | Also makes sure the key is on that GitHub account for signing and, optionally, authentication | Until removed | 1 | +| `github/user` provider (device login) | The same, without a stored token | Until removed | Blocked: core cannot show a device code | +| An AWS identity | Pushes the public key to the instance (EC2 Instance Connect) | About a minute | 4 | +| A Vault or CA identity | Obtains a certificate for the key | Minutes to hours | 4 | + +So there is no "register" command. Enrollment is the identity's login step and it is idempotent. Each row is an implementation of one [authority interface](#authorities-the-enrollment-interface); [Managing keys on GitHub](#managing-keys-on-github) shows the user's side. + +```yaml +plugins: + packages: + ssh: + source: oci://ghcr.io/cloudposse/atmos/plugins/ssh + version: "^0.1.0" + secrets: + source: oci://ghcr.io/cloudposse/atmos/plugins/auth-secrets + version: "^1.1.0" +aliases: + ssh: plugin run ssh -- +stores: + keychain: + type: keychain + secret: true +auth: + providers: + github-token: + kind: secrets/store # a GitHub token with the key-management scopes + spec: + credentials: + token: {store: keychain, path: github/ssh-keys} + identities: + ops: + kind: ssh/key + via: {provider: github-token} + spec: + authority: github # who vouches for the key; `self` when nobody does + store: secure-enclave # secure-enclave | tpm | file + allow: [signing, authentication] + require: unlocked # omit to let the key be used while the screen is locked + user: deploy # SSH login name on the remote side, optional +git: + repositories: + infra: + auth: {identity: ops} + commit: {signing: always} +``` + +The configuration is shared; the key is not. Each person's login creates and enrolls that machine's own key. + +| Field | Meaning | +|---|---| +| `spec.authority` | Which [authority](#authorities-the-enrollment-interface) vouches for the key. It reads its credentials from the upstream's string map; the token is the `token` entry | +| `spec.store` | Where the key is held | +| `spec.allow` | What the key may do: `signing`, `authentication`, or both. It limits the identity's agent socket and decides what is enrolled with the authority | +| `spec.require` | What must hold for the key to be used. See [Key stores](#key-stores) | +| `spec.user` | The SSH login name on the remote side. It is in `spec` because the host validates only one of `spec` and `principal` | +| `spec.rewrite_https` | Hosts whose HTTPS Git URLs are rewritten to SSH | + +Each identity has its own key, named after the identity. `spec.key` can name a shared key instead; identities that share one must agree on `store` and `require`, or validation fails. + +**Authentication runs on every use, so it must be cheap.** When a receipt on disk already matches the key and the `allow` list, the identity makes no network call and needs no upstream interaction; it works offline and while locked. It contacts the authority only when something changed: a new key, a changed `allow` list, or a deleted key to revoke. + +### What the identity provides + +An `environment/v1` credential: + +| Variable | When | Purpose | +|---|---|---| +| `SSH_AUTH_SOCK` | always | The identity's own agent socket | +| `ANSIBLE_REMOTE_USER` | `spec.user` set and `authentication` in `allow` | Ansible's login name | +| `GIT_CONFIG_GLOBAL` | `signing` in `allow`, or `spec.rewrite_https` set | A generated file that first includes the user's own global Git configuration, then sets `gpg.format=ssh`, `gpg.ssh.program`, `user.signingkey=key::`, `commit.gpgsign=true`, `tag.gpgsign=true`, and any `url."ssh://git@/".insteadOf=https:///` rewrites | + +`GIT_CONFIG_COUNT` is not used: the plugin cannot append to entries the caller already has, the host would overwrite them, and short values such as `ssh` would be masked everywhere in Atmos output. Nothing in this environment is secret, but the host masks every value, so these paths appear masked in Atmos output. + +### Consumers + +| Consumer | How it names the identity | Needs | +|---|---|---| +| Terraform, Packer, Helmfile components | `--identity ops` or component auth | Nothing new. The environment reaches the tool, so `terraform init` module sources and provisioners use the key | +| Ansible components | `--identity ops` | Core fix on this branch: the identity's environment was not passed to `ansible-playbook`. Ansible still authenticates only on an explicit `--identity` | +| `atmos git` | `git.repositories..auth.identity` | Core fix on this branch: `atmos git` ignored the configured identity | +| Anything else | `atmos auth exec`, `shell`, `env` | Nothing new | +| `atmos vendor pull`, component `source:`, remote stack imports | `identity:` on the source or import | **Core change below** | + +### Core change: identities on remote reads + +Add `identity:` wherever a remote location is declared, using the flat field that commands, stores, MCP servers, secrets, workflows, and plugin registries already use: + +```yaml +# vendor.yaml +spec: + sources: + - component: vpc + source: git::ssh://git@github.com/acme/modules.git//vpc?ref={{.Version}} + version: 1.4.0 + identity: ops + +# stack manifest +import: + - path: git::ssh://git@github.com/acme/stacks.git//catalog/vpc?ref=v2 + identity: ops +components: + terraform: + vpc: + source: + uri: git::ssh://git@github.com/acme/modules.git//vpc + version: 1.4.0 + identity: ops +``` + +- **Fields:** `AtmosVendorSource.Identity`, `VendorComponentSource.Identity`, and `StackImport.Identity`, plus the JSON schemas in `pkg/datafetcher/schema/`. `ConfigImport` is left out; see the limits. +- **Defaults:** `vendor.identity` and `imports.identity` in `atmos.yaml`, so a project names its source identity once. A default applies to every remote go-getter source, not only Git ones. +- **Precedence:** `--identity` flag, then the field on the entry, then the section default, then the existing ambient broker, then the process environment. The flag applies to `atmos vendor pull` only. On component commands it selects the component's own identity, and it is not reachable during stack processing, so sources and imports there use the field and the default. +- **One seam.** The downloader gains a per-fetch environment (`downloader.WithEnvironment(ctx, env)`), composed with the existing `git.ComposeEnvironment` and `EnsureIdentityEnvironment`. The Git getter uses it in place of the bare process environment. Nothing is exported with `os.Setenv`, so two sources fetched concurrently can use different identities. +- **Not SSH-specific.** Any identity works: an AWS identity for an S3 source, `oci/user` for an OCI source, a `github/sts`-linked identity for HTTPS. SSH identities are one beneficiary. +- **Same file, laptop and CI.** The identity name is stable; its kind can differ by profile. `source-reader` is an `ssh/key` on a workstation and an Atmos Pro identity with `github/sts` in CI, and `vendor.yaml` does not change. +- **Limits:** + - Module sources inside `.tf` files cannot name an identity; they use the component's. + - A remote `import:` in `atmos.yaml` cannot name an identity. When imports resolve, only the importing file's own `auth:` section is loaded, so an identity defined in another fragment or profile is not visible. Supporting it means reworking configuration loading. + - `atmos vendor update` and `vendor diff` do not use the identity when listing versions; only the fetch does. + - Only Git subprocesses receive the identity's environment. S3 and OCI sources authenticate as before. + - Outside CI, a named identity with no valid session fails with a hint to run `atmos auth login`; it never prompts mid-fetch. In CI the identity authenticates directly, because providers such as OIDC have nothing stored. +- **Replaces:** the `sshkey=` go-getter URL parameter, which puts a private key in configuration. + +### Plugin package + +`plugins/ssh/`, published as `oci://ghcr.io/cloudposse/atmos/plugins/ssh`, name `cloudposse/ssh`, starting at `0.1.0`. Its own Go module, added to `plugins/releases.yaml`. One executable with two roles: + +1. **Plugin server**, started by Atmos: the auth exports and the command tree. +2. **Agent daemon**, started by launchd or systemd and never by Atmos: invoked directly as `plugin agent-serve --config `, it skips the plugin handshake and serves sockets. The service definition points at a stable copy of the executable, because the installed path changes with every release. + +Exports: + +| Export | Kind | Role | +|---|---|---| +| Provider | `ssh/self` | No authority; the key is self-asserted | +| Provider | `github/user` | Deferred. Device login to a GitHub user account; needs core to display a device code | +| Identity | `ssh/key` | The key, its enrollment with the upstream, and its environment | +| Command | `ssh` | Key and agent management | + +All six platforms ship from the first release so a mixed-OS team can activate the package everywhere. Hardware key stores report "not supported on this platform" where they don't apply. + +### Authorities: the enrollment interface + +An **authority** is a party that can vouch for a public key. The `ssh/key` identity never talks to GitHub directly; it hands the key to the authority named in `spec.authority`. GitHub is the first implementation. + +```go +// Authority vouches for a public key using an upstream credential. +type Authority interface { + // Name is the registry key, such as "github". + Name() string + // Supports lists the uses it can vouch for. + Supports() []Use // UseSigning, UseAuthentication + // Ensure makes the authority trust the key for the requested uses. It is idempotent: + // a key that is already trusted returns the existing receipt and changes nothing. + Ensure(ctx context.Context, credentials map[string]string, request Request) (Receipt, error) + // Revoke withdraws an enrollment. It is idempotent: a missing enrollment is success. + Revoke(ctx context.Context, credentials map[string]string, receipt Receipt) error +} + +type Request struct { + Key ssh.PublicKey + Uses []Use // from the identity's `allow` + Title string // "@ (Atmos)" + Principals []string // remote usernames, for authorities that issue certificates + Target string // host or instance, for authorities that enroll per host +} + +type Receipt struct { + Authority string // "github" + Subject string // the account, role, or host that now trusts the key + References map[Use]string // the authority's own IDs, needed to revoke + Certificate []byte // set by authorities that issue certificates + ExpiresAt time.Time // zero means trusted until revoked +} +``` + +- **Registry.** Authorities register by name (`authority.Register`). The platform has one usable credential type between a provider and an identity, a string map, so the authority cannot be chosen by credential type. The identity fails before any network call when the named authority is unknown or a requested use is unsupported. +- **Receipts** are JSON files on disk beside the key, one per key and authority. They hold no secret. They cannot live in the plugin's private state, which the host wipes on logout. +- **One contract for long-lived and short-lived trust.** A receipt with no expiry is an enrollment (GitHub). A receipt with an expiry and a certificate is an issuance (Vault). The identity's credential expires when its receipt does. + +The auth lifecycle maps onto the interface, so no authority needs its own commands: + +| Atmos action | What the identity does | +|---|---| +| `atmos auth login --identity `, and every other use | Creates the key if missing. If a receipt matches the key and its `allow` list, nothing else. Otherwise `Ensure`, then `Revoke` for this machine's receipts whose key no longer exists. Makes sure the agent is running | +| `atmos auth logout ` | `Revoke` for this machine's receipts whose key no longer exists, using the token the identity saved in its own private state when it last authenticated. A live key stays enrolled | + +A short-lived receipt (a certificate) is renewed by the same step when it has expired, which works unattended only if the upstream can authenticate without interaction. + +The rule for removal is: **an enrollment is revoked once its key is gone.** Logging out alone never removes a key from the authority, so a routine `atmos auth logout --all` cannot strip a working machine's keys. + +**Adding an authority** + +1. Implement `Authority` in `plugins/ssh/authority//` and register it. +2. If no existing Atmos provider or identity yields a credential it can use, add a provider kind to the package (`gitlab/user`, for example). +3. Pass the shared conformance suite, which runs every registered authority against a fake server: `Ensure` twice changes nothing, each use is enrolled separately, `Revoke` twice succeeds, a credential without the needed permission produces an error naming it, and receipts contain no secret. +4. Add a row to the table below and to the docs. + +The identity, the agent, the configuration schema, and the commands do not change. A test-only fake authority proves the identity has no GitHub-specific code. + +| Authority | Upstream credential | Uses | Receipt | Phase | +|---|---|---|---|---| +| `github` | Any provider that yields a `token`: `secrets/store` today, `github/user` later | signing, authentication | No expiry | 1 | +| `gitlab` | Any provider that yields a `token` | signing, authentication | No expiry | later | +| `ec2-instance-connect` | A built-in AWS identity | authentication | About a minute, per instance | 4 | +| `vault-ssh` | A Vault identity | authentication | Certificate, minutes to hours | 4 | + +Limit: authorities are compiled into this package. Another plugin can supply the upstream login, but not a new authority, until plugins can call each other. + +### Managing keys on GitHub + +The identity is configured once, in the shared `atmos.yaml` shown above. Everything a user does afterwards is one of these: + +| I want to | Run | What happens on GitHub | +|---|---|---| +| Set up this machine | `atmos auth login --identity ops` | A new signing key titled `ops@ (Atmos)` | +| See what this machine has enrolled, and the last error | `atmos ssh key status` | Nothing; reads local receipts | +| Also push over SSH | Add `authentication` to `allow`, then `atmos auth login --identity ops` | The same key is added as an authentication key | +| Set up a second machine | `atmos auth login --identity ops` there | A second key; each machine has its own | +| Rotate this machine's key | `atmos ssh key delete ops --yes`, then `atmos auth login --identity ops` | The new key is added and the old one removed, in one login | +| Retire this machine | `atmos ssh key delete ops --yes`, then `atmos auth logout ops` | This machine's key is removed | +| Deal with a lost machine | Delete `ops@ (Atmos)` under GitHub Settings, SSH and GPG keys | Removed by hand; the lost machine cannot be asked to do it | + +First login on a machine. The host shows no text from a plugin during authentication, so the detail comes from `key status`, which also reports the last failure: + +```console +$ atmos auth login --identity ops +$ atmos ssh key status +KEY STORE REQUIRE AUTHORITY SUBJECT ALLOW SINCE EXPIRES LAST RESULT +ops secure-enclave unlocked github osterman signing 2026-11-12 never enrolled +``` + +GitHub specifics, all inside the `github` authority: + +- **API:** `POST /user/ssh_signing_keys` for signing and `POST /user/keys` for authentication; the matching `DELETE` calls to revoke. A key GitHub reports as already present on the account is adopted, not duplicated. +- **Token:** a GitHub token stored in any Atmos secret store and read through `secrets/store`. It needs `write:ssh_signing_key`, plus `write:public_key` when `allow` includes `authentication`. A token lacking a scope produces an error naming it, visible in `key status`. +- **Device login (`github/user`) is deferred.** It needs two things that do not exist: a client ID that Atmos owns (embedding the GitHub CLI's is not an option, because the consent screen and audit log would name GitHub CLI for a request Atmos made), and a way for a plugin to show the user a code. +- **Without a token:** the identity uses `authority: self` with the `ssh/self` provider, and `key show --format github` prints the public key, the settings URL, and the equivalent `gh ssh-key add --type signing` command. +- **Bots:** the endpoints are per user. A machine-user account can hold a signing key; a GitHub App cannot. + +### Key stores + +Each returns a `crypto.Signer` plus an OpenSSH public key: + +| Store | Platforms | Phase | Notes | +|---|---|---|---| +| `secure-enclave` | darwin | 1 | `Security.framework` through `ebitengine/purego`. P-256. With no `require` the key is created as `AfterFirstUnlockThisDeviceOnly`; with `require: unlocked` as `WhenUnlockedThisDeviceOnly`. Stored as a wrapped blob, mode 0600, under the Atmos XDG data directory | +| `file` | all | 1 | Ordinary OpenSSH private key file. For tests and platforms without hardware support. Documented as not hardware-protected | +| `tpm` | linux, windows | 3 | `google/go-tpm` (pure Go). Needs `/dev/tpmrm0` access on Linux | + +Existing keys cannot be imported into the Secure Enclave; each machine generates its own. + +**`require`** lists what must hold for the key to be used. It takes one value or a list, and every term is enforced by the key store. + +| `require` | Screen locked | After a reboot, before first login | +|---|---|---| +| omitted | Usable | Refused | +| `unlocked` | Refused | Refused | + +- **The key store enforces it, not the plugin.** The agent contains no lock-state check that grants or refuses signing. In the proof of concept a separate process that loaded the key blob directly, bypassing the agent, was refused by the enclave while locked. So `require: unlocked` holds even against other processes running as the user. +- **It is fixed when the key is created** and cannot be changed afterwards. When the configured value differs from the existing key's, login fails and says to rotate the key (`key delete`, then log in). It never keeps the old setting silently. +- **A store that cannot enforce a term rejects it** at validation instead of accepting a promise it cannot keep. `tpm` and `file` reject `unlocked`. +- **Omitting it is not "always".** The key still does not work after a reboot until the first login. +- `key status` shows what each key was created with. +- **Later terms** must also be hardware-enforced. `presence` (Touch ID per signature) is the candidate; it is deferred because it has not been tested from a LaunchAgent. + +Why this word: + +- **Not `when`.** A `when` on imports, tasks, and workflow steps is a condition Atmos evaluates at run time and that can be edited freely. This is a constraint the hardware enforces and that is fixed at creation. Conditions Atmos evaluates, such as `local`, are out of scope; they would be a top-level `when` on identities, a separate core change. +- **Same meaning as `version.require`:** a constraint that must hold or the operation fails. +- **Not `policy`,** which means admission rules in Atmos (`policy.d/`, `plugins.policy`). +- **Not the same as `required: true`** at the top level of an identity, which asks Atmos to authenticate it automatically. + +### Agent + +`golang.org/x/crypto/ssh/agent` over Unix sockets (a named pipe on Windows in phase 3). + +- **One socket per identity.** The socket exposes only that identity's key and signs only what `allow` permits. A Git signature request is an SSHSIG blob that begins with the magic bytes `SSHSIG` and carries the namespace `git`; an SSH login request does not. A signing-only socket therefore cannot be used to log in to a server, and an authentication-only socket cannot sign commits. +- **Peer check.** Connections from another uid are rejected. +- **Audit log.** One line per request: time, identity, request type, result, screen lock state. +- **Lifecycle.** `agent start` writes a LaunchAgent plist (systemd user unit on Linux) pointing at the installed executable and loads it with `/bin/launchctl bootstrap gui/`. The daemon is launchd's child, in the user's login session, and survives Atmos exiting. Using an identity starts it when it is not running. + +### Commands + +Invoked as `atmos plugin run ssh -- `, or `atmos ssh ` with the alias above: + +``` +key list +key show [--format authorized-key|allowed-signers|github|json] +key status [] # this machine's keys and receipts: store, require, authority, subject, allow, expiry +key delete --yes # local only; the authority is updated at the next login or logout +agent start | stop | status | log +test [--soak ] # sign and verify in a scratch repo; reports lock state +``` + +Keys are created by `atmos auth login`. Destructive commands need `--yes` because commands cannot prompt. Commands never call an authority: they have no access to auth credentials, so every remote change goes through login or logout. + +## Security model + +- **Protected:** the key cannot leave the machine. A stolen disk, backup, or copied blob is useless elsewhere. After a reboot nothing signs until the first login. +- **Not protected:** other processes running as the same user. They can read the blob and sign directly, bypassing the socket's `allow` limits. Those limits are a real boundary only for callers that cannot read the key directory: containers and sandboxes given just the socket. +- **Documented rule:** never mount the key directory into a sandbox. +- **Closing the gap** needs keychain-held keys restricted to a signed app, which needs an Apple Developer Program membership, a provisioning profile, and an app bundle. The plugin release format cannot carry those today. Deferred. +- **Scoped use.** A source or import uses a key only when it names the identity; nothing is exported process-wide. +- **Authority token.** The GitHub token should carry only key-management scopes. It lives in the secret store the user chose. The identity keeps a copy in its own private state so logout can revoke enrollments; the host removes that copy at logout. +- The daemon is started by launchd directly, so Atmos's re-verification of cached plugin files does not run before it starts. Accepted for now, given the same-user boundary above. + +## Delivery + +The plugin and the core change are independent and can land in either order. One pull request each. + +**Phase 0: spikes** + +| # | Question | Result | +|---|---|---| +| 1 | Can Go with `purego` and cgo off create, reload, and sign with a Secure Enclave key, and load the POC's blob? | **Yes.** Verified on hardware; builds for all six platforms | +| 2 | Does that binary sign while the screen is locked, as a LaunchAgent, for a Git signature and an SSH login? | Not run yet. Needs a person to lock the screen | +| 3 | Can a plugin run `/bin/launchctl` and find the home directory with an empty environment? | **Yes.** `os/user` returns it with cgo off; `launchctl` runs by absolute path | +| 4 | Can a plugin identity return a credential that does not expire? | **No.** An expiry is required; a far-future one is accepted | +| 5 | Does `atmos auth login` re-run the identity's authentication? | **Yes, and so does every other use.** Authentication must be cheap and idempotent | +| 6 | Can logout reach the authority? | **Yes,** if the identity saved the token in its own private state when it authenticated | +| 7 | Does an Ansible run with `--identity` receive the identity's environment? | **It did not; fixed.** A test reproduced the missing environment, and the executor now passes it the way Helmfile and Packer do | +| 8 | Does a stored `environment/v1` credential work in a fresh process (`atmos auth exec`, `atmos git`)? | **It did not; fixed.** A two-manager test against the real `auth-secrets` plugin failed on all three passive paths. The manager now hands the stored credential back to the identity | +| 9 | Does `atmos git` use `git.repositories..auth.identity`? | **It did not; fixed.** `cmd/git` passed no identity provider. It now resolves the flag, then the repository's identity | + +Rows 4 to 6 come from reading the platform code, not from running it. Row 2 is the one acceptance step still open. + +**Phase 1a: plugin, macOS MVP (`0.1.0`)** +- Package scaffold from `atmos plugin create`, manifest, schemas, `releases.yaml` entry. +- `ssh/self` provider, `ssh/key` identity, `secure-enclave` and `file` stores, daemon with per-identity sockets and audit log, commands. GitHub enrollment uses a token from `secrets/store`; the `github/user` device login waits on core. +- The `Authority` interface, its registry, receipts, the `github` authority, and the conformance suite. +- Tests on Linux CI against the `file` store: agent protocol, SSHSIG parsing, `allow` limits (allowed and refused requests), environment composition, the conformance suite against a fake GitHub API, and the login and logout reconciliation rules against a fake authority. `plugin.tests.yaml` fixtures for the auth exports. +- Manual acceptance on a real Mac: the POC's soak test, ported to `test --soak`. GitHub's macOS runners are virtual machines without a Secure Enclave, so this cannot run in CI. +- Docs: package README, website page, blog post, roadmap entry. + +**Phase 1b: core** +- Identities on remote reads: the three fields, section defaults, schemas, precedence, and the downloader's per-fetch environment. +- Tests with a mock identity environment provider: per-source environment reaches the Git subprocess, two concurrent sources with different identities stay isolated, precedence, and the no-identity path is unchanged. +- Make `atmos git` use the configured identity. +- Pass an identity's environment to `ansible-playbook`. +- Make a stored `environment/v1` credential usable in a fresh process, if the reproducing test confirms the defect. +- Docs for vendoring, `source:`, and imports. + +**Phase 1c: core, what a plugin can tell the user** (needs a decision; see open questions) +- Display a provider's interaction event, so a device code can be shown. +- Surface a bounded, masked error message from auth calls. +- Let a plugin mark environment values as not sensitive. + +**Phase 2: sandboxes** +- Mount an identity's socket into devcontainers and containers. +- Spike first: Docker Desktop on macOS does not reliably pass arbitrary host Unix sockets into containers. + +**Phase 3: Linux and Windows** +- `tpm` key store, systemd user unit, Windows named pipe and service. + +**Phase 4: more authorities** +- `ssh/key` chained `via` an AWS identity (EC2 Instance Connect) or a Vault or CA identity (certificates), so hosts trust an authority instead of each machine. +- Host restrictions per identity, using the destination information newer OpenSSH sends to agents. + +**Deferred** +- Top-level `atmos ssh` without an alias (waits on plugin command shortcuts in core). +- Touch ID keys; keychain-restricted keys. +- Secure Enclave as a secrets store or SOPS age recipient (waits on the stores extension). + +## Acceptance criteria + +**Plugin (phase 1a)** +- `atmos auth login --identity ops` on a fresh Mac creates the key and enrolls it on GitHub; a second run changes nothing. +- Every row of the [Managing keys on GitHub](#managing-keys-on-github) table works as described: adding a use, a second machine, rotation, and retiring a machine. +- `atmos auth logout` with a live key leaves it enrolled. +- A second authority can be added without changing the identity, the agent, the configuration schema, or the commands. A test-only fake authority demonstrates this and passes the conformance suite. +- With the screen locked and no prompt: `atmos git commit` produces a commit `git verify-commit` accepts, even with 1Password's signer configured globally; and `atmos ansible playbook --identity ops` connects to a test host. +- A `require: unlocked` key is refused while locked, and `key status` and the audit log name the cause. +- Changing `require` for an existing key makes login fail with instructions to rotate; `require: unlocked` on the `file` store and any unknown term fail validation. +- A signing-only identity cannot authenticate an SSH login; an authentication-only identity cannot sign a commit. +- `agent status` reports when the running daemon is a different version from the installed plugin. +- On Linux and Windows the package installs and activates; `secure-enclave` fails with a clear "not supported" error; the `file` store works. +- The release is built on Linux with cgo disabled and passes `atmos plugin release --check`. + +**Core (phase 1b)** +- A vendored source, a component `source:`, and a remote stack import that each name an identity are fetched with that identity's environment and nothing is written to the process environment. +- Two sources with different identities in one `atmos vendor pull` each use their own. +- Sources without an identity behave exactly as before, including the Atmos Pro broker in CI. +- An identity that needs interactive login fails with the `atmos auth login` hint. + +## Open questions + +1. **What a plugin can tell the user.** The host discards plugin text during authentication, deliberately, because it could contain secrets. That blocks the device login, makes failures generic, and masks harmless values. Phase 1c lists the three core changes that would lift it. Until then enrollment needs a stored token, and diagnostics live in `atmos ssh key status`. +2. **GitHub app.** A device login also needs a client ID that Atmos owns: register an OAuth app, or check whether the existing Atmos Pro GitHub App can be granted the signing-keys account permission. +3. **Two identities in one run.** A run uses one identity's environment. A playbook that needs cloud credentials for dynamic inventory as well as the SSH identity has to chain them, which depends on question 5. +4. **Signing with a different push identity.** A repository has one `auth.identity`. If pushes use a `github/sts` App token on another identity, the SSH identity cannot also be named there. A core `git/signing` integration, or a separate `commit.identity` field, would solve it. +5. **Chaining from built-in identities.** A plugin identity can accept AWS credentials as its upstream (`aws/v1`). No other built-in credential family can be passed to a plugin today, so a Vault upstream needs core work. +6. **Old commits after a key is removed.** Whether GitHub keeps showing earlier commits as Verified once their signing key is deleted needs confirming; it decides how freely rotation should be recommended. +7. **Removing a lost machine's key from another machine.** Today that is done by hand on GitHub. An authority could list and revoke enrollments it did not create, at the cost of a wider interface. +8. **Per-user daemon, per-project plugin.** Two projects can lock different plugin versions while sharing one daemon and key directory. Proposal: the newest started daemon wins and `agent status` reports the mismatch. +9. **Explicit `require`.** Omitting it lets the key be used while locked, which is the purpose of this package. The cautious alternative is to make the field mandatory on `ssh/key` identities, with an empty list meaning no requirement, so locked-screen use is always a written choice. +10. **Developer Program.** Join it to close the same-user gap, and extend the plugin release format to carry a signed macOS helper? diff --git a/errors/errors.go b/errors/errors.go index 7a989a94a6e..0c742f1bf9f 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1460,6 +1460,13 @@ var ( ErrScaffoldComputedFieldNotSettable = errors.New("computed field cannot be set") ErrScaffoldFileDelimitersInvalid = errors.New("spec.files[].delimiters must be exactly two non-empty strings") + // Remote-read identity errors (identity: on vendor sources, component sources, and remote imports). + ErrRemoteIdentityLoginRequired = errors.New("remote read identity is not logged in") + ErrRemoteIdentityResolverUnavailable = errors.New("remote read identity cannot be resolved") + // ErrRemoteImportIdentity marks a remote import that failed because its identity could not be + // resolved, as opposed to a source that is missing. `skip_if_missing` never skips it. + ErrRemoteImportIdentity = errors.New("failed to resolve the identity for a remote import") + // Source provisioner errors. ErrSourceProvision = errors.New("source provisioning failed") ErrSourceNotFound = errors.New("source not found") diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index 70556c20359..66519d26c6d 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -1659,9 +1659,11 @@ func processYAMLConfigFileWithContextInternal( } // Download the remote import. log.Debug("Downloading remote stack import", "uri", imp, "file", relativeFilePath, "nested_imports", nestedImports, "ttl", importTTL) - remoteMatches, err := stackimports.ResolveRemoteImportNested(atmosConfig, imp, nestedImports, importTTL) + remoteMatches, err := stackimports.ResolveRemoteImportNestedIdentity(atmosConfig, imp, nestedImports, importTTL, importStruct.Identity) if err != nil { - if importStruct.SkipIfMissing { + // skip_if_missing covers a source that is not there, never an identity that could + // not be resolved (see stackimports.SkippableWhenMissing). + if importStruct.SkipIfMissing && stackimports.SkippableWhenMissing(err) { log.Debug("Skipping missing remote import", "uri", imp) continue } diff --git a/internal/exec/utils_auth.go b/internal/exec/utils_auth.go index 749a9685926..34627ea08d1 100644 --- a/internal/exec/utils_auth.go +++ b/internal/exec/utils_auth.go @@ -340,3 +340,12 @@ func prepareComponentAuthEnvironment(authManager auth.AuthManager, identity stri } return preparedEnv, nil } + +// PrepareComponentAuthEnvironment is the exported form of prepareComponentAuthEnvironment for +// component packages (such as Ansible) that run their own subprocess. It layers the resolved +// identity's prepared environment over envVars, exactly as the Helmfile and Packer paths do. +func PrepareComponentAuthEnvironment(authManager auth.AuthManager, identity string, envVars []string) ([]string, error) { + defer perf.Track(nil, "exec.PrepareComponentAuthEnvironment")() + + return prepareComponentAuthEnvironment(authManager, identity, envVars) +} diff --git a/internal/exec/vendor.go b/internal/exec/vendor.go index 8eda24a41b7..e0b9f3013bb 100644 --- a/internal/exec/vendor.go +++ b/internal/exec/vendor.go @@ -162,7 +162,7 @@ func PlanVendorPull(cmd *cobra.Command, args []string) (*VendorPullPlan, error) return nil, err } atmosConfig.Vendor.MaxConcurrency = n - plan := &VendorPullPlan{Config: atmosConfig, Options: install.InstallOptions{DryRun: vendorFlags.DryRun, RefreshLock: vendorFlags.RefreshLock, LockEnforcement: vendorFlags.LockEnforcement, MaxConcurrency: n}} + plan := &VendorPullPlan{Config: atmosConfig, Options: install.InstallOptions{DryRun: vendorFlags.DryRun, RefreshLock: vendorFlags.RefreshLock, LockEnforcement: vendorFlags.LockEnforcement, MaxConcurrency: n, Identity: info.Identity}} vendorFlags.ctx = ctx vendorFlags.collect = &plan.Packages if err := handleVendorConfig(&plan.Config, &vendorFlags, args); err != nil { diff --git a/pkg/auth/broker/broker.go b/pkg/auth/broker/broker.go index 9e26c6cc7df..991191eb606 100644 --- a/pkg/auth/broker/broker.go +++ b/pkg/auth/broker/broker.go @@ -108,7 +108,11 @@ func runEnabledBrokers(ctx context.Context, atmosConfig *schema.AtmosConfigurati // downstream auth-retry would be wrong. exported := false for key, value := range env { + // Remember what the variable was before, so a remote read that names its own identity + // can run without the broker's credentials (see WithoutBrokeredEnvironment). + recordExport(key) if err := os.Setenv(key, value); err != nil { + forgetExport(key) log.Warn("Failed to export credential broker variable", "broker", p.Name(), "key", key, "error", err) continue } @@ -162,11 +166,12 @@ func SwapRegistryForTest() func() { } } -// resetBrokeredForTest clears the brokered-credentials flag so a test observes only what -// its own registered providers provision. Pairs with SwapRegistryForTest. +// resetBrokeredForTest clears the brokered-credentials flag and the record of exported variables so +// a test observes only what its own registered providers provision. Pairs with SwapRegistryForTest. func resetBrokeredForTest() { brokeredMu.Lock() brokeredCredentials = false + exportedEnv = map[string]priorVariable{} brokeredMu.Unlock() } diff --git a/pkg/auth/broker/exported.go b/pkg/auth/broker/exported.go new file mode 100644 index 00000000000..7c047e240e2 --- /dev/null +++ b/pkg/auth/broker/exported.go @@ -0,0 +1,99 @@ +package broker + +import ( + "os" + "runtime" + "strings" + + "github.com/cloudposse/atmos/pkg/perf" +) + +// envSeparator separates a variable name from its value in a "KEY=value" entry. +const envSeparator = "=" + +// priorVariable is the state a process environment variable had before a broker replaced it. +type priorVariable struct { + value string + set bool +} + +// The exportedEnv map records every variable a broker exported into the process environment this +// process, keyed by its normalized name, with the state it had before. Guarded by brokeredMu. +// +// A remote read that names an Atmos Auth identity uses that identity's credentials, never the +// ambient broker's. The broker exports into the process environment, which every later subprocess +// inherits, so such a read must be able to see the environment as it was before the broker ran. +var exportedEnv = map[string]priorVariable{} + +// recordExport remembers the state of key before a broker replaces it. Only the first export of a +// variable is recorded, so the remembered state is the one from before any broker ran. +func recordExport(key string) { + normalized := normalizeEnvKey(key) + + brokeredMu.Lock() + defer brokeredMu.Unlock() + if _, recorded := exportedEnv[normalized]; recorded { + return + } + // A snapshot of the live process variable a broker is about to replace. + value, set := os.LookupEnv(key) + exportedEnv[normalized] = priorVariable{value: value, set: set} +} + +// forgetExport drops the record for a variable whose export failed, so it is not treated as +// broker-owned. +func forgetExport(key string) { + brokeredMu.Lock() + defer brokeredMu.Unlock() + delete(exportedEnv, normalizeEnvKey(key)) +} + +// PreBrokerValue reports the value a variable had before a broker exported it. The second result +// is false when no broker exported the variable, in which case the process environment is +// authoritative. A variable that was unset before the broker ran reports "" and true. +func PreBrokerValue(name string) (string, bool) { + defer perf.Track(nil, "auth.broker.PreBrokerValue")() + + brokeredMu.Lock() + defer brokeredMu.Unlock() + prior, exported := exportedEnv[normalizeEnvKey(name)] + return prior.value, exported +} + +// WithoutBrokeredEnvironment returns env ("KEY=value" entries) with every broker-exported variable +// restored to the state it had before the broker ran: removed when it was unset, or set back to +// its earlier value. Variables no broker exported are returned unchanged, and env is not modified. +func WithoutBrokeredEnvironment(env []string) []string { + defer perf.Track(nil, "auth.broker.WithoutBrokeredEnvironment")() + + brokeredMu.Lock() + defer brokeredMu.Unlock() + if len(exportedEnv) == 0 { + return env + } + + restored := make([]string, 0, len(env)) + for _, entry := range env { + key, _, found := strings.Cut(entry, envSeparator) + if !found { + restored = append(restored, entry) + continue + } + prior, exported := exportedEnv[normalizeEnvKey(key)] + switch { + case !exported: + restored = append(restored, entry) + case prior.set: + restored = append(restored, key+envSeparator+prior.value) + } + } + return restored +} + +// normalizeEnvKey folds case on Windows, where environment names are case-insensitive. +func normalizeEnvKey(key string) string { + if runtime.GOOS == "windows" { + return strings.ToUpper(key) + } + return key +} diff --git a/pkg/auth/broker/exported_test.go b/pkg/auth/broker/exported_test.go new file mode 100644 index 00000000000..6660e98760e --- /dev/null +++ b/pkg/auth/broker/exported_test.go @@ -0,0 +1,83 @@ +package broker + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/schema" +) + +// unsetForTest registers a restore for key with t.Setenv, then removes it for the test. +func unsetForTest(t *testing.T, key string) { + t.Helper() + t.Setenv(key, "") + require.NoError(t, os.Unsetenv(key)) +} + +func TestWithoutBrokeredEnvironment_RestoresPreBrokerState(t *testing.T) { + resetRegistry() + t.Cleanup(resetRegistry) + + const ( + newKey = "ATMOS_BROKER_TEST_NEW" + replacedKey = "ATMOS_BROKER_TEST_REPLACED" + untouched = "ATMOS_BROKER_TEST_UNTOUCHED" + ) + unsetForTest(t, newKey) + t.Setenv(replacedKey, "before") + t.Setenv(untouched, "mine") + + Register(&fakeProvider{name: "fake", enabled: true, env: map[string]string{newKey: "minted", replacedKey: "after"}}) + runEnabledBrokers(t.Context(), &schema.AtmosConfiguration{}) + require.Equal(t, "minted", os.Getenv(newKey), "prerequisite: the broker exported into the process") + require.Equal(t, "after", os.Getenv(replacedKey)) + + input := []string{"PATH=/bin", newKey + "=minted", replacedKey + "=after", untouched + "=mine", "not-a-variable"} + restored := WithoutBrokeredEnvironment(input) + + assert.Equal(t, []string{"PATH=/bin", replacedKey + "=before", untouched + "=mine", "not-a-variable"}, restored, + "a variable the broker created is removed and one it replaced gets its earlier value back") + assert.Equal(t, newKey+"=minted", input[1], "the input is not modified") + + value, exported := PreBrokerValue(newKey) + assert.True(t, exported) + assert.Empty(t, value, "the variable was unset before the broker ran") + value, exported = PreBrokerValue(replacedKey) + assert.True(t, exported) + assert.Equal(t, "before", value) + _, exported = PreBrokerValue(untouched) + assert.False(t, exported, "a variable no broker exported is not broker-owned") +} + +// Without a broker run there is nothing to restore: the environment passes through untouched. +func TestWithoutBrokeredEnvironment_NoBrokerRunIsANoOp(t *testing.T) { + resetRegistry() + t.Cleanup(resetRegistry) + + input := []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=url.https://example.com/.insteadOf"} + + assert.Equal(t, input, WithoutBrokeredEnvironment(input)) + _, exported := PreBrokerValue("GIT_CONFIG_COUNT") + assert.False(t, exported) +} + +// Two brokers exporting the same variable: the remembered state is the one from before either ran. +func TestRecordExport_KeepsTheEarliestState(t *testing.T) { + resetRegistry() + t.Cleanup(resetRegistry) + + const key = "ATMOS_BROKER_TEST_SHARED" + t.Setenv(key, "original") + + Register(&fakeProvider{name: "first", enabled: true, env: map[string]string{key: "first"}}) + Register(&fakeProvider{name: "second", enabled: true, env: map[string]string{key: "second"}}) + runEnabledBrokers(t.Context(), &schema.AtmosConfiguration{}) + require.Equal(t, "second", os.Getenv(key)) + + value, exported := PreBrokerValue(key) + assert.True(t, exported) + assert.Equal(t, "original", value) +} diff --git a/pkg/auth/gitenv/gitenv.go b/pkg/auth/gitenv/gitenv.go new file mode 100644 index 00000000000..e66fe28b718 --- /dev/null +++ b/pkg/auth/gitenv/gitenv.go @@ -0,0 +1,117 @@ +// Package gitenv composes the environment `atmos git` runs its Git subprocesses with. +// +// The environment is the process environment plus the environment of the Atmos Auth identity in +// effect. The identity is chosen with this precedence: the global `--identity` flag (or +// `ATMOS_IDENTITY`), then `git.repositories..auth.identity`, then nothing, in which case the +// process environment is returned unchanged and ambient credentials apply. +// +// `atmos git` is a command the user runs on purpose, so unlike a remote read (see +// pkg/auth/remoteread) it may authenticate interactively when the identity has no session. +package gitenv + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -source=$GOFILE -destination=mock_gitenv.go -package=$GOPACKAGE + +import ( + "context" + "fmt" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/auth" + cfg "github.com/cloudposse/atmos/pkg/config" + atmosgit "github.com/cloudposse/atmos/pkg/git" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// IdentityManager is the part of the Atmos Auth manager `atmos git` needs. +type IdentityManager interface { + GetDefaultIdentity(forceSelect bool) (string, error) + EnsureIdentityEnvironment(ctx context.Context, identityName string) (map[string]string, error) +} + +// Composer builds Git subprocess environments from Atmos Auth identities. +type Composer struct { + newManager func(*schema.AtmosConfiguration) (IdentityManager, error) +} + +// Option configures a Composer. +type Option func(*Composer) + +// WithManagerFactory replaces how the auth manager is built from the configuration (for tests). +func WithManagerFactory(factory func(*schema.AtmosConfiguration) (IdentityManager, error)) Option { + return func(c *Composer) { c.newManager = factory } +} + +// NewComposer creates a Composer backed by the Atmos Auth manager built from atmos.yaml. +func NewComposer(opts ...Option) *Composer { + defer perf.Track(nil, "gitenv.NewComposer")() + + c := &Composer{newManager: defaultManager} + for _, opt := range opts { + opt(c) + } + return c +} + +// defaultManager builds the same manager the remote-read path uses. +func defaultManager(atmosConfig *schema.AtmosConfiguration) (IdentityManager, error) { + if atmosConfig == nil { + return nil, fmt.Errorf("%w: atmos configuration is nil", errUtils.ErrInvalidAuthConfig) + } + return auth.NewDefaultManager(&atmosConfig.Auth, atmosConfig.CliConfigPath) +} + +// Compose returns base with the environment of the effective identity overlaid. +// The flagIdentity argument is the value of the global `--identity` flag (or ATMOS_IDENTITY) and +// wins over repoIdentity, the identity configured for the repository. +// With no effective identity, base is returned unchanged. +func (c *Composer) Compose(ctx context.Context, atmosConfig *schema.AtmosConfiguration, base []string, flagIdentity, repoIdentity string) ([]string, error) { + defer perf.Track(atmosConfig, "gitenv.Composer.Compose")() + + identity := effectiveIdentity(flagIdentity, repoIdentity) + if identity == "" { + return base, nil + } + + provider := &lazyProvider{atmosConfig: atmosConfig, newManager: c.newManager} + return atmosgit.ComposeEnvironment(ctx, base, identity, provider) +} + +// effectiveIdentity applies the precedence: flag, then repository. `--identity=false` disables +// authentication entirely, including the repository's identity. +func effectiveIdentity(flagIdentity, repoIdentity string) string { + flagIdentity = cfg.NormalizeIdentityValue(flagIdentity) + switch flagIdentity { + case cfg.IdentityFlagDisabledValue: + return "" + case "": + return repoIdentity + default: + return flagIdentity + } +} + +// lazyProvider builds the auth manager on first use, resolves an interactive selection +// (`--identity` without a value), then asks the manager for the identity environment. The manager +// authenticates when the identity has no valid session. +type lazyProvider struct { + atmosConfig *schema.AtmosConfiguration + newManager func(*schema.AtmosConfiguration) (IdentityManager, error) +} + +// EnsureIdentityEnvironment satisfies git.IdentityEnvironmentProvider. +func (p *lazyProvider) EnsureIdentityEnvironment(ctx context.Context, identity string) (map[string]string, error) { + manager, err := p.newManager(p.atmosConfig) + if err != nil { + return nil, fmt.Errorf("%w: creating the auth manager for identity %q: %w", errUtils.ErrAuthManager, identity, err) + } + + if identity == cfg.IdentityFlagSelectValue { + identity, err = manager.GetDefaultIdentity(true) + if err != nil { + return nil, fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrNoDefaultIdentity, err) + } + } + + return manager.EnsureIdentityEnvironment(ctx, identity) +} diff --git a/pkg/auth/gitenv/gitenv_test.go b/pkg/auth/gitenv/gitenv_test.go new file mode 100644 index 00000000000..38fb09c840e --- /dev/null +++ b/pkg/auth/gitenv/gitenv_test.go @@ -0,0 +1,99 @@ +package gitenv + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/schema" +) + +// composerFor returns a Composer whose auth manager is the given mock. +func composerFor(m *MockIdentityManager) *Composer { + return NewComposer(WithManagerFactory(func(*schema.AtmosConfiguration) (IdentityManager, error) { return m, nil })) +} + +func TestCompose(t *testing.T) { + base := []string{"PATH=/bin", "SSH_AUTH_SOCK=/ambient.sock"} + envs := map[string]map[string]string{ + "repo-id": {"SSH_AUTH_SOCK": "/repo.sock", "GIT_CONFIG_GLOBAL": "/repo.cfg"}, + "flag-id": {"SSH_AUTH_SOCK": "/flag.sock"}, + } + + tests := []struct { + name string + flag, repo string + wantEnv []string + wantEnsured []string + defaultID string + wantEnsureOf string + }{ + {name: "no identity leaves the process environment exactly", wantEnv: base}, + { + name: "repository identity reaches the subprocess environment", repo: "repo-id", + wantEnv: []string{"PATH=/bin", "GIT_CONFIG_GLOBAL=/repo.cfg", "SSH_AUTH_SOCK=/repo.sock"}, + wantEnsured: []string{"repo-id"}, + }, + { + name: "flag overrides the repository identity", flag: "flag-id", repo: "repo-id", + wantEnv: []string{"PATH=/bin", "SSH_AUTH_SOCK=/flag.sock"}, + wantEnsured: []string{"flag-id"}, + }, + {name: "identity=false disables the repository identity", flag: "false", repo: "repo-id", wantEnv: base}, + { + name: "identity without a value selects the default", flag: cfg.IdentityFlagSelectValue, repo: "repo-id", + defaultID: "flag-id", wantEnv: []string{"PATH=/bin", "SSH_AUTH_SOCK=/flag.sock"}, wantEnsured: []string{"flag-id"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // The mock fails the test on any call that is not expected, so these expectations also + // assert which identity is authenticated, and that none is when no identity applies. + m := NewMockIdentityManager(gomock.NewController(t)) + if tt.defaultID != "" { + m.EXPECT().GetDefaultIdentity(true).Return(tt.defaultID, nil).Times(1) + } + for _, identity := range tt.wantEnsured { + m.EXPECT().EnsureIdentityEnvironment(gomock.Any(), identity).Return(envs[identity], nil).Times(1) + } + + got, err := composerFor(m).Compose(context.Background(), &schema.AtmosConfiguration{}, base, tt.flag, tt.repo) + + require.NoError(t, err) + assert.Equal(t, tt.wantEnv, got) + }) + } +} + +func TestCompose_Errors(t *testing.T) { + t.Run("authentication failure is returned", func(t *testing.T) { + boom := errors.New("boom") + m := NewMockIdentityManager(gomock.NewController(t)) + m.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "repo-id").Return(nil, boom) + _, err := composerFor(m).Compose(context.Background(), &schema.AtmosConfiguration{}, nil, "", "repo-id") + require.ErrorIs(t, err, boom) + }) + t.Run("no default identity", func(t *testing.T) { + m := NewMockIdentityManager(gomock.NewController(t)) + m.EXPECT().GetDefaultIdentity(true).Return("", errors.New("none")) + _, err := composerFor(m).Compose(context.Background(), &schema.AtmosConfiguration{}, nil, cfg.IdentityFlagSelectValue, "") + require.ErrorIs(t, err, errUtils.ErrNoDefaultIdentity) + }) + t.Run("manager construction failure", func(t *testing.T) { + c := NewComposer(WithManagerFactory(func(*schema.AtmosConfiguration) (IdentityManager, error) { + return nil, errors.New("nope") + })) + _, err := c.Compose(context.Background(), &schema.AtmosConfiguration{}, nil, "", "repo-id") + require.ErrorIs(t, err, errUtils.ErrAuthManager) + }) + t.Run("default factory rejects a nil configuration", func(t *testing.T) { + _, err := NewComposer().Compose(context.Background(), nil, nil, "", "repo-id") + require.ErrorIs(t, err, errUtils.ErrInvalidAuthConfig) + }) +} diff --git a/pkg/auth/gitenv/mock_gitenv.go b/pkg/auth/gitenv/mock_gitenv.go new file mode 100644 index 00000000000..e0944039693 --- /dev/null +++ b/pkg/auth/gitenv/mock_gitenv.go @@ -0,0 +1,71 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: gitenv.go +// +// Generated by this command: +// +// mockgen -source=gitenv.go -destination=mock_gitenv.go -package=gitenv +// + +// Package gitenv is a generated GoMock package. +package gitenv + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockIdentityManager is a mock of IdentityManager interface. +type MockIdentityManager struct { + ctrl *gomock.Controller + recorder *MockIdentityManagerMockRecorder + isgomock struct{} +} + +// MockIdentityManagerMockRecorder is the mock recorder for MockIdentityManager. +type MockIdentityManagerMockRecorder struct { + mock *MockIdentityManager +} + +// NewMockIdentityManager creates a new mock instance. +func NewMockIdentityManager(ctrl *gomock.Controller) *MockIdentityManager { + mock := &MockIdentityManager{ctrl: ctrl} + mock.recorder = &MockIdentityManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIdentityManager) EXPECT() *MockIdentityManagerMockRecorder { + return m.recorder +} + +// EnsureIdentityEnvironment mocks base method. +func (m *MockIdentityManager) EnsureIdentityEnvironment(ctx context.Context, identityName string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureIdentityEnvironment", ctx, identityName) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnsureIdentityEnvironment indicates an expected call of EnsureIdentityEnvironment. +func (mr *MockIdentityManagerMockRecorder) EnsureIdentityEnvironment(ctx, identityName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureIdentityEnvironment", reflect.TypeOf((*MockIdentityManager)(nil).EnsureIdentityEnvironment), ctx, identityName) +} + +// GetDefaultIdentity mocks base method. +func (m *MockIdentityManager) GetDefaultIdentity(forceSelect bool) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDefaultIdentity", forceSelect) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDefaultIdentity indicates an expected call of GetDefaultIdentity. +func (mr *MockIdentityManagerMockRecorder) GetDefaultIdentity(forceSelect any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultIdentity", reflect.TypeOf((*MockIdentityManager)(nil).GetDefaultIdentity), forceSelect) +} diff --git a/pkg/auth/manager.go b/pkg/auth/manager.go index 1dc5f0dbd6b..dd3a2ce4177 100644 --- a/pkg/auth/manager.go +++ b/pkg/auth/manager.go @@ -526,6 +526,14 @@ func (m *manager) GetCachedCredentials(ctx context.Context, identityName string) return nil, fmt.Errorf(errFormatWithString, errUtils.ErrExpiredCredentials, fmt.Sprintf(backtickedFmt, identityName)) } + // Hand the stored credential to identities that keep their state only in memory. This path + // skips Authenticate, so without it a fresh process has no environment to serve. + if restorer, ok := m.identities[identityName].(types.CredentialRestorer); ok { + if err := restorer.RestoreCredentials(ctx, creds); err != nil { + return nil, fmt.Errorf("%w: identity=%s: restoring stored credentials: %w", errUtils.ErrNoCredentialsFound, identityName, err) + } + } + return m.buildWhoamiInfo(identityName, creds), nil } diff --git a/pkg/auth/manager_restore_test.go b/pkg/auth/manager_restore_test.go new file mode 100644 index 00000000000..b0e9a752d5e --- /dev/null +++ b/pkg/auth/manager_restore_test.go @@ -0,0 +1,73 @@ +package auth + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/auth/types" + "github.com/cloudposse/atmos/pkg/schema" +) + +// restoringIdentity records the credential the manager hands it on the passive path. +type restoringIdentity struct { + stubIdentity + restored types.ICredentials + calls int + err error +} + +func (r *restoringIdentity) RestoreCredentials(_ context.Context, creds types.ICredentials) error { + r.calls++ + r.restored = creds + return r.err +} + +func TestManager_GetCachedCredentials_RestoresStoredCredential(t *testing.T) { + validTime := time.Now().Add(time.Hour) + expiredTime := time.Now().Add(-time.Hour) + + newManager := func(id types.Identity, creds *testCreds) *manager { + store := &testStore{data: map[string]any{"dev": creds}, expired: map[string]bool{}} + return &manager{ + config: &schema.AuthConfig{Identities: map[string]schema.Identity{"dev": {Kind: "plugin/kind"}}}, + identities: map[string]types.Identity{"dev": id}, + credentialStore: store, + } + } + + t.Run("a valid stored credential is handed to the identity", func(t *testing.T) { + id := &restoringIdentity{stubIdentity: stubIdentity{provider: "p"}} + stored := &testCreds{exp: &validTime} + + _, err := newManager(id, stored).GetCachedCredentials(context.Background(), "dev") + require.NoError(t, err) + assert.Equal(t, 1, id.calls) + assert.Same(t, stored, id.restored) + }) + + t.Run("an expired stored credential is not restored", func(t *testing.T) { + id := &restoringIdentity{stubIdentity: stubIdentity{provider: "p"}} + + _, err := newManager(id, &testCreds{exp: &expiredTime}).GetCachedCredentials(context.Background(), "dev") + require.ErrorIs(t, err, errUtils.ErrExpiredCredentials) + assert.Zero(t, id.calls) + }) + + t.Run("a credential the identity refuses is not a usable cache hit", func(t *testing.T) { + id := &restoringIdentity{stubIdentity: stubIdentity{provider: "p"}, err: errors.New("refused")} + + _, err := newManager(id, &testCreds{exp: &validTime}).GetCachedCredentials(context.Background(), "dev") + require.ErrorIs(t, err, errUtils.ErrNoCredentialsFound) + }) + + t.Run("identities without restore support are unaffected", func(t *testing.T) { + _, err := newManager(stubIdentity{provider: "p"}, &testCreds{exp: &validTime}).GetCachedCredentials(context.Background(), "dev") + require.NoError(t, err) + }) +} diff --git a/pkg/auth/remoteread/remoteread.go b/pkg/auth/remoteread/remoteread.go new file mode 100644 index 00000000000..6484ad0e669 --- /dev/null +++ b/pkg/auth/remoteread/remoteread.go @@ -0,0 +1,221 @@ +// Package remoteread resolves the Atmos Auth identity named on a remote read (a vendored source, a +// component `source:`, or a remote stack import) into the environment its Git subprocess runs with. +// +// It registers itself with pkg/downloader from init(), the same way credential brokers register, +// so the downloader stays free of a dependency on pkg/auth. The command layer blank-imports this +// package. The identity environment is composed with git.ComposeEnvironment, the primitive +// `atmos git` uses, and is handed to the fetch as a per-fetch environment: it is never exported +// with os.Setenv, so concurrent fetches can each use their own identity. +package remoteread + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sync" + + "gopkg.in/yaml.v3" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/auth" + "github.com/cloudposse/atmos/pkg/auth/types" + "github.com/cloudposse/atmos/pkg/downloader" + atmosgit "github.com/cloudposse/atmos/pkg/git" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/telemetry" +) + +func init() { + downloader.RegisterIdentityEnvironmentResolver(defaultResolver.Resolve) +} + +var defaultResolver = NewResolver() + +// identityManager is the part of the Atmos Auth manager a remote read needs. +type identityManager interface { + GetCachedCredentials(ctx context.Context, identityName string) (*types.WhoamiInfo, error) + EnsureIdentityEnvironment(ctx context.Context, identityName string) (map[string]string, error) +} + +// Option configures a Resolver. +type Option func(*Resolver) + +// WithManagerFactory replaces how the auth manager is built from the configuration (for tests). +func WithManagerFactory(factory func(*schema.AtmosConfiguration) (identityManager, error)) Option { + return func(r *Resolver) { r.newManager = factory } +} + +// WithCIDetector replaces CI detection (for tests). +func WithCIDetector(inCI func() bool) Option { + return func(r *Resolver) { r.inCI = inCI } +} + +// Resolver turns identity names into per-fetch environments. A successful resolution is kept for +// the life of the Resolver, so an identity is resolved once however many sources name it and +// however concurrently they are fetched. A failed resolution is not kept: the next fetch that names +// the identity tries again, so one transient error does not fail every later source. +// +// Resolutions are scoped to the auth configuration they were made under (see scopeKey). An +// identity name resolved for one configuration is never served to another, and each configuration +// gets its own auth manager. +type Resolver struct { + newManager func(*schema.AtmosConfiguration) (identityManager, error) + inCI func() bool + + mu sync.Mutex + scopes map[string]*scope +} + +// scope holds what was resolved under one auth configuration. +type scope struct { + // The mu mutex serializes every use of manager. The Atmos Auth manager keeps the chain it is + // authenticating in a field, so two identities must not be resolved through it at once. + mu sync.Mutex + manager identityManager + envs map[string][]string +} + +// NewResolver creates a Resolver backed by the Atmos Auth manager built from atmos.yaml. +func NewResolver(opts ...Option) *Resolver { + defer perf.Track(nil, "remoteread.NewResolver")() + + r := &Resolver{ + newManager: defaultManager, + inCI: telemetry.IsCI, + scopes: map[string]*scope{}, + } + for _, opt := range opts { + opt(r) + } + return r +} + +func defaultManager(atmosConfig *schema.AtmosConfiguration) (identityManager, error) { + if atmosConfig == nil { + return nil, fmt.Errorf("%w: atmos configuration is nil", errUtils.ErrInvalidAuthConfig) + } + return auth.NewDefaultManager(&atmosConfig.Auth, atmosConfig.CliConfigPath) +} + +// Resolve returns the environment overlay for identity, as "KEY=value" entries. +func (r *Resolver) Resolve(ctx context.Context, atmosConfig *schema.AtmosConfiguration, identity string) ([]string, error) { + defer perf.Track(atmosConfig, "remoteread.Resolver.Resolve")() + + sc := r.scopeFor(atmosConfig) + sc.mu.Lock() + defer sc.mu.Unlock() + + if env, ok := sc.envs[identity]; ok { + return append([]string(nil), env...), nil + } + + env, err := r.compose(ctx, sc, atmosConfig, identity) + if err != nil { + return nil, err + } + sc.envs[identity] = env + return append([]string(nil), env...), nil +} + +// scopeFor returns the scope for the auth configuration of atmosConfig, creating it on first use. +// A configuration that cannot be keyed gets a scope of its own that is not kept, so it is resolved +// afresh every time and never shares a manager or an environment with another configuration. +func (r *Resolver) scopeFor(atmosConfig *schema.AtmosConfiguration) *scope { + key, ok := scopeKey(atmosConfig) + if !ok { + return &scope{envs: map[string][]string{}} + } + + r.mu.Lock() + defer r.mu.Unlock() + sc, found := r.scopes[key] + if !found { + sc = &scope{envs: map[string][]string{}} + r.scopes[key] = sc + } + return sc +} + +// scopeKey identifies the auth configuration a resolution belongs to: the `auth` section (which +// carries the realm, providers, and identities) and the directory of the atmos.yaml it came from +// (which the default realm and the profile fallback are derived from). These are exactly the inputs +// the auth manager is built from. +func scopeKey(atmosConfig *schema.AtmosConfiguration) (string, bool) { + if atmosConfig == nil { + return "", false + } + authSection, err := yaml.Marshal(&atmosConfig.Auth) + if err != nil { + return "", false + } + digest := sha256.New() + digest.Write(authSection) + digest.Write([]byte{0}) + digest.Write([]byte(atmosConfig.Auth.RealmSource)) + digest.Write([]byte{0}) + digest.Write([]byte(atmosConfig.CliConfigPath)) + return hex.EncodeToString(digest.Sum(nil)), true +} + +// compose resolves identity through the scope's manager. The caller holds sc.mu. +func (r *Resolver) compose(ctx context.Context, sc *scope, atmosConfig *schema.AtmosConfiguration, identity string) ([]string, error) { + if sc.manager == nil { + manager, err := r.newManager(atmosConfig) + if err != nil { + return nil, fmt.Errorf("%w: creating the auth manager for identity %q: %w", errUtils.ErrAuthManager, identity, err) + } + sc.manager = manager + } + // A nil base keeps the process environment out of the overlay: the downloader applies the + // overlay to the process environment itself, per subprocess. + return atmosgit.ComposeEnvironment(ctx, nil, identity, &noPromptProvider{manager: sc.manager, inCI: r.inCI}) +} + +// noPromptProvider guards identity resolution so a remote read never prompts mid-fetch. Locally an +// identity must already hold a valid session (`atmos auth login`); in CI there is nobody to prompt +// and providers authenticate from the ambient environment (OIDC), so resolution may authenticate. +// Prompting is also disabled on the context for the identity kinds that honor it. +type noPromptProvider struct { + manager identityManager + inCI func() bool +} + +// EnsureIdentityEnvironment satisfies git.IdentityEnvironmentProvider. +func (p *noPromptProvider) EnsureIdentityEnvironment(ctx context.Context, identity string) (map[string]string, error) { + ctx = types.WithAllowPrompts(ctx, false) + + if !p.inCI() { + if _, err := p.manager.GetCachedCredentials(ctx, identity); err != nil { + return nil, loginRequired(identity, err) + } + } + + env, err := p.manager.EnsureIdentityEnvironment(ctx, identity) + if err != nil { + return nil, loginRequired(identity, err) + } + return env, nil +} + +// loginRequired maps an authentication failure to the static sentinel plus the login hint. Errors +// that are not about missing credentials (an unknown identity, an invalid configuration) pass +// through unchanged because logging in would not fix them. +func loginRequired(identity string, cause error) error { + for _, passthrough := range []error{ + errUtils.ErrIdentityNotFound, errUtils.ErrInvalidAuthConfig, errUtils.ErrInvalidIdentityConfig, + errUtils.ErrInvalidIdentityKind, errUtils.ErrProviderNotFound, errUtils.ErrCircularDependency, + } { + if errors.Is(cause, passthrough) { + return cause + } + } + return errUtils.Build(errUtils.ErrRemoteIdentityLoginRequired). + WithCause(cause). + WithExplanation(fmt.Sprintf("Identity `%s` is used for a remote read but has no valid session. Atmos never prompts in the middle of a fetch.", identity)). + WithHintf("Run `atmos auth login --identity %s`, then retry", identity). + WithContext("identity", identity). + Err() +} diff --git a/pkg/auth/remoteread/remoteread_test.go b/pkg/auth/remoteread/remoteread_test.go new file mode 100644 index 00000000000..9d368affffd --- /dev/null +++ b/pkg/auth/remoteread/remoteread_test.go @@ -0,0 +1,330 @@ +package remoteread + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + cockroachErrors "github.com/cockroachdb/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/auth/types" + "github.com/cloudposse/atmos/pkg/downloader" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinel: the tests build the auth configuration through this struct. +var _ = schema.AtmosConfiguration{} + +// Compile-time sentinel: the scoping tests tell configurations apart by this field. +var _ = schema.AuthConfig{Realm: "a"} + +func newTestResolver(t *testing.T, manager identityManager, inCI bool) *Resolver { + t.Helper() + return NewResolver( + WithManagerFactory(func(*schema.AtmosConfiguration) (identityManager, error) { return manager, nil }), + WithCIDetector(func() bool { return inCI }), + ) +} + +func TestResolve_LoggedInIdentityComposesEnvironment(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().GetCachedCredentials(gomock.Any(), "ops").Return(&types.WhoamiInfo{}, nil) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").DoAndReturn( + func(ctx context.Context, _ string) (map[string]string, error) { + assert.False(t, types.AllowPrompts(ctx), "a remote read must never allow prompts") + return map[string]string{"SSH_AUTH_SOCK": "/tmp/ops.sock", "GIT_SSH_COMMAND": "ssh -o IdentitiesOnly=yes"}, nil + }, + ) + + env, err := newTestResolver(t, manager, false).Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + + require.NoError(t, err) + assert.Equal(t, []string{"GIT_SSH_COMMAND=ssh -o IdentitiesOnly=yes", "SSH_AUTH_SOCK=/tmp/ops.sock"}, env, + "only the identity's variables are returned, not the process environment") +} + +func TestResolve_NotLoggedInFailsWithSentinelAndLoginHint(t *testing.T) { + tests := []struct { + name string + err error + }{ + {name: "no cached credentials", err: fmt.Errorf("%w: identity=ops", errUtils.ErrNoCredentialsFound)}, + {name: "expired credentials", err: fmt.Errorf("%w: `ops`", errUtils.ErrExpiredCredentials)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + // No EnsureIdentityEnvironment expectation: calling it would authenticate (and could + // prompt), and gomock fails the test if it happens. + manager.EXPECT().GetCachedCredentials(gomock.Any(), "ops").Return(nil, tt.err) + + env, err := newTestResolver(t, manager, false).Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) + assert.Nil(t, env) + hints := fmt.Sprint(cockroachErrors.GetAllHints(err)) + assert.Contains(t, hints, "atmos auth login --identity ops") + }) + } +} + +func TestResolve_UnknownIdentityIsNotReportedAsLoginRequired(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().GetCachedCredentials(gomock.Any(), "typo").Return(nil, fmt.Errorf("%w: `typo`", errUtils.ErrIdentityNotFound)) + + _, err := newTestResolver(t, manager, false).Resolve(context.Background(), &schema.AtmosConfiguration{}, "typo") + + require.ErrorIs(t, err, errUtils.ErrIdentityNotFound) + assert.NotErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired, "logging in would not fix a misspelled identity") +} + +func TestResolve_InCIAuthenticatesWithoutRequiringACachedSession(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + // In CI there is nobody to prompt; the cached-session pre-check is skipped, so + // GetCachedCredentials is deliberately not expected. + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "source-reader"). + Return(map[string]string{"GIT_CONFIG_COUNT": "1"}, nil) + + env, err := newTestResolver(t, manager, true).Resolve(context.Background(), &schema.AtmosConfiguration{}, "source-reader") + + require.NoError(t, err) + assert.Equal(t, []string{"GIT_CONFIG_COUNT=1"}, env) +} + +func TestResolve_AuthenticationFailureMapsToLoginRequired(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(nil, errors.Join(errUtils.ErrIdentityAuthFailed, errors.New("device flow unavailable"))) + + _, err := newTestResolver(t, manager, true).Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) + require.ErrorIs(t, err, errUtils.ErrIdentityAuthFailed, "the cause stays in the chain") +} + +func TestResolve_ManagerFailure(t *testing.T) { + boom := errors.New("bad auth config") + resolver := NewResolver(WithManagerFactory(func(*schema.AtmosConfiguration) (identityManager, error) { return nil, boom })) + + _, err := resolver.Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + + require.ErrorIs(t, err, errUtils.ErrAuthManager) + require.ErrorIs(t, err, boom) +} + +func TestResolve_SharedIdentityResolvedOnceAcrossConcurrentSources(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().GetCachedCredentials(gomock.Any(), "ops").Return(&types.WhoamiInfo{}, nil).Times(1) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"A": "1"}, nil).Times(1) + manager.EXPECT().GetCachedCredentials(gomock.Any(), "deploy").Return(&types.WhoamiInfo{}, nil).Times(1) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "deploy").Return(map[string]string{"B": "2"}, nil).Times(1) + resolver := newTestResolver(t, manager, false) + + var wg sync.WaitGroup + results := make(chan string, 16) + for i := 0; i < 8; i++ { + for _, identity := range []string{"ops", "deploy"} { + wg.Add(1) + go func() { + defer wg.Done() + env, err := resolver.Resolve(context.Background(), &schema.AtmosConfiguration{}, identity) + if err != nil { + results <- "error: " + err.Error() + return + } + results <- identity + ":" + env[0] + }() + } + } + wg.Wait() + close(results) + + counts := map[string]int{} + for result := range results { + counts[result]++ + } + assert.Equal(t, map[string]int{"ops:A=1": 8, "deploy:B=2": 8}, counts, "each identity keeps its own environment") +} + +func TestResolve_ReturnedEnvironmentIsACopy(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"A": "1"}, nil).Times(1) + resolver := newTestResolver(t, manager, true) + + first, err := resolver.Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + require.NoError(t, err) + first[0] = "A=mutated" + second, err := resolver.Resolve(context.Background(), &schema.AtmosConfiguration{}, "ops") + + require.NoError(t, err) + assert.Equal(t, []string{"A=1"}, second) +} + +func TestResolve_FailedResolutionIsRetriedAndNotCached(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + transient := errors.Join(errUtils.ErrIdentityAuthFailed, errors.New("token endpoint timed out")) + gomock.InOrder( + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(nil, transient), + // The second fetch tries again; the third is served from the successful resolution. + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"A": "1"}, nil), + ) + resolver := newTestResolver(t, manager, true) + atmosConfig := &schema.AtmosConfiguration{} + + _, err := resolver.Resolve(context.Background(), atmosConfig, "ops") + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) + + second, err := resolver.Resolve(context.Background(), atmosConfig, "ops") + require.NoError(t, err, "a failure must not be kept for the life of the process") + assert.Equal(t, []string{"A=1"}, second) + + third, err := resolver.Resolve(context.Background(), atmosConfig, "ops") + require.NoError(t, err) + assert.Equal(t, []string{"A=1"}, third) +} + +func TestResolve_ManagerFailureIsRetried(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"A": "1"}, nil).Times(1) + boom := errors.New("keyring locked") + calls := 0 + resolver := NewResolver( + WithManagerFactory(func(*schema.AtmosConfiguration) (identityManager, error) { + calls++ + if calls == 1 { + return nil, boom + } + return manager, nil + }), + WithCIDetector(func() bool { return true }), + ) + atmosConfig := &schema.AtmosConfiguration{} + + _, err := resolver.Resolve(context.Background(), atmosConfig, "ops") + require.ErrorIs(t, err, boom) + + env, err := resolver.Resolve(context.Background(), atmosConfig, "ops") + require.NoError(t, err) + assert.Equal(t, []string{"A=1"}, env) + assert.Equal(t, 2, calls, "the manager is built again after a failure, then kept") +} + +// The same identity name under two auth configurations must not share an environment or a manager: +// that would let one configuration fetch with the other's credentials. +func TestResolve_ScopedToAuthConfiguration(t *testing.T) { + ctrl := gomock.NewController(t) + managers := map[string]*types.MockAuthManager{"a": types.NewMockAuthManager(ctrl), "b": types.NewMockAuthManager(ctrl)} + managers["a"].EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"REALM": "a"}, nil).Times(1) + managers["b"].EXPECT().EnsureIdentityEnvironment(gomock.Any(), "ops").Return(map[string]string{"REALM": "b"}, nil).Times(1) + built := map[string]int{} + resolver := NewResolver( + WithManagerFactory(func(atmosConfig *schema.AtmosConfiguration) (identityManager, error) { + built[atmosConfig.Auth.Realm]++ + return managers[atmosConfig.Auth.Realm], nil + }), + WithCIDetector(func() bool { return true }), + ) + configA := func() *schema.AtmosConfiguration { + return &schema.AtmosConfiguration{Auth: schema.AuthConfig{Realm: "a"}} + } + configB := &schema.AtmosConfiguration{Auth: schema.AuthConfig{Realm: "b"}} + + fromA, err := resolver.Resolve(context.Background(), configA(), "ops") + require.NoError(t, err) + fromB, err := resolver.Resolve(context.Background(), configB, "ops") + require.NoError(t, err) + // A second copy of configuration A (a different pointer, the same content) reuses A's scope. + againA, err := resolver.Resolve(context.Background(), configA(), "ops") + require.NoError(t, err) + + assert.Equal(t, []string{"REALM=a"}, fromA) + assert.Equal(t, []string{"REALM=b"}, fromB, "configuration B must not receive configuration A's environment") + assert.Equal(t, []string{"REALM=a"}, againA) + assert.Equal(t, map[string]int{"a": 1, "b": 1}, built, "one manager per configuration, reused for equal configurations") +} + +// The same auth section loaded from another directory is another configuration: the default realm +// and the profile fallback both derive from that directory. +func TestResolve_ScopedToConfigurationDirectory(t *testing.T) { + first, ok := scopeKey(&schema.AtmosConfiguration{CliConfigPath: "repo-one"}) + require.True(t, ok) + second, ok := scopeKey(&schema.AtmosConfiguration{CliConfigPath: "repo-two"}) + require.True(t, ok) + same, ok := scopeKey(&schema.AtmosConfiguration{CliConfigPath: "repo-one"}) + require.True(t, ok) + + assert.NotEqual(t, first, second) + assert.Equal(t, first, same) + + _, ok = scopeKey(nil) + assert.False(t, ok, "a nil configuration has no scope to share") +} + +// The auth manager keeps the chain it is authenticating in a field, so the resolver must never run +// two identities through one manager at the same time. +func TestResolve_ManagerIsNeverUsedConcurrently(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + var inFlight, maxInFlight atomic.Int32 + ensure := func(_ context.Context, identity string) (map[string]string, error) { + current := inFlight.Add(1) + defer inFlight.Add(-1) + for { + seen := maxInFlight.Load() + if current <= seen || maxInFlight.CompareAndSwap(seen, current) { + break + } + } + time.Sleep(5 * time.Millisecond) + return map[string]string{"ID": identity}, nil + } + identities := []string{"one", "two", "three", "four"} + for _, identity := range identities { + manager.EXPECT().EnsureIdentityEnvironment(gomock.Any(), identity).DoAndReturn(ensure).Times(1) + } + resolver := newTestResolver(t, manager, true) + atmosConfig := &schema.AtmosConfiguration{} + + var wg sync.WaitGroup + results := make([][]string, len(identities)) + for i, identity := range identities { + wg.Add(1) + go func() { + defer wg.Done() + env, err := resolver.Resolve(context.Background(), atmosConfig, identity) + if err == nil { + results[i] = env + } + }() + } + wg.Wait() + + assert.Equal(t, int32(1), maxInFlight.Load(), "identities must be resolved one at a time") + assert.Equal(t, []string{"ID=one"}, results[0]) + assert.Equal(t, []string{"ID=four"}, results[len(results)-1]) +} + +// Importing this package registers the resolver the downloader uses, so a named identity reaches the +// auth layer instead of failing with "no resolver registered". +func TestInit_RegistersResolverWithDownloader(t *testing.T) { + _, err := downloader.WithIdentity(context.Background(), &schema.AtmosConfiguration{}, "not-configured", "") + + require.Error(t, err, "an identity that is not configured cannot resolve") + assert.NotErrorIs(t, err, errUtils.ErrRemoteIdentityResolverUnavailable) +} diff --git a/pkg/auth/types/persistence.go b/pkg/auth/types/persistence.go index d5e05b1a611..e949cb874d8 100644 --- a/pkg/auth/types/persistence.go +++ b/pkg/auth/types/persistence.go @@ -1,5 +1,7 @@ package types +import "context" + // CredentialPersistence is the optional interface implemented by identities whose // credentials must never be written to the Atmos credential store (the keyring). // @@ -22,3 +24,12 @@ func PersistsCredentialsInKeyring(identity Identity) bool { } return true } + +// CredentialRestorer is implemented by identities whose runtime state is derived from the stored +// credential and is kept only in memory (for example a credential made of environment variables). +// The auth manager calls it when a valid stored credential is reused without authenticating, so +// the identity can serve Environment and PrepareEnvironment in a fresh process. A non-nil error +// means the stored credential cannot be used and the caller should authenticate instead. +type CredentialRestorer interface { + RestoreCredentials(ctx context.Context, creds ICredentials) error +} diff --git a/pkg/component/ansible/executor.go b/pkg/component/ansible/executor.go index 2826b8fcbd6..07682466b2e 100644 --- a/pkg/component/ansible/executor.go +++ b/pkg/component/ansible/executor.go @@ -24,8 +24,9 @@ import ( ) var ( - setupComponentAuthForCLI = e.SetupComponentAuthForCLI - processStacks = e.ProcessStacks + setupComponentAuthForCLI = e.SetupComponentAuthForCLI + processStacks = e.ProcessStacks + prepareComponentAuthEnvVars = e.PrepareComponentAuthEnvironment ) // Flags represents Ansible command-line flags. @@ -49,7 +50,10 @@ func checkConfig(atmosConfig *schema.AtmosConfiguration) error { return nil } -func processStacksWithAuth(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) error { +// processStacksWithAuth processes the stack and returns the auth manager used for it (nil when no +// explicit identity was given), so the caller can also hand the identity's environment to the +// playbook subprocess. +func processStacksWithAuth(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (auth.AuthManager, error) { var authManager auth.AuthManager if info.Identity != "" { // Ansible owns -i for inventory, so only explicit --identity/ATMOS_IDENTITY @@ -57,17 +61,29 @@ func processStacksWithAuth(atmosConfig *schema.AtmosConfiguration, info *schema. var err error authManager, err = setupComponentAuthForCLI(atmosConfig, info) if err != nil { - return err + return nil, err } } processedInfo, err := processStacks(atmosConfig, *info, true, true, true, nil, authManager) if err != nil { - return err + return nil, err } *info = processedInfo - return nil + return authManager, nil +} + +// playbookEnvVars builds the environment variables for the ansible subprocess: the component +// environment plus the Atmos paths, with the identity's prepared environment layered on top when +// an auth manager is present. This is how Helmfile and Packer deliver identity environments, so a +// credential made of plain environment variables (for example SSH_AUTH_SOCK) reaches Ansible. +func playbookEnvVars(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo, authManager auth.AuthManager) ([]string, error) { + envVars, err := prepareEnvVars(atmosConfig, info.ComponentEnvList) + if err != nil { + return nil, err + } + return prepareComponentAuthEnvVars(authManager, info.Identity, envVars) } // ExecutePlaybook executes an Ansible playbook command. @@ -97,7 +113,8 @@ func ExecutePlaybook( } } - if err := processStacksWithAuth(&atmosConfig, info); err != nil { + authManager, err := processStacksWithAuth(&atmosConfig, info) + if err != nil { return err } @@ -219,11 +236,12 @@ func ExecutePlaybook( e.ConvertComponentEnvSectionToList(info) // Prepare ENV vars. - envVars, err := prepareEnvVars(&atmosConfig, info.ComponentEnvList) + envVars, err := playbookEnvVars(&atmosConfig, info, authManager) if err != nil { return err } - log.Debug("Using ENV", "variables", envVars) + // Names only: the list carries the identity's environment, which can hold credentials. + log.Debug("Using ENV", "variables", envVarNames(envVars)) // In dry-run mode, print the command that would be executed. if info.DryRun { @@ -619,3 +637,14 @@ func prepareEnvVars(atmosConfig *schema.AtmosConfiguration, envList []string) ([ envVars = append(envVars, fmt.Sprintf("ATMOS_BASE_PATH=%s", basePath)) return envVars, nil } + +// envVarNames returns the variable names of "KEY=value" entries, for logging an environment +// without its values. +func envVarNames(envVars []string) []string { + names := make([]string, 0, len(envVars)) + for _, entry := range envVars { + name, _, _ := strings.Cut(entry, "=") + names = append(names, name) + } + return names +} diff --git a/pkg/component/ansible/executor_test.go b/pkg/component/ansible/executor_test.go index dd40c25bdf1..2f900bcb2b4 100644 --- a/pkg/component/ansible/executor_test.go +++ b/pkg/component/ansible/executor_test.go @@ -81,8 +81,9 @@ func TestProcessStacksWithAuthPassesAuthManager(t *testing.T) { Identity: "terraform", } - err := processStacksWithAuth(&schema.AtmosConfiguration{}, info) + gotManager, err := processStacksWithAuth(&schema.AtmosConfiguration{}, info) require.NoError(t, err) + assert.Equal(t, mockAuthManager, gotManager) assert.True(t, info.ComponentIsEnabled) } @@ -111,8 +112,9 @@ func TestProcessStacksWithAuthSkipsAuthWhenIdentityIsEmpty(t *testing.T) { Stack: "cicd-prd", } - err := processStacksWithAuth(&schema.AtmosConfiguration{}, info) + gotManager, err := processStacksWithAuth(&schema.AtmosConfiguration{}, info) require.NoError(t, err) + assert.Nil(t, gotManager) assert.True(t, info.ComponentIsEnabled) } @@ -1306,3 +1308,60 @@ func TestExecutePlaybookCommandWithRetry_NonMatchingError_FailsFast(t *testing.T require.NoError(t, readErr) assert.Len(t, counterBytes, 1, "non-matching error must fail fast on the first attempt") } + +// TestPlaybookEnvVars_DeliversIdentityEnvironment proves the identity's prepared environment +// (for example SSH_AUTH_SOCK from an environment credential) reaches the ansible subprocess +// environment, the way it does for Helmfile and Packer. +func TestPlaybookEnvVars_DeliversIdentityEnvironment(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{CliConfigPath: filepath.Join("cfg", "atmos.yaml"), BasePath: "."} + + t.Run("identity environment reaches the subprocess", func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT(). + PrepareShellEnvironment(gomock.Any(), "ssh-id", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, current []string) ([]string, error) { + return append(slices.Clone(current), "SSH_AUTH_SOCK=/run/agent.sock"), nil + }) + info := &schema.ConfigAndStacksInfo{Identity: "ssh-id", ComponentEnvList: []string{"FOO=bar"}} + + env, err := playbookEnvVars(atmosConfig, info, manager) + require.NoError(t, err) + assert.Contains(t, env, "SSH_AUTH_SOCK=/run/agent.sock") + assert.Contains(t, env, "FOO=bar") + assert.Contains(t, env, "ATMOS_CLI_CONFIG_PATH="+atmosConfig.CliConfigPath) + }) + + t.Run("no auth manager leaves the environment unchanged", func(t *testing.T) { + info := &schema.ConfigAndStacksInfo{ComponentEnvList: []string{"FOO=bar"}} + + env, err := playbookEnvVars(atmosConfig, info, nil) + require.NoError(t, err) + assert.Equal(t, "FOO=bar", env[0]) + for _, entry := range env { + assert.False(t, strings.HasPrefix(entry, "SSH_AUTH_SOCK=")) + } + }) + + t.Run("preparation failure is returned", func(t *testing.T) { + ctrl := gomock.NewController(t) + manager := types.NewMockAuthManager(ctrl) + manager.EXPECT().PrepareShellEnvironment(gomock.Any(), "ssh-id", gomock.Any()).Return(nil, errUtils.ErrAuthenticationFailed) + info := &schema.ConfigAndStacksInfo{Identity: "ssh-id"} + + _, err := playbookEnvVars(atmosConfig, info, manager) + require.ErrorIs(t, err, errUtils.ErrAuthenticationFailed) + }) +} + +// The debug log lists the environment a playbook runs with. It carries the identity's +// environment, so only the names may be logged. +func TestEnvVarNames_DropsValues(t *testing.T) { + names := envVarNames([]string{"AWS_SECRET_ACCESS_KEY=hunter2", "SSH_AUTH_SOCK=/tmp/ops.sock", "URL=a=b", "FLAG"}) + + assert.Equal(t, []string{"AWS_SECRET_ACCESS_KEY", "SSH_AUTH_SOCK", "URL", "FLAG"}, names) + for _, name := range names { + assert.NotContains(t, name, "hunter2") + assert.NotContains(t, name, "=") + } +} diff --git a/pkg/datafetcher/schema/atmos/config/1.0.json b/pkg/datafetcher/schema/atmos/config/1.0.json index a7dbe774c6e..d66bb9b7001 100644 --- a/pkg/datafetcher/schema/atmos/config/1.0.json +++ b/pkg/datafetcher/schema/atmos/config/1.0.json @@ -7241,6 +7241,17 @@ } ], "description": "TTL is the default cache duration for the cloned source repos of remote (git)\nimports, applied when a per-import `ttl` is not set. See StackImport.TTL for the\ncaching semantics and accepted formats." + }, + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity is the default Atmos Auth identity for remote imports that do not name their own.\nSee StackImport.Identity." } }, "type": "object", @@ -15145,6 +15156,17 @@ } ], "description": "Lock configures how `atmos vendor pull` reacts to vendor.lock.yaml drift." + }, + "identity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Identity is the default Atmos Auth identity for vendored sources that do not name their own." } }, "type": "object" diff --git a/pkg/datafetcher/schema/atmos/manifest/1.0.json b/pkg/datafetcher/schema/atmos/manifest/1.0.json index 5c108b606f5..894db12a300 100644 --- a/pkg/datafetcher/schema/atmos/manifest/1.0.json +++ b/pkg/datafetcher/schema/atmos/manifest/1.0.json @@ -396,6 +396,10 @@ "type": "string", "description": "Cache duration for the cloned source repo of a remote (git) import. When set, the clone is reused across Atmos invocations until it expires (e.g. with a warm CI cache). Within a single invocation a source repo is cloned at most once regardless of TTL. Examples: '0s', '5m', '1h', '7d', 'daily'." }, + "identity": { + "type": "string", + "description": "Atmos Auth identity used to fetch this remote import. Overrides imports.identity and is overridden by the global --identity flag. Ignored for local imports." + }, "when": { "$ref": "#/definitions/import_condition", "description": "Optional condition gating whether this import is processed. Evaluated against CI detection, the process environment (env), and native stack labels and tags. An unset condition always imports." @@ -2063,6 +2067,10 @@ "ttl": { "type": "string", "description": "Cache time-to-live as a Go duration string (e.g., '24h'). Overrides the global default; a JIT-vendored source cache older than this is re-fetched." + }, + "identity": { + "type": "string", + "description": "Atmos Auth identity used to fetch this source. Overrides vendor.identity and is overridden by the global --identity flag." } }, "required": [ diff --git a/pkg/datafetcher/schema/stacks/stack-config/1.0.json b/pkg/datafetcher/schema/stacks/stack-config/1.0.json index ce7469c0359..5ec9f942468 100644 --- a/pkg/datafetcher/schema/stacks/stack-config/1.0.json +++ b/pkg/datafetcher/schema/stacks/stack-config/1.0.json @@ -464,6 +464,10 @@ "type": "string", "description": "Cache duration for the cloned source repo of a remote (git) import. When set, the clone is reused across Atmos invocations until it expires (e.g. with a warm CI cache). Within a single invocation a source repo is cloned at most once regardless of TTL. Examples: '0s', '5m', '1h', '7d', 'daily'." }, + "identity": { + "type": "string", + "description": "Atmos Auth identity used to fetch this remote import. Overrides imports.identity and is overridden by the global --identity flag. Ignored for local imports." + }, "when": { "$ref": "#/definitions/import_condition", "description": "Optional condition gating whether this import is processed. Evaluated against CI detection, the process environment (env), and native stack labels and tags. An unset condition always imports." diff --git a/pkg/datafetcher/schema/vendor/package/1.0.json b/pkg/datafetcher/schema/vendor/package/1.0.json index 15900464022..3c255147062 100644 --- a/pkg/datafetcher/schema/vendor/package/1.0.json +++ b/pkg/datafetcher/schema/vendor/package/1.0.json @@ -137,6 +137,10 @@ } }, "additionalProperties": false + }, + "identity": { + "type": "string", + "description": "Atmos Auth identity used to fetch this source. Overrides vendor.identity and is overridden by the global --identity flag." } }, "required": ["component", "source", "version", "targets"] diff --git a/pkg/downloader/broker_insteadof_filemode_test.go b/pkg/downloader/broker_insteadof_filemode_test.go index d34e4f0e6bf..db7ffa305c7 100644 --- a/pkg/downloader/broker_insteadof_filemode_test.go +++ b/pkg/downloader/broker_insteadof_filemode_test.go @@ -1,6 +1,7 @@ package downloader import ( + "context" "net/url" "os" "path/filepath" @@ -117,7 +118,7 @@ func TestBrokerInsteadOfMatchesURL_FileMode(t *testing.T) { parsed, err := url.Parse(tt.urlStr) require.NoError(t, err) - got := brokerInsteadOfMatchesURL(parsed, strings.ToLower(parsed.Hostname())) + got := brokerInsteadOfMatchesURLContext(context.Background(), parsed, strings.ToLower(parsed.Hostname())) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/downloader/broker_insteadof_helper_test.go b/pkg/downloader/broker_insteadof_helper_test.go index b5f091b56e0..74a429562e5 100644 --- a/pkg/downloader/broker_insteadof_helper_test.go +++ b/pkg/downloader/broker_insteadof_helper_test.go @@ -1,6 +1,7 @@ package downloader import ( + "context" "net/url" "strings" "testing" @@ -80,7 +81,7 @@ func TestBrokerInsteadOfMatchesURL(t *testing.T) { parsed, err := url.Parse(tt.urlStr) require.NoError(t, err) - got := brokerInsteadOfMatchesURL(parsed, strings.ToLower(parsed.Hostname())) + got := brokerInsteadOfMatchesURLContext(context.Background(), parsed, strings.ToLower(parsed.Hostname())) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/downloader/custom_git_detector.go b/pkg/downloader/custom_git_detector.go index dadcfc7c64a..0f0d6a9fcee 100644 --- a/pkg/downloader/custom_git_detector.go +++ b/pkg/downloader/custom_git_detector.go @@ -295,12 +295,22 @@ func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host, rawHost string // If an auth broker (e.g. github/sts) has already exported a GIT_CONFIG_* insteadOf rewrite that // matches this host/owner, skip URL injection: injecting userinfo here would prevent git's // insteadOf left-side from matching, silently shadowing the broker's correct minted token. - if brokerInsteadOfMatchesURL(parsedURL, host) { + if brokerInsteadOfMatchesURLContext(d.ctx, parsedURL, host) { maskedURL, _ := maskBasicAuth(parsedURL.String()) log.Debug("Skipping token injection: a broker GIT_CONFIG insteadOf already covers this host/owner; letting git's rewrite win", keyURL, maskedURL) return } + // A fetch that names an Atmos Auth identity authenticates as that identity. URL userinfo takes + // precedence over everything Git would otherwise use (an askpass helper, a credential helper, + // an SSH key), so injecting the ambient token here would silently replace the identity the + // source asked for. + if identity := IdentityFromContext(d.ctx); identity != "" { + maskedURL, _ := maskBasicAuth(parsedURL.String()) + log.Debug("Skipping token injection: the fetch names an identity", "identity", identity, keyURL, maskedURL) + return + } + token, tokenSource := d.resolveToken(host, rawHost) if token != "" { defaultUsername := d.getDefaultUsername(host, rawHost) @@ -312,14 +322,17 @@ func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host, rawHost string } } -// brokerInsteadOfMatchesURL reports whether a live GIT_CONFIG_* insteadOf rewrite (exported by an +// brokerInsteadOfMatchesURLContext reports whether a live GIT_CONFIG_* insteadOf rewrite (exported by an // auth broker, e.g. github/sts) would match this URL's host and owner — "match" being git's own // term for insteadOf: any URL beginning with the rewrite's value is rewritten (git help config, // url..insteadOf). When it matches, URL token injection must be skipped so git's rewrite — // carrying the correct minted token — wins instead of being shadowed by userinfo in the URL. -func brokerInsteadOfMatchesURL(parsedURL *url.URL, host string) bool { - //nolint:forbidigo // Live env read of broker-exported GIT_CONFIG_*; mirrors pkg/http/client.go. - count, err := strconv.Atoi(os.Getenv("GIT_CONFIG_COUNT")) +// +// It is evaluated per fetch (the ctx parameter): it reads GIT_CONFIG_* from the fetch's per-fetch +// environment (see WithEnvironment) first, then from the process environment. A named identity's `insteadOf` rewrite is exported to its Git subprocesses through +// that per-fetch environment, so it must be visible here too or URL token injection would shadow it. +func brokerInsteadOfMatchesURLContext(ctx context.Context, parsedURL *url.URL, host string) bool { + count, err := strconv.Atoi(fetchEnvironmentValue(ctx, "GIT_CONFIG_COUNT")) if err != nil || count <= 0 { return false } @@ -330,7 +343,7 @@ func brokerInsteadOfMatchesURL(parsedURL *url.URL, host string) bool { } for i := 0; i < count; i++ { - if gitConfigInsteadOfMatches(i, host, owner) { + if gitConfigInsteadOfMatches(ctx, i, host, owner) { return true } } @@ -345,12 +358,10 @@ func brokerInsteadOfMatchesURL(parsedURL *url.URL, host string) bool { // rewrites live inside the file (so the inline-key check below can never see them). // // Only an https rewrite matches an https clone — an ssh-only rewrite must not suppress injection. -func gitConfigInsteadOfMatches(i int, host, owner string) bool { +func gitConfigInsteadOfMatches(ctx context.Context, i int, host, owner string) bool { idx := strconv.Itoa(i) - //nolint:forbidigo // Live env read of broker-exported GIT_CONFIG_*; mirrors pkg/http/client.go. - key := os.Getenv("GIT_CONFIG_KEY_" + idx) - //nolint:forbidigo // Live env read of broker-exported GIT_CONFIG_*; mirrors pkg/http/client.go. - value := os.Getenv("GIT_CONFIG_VALUE_" + idx) + key := fetchEnvironmentValue(ctx, "GIT_CONFIG_KEY_"+idx) + value := fetchEnvironmentValue(ctx, "GIT_CONFIG_VALUE_"+idx) // File mode: the rewrites are inside the included gitconfig, not in this env entry. if key == gitConfigIncludePath { diff --git a/pkg/downloader/environment.go b/pkg/downloader/environment.go new file mode 100644 index 00000000000..24316819280 --- /dev/null +++ b/pkg/downloader/environment.go @@ -0,0 +1,146 @@ +package downloader + +import ( + "context" + "os" + "os/exec" + "runtime" + "strings" + + "github.com/cloudposse/atmos/pkg/auth/broker" + "github.com/cloudposse/atmos/pkg/perf" +) + +// envSeparator separates a variable name from its value in a "KEY=value" entry. +const envSeparator = "=" + +type fetchEnvironmentKey struct{} + +// WithEnvironment scopes an environment overlay to the fetches made with the returned context. +// Every Git subprocess a fetch spawns (clone, fetch, checkout, submodules) runs with the process +// environment plus this overlay, with overlay entries replacing process entries of the same name. +// The process environment is never modified, so concurrent fetches can use different overlays +// (for example, one Atmos Auth identity per vendored source). +// +// Entries are "KEY=value" strings, the form returned by git.ComposeEnvironment. Calling it again on +// a derived context merges the new entries over the earlier ones. An empty overlay returns ctx. +func WithEnvironment(ctx context.Context, env []string) context.Context { + defer perf.Track(nil, "downloader.WithEnvironment")() + + if len(env) == 0 { + return ctx + } + merged := overlayEnvironment(EnvironmentFromContext(ctx), env) + return context.WithValue(ctx, fetchEnvironmentKey{}, merged) +} + +// EnvironmentFromContext returns a copy of the per-fetch environment overlay carried by ctx, or nil. +func EnvironmentFromContext(ctx context.Context) []string { + defer perf.Track(nil, "downloader.EnvironmentFromContext")() + + if ctx == nil { + return nil + } + env, _ := ctx.Value(fetchEnvironmentKey{}).([]string) + if len(env) == 0 { + return nil + } + return append([]string(nil), env...) +} + +// fetchEnvironmentValue looks a variable up in the per-fetch overlay first and falls back to the +// process environment, mirroring how a Git subprocess of this fetch would see it. +func fetchEnvironmentValue(ctx context.Context, name string) string { + overlay := EnvironmentFromContext(ctx) + for i := len(overlay) - 1; i >= 0; i-- { + key, value, found := strings.Cut(overlay[i], envSeparator) + if found && envKeysEqual(key, name) { + return value + } + } + // A named identity never sees what an ambient broker exported: see applyFetchEnvironment. + if IdentityFromContext(ctx) != "" { + if prior, exported := broker.PreBrokerValue(name); exported { + return prior + } + } + //nolint:forbidigo // Live env read of broker-exported GIT_CONFIG_*; mirrors pkg/http/client.go. + return os.Getenv(name) +} + +// applyFetchEnvironment gives cmd the process environment overlaid with the per-fetch environment +// carried by ctx. It is a no-op for a fetch that names no identity and carries no overlay, so such +// a fetch keeps inheriting the process environment exactly as before. +// +// A fetch that names an identity runs with that identity's credentials only. An ambient credential +// broker exports its Git rewrites (GIT_CONFIG_*) and tokens into the process environment, where an +// earlier fetch in the same process may have put them. Left in place, Git would rewrite this +// fetch's URL to carry the broker's token instead of using the identity. So for a named identity +// every broker-exported variable is first restored to what it was before the broker ran, and the +// identity's own environment is applied on top. +func applyFetchEnvironment(ctx context.Context, cmd *exec.Cmd) { + overlay := EnvironmentFromContext(ctx) + namedIdentity := IdentityFromContext(ctx) != "" + if len(overlay) == 0 && !namedIdentity { + return + } + base := cmd.Env + if base == nil { + base = os.Environ() + } + if namedIdentity { + base = broker.WithoutBrokeredEnvironment(base) + } + cmd.Env = overlayEnvironment(base, overlay) +} + +// overlayEnvironment returns base with every overlay entry applied: an overlay entry replaces +// base entries of the same name, and later overlay entries win over earlier ones. Neither input +// is modified. +func overlayEnvironment(base, overlay []string) []string { + // The index of the last overlay entry for each name. + last := make(map[string]int, len(overlay)) + for i, entry := range overlay { + if key, ok := envEntryKey(entry); ok { + last[key] = i + } + } + + merged := make([]string, 0, len(base)+len(last)) + for _, entry := range base { + if key, ok := envEntryKey(entry); ok { + if _, replaced := last[key]; replaced { + continue + } + } + merged = append(merged, entry) + } + for i, entry := range overlay { + if key, ok := envEntryKey(entry); ok && last[key] == i { + merged = append(merged, entry) + } + } + return merged +} + +// envEntryKey returns the normalized variable name of a "KEY=value" entry. Entries without an +// equals sign are not variables and report false. +func envEntryKey(entry string) (string, bool) { + key, _, found := strings.Cut(entry, envSeparator) + if !found { + return "", false + } + return normalizeEnvKey(key), true +} + +// normalizeEnvKey folds case on Windows, where environment names are case-insensitive. +func normalizeEnvKey(key string) string { + if runtime.GOOS == "windows" { + return strings.ToUpper(key) + } + return key +} + +func envKeysEqual(a, b string) bool { + return normalizeEnvKey(a) == normalizeEnvKey(b) +} diff --git a/pkg/downloader/environment_test.go b/pkg/downloader/environment_test.go new file mode 100644 index 00000000000..d7d005af01a --- /dev/null +++ b/pkg/downloader/environment_test.go @@ -0,0 +1,97 @@ +package downloader + +import ( + "context" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOverlayEnvironment(t *testing.T) { + tests := []struct { + name string + base []string + overlay []string + want []string + }{ + {name: "empty overlay copies base", base: []string{"A=1", "B=2"}, overlay: nil, want: []string{"A=1", "B=2"}}, + {name: "overlay replaces same name and appends new", base: []string{"A=1", "B=2"}, overlay: []string{"B=3", "C=4"}, want: []string{"A=1", "B=3", "C=4"}}, + {name: "later overlay entry wins", base: []string{"A=1"}, overlay: []string{"A=2", "A=3"}, want: []string{"A=3"}}, + {name: "value may contain equals", base: nil, overlay: []string{"GIT_SSH_COMMAND=ssh -o X=y"}, want: []string{"GIT_SSH_COMMAND=ssh -o X=y"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + base := append([]string(nil), tt.base...) + overlay := append([]string(nil), tt.overlay...) + + got := overlayEnvironment(tt.base, tt.overlay) + + assert.Equal(t, tt.want, got) + // Neither input is modified. + assert.Equal(t, base, append([]string(nil), tt.base...)) + assert.Equal(t, overlay, append([]string(nil), tt.overlay...)) + }) + } +} + +func TestWithEnvironment_MergesAndIsolatesCopies(t *testing.T) { + assert.Nil(t, EnvironmentFromContext(context.Background())) + assert.Nil(t, EnvironmentFromContext(nil)) //nolint:staticcheck // A nil context must not panic. + + parent := context.Background() + assert.Equal(t, parent, WithEnvironment(parent, nil), "an empty overlay returns the context unchanged") + + ctx := WithEnvironment(parent, []string{"A=1", "B=2"}) + child := WithEnvironment(ctx, []string{"B=3"}) + + assert.Equal(t, []string{"A=1", "B=2"}, EnvironmentFromContext(ctx)) + assert.Equal(t, []string{"A=1", "B=3"}, EnvironmentFromContext(child)) + + // Mutating a returned copy never changes what the context carries. + got := EnvironmentFromContext(ctx) + got[0] = "A=mutated" + assert.Equal(t, []string{"A=1", "B=2"}, EnvironmentFromContext(ctx)) +} + +func TestApplyFetchEnvironment(t *testing.T) { + t.Run("no overlay leaves the command environment untouched", func(t *testing.T) { + cmd := exec.Command("git", "version") + applyFetchEnvironment(context.Background(), cmd) + assert.Nil(t, cmd.Env, "a nil Env means the subprocess inherits the process environment, as before") + }) + + t.Run("overlay starts from the process environment", func(t *testing.T) { + t.Setenv("ATMOS_TEST_FETCH_ENV_BASE", "process") + t.Setenv("ATMOS_TEST_FETCH_ENV_SHADOWED", "process") + cmd := exec.Command("git", "version") + + applyFetchEnvironment(WithEnvironment(context.Background(), []string{"ATMOS_TEST_FETCH_ENV_SHADOWED=identity", "ATMOS_TEST_FETCH_ENV_NEW=identity"}), cmd) + + require.NotNil(t, cmd.Env) + assert.Contains(t, cmd.Env, "ATMOS_TEST_FETCH_ENV_BASE=process") + assert.Contains(t, cmd.Env, "ATMOS_TEST_FETCH_ENV_SHADOWED=identity") + assert.NotContains(t, cmd.Env, "ATMOS_TEST_FETCH_ENV_SHADOWED=process") + assert.Contains(t, cmd.Env, "ATMOS_TEST_FETCH_ENV_NEW=identity") + }) + + t.Run("overlay layers over an environment setupGitEnv already built", func(t *testing.T) { + cmd := exec.Command("git", "version") + cmd.Env = []string{"KEEP=1", "GIT_SSH_COMMAND=ssh -i key"} + + applyFetchEnvironment(WithEnvironment(context.Background(), []string{"SSH_AUTH_SOCK=/tmp/agent.sock"}), cmd) + + assert.Equal(t, []string{"KEEP=1", "GIT_SSH_COMMAND=ssh -i key", "SSH_AUTH_SOCK=/tmp/agent.sock"}, cmd.Env) + }) +} + +func TestFetchEnvironmentValue_PrefersFetchEnvironmentOverProcess(t *testing.T) { + t.Setenv("ATMOS_TEST_FETCH_VALUE", "process") + t.Setenv("ATMOS_TEST_FETCH_VALUE_ONLY_PROCESS", "process-only") + ctx := WithEnvironment(context.Background(), []string{"ATMOS_TEST_FETCH_VALUE=identity"}) + + assert.Equal(t, "identity", fetchEnvironmentValue(ctx, "ATMOS_TEST_FETCH_VALUE")) + assert.Equal(t, "process-only", fetchEnvironmentValue(ctx, "ATMOS_TEST_FETCH_VALUE_ONLY_PROCESS")) + assert.Equal(t, "process", fetchEnvironmentValue(context.Background(), "ATMOS_TEST_FETCH_VALUE")) +} diff --git a/pkg/downloader/get_git.go b/pkg/downloader/get_git.go index 3b835a94176..255939ecb10 100644 --- a/pkg/downloader/get_git.go +++ b/pkg/downloader/get_git.go @@ -310,6 +310,11 @@ func defaultBrokeredAuthRetryConfig() *schema.RetryConfig { // getRunCommandWithRetry wraps getRunCommand with retry logic for transient errors. func (g *CustomGitGetter) getRunCommandWithRetry(ctx context.Context, cmd *exec.Cmd) error { + // Every git subprocess of a fetch (clone, fetch, checkout, submodules) passes through here, so + // this is the one place the per-fetch environment (see WithEnvironment) is applied. Retries copy + // cmd.Env, so they inherit it. + applyFetchEnvironment(ctx, cmd) + cfg := g.RetryConfig shouldRetry := isRetryableGitError diff --git a/pkg/downloader/gogetter_downloader.go b/pkg/downloader/gogetter_downloader.go index d6c7e102e4b..0969ad3c8f5 100644 --- a/pkg/downloader/gogetter_downloader.go +++ b/pkg/downloader/gogetter_downloader.go @@ -79,7 +79,13 @@ func (f *goGetterClientFactory) NewClient(ctx context.Context, src, dest string, // the process env, which the git subprocess this client spawns inherits. Process-once and // gated (CI + configured), so local-only fetches and non-Pro repos pay nothing. remote := f.atmosConfig != nil && isRemoteSource(src) - if remote { + // A Git source that names an identity carries its own per-fetch environment (see + // WithEnvironment), which takes precedence over the ambient broker, so the broker is not + // consulted for it. That holds for Git only: the per-fetch environment reaches Git subprocesses + // and nothing else. An HTTP source reads its token from the process environment, which is + // where the broker exports to, so it still needs the broker even when an identity is named. + identityCoversFetch := FetchUsesIdentity(ctx) && isGitSource(src) + if remote && !identityCoversFetch { broker.EnsureCredentials(ctx, f.atmosConfig) } @@ -177,6 +183,23 @@ func isRemoteSource(src string) bool { return isSupportedHost(lowerHost, lowerHost) } +// isGitSource reports whether a go-getter source is fetched by the Git getter, and so runs Git +// subprocesses that receive the per-fetch environment. It recognizes a "git::"-forced source, an +// explicit ssh:// or git:// URL, an SCP-style URL (git@github.com:org/repo), and the scheme-less +// shorthand for a supported host (github.com/org/repo), which CustomGitDetector turns into a Git +// source. An explicit http(s):// URL without "git::" is fetched by the HTTP getter and reports false. +func isGitSource(src string) bool { + if strings.HasPrefix(src, GitPrefix) { + return true + } + if i := strings.Index(src, schemeSeparator); i >= 0 { + scheme := strings.ToLower(src[:i]) + return scheme == "ssh" || scheme == "git" + } + // No scheme: SCP-style and supported-host shorthand are Git; anything else is local. + return isRemoteSource(src) +} + // applyForcedGitDetection re-runs CustomGitDetector against a "git::"-forced source whose inner // URL already carries an explicit scheme (e.g. "git::https://github.com/org/repo.git"). // diff --git a/pkg/downloader/identity.go b/pkg/downloader/identity.go new file mode 100644 index 00000000000..848e09a1028 --- /dev/null +++ b/pkg/downloader/identity.go @@ -0,0 +1,181 @@ +package downloader + +import ( + "context" + "fmt" + "strings" + "sync" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +// The sentinel values of the global --identity flag (mirrors pkg/config, which this package cannot +// import without a cycle): "--identity=false" disables identities and a bare "--identity" asks for +// an interactive selection, which a remote read never does. +const ( + identityFlagDisabled = "__DISABLED__" + identityFlagSelect = "__SELECT__" +) + +// IdentityEnvironmentResolver turns an Atmos Auth identity name into the environment ("KEY=value" +// entries) its Git subprocess needs. Implementations must not modify the process environment and +// must never prompt: a remote read happens in the middle of a command. +type IdentityEnvironmentResolver func(ctx context.Context, atmosConfig *schema.AtmosConfiguration, identity string) ([]string, error) + +type identityOverrideKey struct{} + +type identityNameKey struct{} + +var ( + identityResolverMu sync.RWMutex + identityResolver IdentityEnvironmentResolver +) + +// RegisterIdentityEnvironmentResolver installs the resolver WithIdentity uses. The auth layer +// registers it from init(), the same way credential brokers register, so this package stays free +// of a dependency on pkg/auth. +func RegisterIdentityEnvironmentResolver(resolver IdentityEnvironmentResolver) { + defer perf.Track(nil, "downloader.RegisterIdentityEnvironmentResolver")() + + identityResolverMu.Lock() + defer identityResolverMu.Unlock() + identityResolver = resolver +} + +// WithIdentityOverride records the global --identity flag value on ctx. A non-empty value takes +// precedence over every identity: field and section default (see ResolveIdentityName). +func WithIdentityOverride(ctx context.Context, identity string) context.Context { + defer perf.Track(nil, "downloader.WithIdentityOverride")() + + if identity == "" { + return ctx + } + return context.WithValue(ctx, identityOverrideKey{}, identity) +} + +// ResolveIdentityName applies the remote-read precedence, matching `atmos git`: the global +// --identity flag (carried on ctx by WithIdentityOverride), then the entry's own identity field, +// then the section default. It returns "" when none applies (or when the flag disables identities), +// which keeps today's behavior: the ambient credential broker, then the process environment. +func ResolveIdentityName(ctx context.Context, entry, sectionDefault string) string { + defer perf.Track(nil, "downloader.ResolveIdentityName")() + + override, _ := ctx.Value(identityOverrideKey{}).(string) + switch override { + case identityFlagDisabled: + return "" + case "", identityFlagSelect: + // No usable flag value: fall through to the configuration. + default: + return override + } + if entry = strings.TrimSpace(entry); entry != "" { + return entry + } + return strings.TrimSpace(sectionDefault) +} + +// WithIdentity resolves which identity (if any) applies to one remote read, turns it into an +// environment through the registered resolver, and returns a context whose fetches run with that +// environment (see WithEnvironment) and are attributed to that identity (see IdentityFromContext). +// When no identity applies it returns ctx unchanged. +// +// Prefer WithIdentityForSource when the source is known: an identity only applies to the +// transports that can use it. +func WithIdentity(ctx context.Context, atmosConfig *schema.AtmosConfiguration, entry, sectionDefault string) (context.Context, error) { + defer perf.Track(atmosConfig, "downloader.WithIdentity")() + + name := ResolveIdentityName(ctx, entry, sectionDefault) + if name == "" { + return ctx, nil + } + + identityResolverMu.RLock() + resolver := identityResolver + identityResolverMu.RUnlock() + if resolver == nil { + return ctx, errUtils.Build(errUtils.ErrRemoteIdentityResolverUnavailable). + WithExplanation(fmt.Sprintf("Identity `%s` is named for a remote read, but no identity resolver is registered.", name)). + WithHint("This is a build problem: the auth layer registers the resolver at startup."). + Err() + } + + env, err := resolver(ctx, atmosConfig, name) + if err != nil { + return ctx, err + } + return WithEnvironment(context.WithValue(ctx, identityNameKey{}, name), env), nil +} + +// WithIdentityForSource is WithIdentity for one known source. The identity is resolved only when +// the source's transport can use it (see SourceUsesIdentity). For any other source it returns ctx +// unchanged, so a registry, bucket, or local read is never refused because an identity it would +// not have used has no session. +func WithIdentityForSource(ctx context.Context, atmosConfig *schema.AtmosConfiguration, src, entry, sectionDefault string) (context.Context, error) { + defer perf.Track(atmosConfig, "downloader.WithIdentityForSource")() + + if !SourceUsesIdentity(src) { + return ctx, nil + } + return WithIdentity(ctx, atmosConfig, entry, sectionDefault) +} + +// IdentityFromContext returns the name of the Atmos Auth identity the fetches made with ctx run as +// (set by WithIdentity), or "" when they name none and ambient credentials apply. +func IdentityFromContext(ctx context.Context) string { + defer perf.Track(nil, "downloader.IdentityFromContext")() + + if ctx == nil { + return "" + } + name, _ := ctx.Value(identityNameKey{}).(string) + return name +} + +// FetchUsesIdentity reports whether the fetches made with ctx run with a named identity or a +// per-fetch environment of their own, as opposed to ambient credentials. +func FetchUsesIdentity(ctx context.Context) bool { + defer perf.Track(nil, "downloader.FetchUsesIdentity")() + + return IdentityFromContext(ctx) != "" || len(EnvironmentFromContext(ctx)) > 0 +} + +// identitySkippedPrefixes are the source forms whose getters authenticate through their own +// credential chain (a registry keychain, a cloud SDK) or need no credentials at all. An identity +// environment never reaches them. +var identitySkippedPrefixes = []string{ + "oci://", "oci::", + "s3://", "s3::", + "gcs::", "gs://", + "file://", "file::", +} + +// identitySkippedHosts mark the scheme-less bucket URLs go-getter's own detectors turn into S3 and +// GCS sources. +var identitySkippedHosts = []string{".amazonaws.com/", "googleapis.com/"} + +// SourceUsesIdentity reports whether an identity named for src can take part in fetching it. An +// identity is delivered as an environment for the fetch, which Git subprocesses run with. Sources +// fetched from an OCI registry, an S3 or GCS bucket, or the local filesystem authenticate on their +// own and report false. Everything else (Git in any form, and HTTP) reports true, so an identity +// named for it must resolve. +func SourceUsesIdentity(src string) bool { + defer perf.Track(nil, "downloader.SourceUsesIdentity")() + + lowered := strings.ToLower(strings.TrimSpace(src)) + for _, prefix := range identitySkippedPrefixes { + if strings.HasPrefix(lowered, prefix) { + return false + } + } + if !strings.Contains(lowered, schemeSeparator) && !strings.HasPrefix(lowered, strings.ToLower(GitPrefix)) { + for _, host := range identitySkippedHosts { + if strings.Contains(lowered, host) { + return false + } + } + } + return true +} diff --git a/pkg/downloader/identity_git_test.go b/pkg/downloader/identity_git_test.go new file mode 100644 index 00000000000..d19bc0e553b --- /dev/null +++ b/pkg/downloader/identity_git_test.go @@ -0,0 +1,194 @@ +package downloader + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/tests/testhelpers/gitscrub" +) + +const ( + // The fake remote is never resolvable: a fetch of it can only succeed through an `insteadOf` + // rewrite delivered in the fetch's own environment. + fakeRemote = "https://git.example.invalid/acme/shared.git" + markerFile = "owner.txt" + fetchWindow = 2 * time.Minute +) + +// gitTestRepo creates a local repository with one commit and returns its file:// URL. +func gitTestRepo(t *testing.T, owner string) string { + t.Helper() + if _, err := exec.LookPath(gitCommand); err != nil { + t.Skip("git is not available") + } + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command(gitCommand, args...) + cmd.Dir = dir + // A caller's GIT_DIR, GIT_WORK_TREE, or GIT_INDEX_FILE would point these commands at the + // caller's repository instead of the temporary one, despite cmd.Dir. + cmd.Env = gitscrub.Environ() + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + } + run("init", "--quiet") + require.NoError(t, os.WriteFile(filepath.Join(dir, markerFile), []byte(owner), 0o600)) + run("add", markerFile) + run("-c", "user.name=test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "init") + + repoPath := filepath.ToSlash(dir) + if !strings.HasPrefix(repoPath, "/") { + repoPath = "/" + repoPath + } + return (&url.URL{Scheme: "file", Path: repoPath}).String() +} + +// identityEnv is the environment an identity would hand to a fetch: a git `insteadOf` rewrite +// pointing fakeRemote at repoURL, in the "KEY=value" form git.ComposeEnvironment returns. +func identityEnv(repoURL string) []string { + return []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url." + repoURL + ".insteadOf", + "GIT_CONFIG_VALUE_0=" + fakeRemote, + } +} + +// gitConfigBaseline is the GIT_CONFIG_COUNT every fetch test starts from: no inherited entries. +const gitConfigBaseline = "0" + +// isolateGitConfig keeps the developer's own git config out of the fetch. It also pins +// GIT_CONFIG_COUNT to a known baseline, so rewrite entries inherited from the caller cannot reach +// the fetches and a test can assert the process environment ended as it began. +func isolateGitConfig(t *testing.T) { + t.Helper() + t.Setenv("GIT_CONFIG_GLOBAL", filepath.Join(t.TempDir(), "gitconfig")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_TERMINAL_PROMPT", "0") + t.Setenv("GIT_CONFIG_COUNT", gitConfigBaseline) +} + +func fetchGit(ctx context.Context, dest string) error { + dl := NewGoGetterDownloader(&schema.AtmosConfiguration{}).(ContextFileDownloader) + _, err := dl.FetchWithMetadataContext(ctx, "git::"+fakeRemote, dest, ClientModeDir, fetchWindow) + return err +} + +func TestCustomGitGetter_PerFetchEnvironmentReachesGitSubprocess(t *testing.T) { + isolateGitConfig(t) + repoURL := gitTestRepo(t, "identity-repo") + + t.Run("without a per-fetch environment the unresolvable remote fails", func(t *testing.T) { + require.Error(t, fetchGit(context.Background(), filepath.Join(t.TempDir(), "dst"))) + }) + + t.Run("with a per-fetch environment the git subprocess sees it", func(t *testing.T) { + dest := filepath.Join(t.TempDir(), "dst") + ctx := WithEnvironment(context.Background(), identityEnv(repoURL)) + + require.NoError(t, fetchGit(ctx, dest)) + + got, err := os.ReadFile(filepath.Join(dest, markerFile)) + require.NoError(t, err) + assert.Equal(t, "identity-repo", string(got)) + }) + + t.Run("the process environment is never modified", func(t *testing.T) { + assert.Equal(t, gitConfigBaseline, os.Getenv("GIT_CONFIG_COUNT"), + "the identity's GIT_CONFIG_COUNT must not replace the process environment's") + }) +} + +// Two sources fetched concurrently, each with its own identity environment, must each see only +// their own: the same URL resolves to a different repository per context. +func TestCustomGitGetter_ConcurrentFetchesWithDifferentIdentitiesStayIsolated(t *testing.T) { + isolateGitConfig(t) + owners := []string{"alpha", "bravo"} + repoURLs := make(map[string]string, len(owners)) + for _, owner := range owners { + repoURLs[owner] = gitTestRepo(t, owner) + } + + const fetchesPerIdentity = 4 + var wg sync.WaitGroup + results := make(chan string, len(owners)*fetchesPerIdentity) + errs := make(chan error, len(owners)*fetchesPerIdentity) + for i := 0; i < fetchesPerIdentity; i++ { + for _, owner := range owners { + wg.Add(1) + go func() { + defer wg.Done() + dest := filepath.Join(t.TempDir(), "dst") + if err := fetchGit(WithEnvironment(context.Background(), identityEnv(repoURLs[owner])), dest); err != nil { + errs <- fmt.Errorf("%s: %w", owner, err) + return + } + got, err := os.ReadFile(filepath.Join(dest, markerFile)) + if err != nil { + errs <- err + return + } + results <- owner + "=" + string(got) + }() + } + } + wg.Wait() + close(results) + close(errs) + + for err := range errs { + require.NoError(t, err) + } + count := 0 + for result := range results { + owner, got, _ := strings.Cut(result, "=") + assert.Equal(t, owner, got, "a fetch must see only its own identity's environment") + count++ + } + assert.Equal(t, len(owners)*fetchesPerIdentity, count) +} + +// An identity's insteadOf rewrite lives only in the per-fetch environment. URL token injection +// must see it there, or it would put userinfo into the URL and shadow the rewrite. +func TestBrokerInsteadOfMatchesURLContext_ReadsPerFetchEnvironment(t *testing.T) { + t.Setenv("GIT_CONFIG_COUNT", "") + parsed, err := url.Parse("https://github.com/acme/modules.git") + require.NoError(t, err) + rewrite := []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url.https://x-access-token:token@github.com/acme/.insteadOf", + "GIT_CONFIG_VALUE_0=https://github.com/acme/", + } + // The key above is an inline `url..insteadOf` rewrite whose value is the match target. + + tests := []struct { + name string + ctx context.Context + want bool + }{ + {name: "no per-fetch environment and no process rewrite", ctx: context.Background(), want: false}, + {name: "per-fetch rewrite for the same owner", ctx: WithEnvironment(context.Background(), rewrite), want: true}, + {name: "per-fetch rewrite for another owner", ctx: WithEnvironment(context.Background(), []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url.https://x-access-token:token@github.com/other/.insteadOf", + "GIT_CONFIG_VALUE_0=https://github.com/other/", + }), want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, brokerInsteadOfMatchesURLContext(tt.ctx, parsed, "github.com")) + }) + } +} diff --git a/pkg/downloader/identity_scope_test.go b/pkg/downloader/identity_scope_test.go new file mode 100644 index 00000000000..6f049ab014a --- /dev/null +++ b/pkg/downloader/identity_scope_test.go @@ -0,0 +1,282 @@ +package downloader + +import ( + "context" + "errors" + "net/url" + "os" + "os/exec" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/auth/broker" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinel: the injection tests configure tokens through these fields. +var _ = schema.AtmosSettings{InjectGithubToken: true, GithubToken: "x"} + +// testIdentity is the identity the fetches in these tests run as. +const testIdentity = "ops" + +// identityContext returns a context whose fetches run as testIdentity, with env as its environment. +func identityContext(t *testing.T, env []string) context.Context { + t.Helper() + registerTestResolver(t, func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return env, nil + }) + ctx, err := WithIdentity(context.Background(), &schema.AtmosConfiguration{}, testIdentity, "") + require.NoError(t, err) + return ctx +} + +// brokerExport runs a fake credential broker that exports env into the process environment, the +// way the Atmos Pro broker does on the first ambient remote read. +func brokerExport(t *testing.T, env map[string]string) { + t.Helper() + for key := range env { + // Registers the restore of whatever the broker is about to set, leaving the variable in + // the state the test started with (unset stays unset). + value, wasSet := os.LookupEnv(key) + t.Setenv(key, value) + if !wasSet { + require.NoError(t, os.Unsetenv(key)) + } + } + t.Cleanup(broker.SwapRegistryForTest()) + broker.Register(fakeBroker{env: env}) + broker.EnsureCredentials(context.Background(), &schema.AtmosConfiguration{}) + for key, value := range env { + require.Equal(t, value, os.Getenv(key), "prerequisite: the broker exported %s into the process", key) + } +} + +type fakeBroker struct{ env map[string]string } + +func (fakeBroker) Name() string { return "fake" } +func (fakeBroker) Enabled(*schema.AtmosConfiguration) bool { return true } +func (b fakeBroker) Provision(context.Context, *schema.AtmosConfiguration) (map[string]string, error) { + return b.env, nil +} + +func TestSourceUsesIdentity(t *testing.T) { + tests := []struct { + src string + want bool + }{ + {src: "git::https://github.com/acme/modules.git", want: true}, + {src: "github.com/acme/modules//vpc", want: true}, + {src: "git@github.com:acme/modules.git", want: true}, + {src: "ssh://git@git.example.com/acme/modules.git", want: true}, + {src: "https://example.com/stacks/catalog.yaml", want: true}, + {src: "git::https://bucket.s3.amazonaws.com/repo.git", want: true}, + {src: "oci://ghcr.io/acme/stacks:v1//catalog/vpc.yaml", want: false}, + {src: "OCI://ghcr.io/acme/stacks:v1", want: false}, + {src: "s3::https://s3.amazonaws.com/bucket/module.zip", want: false}, + {src: "s3://bucket/module.zip", want: false}, + {src: "bucket.s3.amazonaws.com/module.zip", want: false}, + {src: "gcs::https://www.googleapis.com/storage/v1/bucket/module.zip", want: false}, + {src: "www.googleapis.com/storage/v1/bucket/module.zip", want: false}, + {src: "file:///tmp/module", want: false}, + } + for _, tt := range tests { + t.Run(tt.src, func(t *testing.T) { + assert.Equal(t, tt.want, SourceUsesIdentity(tt.src)) + }) + } +} + +func TestIsGitSource(t *testing.T) { + tests := []struct { + src string + want bool + }{ + {src: "git::https://github.com/acme/modules.git", want: true}, + {src: "github.com/acme/modules//vpc", want: true}, + {src: "git@github.com:acme/modules.git", want: true}, + {src: "ssh://git@git.example.com/acme/modules.git", want: true}, + {src: "https://github.com/acme/modules/archive/main.tar.gz", want: false}, + {src: "https://example.com/stacks/catalog.yaml", want: false}, + {src: "./local/module", want: false}, + } + for _, tt := range tests { + t.Run(tt.src, func(t *testing.T) { + assert.Equal(t, tt.want, isGitSource(tt.src)) + }) + } +} + +// An identity that cannot be resolved must not stop a source it would never have authenticated. +func TestWithIdentityForSource_ResolvesOnlyForTransportsThatUseIt(t *testing.T) { + notLoggedIn := errors.New("no session") + var calls atomic.Int32 + registerTestResolver(t, func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + calls.Add(1) + return nil, notLoggedIn + }) + background := context.Background() + + for _, src := range []string{"oci://ghcr.io/acme/stacks:v1//catalog/vpc.yaml", "s3::https://s3.amazonaws.com/bucket/module.zip"} { + ctx, err := WithIdentityForSource(background, &schema.AtmosConfiguration{}, src, "ops", "") + require.NoError(t, err, src) + assert.Equal(t, background, ctx, "the context is returned unchanged for %s", src) + assert.Empty(t, IdentityFromContext(ctx)) + } + assert.Zero(t, calls.Load(), "the identity is not resolved for a transport that cannot use it") + + _, err := WithIdentityForSource(background, &schema.AtmosConfiguration{}, "github.com/acme/modules//vpc", "ops", "") + require.ErrorIs(t, err, notLoggedIn, "a Git source that names the identity still requires it") + assert.Equal(t, int32(1), calls.Load()) +} + +func TestWithIdentity_RecordsTheIdentityOnTheContext(t *testing.T) { + assert.Empty(t, IdentityFromContext(context.Background())) + assert.False(t, FetchUsesIdentity(context.Background())) + assert.True(t, FetchUsesIdentity(WithEnvironment(context.Background(), []string{"A=1"}))) + + // An identity whose environment is empty is still the identity the fetch runs as. + ctx := identityContext(t, nil) + + assert.Equal(t, testIdentity, IdentityFromContext(ctx)) + assert.True(t, FetchUsesIdentity(ctx)) + assert.Empty(t, EnvironmentFromContext(ctx)) +} + +// A broker's Git rewrite, exported into the process by an earlier ambient fetch, must not reach a +// later fetch that names its own identity: Git would rewrite the URL to carry the broker's token. +func TestApplyFetchEnvironment_NamedIdentityDoesNotInheritBrokerCredentials(t *testing.T) { + t.Setenv("ATMOS_TEST_USER_SETTING", "kept") + for _, name := range []string{"GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0"} { + t.Setenv(name, "") + require.NoError(t, os.Unsetenv(name)) + } + brokerExport(t, map[string]string{ + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "url.https://x-access-token:broker@github.com/acme/.insteadOf", + "GIT_CONFIG_VALUE_0": "https://github.com/acme/", + }) + + t.Run("a named identity with no rewrite of its own runs without the broker's", func(t *testing.T) { + cmd := exec.Command("git", "version") + applyFetchEnvironment(identityContext(t, []string{"SSH_AUTH_SOCK=/tmp/ops.sock"}), cmd) + + for _, entry := range cmd.Env { + assert.NotContains(t, entry, "broker@", "the broker's token must not be inherited") + for _, exported := range []string{"GIT_CONFIG_COUNT=", "GIT_CONFIG_KEY_0=", "GIT_CONFIG_VALUE_0="} { + assert.NotContains(t, entry, exported, "broker-exported Git configuration must not be inherited") + } + } + assert.Contains(t, cmd.Env, "SSH_AUTH_SOCK=/tmp/ops.sock") + assert.Contains(t, cmd.Env, "ATMOS_TEST_USER_SETTING=kept", "the rest of the process environment is inherited") + }) + + t.Run("a named identity's own rewrite replaces the broker's", func(t *testing.T) { + cmd := exec.Command("git", "version") + applyFetchEnvironment(identityContext(t, []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url.https://x-access-token:ops@github.com/acme/.insteadOf", + "GIT_CONFIG_VALUE_0=https://github.com/acme/", + }), cmd) + + assert.Contains(t, cmd.Env, "GIT_CONFIG_KEY_0=url.https://x-access-token:ops@github.com/acme/.insteadOf") + assert.NotContains(t, cmd.Env, "GIT_CONFIG_KEY_0=url.https://x-access-token:broker@github.com/acme/.insteadOf") + }) + + t.Run("a fetch that names no identity keeps inheriting the broker's credentials", func(t *testing.T) { + cmd := exec.Command("git", "version") + applyFetchEnvironment(context.Background(), cmd) + + assert.Nil(t, cmd.Env, "the subprocess inherits the process environment, broker rewrite included") + }) +} + +// The detector must see what the fetch's Git subprocess will see: for a named identity, not the +// broker's rewrite. +func TestFetchEnvironmentValue_NamedIdentityIgnoresBrokerExports(t *testing.T) { + t.Setenv("ATMOS_TEST_FETCH_VALUE", "before") + brokerExport(t, map[string]string{"ATMOS_TEST_FETCH_VALUE": "brokered"}) + named := identityContext(t, nil) + + assert.Equal(t, "brokered", fetchEnvironmentValue(context.Background(), "ATMOS_TEST_FETCH_VALUE"), + "an ambient fetch reads the broker's value") + assert.Equal(t, "before", fetchEnvironmentValue(named, "ATMOS_TEST_FETCH_VALUE"), + "a named identity reads the value from before the broker ran") + assert.Equal(t, "own", fetchEnvironmentValue(identityContext(t, []string{"ATMOS_TEST_FETCH_VALUE=own"}), "ATMOS_TEST_FETCH_VALUE"), + "the identity's own value wins") +} + +// URL userinfo beats every other Git credential, so an injected ambient token would silently +// replace the identity the source named. +func TestInjectToken_NamedIdentityNeverReceivesAnAmbientToken(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{Settings: schema.AtmosSettings{InjectGithubToken: true, GithubToken: "ambient-token"}} + parse := func() *url.URL { + parsed, err := url.Parse("https://github.com/acme/modules.git") + require.NoError(t, err) + return parsed + } + + t.Run("ambient fetch gets the configured token", func(t *testing.T) { + parsed := parse() + NewCustomGitDetectorContext(context.Background(), atmosConfig, "").injectToken(parsed, "github.com", "github.com") + + require.NotNil(t, parsed.User, "prerequisite: injection is active for this host") + password, _ := parsed.User.Password() + assert.Equal(t, "ambient-token", password) + }) + + t.Run("named identity keeps the URL free of credentials", func(t *testing.T) { + // The identity authenticates through an askpass helper: no insteadOf rewrite to detect. + ctx := identityContext(t, []string{"GIT_ASKPASS=/usr/local/bin/ops-askpass"}) + parsed := parse() + NewCustomGitDetectorContext(ctx, atmosConfig, "").injectToken(parsed, "github.com", "github.com") + + assert.Nil(t, parsed.User, "the ambient token must not be injected for a named identity") + }) +} + +// countingBroker records how many times it was asked to provision. +type countingBroker struct{ calls *atomic.Int32 } + +func (countingBroker) Name() string { return "counting" } +func (countingBroker) Enabled(*schema.AtmosConfiguration) bool { return true } +func (b countingBroker) Provision(context.Context, *schema.AtmosConfiguration) (map[string]string, error) { + b.calls.Add(1) + return nil, nil +} + +// The per-fetch environment only reaches Git subprocesses. An HTTP source reads its token from the +// process environment, which is where the broker exports to, so it needs the broker even when an +// identity is named. A Git source that names an identity does not. +func TestNewClient_BrokerIsSkippedOnlyForGitSourcesThatNameAnIdentity(t *testing.T) { + tests := []struct { + name string + src string + named bool + wantRun bool + }{ + {name: "git source with an identity", src: "git::https://github.com/acme/modules.git", named: true, wantRun: false}, + {name: "http source with an identity", src: "https://example.com/stacks/catalog.yaml", named: true, wantRun: true}, + {name: "git source without an identity", src: "git::https://github.com/acme/modules.git", named: false, wantRun: true}, + {name: "http source without an identity", src: "https://example.com/stacks/catalog.yaml", named: false, wantRun: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Cleanup(broker.SwapRegistryForTest()) + var calls atomic.Int32 + broker.Register(countingBroker{calls: &calls}) + ctx := context.Background() + if tt.named { + ctx = identityContext(t, []string{"SSH_AUTH_SOCK=/tmp/ops.sock"}) + } + factory := &goGetterClientFactory{atmosConfig: &schema.AtmosConfiguration{}} + + _, err := factory.NewClient(ctx, tt.src, t.TempDir(), ClientModeAny) + + require.NoError(t, err) + assert.Equal(t, tt.wantRun, calls.Load() == 1) + }) + } +} diff --git a/pkg/downloader/identity_test.go b/pkg/downloader/identity_test.go new file mode 100644 index 00000000000..ed6df3e1d84 --- /dev/null +++ b/pkg/downloader/identity_test.go @@ -0,0 +1,120 @@ +package downloader + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinel: the tests below rely on the AtmosConfiguration struct. +var _ = schema.AtmosConfiguration{} + +// registerTestResolver installs resolver for the test and restores the previous one afterwards. +func registerTestResolver(t *testing.T, resolver IdentityEnvironmentResolver) { + t.Helper() + identityResolverMu.RLock() + previous := identityResolver + identityResolverMu.RUnlock() + RegisterIdentityEnvironmentResolver(resolver) + t.Cleanup(func() { RegisterIdentityEnvironmentResolver(previous) }) +} + +func TestResolveIdentityName_Precedence(t *testing.T) { + tests := []struct { + name string + override string + entry string + sectionDefault string + want string + }{ + {name: "nothing configured", want: ""}, + {name: "section default only", sectionDefault: "default-id", want: "default-id"}, + {name: "entry beats section default", entry: "entry-id", sectionDefault: "default-id", want: "entry-id"}, + {name: "flag beats entry and section default", override: "flag-id", entry: "entry-id", sectionDefault: "default-id", want: "flag-id"}, + {name: "flag alone", override: "flag-id", want: "flag-id"}, + {name: "disabled flag turns identities off", override: identityFlagDisabled, entry: "entry-id", sectionDefault: "default-id", want: ""}, + {name: "bare --identity never selects interactively", override: identityFlagSelect, entry: "entry-id", sectionDefault: "default-id", want: "entry-id"}, + {name: "bare --identity falls through to the default", override: identityFlagSelect, sectionDefault: "default-id", want: "default-id"}, + {name: "whitespace only entry is unset", entry: " ", sectionDefault: "default-id", want: "default-id"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := WithIdentityOverride(context.Background(), tt.override) + assert.Equal(t, tt.want, ResolveIdentityName(ctx, tt.entry, tt.sectionDefault)) + }) + } +} + +func TestWithIdentity_NoIdentityLeavesContextAndSkipsResolver(t *testing.T) { + var calls atomic.Int32 + registerTestResolver(t, func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + calls.Add(1) + return []string{"X=1"}, nil + }) + + parent := context.Background() + got, err := WithIdentity(parent, &schema.AtmosConfiguration{}, "", "") + + require.NoError(t, err) + assert.Equal(t, parent, got, "a source without an identity behaves exactly as before") + assert.Nil(t, EnvironmentFromContext(got)) + assert.Zero(t, calls.Load()) +} + +func TestWithIdentity_ResolvesEffectiveIdentityIntoFetchEnvironment(t *testing.T) { + var seen []string + registerTestResolver(t, func(_ context.Context, _ *schema.AtmosConfiguration, identity string) ([]string, error) { + seen = append(seen, identity) + return []string{"IDENTITY_NAME=" + identity}, nil + }) + + tests := []struct { + name string + override string + entry string + sectionDefault string + want string + }{ + {name: "section default", sectionDefault: "default-id", want: "default-id"}, + {name: "entry", entry: "entry-id", sectionDefault: "default-id", want: "entry-id"}, + {name: "flag", override: "flag-id", entry: "entry-id", sectionDefault: "default-id", want: "flag-id"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + seen = nil + ctx := WithIdentityOverride(context.Background(), tt.override) + + got, err := WithIdentity(ctx, &schema.AtmosConfiguration{}, tt.entry, tt.sectionDefault) + + require.NoError(t, err) + assert.Equal(t, []string{tt.want}, seen) + assert.Equal(t, []string{"IDENTITY_NAME=" + tt.want}, EnvironmentFromContext(got)) + }) + } +} + +func TestWithIdentity_ResolverFailureIsReturned(t *testing.T) { + boom := errors.New("not logged in") + registerTestResolver(t, func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return nil, boom + }) + + _, err := WithIdentity(context.Background(), &schema.AtmosConfiguration{}, "ops", "") + + require.ErrorIs(t, err, boom) +} + +func TestWithIdentity_NoResolverRegistered(t *testing.T) { + registerTestResolver(t, nil) + + _, err := WithIdentity(context.Background(), &schema.AtmosConfiguration{}, "ops", "") + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityResolverUnavailable) +} diff --git a/pkg/plugin/adapter/auth/environment.go b/pkg/plugin/adapter/auth/environment.go index 478d248dca4..eaba07f00cf 100644 --- a/pkg/plugin/adapter/auth/environment.go +++ b/pkg/plugin/adapter/auth/environment.go @@ -33,9 +33,35 @@ func (m *environmentMaterializer) Apply(ctx context.Context, params *types.PostA if params == nil { return errUtils.ErrPluginProtocol } - credentials, ok := params.Credentials.(*types.PluginCredentials) + environment, err := m.adopt(params.Credentials) + if err != nil { + return err + } + if params.AuthContext != nil { + params.AuthContext.Environment = maps.Clone(environment) + } + return nil +} + +// Restore populates the materializer from a credential read back from the credential store. A +// fresh process that finds a still-valid stored credential skips authentication, so nothing else +// would call Apply. The same validation and secret masking as Apply apply; an expired or +// malformed credential is refused, never adopted. +func (m *environmentMaterializer) Restore(ctx context.Context, stored types.ICredentials) error { + defer perf.Track(nil, "auth.environmentMaterializer.Restore")() + if err := ctx.Err(); err != nil { + return err + } + _, err := m.adopt(stored) + return err +} + +// adopt validates an environment/v1 credential, registers every value with the masker, and keeps a +// private copy. It returns the environment it adopted. +func (m *environmentMaterializer) adopt(value types.ICredentials) (map[string]string, error) { + credentials, ok := value.(*types.PluginCredentials) if !ok || credentials.Type != "environment/v1" || credentials.IsExpired() { - return errUtils.ErrPluginProtocol + return nil, errUtils.ErrPluginProtocol } clone := *credentials clone.Data = maps.Clone(credentials.Data) @@ -44,18 +70,15 @@ func (m *environmentMaterializer) Apply(ctx context.Context, params *types.PostA for name, value := range clone.Data { text, ok := value.(string) if !ok { - return errUtils.ErrPluginProtocol + return nil, errUtils.ErrPluginProtocol } atmosio.RegisterSecret(text) environment[name] = text } - if params.AuthContext != nil { - params.AuthContext.Environment = maps.Clone(environment) - } m.mu.Lock() defer m.mu.Unlock() m.credentials = &clone - return nil + return environment, nil } func (m *environmentMaterializer) Environment() (map[string]string, error) { diff --git a/pkg/plugin/adapter/auth/environment_test.go b/pkg/plugin/adapter/auth/environment_test.go index 13b6a3a9dad..a313dce7cf5 100644 --- a/pkg/plugin/adapter/auth/environment_test.go +++ b/pkg/plugin/adapter/auth/environment_test.go @@ -104,3 +104,60 @@ func TestEnvironmentMaterializerMasksComposedCredentials(t *testing.T) { require.Equal(t, "access="+atmosio.MaskReplacement+" secret="+atmosio.MaskReplacement, atmosio.MaskString("access=private-access secret=private-secret")) } + +func TestEnvironmentMaterializerRestore(t *testing.T) { + valid := func() *types.PluginCredentials { + return &types.PluginCredentials{Type: "environment/v1", ExpiresAt: time.Now().Add(time.Hour), Data: map[string]any{"TOKEN": "restored-private-token"}} + } + t.Run("a stored credential supplies the environment and is masked", func(t *testing.T) { + m := EnvironmentMaterializer("identity", nil) + stored := valid() + require.NoError(t, m.(*environmentMaterializer).Restore(t.Context(), stored)) + values, err := m.Environment() + require.NoError(t, err) + require.Equal(t, map[string]string{"TOKEN": "restored-private-token"}, values) + require.NotContains(t, atmosio.MaskString("value restored-private-token end"), "restored-private-token") + stored.Data["TOKEN"] = "mutated" + values, err = m.Environment() + require.NoError(t, err) + require.Equal(t, "restored-private-token", values["TOKEN"], "restore must copy the stored credential") + }) + tests := []struct { + name string + stored types.ICredentials + }{ + {"expired credential is refused", &types.PluginCredentials{Type: "environment/v1", ExpiresAt: time.Now().Add(-time.Hour), Data: map[string]any{"TOKEN": "x"}}}, + {"missing expiry is refused", &types.PluginCredentials{Type: "environment/v1", Data: map[string]any{"TOKEN": "x"}}}, + {"wrong credential type is refused", &types.PluginCredentials{Type: "secret-values/v1", ExpiresAt: time.Now().Add(time.Hour), Data: map[string]any{"TOKEN": "x"}}}, + {"non-string value is refused", &types.PluginCredentials{Type: "environment/v1", ExpiresAt: time.Now().Add(time.Hour), Data: map[string]any{"TOKEN": 1}}}, + {"other credential kinds are refused", &types.AWSCredentials{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := EnvironmentMaterializer("identity", nil) + require.ErrorIs(t, m.(*environmentMaterializer).Restore(t.Context(), tt.stored), errUtils.ErrPluginProtocol) + _, err := m.Environment() + require.ErrorIs(t, err, errUtils.ErrAuthenticationFailed, "a refused credential must not be adopted") + }) + } + t.Run("cancelled context is refused", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, EnvironmentMaterializer("identity", nil).(*environmentMaterializer).Restore(ctx, valid()), context.Canceled) + }) +} + +func TestIdentityRestoreCredentials(t *testing.T) { + stored := &types.PluginCredentials{Type: "environment/v1", ExpiresAt: time.Now().Add(time.Hour), Data: map[string]any{"TOKEN": "id-restored-token"}} + t.Run("delegates to a restoring materializer", func(t *testing.T) { + id := &identity{materializer: EnvironmentMaterializer("identity", nil)} + require.NoError(t, id.RestoreCredentials(t.Context(), stored)) + values, err := id.Environment() + require.NoError(t, err) + require.Equal(t, map[string]string{"TOKEN": "id-restored-token"}, values) + }) + t.Run("file-backed materializers need no restore", func(t *testing.T) { + id := &identity{materializer: AWSMaterializer("identity", &schema.Identity{})} + require.NoError(t, id.RestoreCredentials(t.Context(), stored)) + }) +} diff --git a/pkg/plugin/adapter/auth/identity.go b/pkg/plugin/adapter/auth/identity.go index 697f9df71fc..cfc1a2a65fb 100644 --- a/pkg/plugin/adapter/auth/identity.go +++ b/pkg/plugin/adapter/auth/identity.go @@ -132,6 +132,25 @@ func (i *identity) PostAuthenticate(ctx context.Context, params *types.PostAuthe return i.materializer.Apply(ctx, params) } +// materializerRestorer is implemented by materializers whose state lives only in memory, so a +// credential read back from the credential store must be handed to them explicitly. +type materializerRestorer interface { + Restore(context.Context, types.ICredentials) error +} + +// RestoreCredentials implements types.CredentialRestorer. It hands a stored credential to an +// in-memory materializer so the passive path, which skips Authenticate, can still produce the +// identity's environment. Materializers backed by files need nothing. +func (i *identity) RestoreCredentials(ctx context.Context, stored types.ICredentials) error { + defer perf.Track(nil, "auth.identity.RestoreCredentials")() + + restorer, ok := i.materializer.(materializerRestorer) + if !ok { + return nil + } + return restorer.Restore(ctx, stored) +} + //nolint:lintroller // Trivial accessor, pure lookup, factory option, or delegation; exempt under repository tracking policy. func (i *identity) Environment() (map[string]string, error) { return i.materializer.Environment() } diff --git a/pkg/plugin/bootstrap/environment_passive_integration_test.go b/pkg/plugin/bootstrap/environment_passive_integration_test.go new file mode 100644 index 00000000000..9c855ee16f1 --- /dev/null +++ b/pkg/plugin/bootstrap/environment_passive_integration_test.go @@ -0,0 +1,76 @@ +package bootstrap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/cloudposse/atmos/pkg/auth" + authcredentials "github.com/cloudposse/atmos/pkg/auth/credentials" + "github.com/cloudposse/atmos/pkg/auth/validation" + "github.com/cloudposse/atmos/pkg/plugin/config" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/pkg/store" +) + +// TestNativeEnvironmentIdentityPassivePathInFreshProcess models two processes: one authenticates +// and stores the identity credential, a second builds a brand new manager (and so brand new +// adapter instances) over the same credential store and only uses the passive paths. The plugin +// must not run again, and the stored credential must supply the environment. +func TestNativeEnvironmentIdentityPassivePathInFreshProcess(t *testing.T) { + if testing.Short() { + t.Skip("builds and executes the auth-secrets native plugin") + } + packageDir := buildSecretPlugin(t) + t.Setenv("ATMOS_KEYRING_TYPE", "memory") + t.Setenv("ATMOS_XDG_CACHE_HOME", t.TempDir()) + + backend := store.NewMockFieldStore(gomock.NewController(t)) + // The provider secret is read once: by the first process. A second read means the plugin re-ran. + backend.EXPECT().GetKeyField("service/account", "api_token").Return("passive-private-token", nil).Times(1) + providerConfig := schema.Provider{Kind: "secrets/store", Spec: map[string]any{"credentials": map[string]any{"api-token": map[string]any{"store": "vault", "path": "service/account", "field": "api_token"}}}} + identityConfig := schema.Identity{Kind: "secrets/env", Via: &schema.IdentityVia{Provider: "vault-provider"}, Spec: map[string]any{"env": map[string]any{"SERVICE_TOKEN": `{{ index .credentials "api-token" }}`}}} + atmos := &schema.AtmosConfiguration{ + Auth: schema.AuthConfig{Providers: map[string]schema.Provider{"vault-provider": providerConfig}, Identities: map[string]schema.Identity{"service": identityConfig}}, + Stores: store.StoreRegistry{"vault": backend}, StoresConfig: store.StoresConfig{"vault": {Secret: true}}, + } + cleanup, err := registerSelected(t.Context(), atmos, &config.Selections{Packages: map[string]config.Selection{"secrets": {Path: packageDir, Local: true}}}) + require.NoError(t, err) + t.Cleanup(cleanup) + + // Process one: authenticate, which stores the identity credential. + sharedStore := authcredentials.NewCredentialStoreWithConfig(&atmos.Auth) + first, err := auth.NewAuthManager(&atmos.Auth, sharedStore, validation.NewValidator(), nil, "") + require.NoError(t, err) + _, err = first.Authenticate(t.Context(), "service") + require.NoError(t, err) + + // Process two: a new manager over the same stored credentials, never calling Authenticate. + newProcess := func(t *testing.T) auth.AuthManager { + t.Helper() + manager, err := auth.NewAuthManager(&atmos.Auth, sharedStore, validation.NewValidator(), nil, "") + require.NoError(t, err) + _, err = manager.GetCachedCredentials(t.Context(), "service") + require.NoError(t, err, "the stored credential must be found on the passive path") + return manager + } + + want := map[string]string{"SERVICE_TOKEN": "passive-private-token"} + t.Run("GetEnvironmentVariables", func(t *testing.T) { + environment, err := newProcess(t).GetEnvironmentVariables("service") + require.NoError(t, err) + assert.Equal(t, want, environment) + }) + t.Run("PrepareShellEnvironment", func(t *testing.T) { + shell, err := newProcess(t).PrepareShellEnvironment(t.Context(), "service", []string{"OTHER=preserved"}) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"SERVICE_TOKEN=passive-private-token", "OTHER=preserved"}, shell) + }) + t.Run("EnsureIdentityEnvironment", func(t *testing.T) { + environment, err := newProcess(t).EnsureIdentityEnvironment(t.Context(), "service") + require.NoError(t, err) + assert.Equal(t, want, environment) + }) +} diff --git a/pkg/provisioner/source/extract.go b/pkg/provisioner/source/extract.go index 24f8aaf26c0..64476dfb87f 100644 --- a/pkg/provisioner/source/extract.go +++ b/pkg/provisioner/source/extract.go @@ -75,6 +75,11 @@ func parseSourceMap(sourceMap map[string]any) (*schema.VendorComponentSource, er spec.Version = v } + // Optional: identity (the Atmos Auth identity used to fetch the source). + if identity, ok := sourceMap["identity"].(string); ok { + spec.Identity = identity + } + // Optional: included_paths. if paths, ok := sourceMap["included_paths"].([]any); ok { spec.IncludedPaths = toStringSlice(paths) diff --git a/pkg/provisioner/source/identity_test.go b/pkg/provisioner/source/identity_test.go new file mode 100644 index 00000000000..304e6ca56aa --- /dev/null +++ b/pkg/provisioner/source/identity_test.go @@ -0,0 +1,145 @@ +package source + +import ( + "context" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/downloader" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/tests/testhelpers/gitscrub" +) + +// Compile-time sentinels for the schema fields the tests set. +var ( + _ = schema.VendorComponentSource{Identity: "x"} + _ = schema.Vendor{Identity: "x"} +) + +func TestExtractSource_Identity(t *testing.T) { + tests := []struct { + name string + source any + want string + }{ + {name: "map form with identity", source: map[string]any{"uri": "github.com/acme/modules//vpc", "identity": "ops"}, want: "ops"}, + {name: "map form without identity", source: map[string]any{"uri": "github.com/acme/modules//vpc"}, want: ""}, + {name: "string form has no identity", source: "github.com/acme/modules//vpc", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec, err := ExtractSource(map[string]any{"source": tt.source}) + require.NoError(t, err) + require.NotNil(t, spec) + assert.Equal(t, tt.want, spec.Identity) + }) + } +} + +func identityTestRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not available") + } + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + // A caller's GIT_DIR, GIT_WORK_TREE, or GIT_INDEX_FILE would point these commands at the + // caller's repository instead of the temporary one, despite cmd.Dir. + cmd.Env = gitscrub.Environ() + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + } + run("init", "--quiet") + require.NoError(t, os.WriteFile(filepath.Join(dir, "main.tf"), []byte("# identity repo\n"), 0o600)) + run("add", "main.tf") + run("-c", "user.name=test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "init") + repoPath := filepath.ToSlash(dir) + if !strings.HasPrefix(repoPath, "/") { + repoPath = "/" + repoPath + } + return (&url.URL{Scheme: "file", Path: repoPath}).String() +} + +func TestDownloadSource_IdentityEnvironmentReachesGit(t *testing.T) { + t.Setenv("GIT_CONFIG_GLOBAL", filepath.Join(t.TempDir(), "gitconfig")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_TERMINAL_PROMPT", "0") + repo := identityTestRepo(t) + const remote = "git::https://git.example.invalid/acme/modules.git" + var resolved []string + downloader.RegisterIdentityEnvironmentResolver(func(_ context.Context, _ *schema.AtmosConfiguration, identity string) ([]string, error) { + resolved = append(resolved, identity) + return []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=url." + repo + ".insteadOf", "GIT_CONFIG_VALUE_0=https://git.example.invalid/acme/modules.git"}, nil + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + + tests := []struct { + name string + spec *schema.VendorComponentSource + section string + resolved []string + }{ + {name: "source identity", spec: &schema.VendorComponentSource{Identity: "ops"}, section: "default", resolved: []string{"ops"}}, + {name: "vendor.identity default", spec: &schema.VendorComponentSource{}, section: "default", resolved: []string{"default"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolved = nil + dest := filepath.Join(t.TempDir(), "dst") + atmosConfig := &schema.AtmosConfiguration{Vendor: schema.Vendor{Identity: tt.section}} + + err := downloadSource(context.Background(), atmosConfig, tt.spec, remote, dest) + + require.NoError(t, err) + assert.FileExists(t, filepath.Join(dest, "main.tf")) + assert.Equal(t, tt.resolved, resolved) + }) + } +} + +func TestDownloadSource_IdentityNeedingLoginFails(t *testing.T) { + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return nil, errUtils.ErrRemoteIdentityLoginRequired + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + + err := downloadSource(context.Background(), &schema.AtmosConfiguration{}, &schema.VendorComponentSource{Identity: "ops"}, "git::https://git.example.invalid/acme/modules.git", filepath.Join(t.TempDir(), "dst")) + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) +} + +// An OCI or S3 source authenticates through its own credential chain. The identity it would never +// use must not be resolved, so a missing session cannot stop the download. +func TestDownloadSource_IdentityIsNotResolvedForTransportsThatCannotUseIt(t *testing.T) { + resolved := 0 + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + resolved++ + return nil, errUtils.ErrRemoteIdentityLoginRequired + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + atmosConfig := &schema.AtmosConfiguration{Vendor: schema.Vendor{Identity: "default"}} + // A canceled context stops the fetch itself at once, without the network. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for _, uri := range []string{"oci://ghcr.io/acme/modules:v1", "s3::https://s3.amazonaws.com/bucket/module.zip"} { + t.Run(uri, func(t *testing.T) { + err := downloadSource(ctx, atmosConfig, &schema.VendorComponentSource{Identity: "ops"}, uri, filepath.Join(t.TempDir(), "dst")) + + require.Error(t, err, "the canceled fetch fails") + assert.NotErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired, "but not because of the identity") + }) + } + assert.Zero(t, resolved, "the identity is never resolved for these transports") +} diff --git a/pkg/provisioner/source/vendor.go b/pkg/provisioner/source/vendor.go index 5c26976d98b..f8ad42791af 100644 --- a/pkg/provisioner/source/vendor.go +++ b/pkg/provisioner/source/vendor.go @@ -231,9 +231,22 @@ func VendorSource( // (oci://) need no special handling: the downloader registers an OCI getter alongside the other // protocols. func downloadSource(ctx context.Context, atmosConfig *schema.AtmosConfiguration, sourceSpec *schema.VendorComponentSource, uri, tempDir string) error { + // The source's own identity, then vendor.identity. The identity's environment is scoped to this + // download's context (never the process environment). It is resolved only for a transport that + // can use it: an OCI or S3 source keeps authenticating through its own credential chain and is + // not refused because the identity has no session. + sectionDefault := "" + if atmosConfig != nil { + sectionDefault = atmosConfig.Vendor.Identity + } + ctx, err := downloader.WithIdentityForSource(ctx, atmosConfig, uri, sourceSpec.Identity, sectionDefault) + if err != nil { + return err + } + downloadOpts := []downloader.GoGetterOption{downloader.WithRetryConfig(effectiveRetryConfig(sourceSpec))} dl := downloader.NewGoGetterDownloader(atmosConfig, downloadOpts...) - _, err := dl.(downloader.ContextFileDownloader).FetchWithMetadataContext(ctx, uri, tempDir, downloader.ClientModeAny, DefaultVendorTimeout) + _, err = dl.(downloader.ContextFileDownloader).FetchWithMetadataContext(ctx, uri, tempDir, downloader.ClientModeAny, DefaultVendorTimeout) if err != nil { return errUtils.Build(errUtils.ErrSourceProvision). WithCause(err). diff --git a/pkg/schema/remote_identity_test.go b/pkg/schema/remote_identity_test.go new file mode 100644 index 00000000000..07b3e1b20be --- /dev/null +++ b/pkg/schema/remote_identity_test.go @@ -0,0 +1,45 @@ +package schema + +import ( + "testing" + + "github.com/go-viper/mapstructure/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// TestRemoteReadIdentityFields verifies the flat `identity:` field decodes from YAML and from the +// map form Viper and the stack processor hand to mapstructure, on every remote-read declaration. +func TestRemoteReadIdentityFields(t *testing.T) { + tests := []struct { + name string + yaml string + target any + get func(any) string + }{ + {name: "vendor.yaml source", yaml: "component: vpc\nsource: github.com/acme/modules\nidentity: ops\n", target: &AtmosVendorSource{}, get: func(v any) string { return v.(*AtmosVendorSource).Identity }}, + {name: "component source", yaml: "uri: github.com/acme/modules\nidentity: ops\n", target: &VendorComponentSource{}, get: func(v any) string { return v.(*VendorComponentSource).Identity }}, + {name: "stack import", yaml: "path: github.com/acme/stacks//catalog\nidentity: ops\n", target: &StackImport{}, get: func(v any) string { return v.(*StackImport).Identity }}, + {name: "vendor section default", yaml: "identity: ops\n", target: &Vendor{}, get: func(v any) string { return v.(*Vendor).Identity }}, + {name: "imports section default", yaml: "identity: ops\n", target: &ImportsSettings{}, get: func(v any) string { return v.(*ImportsSettings).Identity }}, + } + for _, tt := range tests { + t.Run(tt.name+" from yaml", func(t *testing.T) { + require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), tt.target)) + assert.Equal(t, "ops", tt.get(tt.target)) + }) + t.Run(tt.name+" from mapstructure", func(t *testing.T) { + var raw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(tt.yaml), &raw)) + require.NoError(t, mapstructure.Decode(raw, tt.target)) + assert.Equal(t, "ops", tt.get(tt.target)) + }) + } +} + +func TestRemoteReadIdentityOmittedWhenEmpty(t *testing.T) { + out, err := yaml.Marshal(&AtmosVendorSource{Component: "vpc"}) + require.NoError(t, err) + assert.NotContains(t, string(out), "identity") +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 2f88b3f1f0e..c55d2197638 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -2570,6 +2570,9 @@ type ImportsSettings struct { // imports, applied when a per-import `ttl` is not set. See StackImport.TTL for the // caching semantics and accepted formats. TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty" mapstructure:"ttl"` + // Identity is the default Atmos Auth identity for remote imports that do not name their own. + // See StackImport.Identity. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` } type StackImport struct { @@ -2595,6 +2598,10 @@ type StackImport struct { // regardless of TTL. When unset, the source is re-cloned once per invocation. // Examples: "0s" (always re-fetch), "5m", "1h", "7d", or keywords like "daily". TTL string `yaml:"ttl,omitempty" json:"ttl,omitempty" mapstructure:"ttl"` + // Identity names the Atmos Auth identity used to fetch this remote import. It overrides + // `imports.identity` and is overridden by the global `--identity` flag. Ignored for + // local imports. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` } // Dependencies @@ -2663,6 +2670,9 @@ type AtmosVendorSource struct { Tags []string `yaml:"tags" json:"tags" mapstructure:"tags"` Retry *RetryConfig `yaml:"retry,omitempty" json:"retry,omitempty" mapstructure:"retry"` Constraints *VendorConstraints `yaml:"constraints,omitempty" json:"constraints,omitempty" mapstructure:"constraints"` + // Identity names the Atmos Auth identity used to fetch this source. It overrides + // `vendor.identity` and is overridden by the global `--identity` flag. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` } // VendorConstraints controls which upstream versions `atmos vendor update` may @@ -2721,6 +2731,8 @@ type Vendor struct { CI VendorCIConfig `yaml:"ci,omitempty" json:"ci,omitempty" mapstructure:"ci"` // Lock configures how `atmos vendor pull` reacts to vendor.lock.yaml drift. Lock VendorLockConfig `yaml:"lock,omitempty" json:"lock,omitempty" mapstructure:"lock"` + // Identity is the default Atmos Auth identity for vendored sources that do not name their own. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` } type ChromaStyle struct { diff --git a/pkg/schema/vendor_component.go b/pkg/schema/vendor_component.go index 015d748bdab..e65f28bf214 100644 --- a/pkg/schema/vendor_component.go +++ b/pkg/schema/vendor_component.go @@ -17,6 +17,9 @@ type VendorComponentSource struct { // Constraints controls which upstream versions `atmos vendor update` may select for this // source. Mirrors AtmosVendorSource.Constraints (vendor.yaml). Constraints *VendorConstraints `yaml:"constraints,omitempty" json:"constraints,omitempty" mapstructure:"constraints"` + // Identity names the Atmos Auth identity used to fetch this source (and a component.yaml's + // mixins). It overrides `vendor.identity` and is overridden by the global `--identity` flag. + Identity string `yaml:"identity,omitempty" json:"identity,omitempty" mapstructure:"identity"` } type VendorComponentMixins struct { diff --git a/pkg/stack/imports/remote.go b/pkg/stack/imports/remote.go index b3d3000c25b..01c9b1e4f84 100644 --- a/pkg/stack/imports/remote.go +++ b/pkg/stack/imports/remote.go @@ -167,7 +167,14 @@ func (r *RemoteImporter) writeDownloadMeta(uri string) { // uses for git-subdir imports, so the same setting governs both caching mechanisms. func (r *RemoteImporter) Download(uri string) (result string, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.Download")() - if err := guard.Source(context.Background(), "import", uri, ""); err != nil { + + return r.download(context.Background(), uri) +} + +// download is Download for one caller context: the context carries the per-fetch environment of a +// named identity (see downloader.WithIdentity). +func (r *RemoteImporter) download(ctx context.Context, uri string) (result string, resultErr error) { + if err := guard.Source(ctx, "import", uri, ""); err != nil { return "", err } defer func() { @@ -175,10 +182,10 @@ func (r *RemoteImporter) Download(uri string) (result string, resultErr error) { guard.RegisterRemote(result, uri) } }() - return r.downloadUnguarded(uri) + return r.downloadUnguarded(ctx, uri) } -func (r *RemoteImporter) downloadUnguarded(uri string) (result string, resultErr error) { +func (r *RemoteImporter) downloadUnguarded(ctx context.Context, uri string) (result string, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.Download")() if !IsRemote(uri) { @@ -193,22 +200,25 @@ func (r *RemoteImporter) downloadUnguarded(uri string) (result string, resultErr ttl = r.atmosConfig.Imports.TTL } - if cachedPath := r.cachedDownload(uri); cachedPath != "" { + // Every cache below is keyed by the identity the fetch runs as (see scopedKey). + key := scopedKey(ctx, uri) + + if cachedPath := r.cachedDownload(key); cachedPath != "" { return cachedPath, nil } // A ttl is configured but the cached content (if any) is stale: evict it so // GetOrFetch below treats this as a miss instead of reusing stale content. - if ttl != "" && !r.downloadCacheFresh(uri, ttl) { - _ = r.cache.Delete(uri) + if ttl != "" && !r.downloadCacheFresh(key, ttl) { + _ = r.cache.Delete(key) } // Use GetOrFetch to properly handle file locking and atomic writes. // This ensures safe concurrent access across multiple atmos processes. fetched := false - _, err := r.cache.GetOrFetch(uri, func() ([]byte, error) { + _, err := r.cache.GetOrFetch(key, func() ([]byte, error) { fetched = true - return r.fetchDownloadBytes(uri) + return r.fetchDownloadBytes(ctx, uri) }) if err != nil { return "", errUtils.Build(errUtils.ErrDownloadRemoteImport). @@ -221,15 +231,15 @@ func (r *RemoteImporter) downloadUnguarded(uri string) (result string, resultErr // Record freshness only when a real fetch just happened, matching fetchSourceDir's // write-only-on-fetch semantics (not sliding expiration on every cache hit). if fetched && ttl != "" { - r.writeDownloadMeta(uri) + r.writeDownloadMeta(key) } // Get the final cached path. - destPath, _ := r.cache.GetPath(uri) + destPath, _ := r.cache.GetPath(key) // Add to memory cache. r.memMu.Lock() - r.memCache[uri] = destPath + r.memCache[key] = destPath r.memMu.Unlock() return destPath, nil @@ -248,14 +258,14 @@ func (r *RemoteImporter) Resolve(uri string) ([]RemoteImportMatch, error) { ttl = r.atmosConfig.Imports.TTL } - return r.resolve(uri, ttl) + return r.resolve(context.Background(), uri, ttl) } // resolve fetches a remote import and returns all local stack files it resolves to. // The ttl controls cross-run reuse of the cloned source repo for git subdir imports // (see ensureSourceDir); an empty ttl refreshes the clone once per invocation. -func (r *RemoteImporter) resolve(uri, ttl string) (result []RemoteImportMatch, resultErr error) { - if err := guard.Source(context.Background(), "import", uri, ""); err != nil { +func (r *RemoteImporter) resolve(ctx context.Context, uri, ttl string) (result []RemoteImportMatch, resultErr error) { + if err := guard.Source(ctx, "import", uri, ""); err != nil { return nil, err } defer func() { @@ -268,13 +278,13 @@ func (r *RemoteImporter) resolve(uri, ttl string) (result []RemoteImportMatch, r } } }() - return r.resolveUnguarded(uri, ttl) + return r.resolveUnguarded(ctx, uri, ttl) } // resolveUnguarded resolves a remote import to local stack files without the source-policy guard. // Git and oci:// imports with a "//" path are resolved from a shared source directory (file, directory, or // glob); any other remote URI is downloaded as a single file. -func (r *RemoteImporter) resolveUnguarded(uri, ttl string) (result []RemoteImportMatch, resultErr error) { +func (r *RemoteImporter) resolveUnguarded(ctx context.Context, uri, ttl string) (result []RemoteImportMatch, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.resolve")() if !IsRemote(uri) { @@ -284,19 +294,13 @@ func (r *RemoteImporter) resolveUnguarded(uri, ttl string) (result []RemoteImpor Err() } - if matches, ok := r.cachedMatches(uri); ok { + if matches, ok := r.cachedMatches(ctx, uri); ok { return matches, nil } isOCI := IsOCIURI(uri) if !isOCI && !IsGitURI(uri) { - path, err := r.Download(uri) - if err != nil { - return nil, err - } - matches := []RemoteImportMatch{{Path: path, Key: uri}} - r.storeMatches(uri, matches) - return matches, nil + return r.downloadMatches(ctx, uri) } sourceURI, subdir := getter.SourceDirSubdir(uri) @@ -309,26 +313,33 @@ func (r *RemoteImporter) resolveUnguarded(uri, ttl string) (result []RemoteImpor Err() } if subdir == "" { - path, err := r.Download(uri) - if err != nil { - return nil, err - } - matches := []RemoteImportMatch{{Path: path, Key: uri}} - r.storeMatches(uri, matches) - return matches, nil + return r.downloadMatches(ctx, uri) } - matches, err := r.resolveSourceSubdir(uri, sourceURI, subdir, ttl) + matches, err := r.resolveSourceSubdir(ctx, uri, sourceURI, subdir, ttl) + if err != nil { + return nil, err + } + r.storeMatches(ctx, uri, matches) + return matches, nil +} + +// downloadMatches downloads uri as a single file and returns it as the import's one match. +func (r *RemoteImporter) downloadMatches(ctx context.Context, uri string) ([]RemoteImportMatch, error) { + path, err := r.download(ctx, uri) if err != nil { return nil, err } - r.storeMatches(uri, matches) + matches := []RemoteImportMatch{{Path: path, Key: uri}} + r.storeMatches(ctx, uri, matches) return matches, nil } -func (r *RemoteImporter) cachedMatches(uri string) ([]RemoteImportMatch, bool) { +// cachedMatches returns the matches stored for uri by a fetch made as the same identity as ctx. +func (r *RemoteImporter) cachedMatches(ctx context.Context, uri string) ([]RemoteImportMatch, bool) { + key := scopedKey(ctx, uri) r.memMu.RLock() - cached, ok := r.matchCache[uri] + cached, ok := r.matchCache[key] r.memMu.RUnlock() if !ok { return nil, false @@ -338,9 +349,9 @@ func (r *RemoteImporter) cachedMatches(uri string) ([]RemoteImportMatch, bool) { continue } r.memMu.Lock() - delete(r.matchCache, uri) + delete(r.matchCache, key) for _, stale := range cached { - delete(r.memCache, stale.Key) + delete(r.memCache, scopedKey(ctx, stale.Key)) } r.memMu.Unlock() return nil, false @@ -348,11 +359,12 @@ func (r *RemoteImporter) cachedMatches(uri string) ([]RemoteImportMatch, bool) { return cloneMatches(cached), true } -func (r *RemoteImporter) storeMatches(uri string, matches []RemoteImportMatch) { +// storeMatches records the matches for uri under the identity ctx fetched them as. +func (r *RemoteImporter) storeMatches(ctx context.Context, uri string, matches []RemoteImportMatch) { r.memMu.Lock() defer r.memMu.Unlock() - r.matchCache[uri] = cloneMatches(matches) + r.matchCache[scopedKey(ctx, uri)] = cloneMatches(matches) } func cloneMatches(matches []RemoteImportMatch) []RemoteImportMatch { @@ -370,8 +382,8 @@ func cloneMatches(matches []RemoteImportMatch) []RemoteImportMatch { // // Ambient credential brokers (e.g. Atmos Pro github/sts) are provisioned inside // ensureSourceDir, before its detectGitSource call — see the note there. -func (r *RemoteImporter) resolveSourceSubdir(originalURI, sourceURI, subdir, ttl string) ([]RemoteImportMatch, error) { - sourceRoot, err := r.ensureSourceDir(sourceURI, ttl) +func (r *RemoteImporter) resolveSourceSubdir(ctx context.Context, originalURI, sourceURI, subdir, ttl string) ([]RemoteImportMatch, error) { + sourceRoot, err := r.ensureSourceDir(ctx, sourceURI, ttl) if err != nil { return nil, err } @@ -388,7 +400,7 @@ func (r *RemoteImporter) resolveSourceSubdir(originalURI, sourceURI, subdir, ttl return nil, err } key := originalURI + "#" + filepath.ToSlash(rel) - cachedPath, err := r.cacheFile(key, file) + cachedPath, err := r.cacheFile(scopedKey(ctx, key), file) if err != nil { return nil, err } @@ -398,12 +410,12 @@ func (r *RemoteImporter) resolveSourceSubdir(originalURI, sourceURI, subdir, ttl return matches, nil } -func (r *RemoteImporter) detectGitSource(sourceURI string) string { +func (r *RemoteImporter) detectGitSource(ctx context.Context, sourceURI string) string { atmosConfig := r.atmosConfig if atmosConfig == nil { atmosConfig = &schema.AtmosConfiguration{} } - detector := downloader.NewCustomGitDetector(atmosConfig, sourceURI) + detector := downloader.NewCustomGitDetectorContext(ctx, atmosConfig, sourceURI) detected, ok, err := detector.Detect(sourceURI, "") if err != nil || !ok { return sourceURI @@ -652,11 +664,11 @@ func (r *RemoteImporter) cachedDownload(uri string) string { return "" } -func (r *RemoteImporter) fetchDownloadBytes(uri string) ([]byte, error) { +func (r *RemoteImporter) fetchDownloadBytes(ctx context.Context, uri string) ([]byte, error) { // The cache owns locking and the final atomic write. - tempPath := filepath.Join(r.cache.BaseDir(), uriToTempName(uri)) + tempPath := filepath.Join(r.cache.BaseDir(), uriToTempName(scopedKey(ctx, uri))) defer os.Remove(tempPath) - if err := r.downloader.Fetch(uri, tempPath, downloader.ClientModeFile, defaultDownloadTimeout); err != nil { + if err := r.fetch(ctx, uri, tempPath, downloader.ClientModeFile); err != nil { return nil, err } return os.ReadFile(tempPath) diff --git a/pkg/stack/imports/remote_cache_test.go b/pkg/stack/imports/remote_cache_test.go index 408b88cf910..777798de328 100644 --- a/pkg/stack/imports/remote_cache_test.go +++ b/pkg/stack/imports/remote_cache_test.go @@ -1,6 +1,7 @@ package imports import ( + "context" "encoding/json" "os" "path/filepath" @@ -120,19 +121,19 @@ func TestRemoteImporter_GitSubdir_TTLCrossRunReuse(t *testing.T) { // Run 1: cold cache -> fetch once. var c1 atomic.Int32 - _, err := newImporterWithCache(t, baseDir, &c1).resolveNested(importURI, "", "1h") + _, err := newImporterWithCache(t, baseDir, &c1).resolveNested(context.Background(), importURI, "", "1h") require.NoError(t, err) assert.Equal(t, int32(1), c1.Load(), "cold cache must fetch") // Run 2: warm cache, fresh TTL -> reuse, no fetch. var c2 atomic.Int32 - _, err = newImporterWithCache(t, baseDir, &c2).resolveNested(importURI, "", "1h") + _, err = newImporterWithCache(t, baseDir, &c2).resolveNested(context.Background(), importURI, "", "1h") require.NoError(t, err) assert.Equal(t, int32(0), c2.Load(), "fresh persisted clone should be reused across runs within TTL") // Run 3: no TTL -> always refresh. var c3 atomic.Int32 - _, err = newImporterWithCache(t, baseDir, &c3).resolveNested(importURI, "", "") + _, err = newImporterWithCache(t, baseDir, &c3).resolveNested(context.Background(), importURI, "", "") require.NoError(t, err) assert.Equal(t, int32(1), c3.Load(), "no TTL should refresh the clone each run") @@ -140,7 +141,7 @@ func TestRemoteImporter_GitSubdir_TTLCrossRunReuse(t *testing.T) { destDir := filepath.Join(baseDir, uriToTempName(testSourceURI)+".source") backdateSourceMetadata(t, destDir, 2*time.Hour) var c4 atomic.Int32 - _, err = newImporterWithCache(t, baseDir, &c4).resolveNested(importURI, "", "1h") + _, err = newImporterWithCache(t, baseDir, &c4).resolveNested(context.Background(), importURI, "", "1h") require.NoError(t, err) assert.Equal(t, int32(1), c4.Load(), "expired clone should be refreshed") } diff --git a/pkg/stack/imports/remote_fetch.go b/pkg/stack/imports/remote_fetch.go new file mode 100644 index 00000000000..1d41d769174 --- /dev/null +++ b/pkg/stack/imports/remote_fetch.go @@ -0,0 +1,36 @@ +package imports + +import ( + "context" + "net/url" + + "github.com/cloudposse/atmos/pkg/downloader" +) + +// scopedKey qualifies a cache key with the identity the fetch runs as. What one identity may read, +// another may not, and two identities can rewrite the same URI to different repositories. So +// nothing fetched as one identity is ever served to another, or to a fetch that names none: the +// in-memory match and path caches, the file cache, and the shared source clones are all keyed by +// it. A fetch that names no identity keeps the plain key, so its cache entries are unchanged. +// +// The identity goes in front of the key because the file cache takes the file extension from the +// end of it. The import key reported to the caller (RemoteImportMatch.Key) is never scoped. +func scopedKey(ctx context.Context, key string) string { + identity := downloader.IdentityFromContext(ctx) + if identity == "" { + return key + } + return "identity=" + url.QueryEscape(identity) + "|" + key +} + +// fetch downloads src through the importer's downloader. When ctx names an identity (or carries a +// per-fetch environment), the fetch runs with it; otherwise this is exactly the plain Fetch call. +func (r *RemoteImporter) fetch(ctx context.Context, src, dest string, mode downloader.ClientMode) error { + if downloader.FetchUsesIdentity(ctx) { + if contextual, ok := r.downloader.(downloader.ContextFileDownloader); ok { + _, err := contextual.FetchWithMetadataContext(ctx, src, dest, mode, defaultDownloadTimeout) + return err + } + } + return r.downloader.Fetch(src, dest, mode, defaultDownloadTimeout) +} diff --git a/pkg/stack/imports/remote_identity_test.go b/pkg/stack/imports/remote_identity_test.go new file mode 100644 index 00000000000..42f9de4c1a9 --- /dev/null +++ b/pkg/stack/imports/remote_identity_test.go @@ -0,0 +1,194 @@ +package imports + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/cache" + "github.com/cloudposse/atmos/pkg/downloader" + "github.com/cloudposse/atmos/pkg/schema" +) + +// Compile-time sentinels for the schema fields the tests set. +var ( + _ = schema.ImportsSettings{Identity: "x"} + _ = schema.StackImport{Identity: "x"} +) + +// contextRecorder is a downloader that also implements ContextFileDownloader, recording the +// per-fetch environment each contextual fetch ran with. Plain Fetch calls go to the generated mock. +type contextRecorder struct { + *downloader.MockFileDownloader + mu sync.Mutex + envs [][]string +} + +func (c *contextRecorder) FetchWithMetadataContext(ctx context.Context, _, dest string, _ downloader.ClientMode, _ time.Duration) (downloader.FetchMetadata, error) { + c.mu.Lock() + c.envs = append(c.envs, downloader.EnvironmentFromContext(ctx)) + c.mu.Unlock() + // The content names the identity that fetched it, so a test can tell whose copy it was served. + content := "vars:\n fetched_as: " + strconv.Quote(downloader.IdentityFromContext(ctx)) + "\n" + return downloader.FetchMetadata{}, os.WriteFile(dest, []byte(content), testFilePerm) +} + +func newRecorderImporter(t *testing.T, atmosConfig *schema.AtmosConfiguration) *contextRecorder { + t.Helper() + testCache, err := cache.NewFileCache("test", cache.WithBaseDir(t.TempDir())) + require.NoError(t, err) + recorder := &contextRecorder{MockFileDownloader: downloader.NewMockFileDownloader(gomock.NewController(t))} + importer, err := NewRemoteImporter(atmosConfig, WithCache(testCache), WithDownloader(recorder)) + require.NoError(t, err) + setGlobalImporter(t, importer) + return recorder +} + +func registerImportResolver(t *testing.T) { + t.Helper() + downloader.RegisterIdentityEnvironmentResolver(func(_ context.Context, _ *schema.AtmosConfiguration, identity string) ([]string, error) { + return []string{"IMPORT_IDENTITY=" + identity}, nil + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) +} + +func TestResolveRemoteImportNestedIdentity_UsesImportOrSectionIdentity(t *testing.T) { + registerImportResolver(t) + + tests := []struct { + name string + entry string + section string + want []string + }{ + {name: "the import's own identity", entry: "ops", section: "default", want: []string{"IMPORT_IDENTITY=ops"}}, + {name: "imports.identity when the import names none", section: "default", want: []string{"IMPORT_IDENTITY=default"}}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recorder := newRecorderImporter(t, &schema.AtmosConfiguration{Imports: schema.ImportsSettings{Identity: tt.section}}) + uri := fmt.Sprintf("https://example.invalid/stacks/catalog-%d.yaml", i) + + matches, err := ResolveRemoteImportNestedIdentity(&schema.AtmosConfiguration{Imports: schema.ImportsSettings{Identity: tt.section}}, uri, "", "", tt.entry) + + require.NoError(t, err) + require.Len(t, matches, 1) + assert.Equal(t, [][]string{tt.want}, recorder.envs) + }) + } +} + +func TestResolveRemoteImportNestedIdentity_NoIdentityUsesPlainFetch(t *testing.T) { + registerImportResolver(t) + recorder := newRecorderImporter(t, &schema.AtmosConfiguration{}) + const uri = "https://example.invalid/stacks/plain.yaml" + recorder.MockFileDownloader.EXPECT(). + Fetch(uri, gomock.Any(), downloader.ClientModeFile, gomock.Any()). + DoAndReturn(func(_, dest string, _ downloader.ClientMode, _ time.Duration) error { + return os.WriteFile(dest, []byte("vars:\n a: true\n"), testFilePerm) + }).Times(1) + + matches, err := ResolveRemoteImportNestedIdentity(&schema.AtmosConfiguration{}, uri, "", "", "") + + require.NoError(t, err) + require.Len(t, matches, 1) + assert.Empty(t, recorder.envs, "an import without an identity keeps using the plain downloader call") +} + +func TestResolveRemoteImportNestedIdentity_LoginRequiredFailsWithoutFetching(t *testing.T) { + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return nil, errUtils.ErrRemoteIdentityLoginRequired + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + recorder := newRecorderImporter(t, &schema.AtmosConfiguration{}) + + _, err := ResolveRemoteImportNestedIdentity(&schema.AtmosConfiguration{}, "https://example.invalid/stacks/private.yaml", "", "", "ops") + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) + require.ErrorIs(t, err, errUtils.ErrRemoteImportIdentity) + assert.False(t, SkippableWhenMissing(err), "skip_if_missing must not hide an identity that is not logged in") + assert.Empty(t, recorder.envs) +} + +// skip_if_missing covers a source that is not there, and nothing else. +func TestSkippableWhenMissing(t *testing.T) { + missing := fmt.Errorf("%w: 404", errUtils.ErrDownloadRemoteImport) + identity := fmt.Errorf("%w: %w", errUtils.ErrRemoteImportIdentity, errUtils.ErrIdentityNotFound) + + assert.True(t, SkippableWhenMissing(missing)) + assert.False(t, SkippableWhenMissing(identity)) + assert.False(t, SkippableWhenMissing(nil), "nothing failed, so there is nothing to skip") +} + +// What one identity may read, another may not. The same URI fetched as two identities must be two +// fetches with two results, and neither may be served to a fetch that names no identity. +func TestResolveRemoteImportNestedIdentity_CachesAreSeparatedByIdentity(t *testing.T) { + registerImportResolver(t) + atmosConfig := &schema.AtmosConfiguration{} + recorder := newRecorderImporter(t, atmosConfig) + const uri = "https://example.invalid/stacks/shared.yaml" + recorder.MockFileDownloader.EXPECT(). + Fetch(uri, gomock.Any(), downloader.ClientModeFile, gomock.Any()). + DoAndReturn(func(_, dest string, _ downloader.ClientMode, _ time.Duration) error { + return os.WriteFile(dest, []byte("vars:\n fetched_as: \"\"\n"), testFilePerm) + }).Times(1) + read := func(identity string) (path, content string) { + t.Helper() + matches, err := ResolveRemoteImportNestedIdentity(atmosConfig, uri, "", "", identity) + require.NoError(t, err) + require.Len(t, matches, 1) + assert.Equal(t, uri, matches[0].Key, "the reported import key is the URI, never the scoped cache key") + data, err := os.ReadFile(matches[0].Path) + require.NoError(t, err) + return matches[0].Path, string(data) + } + + opsPath, opsContent := read("ops") + deployPath, deployContent := read("deploy") + plainPath, plainContent := read("") + opsAgainPath, opsAgainContent := read("ops") + + assert.Contains(t, opsContent, `fetched_as: "ops"`) + assert.Contains(t, deployContent, `fetched_as: "deploy"`, "a second identity must not receive the first identity's content") + assert.Contains(t, plainContent, `fetched_as: ""`, "a fetch that names no identity must not receive an identity's content") + assert.NotEqual(t, opsPath, deployPath) + assert.NotEqual(t, opsPath, plainPath) + assert.Equal(t, opsPath, opsAgainPath, "the same identity reuses its own cached copy") + assert.Equal(t, opsContent, opsAgainContent) + assert.Equal(t, [][]string{{"IMPORT_IDENTITY=ops"}, {"IMPORT_IDENTITY=deploy"}}, recorder.envs, + "one fetch per identity: the repeat for ops is served from its cache") +} + +// An oci:// import authenticates through the registry keychain. An identity it would never use +// must not be resolved, so a missing session cannot stop the import. +func TestResolveRemoteImportNestedIdentity_OCIImportDoesNotResolveTheIdentity(t *testing.T) { + resolved := 0 + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + resolved++ + return nil, errUtils.ErrRemoteIdentityLoginRequired + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + atmosConfig := &schema.AtmosConfiguration{Imports: schema.ImportsSettings{Identity: "ops"}} + recorder := newRecorderImporter(t, atmosConfig) + registryDown := errors.New("registry unreachable") + recorder.MockFileDownloader.EXPECT(). + Fetch(gomock.Any(), gomock.Any(), downloader.ClientModeDir, gomock.Any()). + Return(registryDown).Times(1) + + _, err := ResolveRemoteImportNestedIdentity(atmosConfig, "oci://ghcr.io/acme/stacks:v1//catalog/vpc.yaml", "", "", "") + + require.ErrorIs(t, err, registryDown, "the import reached the registry fetch") + assert.NotErrorIs(t, err, errUtils.ErrRemoteImportIdentity) + assert.Zero(t, resolved, "the identity is not resolved for an OCI import") + assert.Empty(t, recorder.envs, "the fetch ran without an identity environment") +} diff --git a/pkg/stack/imports/remote_nested.go b/pkg/stack/imports/remote_nested.go index de57cfc744f..0a2bdf701fb 100644 --- a/pkg/stack/imports/remote_nested.go +++ b/pkg/stack/imports/remote_nested.go @@ -3,6 +3,8 @@ package imports import ( "context" "encoding/json" + "errors" + "fmt" "os" "path/filepath" "strings" @@ -43,13 +45,13 @@ type sourceMetadata struct { func (r *RemoteImporter) ResolveNested(uri, nestedImports string) ([]RemoteImportMatch, error) { defer perf.Track(nil, "imports.RemoteImporter.ResolveNested")() - return r.resolveNested(uri, nestedImports, "") + return r.resolveNested(context.Background(), uri, nestedImports, "") } // resolveNested is the ttl-aware implementation behind ResolveNested. The ttl controls // cross-run reuse of the cloned source repo (see ensureSourceDir). -func (r *RemoteImporter) resolveNested(uri, nestedImports, ttl string) (result []RemoteImportMatch, resultErr error) { - if err := guard.Source(context.Background(), "import", uri, ""); err != nil { +func (r *RemoteImporter) resolveNested(ctx context.Context, uri, nestedImports, ttl string) (result []RemoteImportMatch, resultErr error) { + if err := guard.Source(ctx, "import", uri, ""); err != nil { return nil, err } defer func() { @@ -62,17 +64,17 @@ func (r *RemoteImporter) resolveNested(uri, nestedImports, ttl string) (result [ } } }() - return r.resolveNestedUnguarded(uri, nestedImports, ttl) + return r.resolveNestedUnguarded(ctx, uri, nestedImports, ttl) } // resolveNestedUnguarded resolves a remote import honoring its nested_imports mode without the // source-policy guard. In "remote" mode the import must be a Git or oci:// source with a "//" path that // contains the stack base path, so nested imports can resolve from the remote source. -func (r *RemoteImporter) resolveNestedUnguarded(uri, nestedImports, ttl string) (result []RemoteImportMatch, resultErr error) { +func (r *RemoteImporter) resolveNestedUnguarded(ctx context.Context, uri, nestedImports, ttl string) (result []RemoteImportMatch, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.resolveNested")() if nestedImports == "" || nestedImports == schema.StackImportNestedImportsLocal { - return r.resolve(uri, ttl) + return r.resolve(ctx, uri, ttl) } if nestedImports != schema.StackImportNestedImportsRemote { return nil, errUtils.Build(errUtils.ErrInvalidImport). @@ -102,22 +104,22 @@ func (r *RemoteImporter) resolveNestedUnguarded(uri, nestedImports, ttl string) } cacheKey := uri + "|nested_imports=remote" - if matches, ok := r.cachedMatches(cacheKey); ok { + if matches, ok := r.cachedMatches(ctx, cacheKey); ok { return matches, nil } - matches, err := r.resolveSourceSubdirWithRemoteBase(uri, sourceURI, subdir, ttl) + matches, err := r.resolveSourceSubdirWithRemoteBase(ctx, uri, sourceURI, subdir, ttl) if err != nil { return nil, err } - r.storeMatches(cacheKey, matches) + r.storeMatches(ctx, cacheKey, matches) return matches, nil } // resolveSourceSubdirWithRemoteBase resolves a Git or oci:// subdirectory import in place from the shared // source directory, returning each file with the remote stack base path inferred from the import path. -func (r *RemoteImporter) resolveSourceSubdirWithRemoteBase(originalURI, sourceURI, subdir, ttl string) ([]RemoteImportMatch, error) { - sourceRoot, err := r.ensureSourceDir(sourceURI, ttl) +func (r *RemoteImporter) resolveSourceSubdirWithRemoteBase(ctx context.Context, originalURI, sourceURI, subdir, ttl string) ([]RemoteImportMatch, error) { + sourceRoot, err := r.ensureSourceDir(ctx, sourceURI, ttl) if err != nil { return nil, err } @@ -156,39 +158,45 @@ func (r *RemoteImporter) resolveSourceSubdirWithRemoteBase(originalURI, sourceUR // - Across runs: the persisted clone is reused only when ttl is set and the clone is // still fresh per its recorded UpdatedAt. With no ttl, the clone is re-fetched once // per run (mutable refs like `main` stay fresh) while still deduped within the run. -func (r *RemoteImporter) ensureSourceDir(sourceURI, ttl string) (string, error) { +func (r *RemoteImporter) ensureSourceDir(ctx context.Context, sourceURI, ttl string) (string, error) { // OCI sources authenticate through go-containerregistry (docker keychain, ghcr.io token), never // through the git/GitHub credential brokers or the git detector, so they skip that setup. if !IsOCIURI(sourceURI) { - sourceURI = r.prepareGitSource(sourceURI) + sourceURI = r.prepareGitSource(ctx, sourceURI) } - destDir := filepath.Join(r.cache.BaseDir(), uriToTempName(sourceURI)+".source") - return r.reuseOrFetchSourceDir(sourceURI, destDir, ttl) + // The clone is fetched as one identity, so it is shared only by imports that use that identity. + destDir := filepath.Join(r.cache.BaseDir(), uriToTempName(scopedKey(ctx, sourceURI))+".source") + return r.reuseOrFetchSourceDir(ctx, sourceURI, destDir, ttl) } // prepareGitSource provisions credentials for a git source and returns the detected source URI. -func (r *RemoteImporter) prepareGitSource(sourceURI string) string { +func (r *RemoteImporter) prepareGitSource(ctx context.Context, sourceURI string) string { // Provision ambient credential brokers (e.g. Atmos Pro github/sts) before the eager detect below. // detectGitSource runs CustomGitDetector.Detect directly — not through the downloader — so unlike // downloader.Fetch it does not itself trigger the broker. Without provisioning here, the detector // runs before the broker has exported its GIT_CONFIG_* insteadOf rewrite into the process env, // misses the rewrite, and injects the ambient GITHUB_TOKEN — shadowing the minted least-privilege // token and 404ing a cross-repo import. EnsureCredentials is process-once and gated (CI + config). - if r.atmosConfig != nil { - broker.EnsureCredentials(context.Background(), r.atmosConfig) + // + // A source that names an identity carries its own per-fetch environment, which takes precedence + // over the ambient broker, so the broker is not consulted for it. + if r.atmosConfig != nil && !downloader.FetchUsesIdentity(ctx) { + broker.EnsureCredentials(ctx, r.atmosConfig) } - return r.detectGitSource(sourceURI) + return r.detectGitSource(ctx, sourceURI) } // reuseOrFetchSourceDir returns the cached source directory at destDir when it can be reused // (within this run, or across runs while its ttl is fresh) and fetches sourceURI into it otherwise. -func (r *RemoteImporter) reuseOrFetchSourceDir(sourceURI, destDir, ttl string) (string, error) { +func (r *RemoteImporter) reuseOrFetchSourceDir(ctx context.Context, sourceURI, destDir, ttl string) (string, error) { r.sourceMu.Lock() defer r.sourceMu.Unlock() + sessionKey := scopedKey(ctx, sourceURI) + // Within-run dedup: already fetched this process -> reuse, ignoring TTL. - if r.sessionFetched[sourceURI] { + if r.sessionFetched[sessionKey] { if _, err := os.Stat(destDir); err == nil { return destDir, nil } @@ -196,27 +204,27 @@ func (r *RemoteImporter) reuseOrFetchSourceDir(sourceURI, destDir, ttl string) ( // Cross-run reuse: only when a TTL is configured and the persisted clone is fresh. if sourceCacheFresh(destDir, ttl) { - r.sessionFetched[sourceURI] = true + r.sessionFetched[sessionKey] = true return destDir, nil } - if err := r.fetchSourceDir(sourceURI, destDir); err != nil { + if err := r.fetchSourceDir(ctx, sourceURI, destDir); err != nil { return "", err } - r.sessionFetched[sourceURI] = true + r.sessionFetched[sessionKey] = true return destDir, nil } // fetchSourceDir clones sourceURI into a temp dir, writes the freshness marker, and // atomically swaps it into destDir. -func (r *RemoteImporter) fetchSourceDir(sourceURI, destDir string) error { +func (r *RemoteImporter) fetchSourceDir(ctx context.Context, sourceURI, destDir string) error { tempDir, err := os.MkdirTemp(r.cache.BaseDir(), filepath.Base(destDir)+"-") if err != nil { return err } defer os.RemoveAll(tempDir) - if err := r.downloader.Fetch(sourceURI, tempDir, downloader.ClientModeDir, defaultDownloadTimeout); err != nil { + if err := r.fetch(ctx, sourceURI, tempDir, downloader.ClientModeDir); err != nil { return err } if err := writeSourceMetadata(tempDir, sourceURI); err != nil { @@ -330,5 +338,41 @@ func ResolveRemoteImportNested(atmosConfig *schema.AtmosConfiguration, uri, nest if err != nil { return nil, err } - return importer.resolveNested(uri, nestedImports, ttl) + return importer.resolveNested(context.Background(), uri, nestedImports, ttl) +} + +// ResolveRemoteImportNestedIdentity is ResolveRemoteImportNested for one import that may name an +// Atmos Auth identity. The identity (the import's own, else imports.identity) is resolved +// to a per-fetch environment that the import's Git subprocesses run with; nothing is written to +// the process environment. An import that names no identity behaves exactly as before, and so does +// one fetched over a transport the identity cannot serve (an oci:// import authenticates through +// the registry keychain), which is never refused for want of an identity session. +func ResolveRemoteImportNestedIdentity(atmosConfig *schema.AtmosConfiguration, uri, nestedImports, ttl, identity string) ([]RemoteImportMatch, error) { + defer perf.Track(atmosConfig, "imports.ResolveRemoteImportNestedIdentity")() + + sectionDefault := "" + if atmosConfig != nil { + sectionDefault = atmosConfig.Imports.Identity + } + ctx, err := downloader.WithIdentityForSource(context.Background(), atmosConfig, uri, identity, sectionDefault) + if err != nil { + // Marked so a caller can tell "the identity could not be used" from "the source is not + // there": only the second may be skipped (skip_if_missing). + return nil, fmt.Errorf("%w: %w", errUtils.ErrRemoteImportIdentity, err) + } + importer, err := getGlobalImporter(atmosConfig) + if err != nil { + return nil, err + } + return importer.resolveNested(ctx, uri, nestedImports, ttl) +} + +// SkippableWhenMissing reports whether a failed remote import may be skipped under +// `skip_if_missing`. That option covers a source that is not there. An identity that could not be +// resolved (no session, an unknown name) is a different failure: skipping it would load the stack +// without the import and without telling the user to log in, so it is never skippable. +func SkippableWhenMissing(err error) bool { + defer perf.Track(nil, "imports.SkippableWhenMissing")() + + return err != nil && !errors.Is(err, errUtils.ErrRemoteImportIdentity) } diff --git a/pkg/vendoring/install/atmos.go b/pkg/vendoring/install/atmos.go index feba365c3f4..6104408211f 100644 --- a/pkg/vendoring/install/atmos.go +++ b/pkg/vendoring/install/atmos.go @@ -74,6 +74,7 @@ func (p *atmosVendorInstaller) prepare(ctx context.Context, tempDir string, atmo ClientMode: downloader.ClientModeAny, SourceIsLocalFile: p.localFile, Retry: p.source.Retry, + Identity: p.source.Identity, }) if err != nil { return nil, err @@ -114,7 +115,7 @@ func (p *atmosVendorInstaller) dryRunCheck(ctx context.Context, atmosConfig *sch return err } log.Debug("Entering dry-run flow for generic (non component/mixin) vendoring", "package", p.name) - if err := detectIfNeeded(ctx, atmosConfig, p.srcURI); err != nil { + if err := detectIfNeeded(ctx, atmosConfig, p.srcURI, p.pType, p.source.Identity); err != nil { return fmt.Errorf("%w: %w", ErrDryRunDetectionFailed, err) } return nil diff --git a/pkg/vendoring/install/batch.go b/pkg/vendoring/install/batch.go index f61a0586b90..dd532a8fe83 100644 --- a/pkg/vendoring/install/batch.go +++ b/pkg/vendoring/install/batch.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/cloudposse/atmos/pkg/auth/broker" + "github.com/cloudposse/atmos/pkg/downloader" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/schema" "github.com/cloudposse/atmos/pkg/ui/batch" @@ -37,6 +38,7 @@ func InstallBatch(ctx context.Context, config *schema.AtmosConfiguration, packag if err != nil { return nil, err } + ctx = downloader.WithIdentityOverride(ctx, opts.Identity) b := &batchInstaller{ctx: ctx, config: config, packages: packages, options: opts, observer: observer, report: &BatchReport{Results: make([]Result, len(packages))}} for i, pkg := range packages { b.report.Results[i] = Result{Name: pkg.Name, Outcome: "canceled"} diff --git a/pkg/vendoring/install/component.go b/pkg/vendoring/install/component.go index 4c039752330..45307e94b16 100644 --- a/pkg/vendoring/install/component.go +++ b/pkg/vendoring/install/component.go @@ -103,6 +103,7 @@ func (p *componentVendorInstaller) prepareComponent(ctx context.Context, tempDir JoinSanitizedFilename: true, SourceIsLocalFile: p.localFile, Retry: p.retry(), + Identity: p.identity(), }) if err != nil { return nil, err @@ -163,6 +164,7 @@ func (p *componentVendorInstaller) prepareMixin(ctx context.Context, tempDir str ClientMode: downloader.ClientModeFile, Target: filepath.Join(tempDir, p.mixinFile), Retry: p.retry(), + Identity: p.identity(), }) if err != nil { return nil, err @@ -212,7 +214,7 @@ func (p *componentVendorInstaller) dryRunCheck(ctx context.Context, atmosConfig return err } log.Debug("Dry-run mode: custom detection required for component (or mixin) URI", "component", p.name, "uri", p.srcURI) - if err := detectIfNeeded(ctx, atmosConfig, p.srcURI); err != nil { + if err := detectIfNeeded(ctx, atmosConfig, p.srcURI, p.pType, p.identity()); err != nil { return fmt.Errorf("%w for component %s: %w", ErrDryRunDetectionFailed, p.name, err) } return nil @@ -267,6 +269,15 @@ func (p *componentVendorInstaller) retry() *schema.RetryConfig { return p.spec.Source.Retry } +// identity returns the Atmos Auth identity the component's source names, if any. Mixins fetched +// for the same component use it too. +func (p *componentVendorInstaller) identity() string { + if p.spec == nil { + return "" + } + return p.spec.Source.Identity +} + func (p *componentVendorInstaller) includedPaths() []string { if p.spec == nil { return nil diff --git a/pkg/vendoring/install/fetch.go b/pkg/vendoring/install/fetch.go index f7b8812d5f1..501e96653a0 100644 --- a/pkg/vendoring/install/fetch.go +++ b/pkg/vendoring/install/fetch.go @@ -54,6 +54,10 @@ type fetchOptions struct { SourceIsLocalFile bool // Retry carries a source's configured retry policy through to the go-getter downloader. Retry *schema.RetryConfig + // Identity is the Atmos Auth identity the source itself names. The global --identity flag + // (carried on the context) overrides it and vendor.identity is the fallback; see + // downloader.WithIdentity. + Identity string } // fetchToTempDir fetches uri (classified by pType) into tempDir, dispatching to go-getter @@ -70,7 +74,11 @@ type fetchOptions struct { func fetchToTempDir(ctx context.Context, atmosConfig *schema.AtmosConfiguration, uri string, pType PkgType, tempDir string, opts fetchOptions) (string, downloader.FetchMetadata, error) { switch pType { case PkgTypeRemote: - return fetchRemote(ctx, atmosConfig, uri, tempDir, opts) + identityCtx, err := withSourceIdentity(ctx, atmosConfig, uri, opts.Identity) + if err != nil { + return "", downloader.FetchMetadata{}, err + } + return fetchRemote(identityCtx, atmosConfig, uri, tempDir, opts) case PkgTypeOci: // OCI goes through the same downloader as every other remote source. An image is always a // whole directory fetched straight into tempDir, so the single-file and sanitized-name @@ -90,6 +98,20 @@ func fetchToTempDir(ctx context.Context, atmosConfig *schema.AtmosConfiguration, } } +// withSourceIdentity scopes ctx to the identity that applies to one source: the global --identity +// flag, then the source's own identity, then vendor.identity. Fetches made with the returned context +// run their Git subprocesses with that identity's environment and nothing is written to the process +// environment, so sources fetched concurrently can use different identities. A source with no +// identity, or one whose transport an identity cannot serve (see downloader.SourceUsesIdentity), +// gets ctx back unchanged, so it behaves exactly as before. +func withSourceIdentity(ctx context.Context, atmosConfig *schema.AtmosConfiguration, uri, entry string) (context.Context, error) { + sectionDefault := "" + if atmosConfig != nil { + sectionDefault = atmosConfig.Vendor.Identity + } + return downloader.WithIdentityForSource(ctx, atmosConfig, uri, entry, sectionDefault) +} + func fetchRemote(ctx context.Context, atmosConfig *schema.AtmosConfiguration, uri, tempDir string, opts fetchOptions) (string, downloader.FetchMetadata, error) { target := tempDir switch { @@ -192,10 +214,23 @@ func needsCustomDetection(src string) bool { // detectIfNeeded runs the custom Git detector against uri only when needsCustomDetection reports // it's required, matching the pre-unification dry-run behavior for both installer types. -func detectIfNeeded(ctx context.Context, atmosConfig *schema.AtmosConfiguration, uri string) error { +// +// The detector decides how the source would be fetched from the identity in effect: a named +// identity's `insteadOf` rewrites reach it through the per-fetch environment, and it leaves the +// URL alone instead of injecting an ambient token. So the identity is resolved here exactly as +// fetchToTempDir resolves it (for a PkgTypeRemote source only), and a dry run reports what the +// real fetch would do. +func detectIfNeeded(ctx context.Context, atmosConfig *schema.AtmosConfiguration, uri string, pType PkgType, identity string) error { if !needsCustomDetection(uri) { return nil } + if pType == PkgTypeRemote { + identityCtx, err := withSourceIdentity(ctx, atmosConfig, uri, identity) + if err != nil { + return err + } + ctx = identityCtx + } detector := downloader.NewCustomGitDetectorContext(ctx, atmosConfig, "") _, _, err := detector.Detect(uri, "") return errors.Join(err, ctx.Err()) diff --git a/pkg/vendoring/install/identity_test.go b/pkg/vendoring/install/identity_test.go new file mode 100644 index 00000000000..43312541c3d --- /dev/null +++ b/pkg/vendoring/install/identity_test.go @@ -0,0 +1,270 @@ +package install + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/downloader" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/cloudposse/atmos/tests/testhelpers/gitscrub" +) + +// Compile-time sentinels for the schema fields the tests set. +var ( + _ = schema.Vendor{Identity: "x"} + _ = schema.AtmosVendorSource{Identity: "x"} + _ = schema.VendorComponentSource{Identity: "x"} +) + +const ( + // The fake remote cannot be resolved: a fetch of it only succeeds through the identity's + // `insteadOf` rewrite, delivered in the per-fetch environment. + fakeRemote = "git::https://git.example.invalid/acme/modules.git" + markerFile = "owner.txt" +) + +func localGitRepo(t *testing.T, owner string) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not available") + } + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + // A caller's GIT_DIR, GIT_WORK_TREE, or GIT_INDEX_FILE would point these commands at the + // caller's repository instead of the temporary one, despite cmd.Dir. + cmd.Env = gitscrub.Environ() + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + } + run("init", "--quiet") + require.NoError(t, os.WriteFile(filepath.Join(dir, markerFile), []byte(owner), 0o600)) + run("add", markerFile) + run("-c", "user.name=test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "init") + repoPath := filepath.ToSlash(dir) + if !strings.HasPrefix(repoPath, "/") { + repoPath = "/" + repoPath + } + return (&url.URL{Scheme: "file", Path: repoPath}).String() +} + +// registerIdentityRepos registers a resolver that gives each identity an environment rewriting the +// fake remote to that identity's own repository, and records which identities were resolved. +func registerIdentityRepos(t *testing.T, repos map[string]string) (resolved func() []string) { + t.Helper() + t.Setenv("GIT_CONFIG_GLOBAL", filepath.Join(t.TempDir(), "gitconfig")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + t.Setenv("GIT_TERMINAL_PROMPT", "0") + + var mu sync.Mutex + var names []string + downloader.RegisterIdentityEnvironmentResolver(func(_ context.Context, _ *schema.AtmosConfiguration, identity string) ([]string, error) { + mu.Lock() + names = append(names, identity) + mu.Unlock() + repo, ok := repos[identity] + if !ok { + return nil, fmt.Errorf("%w: %q", errUtils.ErrIdentityNotFound, identity) + } + return []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=url." + repo + ".insteadOf", + "GIT_CONFIG_VALUE_0=https://git.example.invalid/acme/modules.git", + }, nil + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + return func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), names...) + } +} + +func fetchOwner(ctx context.Context, t *testing.T, atmosConfig *schema.AtmosConfiguration, entryIdentity string) (string, error) { + t.Helper() + dir, _, err := fetchToTempDir(ctx, atmosConfig, fakeRemote, PkgTypeRemote, filepath.Join(t.TempDir(), "dst"), fetchOptions{ + ClientMode: downloader.ClientModeDir, + Identity: entryIdentity, + }) + if err != nil { + return "", err + } + got, err := os.ReadFile(filepath.Join(dir, markerFile)) + return string(got), err +} + +func TestFetchToTempDir_SourceIdentityEnvironmentReachesGit(t *testing.T) { + registerIdentityRepos(t, map[string]string{"ops": localGitRepo(t, "ops-repo")}) + + got, err := fetchOwner(context.Background(), t, &schema.AtmosConfiguration{}, "ops") + + require.NoError(t, err) + assert.Equal(t, "ops-repo", got) +} + +func TestFetchToTempDir_SourceWithoutIdentityBehavesAsBefore(t *testing.T) { + resolved := registerIdentityRepos(t, map[string]string{"ops": localGitRepo(t, "ops-repo")}) + direct := localGitRepo(t, "direct-repo") + + dir, _, err := fetchToTempDir(context.Background(), &schema.AtmosConfiguration{}, "git::"+direct, PkgTypeRemote, filepath.Join(t.TempDir(), "dst"), fetchOptions{ClientMode: downloader.ClientModeDir}) + + require.NoError(t, err) + got, err := os.ReadFile(filepath.Join(dir, markerFile)) + require.NoError(t, err) + assert.Equal(t, "direct-repo", string(got)) + assert.Empty(t, resolved(), "no identity applies, so the resolver is never consulted") +} + +func TestFetchToTempDir_IdentityPrecedence(t *testing.T) { + repos := map[string]string{ + "flag": localGitRepo(t, "flag-repo"), + "entry": localGitRepo(t, "entry-repo"), + "default": localGitRepo(t, "default-repo"), + } + tests := []struct { + name string + override string + entry string + section string + want string + }{ + {name: "section default applies to a source with no identity", section: "default", want: "default-repo"}, + {name: "the source's own identity beats the section default", entry: "entry", section: "default", want: "entry-repo"}, + {name: "the --identity flag beats both", override: "flag", entry: "entry", section: "default", want: "flag-repo"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolved := registerIdentityRepos(t, repos) + ctx := downloader.WithIdentityOverride(context.Background(), tt.override) + atmosConfig := &schema.AtmosConfiguration{Vendor: schema.Vendor{Identity: tt.section}} + + dir, _, err := fetchToTempDir(ctx, atmosConfig, fakeRemote, PkgTypeRemote, filepath.Join(t.TempDir(), "dst"), fetchOptions{ClientMode: downloader.ClientModeDir, Identity: tt.entry}) + + require.NoError(t, err) + got, err := os.ReadFile(filepath.Join(dir, markerFile)) + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + assert.Len(t, resolved(), 1) + }) + } +} + +// Two sources of one `vendor pull` that name different identities each fetch with their own. +func TestFetchToTempDir_ConcurrentSourcesWithDifferentIdentitiesStayIsolated(t *testing.T) { + registerIdentityRepos(t, map[string]string{ + "alpha": localGitRepo(t, "alpha-repo"), + "bravo": localGitRepo(t, "bravo-repo"), + }) + + const perIdentity = 4 + type outcome struct { + identity, got string + err error + } + outcomes := make(chan outcome, 2*perIdentity) + var wg sync.WaitGroup + for i := 0; i < perIdentity; i++ { + for _, identity := range []string{"alpha", "bravo"} { + wg.Add(1) + go func() { + defer wg.Done() + dst := filepath.Join(t.TempDir(), "dst") + dir, _, err := fetchToTempDir(context.Background(), &schema.AtmosConfiguration{}, fakeRemote, PkgTypeRemote, dst, fetchOptions{ClientMode: downloader.ClientModeDir, Identity: identity}) + if err != nil { + outcomes <- outcome{identity: identity, err: err} + return + } + got, err := os.ReadFile(filepath.Join(dir, markerFile)) + outcomes <- outcome{identity: identity, got: string(got), err: err} + }() + } + } + wg.Wait() + close(outcomes) + + n := 0 + for o := range outcomes { + require.NoError(t, o.err) + assert.Equal(t, o.identity+"-repo", o.got) + n++ + } + assert.Equal(t, 2*perIdentity, n) +} + +func TestFetchToTempDir_IdentityNeedingLoginFailsBeforeFetching(t *testing.T) { + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return nil, errUtils.Build(errUtils.ErrRemoteIdentityLoginRequired).WithHint("Run `atmos auth login --identity ops`").Err() + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + + _, _, err := fetchToTempDir(context.Background(), &schema.AtmosConfiguration{}, fakeRemote, PkgTypeRemote, filepath.Join(t.TempDir(), "dst"), fetchOptions{ClientMode: downloader.ClientModeDir, Identity: "ops"}) + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) +} + +func TestComponentInstallerIdentity(t *testing.T) { + assert.Equal(t, "", (&componentVendorInstaller{}).identity()) + assert.Equal(t, "ops", (&componentVendorInstaller{spec: &schema.VendorComponentSpec{Source: schema.VendorComponentSource{Identity: "ops"}}}).identity()) +} + +// A dry run probes the source with the same detector a real fetch uses, and the detector's result +// depends on the identity in effect. So the dry run must resolve the identity exactly as the fetch +// does: for a go-getter source, with the same precedence, and not at all for an OCI package. +func TestDetectIfNeeded_ResolvesTheSourceIdentityLikeTheRealFetch(t *testing.T) { + const uri = "github.com/acme/modules.git//vpc" + require.True(t, needsCustomDetection(uri), "prerequisite: this source goes through the custom detector") + atmosConfig := &schema.AtmosConfiguration{Vendor: schema.Vendor{Identity: "default"}} + + tests := []struct { + name string + ctx context.Context + pType PkgType + entry string + resolved []string + }{ + {name: "the source's own identity", ctx: context.Background(), pType: PkgTypeRemote, entry: "ops", resolved: []string{"ops"}}, + {name: "vendor.identity when the source names none", ctx: context.Background(), pType: PkgTypeRemote, resolved: []string{"default"}}, + {name: "the --identity flag overrides both", ctx: downloader.WithIdentityOverride(context.Background(), "flag-id"), pType: PkgTypeRemote, entry: "ops", resolved: []string{"flag-id"}}, + {name: "an OCI package resolves none, as its fetch does", ctx: context.Background(), pType: PkgTypeOci, entry: "ops"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resolved []string + downloader.RegisterIdentityEnvironmentResolver(func(_ context.Context, _ *schema.AtmosConfiguration, identity string) ([]string, error) { + resolved = append(resolved, identity) + return []string{"SSH_AUTH_SOCK=/tmp/" + identity + ".sock"}, nil + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + + require.NoError(t, detectIfNeeded(tt.ctx, atmosConfig, uri, tt.pType, tt.entry)) + + assert.Equal(t, tt.resolved, resolved) + }) + } +} + +// A real fetch fails when its identity has no session. A dry run of the same source must report +// that too, instead of passing and leaving the failure for the real run. +func TestDetectIfNeeded_IdentityNeedingLoginFailsTheDryRun(t *testing.T) { + downloader.RegisterIdentityEnvironmentResolver(func(context.Context, *schema.AtmosConfiguration, string) ([]string, error) { + return nil, errUtils.ErrRemoteIdentityLoginRequired + }) + t.Cleanup(func() { downloader.RegisterIdentityEnvironmentResolver(nil) }) + + err := detectIfNeeded(context.Background(), &schema.AtmosConfiguration{}, "github.com/acme/modules.git//vpc", PkgTypeRemote, "ops") + + require.ErrorIs(t, err, errUtils.ErrRemoteIdentityLoginRequired) +} diff --git a/pkg/vendoring/install/install.go b/pkg/vendoring/install/install.go index 95888da73d1..50b647ad473 100644 --- a/pkg/vendoring/install/install.go +++ b/pkg/vendoring/install/install.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/cloudposse/atmos/pkg/downloader" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/schema" @@ -46,6 +47,9 @@ type InstallOptions struct { // LockEnforcement is one of LockEnforcementSilent/Warn/Strict, governing how FilterPending // reacts to a drifted package. Empty is treated as LockEnforcementWarn, the config default. LockEnforcement string + // Identity is the global --identity flag value. When set it overrides the identity a source + // names and vendor.identity; see downloader.WithIdentity. Empty means "not given". + Identity string } // Result reports the outcome of installing a single VendorPackage. @@ -70,6 +74,7 @@ func Install(atmosConfig *schema.AtmosConfiguration, pkg VendorPackage, opts Ins func InstallContext(ctx context.Context, atmosConfig *schema.AtmosConfiguration, pkg VendorPackage, opts InstallOptions) (Result, error) { defer perf.Track(atmosConfig, "install.InstallContext")() + ctx = downloader.WithIdentityOverride(ctx, opts.Identity) if opts.DryRun { if err := pkg.installer.dryRunCheck(ctx, atmosConfig); err != nil { return Result{Name: pkg.Name, Err: err}, nil diff --git a/tests/testhelpers/gitmirror/gitmirror.go b/tests/testhelpers/gitmirror/gitmirror.go index 856073aee2b..b306ad741ea 100644 --- a/tests/testhelpers/gitmirror/gitmirror.go +++ b/tests/testhelpers/gitmirror/gitmirror.go @@ -25,6 +25,7 @@ import ( "github.com/otiai10/copy" "github.com/cloudposse/atmos/tests/testhelpers" + "github.com/cloudposse/atmos/tests/testhelpers/gitscrub" ) // Owner and Repo identify the single upstream this package mirrors today: @@ -201,33 +202,10 @@ func gitEnv() []string { } // filterGitDirEnv is the pure filtering logic behind gitEnv, split out so it can be unit tested -// without touching the real process environment. Env variable names are matched -// case-insensitively (via strings.ToUpper) because Windows resolves environment variable names -// case-insensitively, so a lower/mixed-case GIT_DIR (etc.) would otherwise slip through untouched -// and stay active for git. +// without touching the real process environment. The logic lives in gitscrub, a standard-library +// only package, so tests that cannot import this one (it depends on the downloader) share it. func filterGitDirEnv(env []string) []string { - kept := make([]string, 0, len(env)) - for _, kv := range env { - key, _, _ := strings.Cut(kv, "=") - switch strings.ToUpper(key) { - case "GIT_DIR", - "GIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_OBJECT_DIRECTORY", - "GIT_ALTERNATE_OBJECT_DIRECTORIES", - // GIT_COMMON_DIR can redirect the repository's non-worktree files outside the throwaway - // repository, and GIT_NAMESPACE would make the receive-pack store main under - // refs/namespaces//, where the HTTP backend (which does not inherit it) cannot see it. - "GIT_COMMON_DIR", - "GIT_NAMESPACE", - // GIT_TEMPLATE_DIR would seed the throwaway repositories with the host's template, - // including any executable hooks that could fail or hang the mirror's own commit. - "GIT_TEMPLATE_DIR": - continue - } - kept = append(kept, kv) - } - return kept + return gitscrub.Filter(env) } // buildEnv returns gitEnv, with the caller's global, system, and command-scope git configuration diff --git a/tests/testhelpers/gitscrub/gitscrub.go b/tests/testhelpers/gitscrub/gitscrub.go new file mode 100644 index 00000000000..4a77ed10435 --- /dev/null +++ b/tests/testhelpers/gitscrub/gitscrub.go @@ -0,0 +1,46 @@ +// Package gitscrub removes the environment variables that point git at a repository other than +// the one in its working directory. Tests that spawn git against a throwaway repository use it so +// a caller's GIT_DIR (a git hook, a worktree-aware wrapper) cannot redirect them. +// +// It imports only the standard library, so any package's tests can use it without an import cycle. +package gitscrub + +import ( + "os" + "strings" +) + +// Environ returns the process environment without the repository-location variables (see Filter). +func Environ() []string { + return Filter(os.Environ()) +} + +// Filter returns env without the variables that make git operate on a repository other than the +// one in its working directory. Names are matched case-insensitively because Windows resolves +// environment variable names case-insensitively, so a lower or mixed-case GIT_DIR would otherwise +// slip through and stay active for git. Authentication and configuration variables (GIT_CONFIG_*, +// GIT_SSH_COMMAND, GIT_ASKPASS, and so on) are kept. The input is not modified. +func Filter(env []string) []string { + kept := make([]string, 0, len(env)) + for _, kv := range env { + key, _, _ := strings.Cut(kv, "=") + switch strings.ToUpper(key) { + case "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + // GIT_COMMON_DIR can redirect the repository's non-worktree files outside the throwaway + // repository, and GIT_NAMESPACE would make a receive-pack store main under + // refs/namespaces//, where a backend that does not inherit it cannot see it. + "GIT_COMMON_DIR", + "GIT_NAMESPACE", + // GIT_TEMPLATE_DIR would seed the throwaway repositories with the host's template, + // including any executable hooks that could fail or hang a test's own commit. + "GIT_TEMPLATE_DIR": + continue + } + kept = append(kept, kv) + } + return kept +} diff --git a/tests/testhelpers/gitscrub/gitscrub_test.go b/tests/testhelpers/gitscrub/gitscrub_test.go new file mode 100644 index 00000000000..20e8201ec6b --- /dev/null +++ b/tests/testhelpers/gitscrub/gitscrub_test.go @@ -0,0 +1,40 @@ +package gitscrub + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFilter_DropsRepositoryLocationAndKeepsTheRest(t *testing.T) { + env := []string{ + "PATH=/usr/bin:/bin", + "GIT_DIR=/caller/.git", + "git_work_tree=/caller", + "Git_Index_File=/caller/.git/index", + "GIT_CONFIG_COUNT=1", + "GIT_SSH_COMMAND=ssh -o IdentitiesOnly=yes", + "GIT_ASKPASS=/bin/askpass", + } + + kept := Filter(env) + + assert.Equal(t, []string{ + "PATH=/usr/bin:/bin", + "GIT_CONFIG_COUNT=1", + "GIT_SSH_COMMAND=ssh -o IdentitiesOnly=yes", + "GIT_ASKPASS=/bin/askpass", + }, kept, "repository-location variables go, in any casing; authentication variables stay") + assert.Equal(t, "GIT_DIR=/caller/.git", env[1], "the input is not modified") +} + +// Environ must reflect the live process environment, minus the repository-location variables. +func TestEnviron_ScrubsTheProcessEnvironment(t *testing.T) { + t.Setenv("GIT_DIR", "/caller/.git") + t.Setenv("GITSCRUB_TEST_MARKER", "kept") + + env := Environ() + + assert.NotContains(t, env, "GIT_DIR=/caller/.git") + assert.Contains(t, env, "GITSCRUB_TEST_MARKER=kept") +} diff --git a/website/blog/2026-11-12-identities-for-remote-reads.mdx b/website/blog/2026-11-12-identities-for-remote-reads.mdx new file mode 100644 index 00000000000..53aa5113b70 --- /dev/null +++ b/website/blog/2026-11-12-identities-for-remote-reads.mdx @@ -0,0 +1,105 @@ +--- +slug: identities-for-remote-reads +title: "Use an Atmos Auth Identity to Fetch Private Sources" +authors: [osterman] +tags: [feature, bugfix] +--- + +Vendoring a module from a private repository, importing a stack catalog from one, or provisioning a +component from one has always meant getting credentials into the environment first. Either you exported a +token for the whole process so every command could see it, or you put a private key in the URL with +`sshkey=` and committed that to configuration. Neither tells Atmos which credential belongs to which source. + + + +## The Problem + +When several sources need different credentials, a process-wide environment variable cannot express it. One +token is visible to every fetch, and it is visible to everything else the process runs. The `sshkey=` URL +parameter keeps a private key in your configuration, which is the opposite of what an +[identity](/cli/configuration/auth/identities) is for. + +Identities already describe who you are to a provider and how a session is established. They just were not +usable when Atmos read a remote location. + +## The Fix + +A remote read can now name an Atmos Auth identity. The identity's environment is applied to the `git` +commands for that one fetch, and nothing is exported to the process, so sources fetched concurrently can use +different identities. + +The same `identity` field works in three places: a source in +[`vendor.yaml`](/vendor/config/sources#attributes), a component's +[`source`](/cli/commands/terraform/source#source-fields), and a remote +[stack import](/stacks/imports#private-repositories-and-authentication). + +## How to Use It + +Name an identity on a vendored source: + +```yaml title="vendor.yaml" +spec: + sources: + - component: vpc + source: git::ssh://git@github.com/acme/modules.git//vpc?ref={{.Version}} + version: 1.4.0 + identity: source-reader + targets: + - components/terraform/vpc +``` + +On a component source: + +```yaml +components: + terraform: + vpc: + source: + uri: git::ssh://git@github.com/acme/modules.git//vpc + version: 1.4.0 + identity: source-reader +``` + +On a remote stack import, using the object form of `import`: + +```yaml +import: + - path: git::ssh://git@github.com/acme/stacks.git//catalog/vpc?ref=v2 + identity: source-reader +``` + +To name the identity once for a project, set a default in `atmos.yaml`: + +```yaml title="atmos.yaml" +vendor: + identity: source-reader +imports: + identity: source-reader +``` + +The identity is chosen in this order: the `--identity` flag of [`atmos vendor pull`](/cli/commands/vendor/pull), +then the field on the entry, then the default from `atmos.yaml`. With none of them, nothing changes. The +identity name can stay the same between a laptop and CI while its kind differs by profile, so the same +`vendor.yaml` works in both. + +Outside CI, an identity without a valid session fails with a hint to run +`atmos auth login --identity `. Atmos never prompts for a login in the middle of a fetch. + +A few limits apply: + +- A remote `import` in `atmos.yaml` cannot name an identity. +- `atmos vendor update` and `atmos vendor diff` do not use the identity when they list versions. +- S3 and OCI sources authenticate as before. +- A default applies to every remote source, though only Git sources use the identity's environment. + +## Fixes Along the Way + +- `atmos git` now honors `git.repositories..auth.identity` and the `--identity` flag. Both were ignored. +- `atmos ansible playbook --identity` now passes the identity's environment to `ansible-playbook`. +- A stored plugin credential of kind `environment/v1` now works in a fresh process, so `atmos auth exec`, + `atmos auth shell`, and `atmos git` no longer fail after a previous login. + +## Get Involved + +Try it on a private source and tell us what you hit. Open an issue or a discussion on +[GitHub](https://github.com/cloudposse/atmos). diff --git a/website/docs/cli/commands/ansible/ansible-playbook.mdx b/website/docs/cli/commands/ansible/ansible-playbook.mdx index c17ecad7569..9842f34a16e 100644 --- a/website/docs/cli/commands/ansible/ansible-playbook.mdx +++ b/website/docs/cli/commands/ansible/ansible-playbook.mdx @@ -77,6 +77,12 @@ For more details on the `ansible-playbook` command and options, refer to [Ansibl The command line flag takes precedence. +
`--identity`(optional)
+
+ Atmos Auth identity for this run. Its environment is passed to `ansible-playbook`, so credentials + made of environment variables, such as `SSH_AUTH_SOCK`, reach the playbook. +
+
`--dry-run`(optional)
Perform a dry run without executing the playbook. Shows what commands would be run. diff --git a/website/docs/cli/commands/terraform/source/source.mdx b/website/docs/cli/commands/terraform/source/source.mdx index a5d2b328078..862f2c812f7 100644 --- a/website/docs/cli/commands/terraform/source/source.mdx +++ b/website/docs/cli/commands/terraform/source/source.mdx @@ -133,6 +133,11 @@ source:
`excluded_paths`
Glob patterns for files to exclude. Applied after included_paths filtering.
+
`identity`
+
+ The name of an [Atmos Auth identity](/cli/configuration/auth/identities) used to fetch this source, for example `identity: source-reader`. It overrides [`vendor.identity`](/cli/configuration/vendor) in `atmos.yaml`. On a component command the `--identity` flag selects the identity the component itself runs as and does not change the identity a `source:` is fetched with; it overrides the source identity only on [`atmos vendor pull`](/cli/commands/vendor/pull). The identity's environment is applied only to the `git` commands that fetch this source. Outside CI, an identity without a valid session fails with a hint to run `atmos auth login --identity `. See [`identity` in `vendor.yaml`](/vendor/config/sources#attributes) for the limits. +
+
`ttl`
Cache duration for JIT-vendored sources. Controls how long a cached source is reused before re-pulling from the remote. When set, Atmos checks the source's last update time against this TTL. If expired, the source is re-pulled automatically on the next command invocation. If not set, cached sources are reused indefinitely (only re-pulled on version or URI changes). diff --git a/website/docs/cli/commands/vendor/vendor-pull.mdx b/website/docs/cli/commands/vendor/vendor-pull.mdx index 4da28a3145c..1ef7b5f973a 100644 --- a/website/docs/cli/commands/vendor/vendor-pull.mdx +++ b/website/docs/cli/commands/vendor/vendor-pull.mdx @@ -368,6 +368,9 @@ When executing the `atmos vendor pull` command, Atmos performs the following ste
`--type` / `-t` (optional)
Component type: `terraform`, `helmfile`, or `packer` (`terraform` is default). Pass this flag explicitly to scope a `--stack`/`--labels` selection to one component type. When you omit it, every component type is considered.
+
`--identity` (optional)
+
Atmos Auth identity used to fetch every source in this run. Overrides `identity` on a source and `vendor.identity` in `atmos.yaml`. See [`identity`](/vendor/config/sources#attributes).
+
`--dry-run` (optional)
Dry run.
diff --git a/website/docs/cli/configuration/imports.mdx b/website/docs/cli/configuration/imports.mdx index 412640ed2ff..070525d9a5d 100644 --- a/website/docs/cli/configuration/imports.mdx +++ b/website/docs/cli/configuration/imports.mdx @@ -250,6 +250,8 @@ import: - "git::https://github.com/acme/private-config.git//.atmos.d/ai.yaml?ref=v1.2.3" ``` +A remote `import` in `atmos.yaml` cannot name an Atmos Auth identity. Only the importing file's own `auth:` section is loaded while imports resolve, so an identity defined in another file or profile is not visible yet. Stack manifest imports can name one; see [Private Repositories and Authentication](/stacks/imports#private-repositories-and-authentication). + ### Pin Remote Imports Atmos warns when a `git::` import is pinned to a **mutable** ref - a branch (for example @@ -310,6 +312,7 @@ top-level `import` entries. - Templated imports of Atmos configuration are not supported (unlike stacks) - Only `.yml` and `.yaml` files are processed when using glob patterns - The object entry form does not support a per-import `ttl` (use the global `imports.ttl`) +- A remote entry cannot name an Atmos Auth identity. The `imports.identity` setting applies to remote [stack manifest imports](/stacks/imports#private-repositories-and-authentication) only ## See Also diff --git a/website/docs/cli/configuration/vendor.mdx b/website/docs/cli/configuration/vendor.mdx index c9015124f6c..1a6591fe83d 100644 --- a/website/docs/cli/configuration/vendor.mdx +++ b/website/docs/cli/configuration/vendor.mdx @@ -45,6 +45,9 @@ vendor: max_delay: 30s backoff_strategy: exponential + # Default Atmos Auth identity for sources that do not name their own (optional) + identity: source-reader + # Path to the vendor lock file (optional) lock_file: vendor.lock.yaml @@ -154,6 +157,21 @@ vendor: +## Default Identity + +
+
`identity`
+
+ The name of an [Atmos Auth identity](/cli/configuration/auth/identities) used to fetch every vendored source that does not set its own `identity`. Set it once here instead of repeating it on each source in `vendor.yaml`. + + The identity is chosen in this order: the `--identity` flag of [`atmos vendor pull`](/cli/commands/vendor/pull), then `identity` on the source in [`vendor.yaml`](/vendor/config/sources#attributes) or `component.yaml`, then `vendor.identity`. With none of these, fetching works as before. + + The default applies to every remote go-getter source, including HTTPS and S3 sources, but only Git sources use the identity's environment. `atmos vendor update` and `atmos vendor diff` do not use it when listing versions. + + **Default:** none +
+
+ ## Concurrency `vendor.max_concurrency` sets the maximum number of simultaneous downloads and diff --git a/website/docs/stacks/imports.mdx b/website/docs/stacks/imports.mdx index ec4a686b52b..3f837a51618 100644 --- a/website/docs/stacks/imports.mdx +++ b/website/docs/stacks/imports.mdx @@ -264,6 +264,33 @@ Cached sources, including pulled OCI artifacts, live under the XDG cache directo `ttl` accepts durations like `0s` (always re-fetch), `5m`, `1h`, `7d`, or keywords like `daily`. Pin to an immutable `?ref=` (or commit SHA) with a longer `ttl` for maximum reuse; keep `ttl` short for mutable refs like `main` so the catalog stays fresh. +### Private Repositories and Authentication + +A remote import of a private repository normally relies on the credentials already in your environment. To fetch it with an [Atmos Auth identity](/cli/configuration/auth/identities) instead, use the object form of `import` and set `identity`: + +```yaml +import: + - path: "git::ssh://git@github.com/acme/stacks.git//catalog/vpc?ref=v2" + identity: source-reader +``` + +To name the identity once for every remote import, set it in `atmos.yaml`: + +```yaml title="atmos.yaml" +imports: + identity: source-reader +``` + +The identity is chosen in this order: `identity` on the import, then `imports.identity`. With neither, fetching works as before. The `--identity` flag does not reach stack processing, so it does not change which identity an import uses. + +The identity's environment is applied only to the `git` commands that fetch that import, so imports fetched concurrently can use different identities. Outside CI, an identity without a valid session fails with a hint to run `atmos auth login --identity `. Atmos never prompts for a login in the middle of a fetch. The default applies to every remote import, but only Git sources use the identity's environment. An `oci://` import authenticates through the registry and does not need the identity's session. + +An import that names an identity is fetched with that identity only. Atmos does not add a token from your environment (such as `GITHUB_TOKEN`) to its URL, and credentials that Atmos Pro provisioned for other fetches are not passed to it. Imports fetched with different identities are cached separately. + +`skip_if_missing` skips an import whose source does not exist. It does not skip an import whose identity cannot be used: that still fails with the login hint. + +A remote `import` in `atmos.yaml` cannot name an identity. See [CLI configuration imports](/cli/configuration/imports#private-repositories-and-authentication). + ### Best Practices for Remote Imports 1. **Pin Versions:** Always use `?ref=` with a specific tag or commit SHA for Git imports to ensure reproducible builds. @@ -393,6 +420,9 @@ The `import` section supports the following fields:
`when` - (string, list, or map)
An optional condition gating whether the import is processed at all, for example `ci` to import a file only in a CI pipeline. An unset condition always imports. See [Conditional Imports](#conditional-imports).
+
`identity` - (string)
+
The name of an Atmos Auth identity used to fetch a remote import. Ignored for local imports. Overrides `imports.identity` in `atmos.yaml`. See [Private Repositories and Authentication](#private-repositories-and-authentication).
+
`ttl` - (string)
Cache duration for the cloned source repository of a remote (Git) import. When set, the clone is reused across Atmos invocations until it expires (e.g. with a warm CI cache), skipping the re-clone. Within a single invocation a source repo is always cloned at most once regardless of `ttl`. When unset, the source is re-cloned once per invocation. Accepts durations like `0s`, `5m`, `1h`, `7d`, or keywords like `daily`. Set a default for all imports with the top-level `imports.ttl` setting in `atmos.yaml`. See [Caching Remote Imports](#caching-remote-imports).
diff --git a/website/docs/vendor/component-manifest/source.mdx b/website/docs/vendor/component-manifest/source.mdx index 209462f1415..61a870df109 100644 --- a/website/docs/vendor/component-manifest/source.mdx +++ b/website/docs/vendor/component-manifest/source.mdx @@ -26,6 +26,7 @@ spec: - "**/*.md" excluded_paths: - "**/context.tf" + identity: source-reader ``` ## Attributes @@ -50,6 +51,11 @@ spec: The version of the component to vendor. This value is substituted into `{{.Version}}` placeholders in the `uri`. +
`identity`
+
+ The name of an [Atmos Auth identity](/cli/configuration/auth/identities) used to fetch the source and its [mixins](/vendor/component-manifest/mixins). It overrides [`vendor.identity`](/cli/configuration/vendor) in `atmos.yaml`, and the `--identity` flag of [`atmos vendor pull`](/cli/commands/vendor/pull) overrides both. The identity's environment is applied only to the `git` commands that fetch this component. See [`identity` in `vendor.yaml`](/vendor/config/sources#attributes) for the details and limits. +
+
`included_paths`
Only include the files that match the `included_paths` patterns. diff --git a/website/docs/vendor/config/sources.mdx b/website/docs/vendor/config/sources.mdx index a3deed3c78c..c285b504781 100644 --- a/website/docs/vendor/config/sources.mdx +++ b/website/docs/vendor/config/sources.mdx @@ -29,6 +29,7 @@ spec: - "**/test/**" tags: - networking + identity: source-reader retry: max_attempts: 5 initial_delay: 2s @@ -75,6 +76,30 @@ spec:
+
`identity`
+
+ The name of an [Atmos Auth identity](/cli/configuration/auth/identities) used to fetch this source, for private repositories that need credentials the process does not already have. The identity's environment is applied only to the `git` commands that fetch this source, so sources vendored concurrently can use different identities. + + ```yaml + spec: + sources: + - component: "vpc" + source: "git::ssh://git@github.com/acme/modules.git//vpc?ref={{.Version}}" + version: "1.4.0" + identity: source-reader + targets: + - "components/terraform/vpc" + ``` + + The identity is chosen in this order: the `--identity` flag of [`atmos vendor pull`](/cli/commands/vendor/pull), then `identity` on the source, then [`vendor.identity`](/cli/configuration/vendor) in `atmos.yaml`. With none of these, fetching works as before, with the credentials already in your environment. + + Outside CI, an identity without a valid session fails with a hint to run `atmos auth login --identity `. Atmos never prompts for a login in the middle of a fetch. + + A source that names an identity is fetched with that identity only. Atmos does not add a token from your environment (such as `GITHUB_TOKEN`) to its URL, and credentials that Atmos Pro provisioned for other fetches are not passed to it. + + Only Git sources use the identity's environment. S3 and OCI sources authenticate as before and do not need the identity's session. `atmos vendor update` and `atmos vendor diff` do not use the identity when they list available versions. +
+
`targets`
The `targets` in each source supports absolute paths and relative paths (relative to the `vendor.yaml` file). Note: if the `targets` paths diff --git a/website/src/data/roadmap.js b/website/src/data/roadmap.js index bdcb922de40..db0323ce365 100644 --- a/website/src/data/roadmap.js +++ b/website/src/data/roadmap.js @@ -585,6 +585,7 @@ export const roadmapConfig = { progress: 100, status: 'shipped', milestones: [ + { label: 'Atmos Auth identities for remote reads', status: 'shipped', quarter: 'q4-2026', changelog: 'identities-for-remote-reads', docs: '/vendor/config/sources#attributes', prd: 'ssh-agent-plugin', description: 'Vendored sources, component sources, and remote stack imports can name an Atmos Auth identity (`identity:`), with `vendor.identity` and `imports.identity` defaults. The identity\'s environment is applied to that fetch\'s git commands only.', benefits: 'Private sources no longer need credentials exported process-wide or a private key in the URL.' }, { label: 'Concurrent vendoring with ordered writes', status: 'shipped', quarter: 'q3-2026', changelog: 'concurrent-vendoring', docs: '/cli/configuration/vendor#concurrency', prd: 'concurrent-vendoring', description: 'Bounded concurrent downloads and upstream checks share toolchain-style progress. Destination writes and receipts remain ordered, and older edition pins retain one worker.', benefits: 'Fetch independent components faster while retaining reproducible destination contents and a readable terminal.' }, { label: 'Retry with exponential backoff', status: 'shipped', quarter: 'q3-2025', docs: '/cli/commands/vendor/pull', description: 'Automatic retries with increasing delays for transient network failures and rate limits.', category: 'featured', priority: 'high', benefits: 'Vendoring succeeds despite flaky networks or GitHub rate limits. CI doesn\'t fail on transient errors.' }, { label: 'Version constraints for vendor updates', status: 'shipped', quarter: 'q3-2025', docs: '/cli/configuration/vendor', changelog: 'version-constraint-validation', version: 'v1.201.0', description: 'Semantic versioning constraints to control which versions are pulled during vendor updates.', benefits: 'Control upgrade behavior. Pin to major versions or allow patches automatically.' },