diff --git a/CHANGELOG.md b/CHANGELOG.md index baea9c3c..1341a328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## UNRELEASED IMPROVEMENTS: +* template: Add `vaultRead()` and `vaultKV()` functions to read secrets from HashiCorp Vault with automatic KV v1/v2 detection [[GH-473](https://github.com/hashicorp/nomad-pack/pull/848)] * cli: Improved error message when pack lacks `.nomad.tpl` files to clearly explain naming requirements, show template naming convention, and list found template files [[GH-831](https://github.com/hashicorp/nomad-pack/pull/831)] * cli: Add registry now honors default main/master branch [[GH-843](https://github.com/hashicorp/nomad-pack/pull/843)] * cli: Fixed stale-job reconciliation so nomad-pack run no longer stops active parameterized/periodic child jobs [[GH-855](https://github.com/hashicorp/nomad-pack/pull/855)] diff --git a/docs/functions.md b/docs/functions.md index 346a3ded..73815a9c 100644 --- a/docs/functions.md +++ b/docs/functions.md @@ -130,6 +130,89 @@ The `nomadVariable` function retrieves a specific Nomad Variable by path and nam password = "[[ .Items.password ]]" [[ end ]] +### Vault functions + +#### `vaultRead` + +The `vaultRead` function retrieves secrets from HashiCorp Vault with automatic detection of KV v1 and KV v2 secret engines. This function intelligently handles both versions and returns the secret data directly. + +**NOTE:** This function requires Vault to be configured via the `VAULT_ADDR` and `VAULT_TOKEN` environment variables. If Vault is not configured, the function will not be available in templates. + +##### Parameters + +- 1: `string` - The path to the secret in Vault + +##### Returns + +- `error` or `map[string]interface{}` - The secret data + +##### Example + +Read a secret from Vault KV v2: + +[[ with vaultRead "secret/data/database" ]] +DB_PASSWORD = "[[ .password ]]" +DB_USERNAME = "[[ .username ]]" +[[ end ]] + +Use with fallback values: + +DB_PASSWORD = "[[ try (vaultRead "secret/data/db" | get "password") "default-password" ]]" + +#### `vaultKV` + +The `vaultKV` function retrieves raw secret data from HashiCorp Vault without any automatic version detection or data extraction. This is useful for advanced use cases where you need access to the complete Vault response including metadata. + +**NOTE:** This function requires Vault to be configured via the `VAULT_ADDR` and `VAULT_TOKEN` environment variables. If Vault is not configured, the function will not be available in templates. + +##### Parameters + +- 1: `string` - The path to the secret in Vault + +##### Returns + +- `error` or `map[string]interface{}` - The raw Vault response + +##### Example + +Read raw secret data: + +[[ $secret := vaultKV "secret/data/app" ]] +[[ range $key, $value := $secret ]] +[[ $key ]]: [[ $value ]] +[[ end ]] + +### Vault-backed variable source + +Nomad Pack can populate declared pack variables from Vault KV during command execution. + +Use the following flags: + +- `--var-source=vault` +- `--vault-var-path=` + +Example: + +```bash +nomad-pack render \ + --no-format=true \ + --var-source=vault \ + --vault-var-path=secret/data/myapp \ + ./fixtures/v2/simple_raw_exec_v2 +``` + +When using --var-source=vault, --vault-var-path is required. + +Currently only vault is supported as a variable source. + +Variable precedence is: + +1. external variable source +2. environment variables +3. variable files +4. CLI --var +This means CLI --var values override Vault-sourced values. + ### Region functions #### `nomadRegions` @@ -629,6 +712,8 @@ These are the additional functions supplied by Nomad Pack itself. - [`spewDump`][] - Returns a string representation of a value using `spew.Sdump`. - [`spewPrintf`][] - Returns a formatted string representation of a value using `spew.Sprintf`. - [`toStringList`][] - Converts a value to a string list. +- [`vaultKV`][] - Retrieves raw secret data from HashiCorp Vault. +- [`vaultRead`][] - Retrieves secrets from HashiCorp Vault with automatic KV version detection. - [`tpl`][] - Renders a template string using the current template context. - [`withContinueOnMethod`][] - Sets the `ContinueOnMethod` flag for a `customSpew`. - [`withDisableCapacities`][] - Sets the `DisableCapacities` flag for a `customSpew`. @@ -670,3 +755,6 @@ These are the additional functions supplied by Nomad Pack itself. [`withContinueOnMethod`]: #withContinueOnMethod [`withSortKeys`]: #withSortKeys [`withSpewKeys`]: #withSpewKeys +[`vaultKV`]: #vaultKV +[`vaultRead`]: #vaultRead + diff --git a/go.mod b/go.mod index f669b7ad..fd4d4ac6 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/hashicorp/hcl/v2 v2.23.0 github.com/hashicorp/nomad v1.11.3 github.com/hashicorp/nomad/api v0.0.0-20260304165455-489f8b9d1054 + github.com/hashicorp/vault/api v1.22.0 github.com/kr/text v0.2.0 github.com/lab47/vterm v0.0.0-20211107042118-80c3d2849f9c github.com/mattn/go-isatty v0.0.20 @@ -241,7 +242,6 @@ require ( github.com/hashicorp/raft-autopilot v0.3.0 // indirect github.com/hashicorp/raft-boltdb/v2 v2.3.1 // indirect github.com/hashicorp/serf v0.10.2 // indirect - github.com/hashicorp/vault/api v1.22.0 // indirect github.com/hashicorp/vault/api/auth/kubernetes v0.10.0 // indirect github.com/hashicorp/vic v1.5.1-0.20241121050025-d1d58fa204f5 // indirect github.com/hashicorp/yamux v0.1.2 // indirect diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index c21d6134..2646452e 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -8,6 +8,8 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "os" "path" "path/filepath" @@ -18,6 +20,7 @@ import ( "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/command/agent" + vault "github.com/hashicorp/vault/api" "github.com/mitchellh/cli" "github.com/shoenig/test/must" "github.com/shoenig/test/wait" @@ -52,6 +55,36 @@ const ( testPlanCmdString = "plan --exit-code-no-changes=90 --exit-code-makes-changes=91 --exit-code-error=92" ) +func testVaultClientForCLI(t *testing.T, handler http.HandlerFunc) *vault.Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + cfg := vault.DefaultConfig() + cfg.Address = server.URL + + client, err := vault.NewClient(cfg) + must.NoError(t, err) + + return client +} + +func withEnv(t *testing.T, key, value string) { + t.Helper() + + prev, ok := os.LookupEnv(key) + must.NoError(t, os.Setenv(key, value)) + + t.Cleanup(func() { + if ok { + _ = os.Setenv(key, prev) + return + } + _ = os.Unsetenv(key) + }) +} + func TestCLI_CreateTestRegistry(t *testing.T) { // This test is here to help setup the pack registry cache. It needs to be // the first one in the file and can not be `Parallel()` @@ -88,6 +121,71 @@ func TestCLI_Version(t *testing.T) { must.Zero(t, exitCode) } +func TestCLI_Render_VaultVariableSource(t *testing.T) { + t.Parallel() + + vaultClient := testVaultClientForCLI(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "us-east-1", + "count": float64(3), + }, + }, + })) + }) + + withEnv(t, "VAULT_ADDR", vaultClient.Address()) + + result := runPackCmd(t, []string{ + "render", + "--no-format=true", + "--var-source=vault", + "--vault-var-path=secret/data/myapp", + testfixture.AbsPath(t, "v2/simple_raw_exec_v2"), + }) + + must.Zero(t, result.exitCode, must.Sprintf("stdout:\n%s\n\nstderr:\n%s\n", result.cmdOut.String(), result.cmdErr.String())) + must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) + must.StrContains(t, result.cmdOut.String(), `region = "us-east-1"`) + must.StrContains(t, result.cmdOut.String(), `count = 3`) +} + +func TestCLI_Render_VaultVariableSource_MissingPath(t *testing.T) { + t.Parallel() + + result := runPackCmd(t, []string{ + "render", + "--no-format=true", + "--var-source=vault", + testfixture.AbsPath(t, "v2/simple_raw_exec_v2"), + }) + + must.Eq(t, 1, result.exitCode) + out := result.cmdOut.String() + if !strings.Contains(out, "vault variable source requires a non-empty path") && !strings.Contains(out, "a Vault-backed variable source was requested, but no Vault client is configured") { + t.Fatalf("expected missing-path or missing-client error, got:\n%s", out) + } +} + +func TestCLI_Render_VariableSource_Unsupported(t *testing.T) { + t.Parallel() + + result := runPackCmd(t, []string{ + "render", + "--no-format=true", + "--var-source=consul", + testfixture.AbsPath(t, "v2/simple_raw_exec_v2"), + }) + + must.Eq(t, 1, result.exitCode) + must.StrContains(t, result.cmdOut.String(), `unsupported variable source type "consul"`) + +} + func TestCLI_JobRun(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 205ce86c..6e730a62 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -71,6 +71,12 @@ type baseCommand struct { // for defined input variables varFiles []string + // varSourceType configures an optional external variable source. + varSourceType string + + // vaultVarPath is the Vault path used for external variable sourcing. + vaultVarPath string + // allowUnsetVars suppresses errors from variables with nil values, // i.e. those that are not set and have no default allowUnsetVars bool @@ -292,6 +298,20 @@ func (c *baseCommand) flagSet(bit flagSetBit, f func(*flag.Sets)) *flag.Sets { syntax and can be specified multiple times per command.`, }) + f.StringVar(&flag.StringVar{ + Name: "var-source", + Target: &c.varSourceType, + Default: "", + Usage: `Configures an external variable source. Currently only "vault" is supported.`, + }) + + f.StringVar(&flag.StringVar{ + Name: "vault-var-path", + Target: &c.vaultVarPath, + Default: "", + Usage: `Vault path used to populate declared pack variables from an external variable source.`, + }) + f.StringVar(&flag.StringVar{ Name: "name", Target: &c.deploymentName, diff --git a/internal/cli/helpers.go b/internal/cli/helpers.go index c10ec5e6..94974daf 100644 --- a/internal/cli/helpers.go +++ b/internal/cli/helpers.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/hashicorp/nomad/api" + vault "github.com/hashicorp/vault/api" "github.com/posener/complete" "github.com/hashicorp/nomad-pack/internal/pkg/caching" @@ -18,6 +19,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/manager" "github.com/hashicorp/nomad-pack/internal/pkg/renderer" "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" + parserconfig "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" "github.com/hashicorp/nomad-pack/internal/runner" "github.com/hashicorp/nomad-pack/internal/runner/job" "github.com/hashicorp/nomad-pack/sdk/pack/variables" @@ -42,6 +44,25 @@ func initPackCommand(cfg *caching.PackConfig) (errorContext *errors.UIErrorConte // generatePackManager is used to generate the pack manager for this Nomad Pack run. func generatePackManager(c *baseCommand, client *api.Client, packCfg *caching.PackConfig) *manager.PackManager { // TODO: Refactor to have manager use cache. + + var variableSource *parserconfig.VariableSourceConfig + + switch c.varSourceType { + case "": + // no external variable source configured + case "vault": + variableSource = &parserconfig.VariableSourceConfig{ + Type: "vault", + Vault: &parserconfig.VaultVariableSourceConfig{ + Path: c.vaultVarPath, + }, + } + default: + variableSource = &parserconfig.VariableSourceConfig{ + Type: c.varSourceType, + } + } + cfg := manager.Config{ Path: packCfg.Path, VariableFiles: c.varFiles, @@ -49,8 +70,17 @@ func generatePackManager(c *baseCommand, client *api.Client, packCfg *caching.Pa VariableEnvVars: c.envVars, AllowUnsetVars: c.allowUnsetVars, UseParserV1: c.useParserV1, + VariableSource: variableSource, } - return manager.NewPackManager(&cfg, client) + + // Create Vault client if configured + vaultClient, err := getVaultClient() + if err != nil { + c.ui.ErrorWithContext(err, "failed to create Vault client") + // Continue without Vault; it's optional + vaultClient = nil + } + return manager.NewPackManager(&cfg, client, vaultClient) } // predictPackName is a complete.Predictor that suggests cached pack names. @@ -686,6 +716,37 @@ func limit(s string, length int) string { return s[:length] } +// getVaultClient creates a Vault client from environment variables. +// Returns nil client if Vault is not configured (VAULT_ADDR not set). +// This allows Vault integration to be optional - if VAULT_ADDR is not set, +// the pack will work normally but Vault template functions won't be available. +func getVaultClient() (*vault.Client, error) { + // check if Vault is configured via VAULT_ADDR environment variable + vaultAddr := os.Getenv("VAULT_ADDR") + if vaultAddr == "" { + // Vault not configured - this is OK, return nil without error + return nil, nil + } + + //Create default Vault configuration with sensible defaults + config := vault.DefaultConfig() + config.Address = vaultAddr + + //create Vault client + client, err := vault.NewClient(config) + if err != nil { + return nil, fmt.Errorf("failed to create Vault client: %w", err) + } + + //set authentication token if provided via VAULT_TOKEN environment variable + vaultToken := os.Getenv("VAULT_TOKEN") + if vaultToken != "" { + client.SetToken(vaultToken) + } + + return client, nil +} + // addNoParentTemplatesContext adds error details for missing parent templates // to an existing error context. It lists any .tpl files discovered and provides // naming guidance. diff --git a/internal/pkg/manager/manager.go b/internal/pkg/manager/manager.go index f2d0a664..13d87afe 100644 --- a/internal/pkg/manager/manager.go +++ b/internal/pkg/manager/manager.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" "github.com/hashicorp/nomad-pack/sdk/pack" "github.com/hashicorp/nomad/api" + vault "github.com/hashicorp/vault/api" ) // Config contains all the user specified parameters needed to correctly run @@ -27,23 +28,26 @@ type Config struct { VariableEnvVars map[string]string UseParserV1 bool AllowUnsetVars bool + VariableSource *config.VariableSourceConfig } // PackManager is responsible for loading, parsing, and rendering a Pack and // all dependencies. type PackManager struct { - cfg *Config - client *api.Client - renderer *renderer.Renderer + cfg *Config + client *api.Client + vaultClient *vault.Client + renderer *renderer.Renderer // loadedPack is unavailable until the loadAndValidatePacks func is run. loadedPack *pack.Pack } -func NewPackManager(cfg *Config, client *api.Client) *PackManager { +func NewPackManager(cfg *Config, client *api.Client, vaultClient *vault.Client) *PackManager { return &PackManager{ - cfg: cfg, - client: client, + cfg: cfg, + client: client, + vaultClient: vaultClient, } } @@ -73,6 +77,8 @@ func (pm *PackManager) ProcessVariableFiles() (*parser.ParsedVariables, []*error EnvOverrides: pm.cfg.VariableEnvVars, FileOverrides: pm.cfg.VariableFiles, FlagOverrides: pm.cfg.VariableCLIArgs, + VaultClient: pm.vaultClient, + VariableSource: pm.cfg.VariableSource, } if pm.cfg.UseParserV1 { @@ -142,6 +148,7 @@ func (pm *PackManager) ProcessTemplates(renderAux bool, format bool, ignoreMissi r := new(renderer.Renderer) r.Client = pm.client + r.VaultClient = pm.vaultClient r.PackPath = pm.cfg.Path pm.renderer = r diff --git a/internal/pkg/renderer/funcs.go b/internal/pkg/renderer/funcs.go index 6aecafe0..b614daf5 100644 --- a/internal/pkg/renderer/funcs.go +++ b/internal/pkg/renderer/funcs.go @@ -13,6 +13,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" "github.com/hashicorp/nomad/api" + vault "github.com/hashicorp/vault/api" "golang.org/x/exp/maps" ) @@ -53,6 +54,11 @@ func funcMap(r *Renderer) template.FuncMap { f["nomadVariable"] = nomadVariable(r.Client) } + if r != nil && r.VaultClient != nil { + f["vaultRead"] = vaultRead(r.VaultClient) + f["vaultKV"] = vaultKV(r.VaultClient) + } + if r != nil && r.PackPath != "" { f["packPath"] = func() (string, error) { return r.PackPath, nil @@ -238,3 +244,40 @@ func withSpewKeys(s *spew.ConfigState) any { s.SpewKeys = true return s } + +// vaultRead reads a secret from Vault with automatic KV v1/v2 detection. +// For KV v2, it automatically unwraps nested data structure. +func vaultRead(client *vault.Client) func(string) (map[string]interface{}, error) { + return func(path string) (map[string]interface{}, error) { + secret, err := client.Logical().Read(path) + if err != nil { + return nil, fmt.Errorf("failed to read from Vault path %s: %w", path, err) + } + if secret == nil { + return nil, fmt.Errorf("no secret found at Vault path %s", path) + } + + //for KV v2, data is nested under "data" key + if data, ok := secret.Data["data"].(map[string]interface{}); ok { + return data, nil + } + + //for KV v1 or other engines, return data directly + return secret.Data, nil + } +} + +// vaultKV reads from Vault KV and returns raw data without unwrapping. +// useful when you need access to metadata or want to handle KV versions yourself. +func vaultKV(client *vault.Client) func(string) (map[string]interface{}, error) { + return func(path string) (map[string]interface{}, error) { + secret, err := client.Logical().Read(path) + if err != nil { + return nil, fmt.Errorf("failed to read from Vault KV path %s: %w", path, err) + } + if secret == nil { + return nil, fmt.Errorf("no secret found at Vault KV path %s", path) + } + return secret.Data, nil + } +} diff --git a/internal/pkg/renderer/renderer.go b/internal/pkg/renderer/renderer.go index 495b7847..91265c26 100644 --- a/internal/pkg/renderer/renderer.go +++ b/internal/pkg/renderer/renderer.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/hashicorp/nomad/api" + vault "github.com/hashicorp/vault/api" "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" "github.com/hashicorp/nomad-pack/sdk/pack" @@ -35,6 +36,10 @@ type Renderer struct { // when accessing it. Client *api.Client + //VaultClient is the Vault API client used when running Vault template functions. + //It can potentially be nil, therefore care should be taken when accessing it. + VaultClient *vault.Client + // RenderAuxFiles determines whether we should render auxiliary files found // in template/ or not RenderAuxFiles bool diff --git a/internal/pkg/variable/parser/config/config.go b/internal/pkg/variable/parser/config/config.go index 0a6980bf..17add480 100644 --- a/internal/pkg/variable/parser/config/config.go +++ b/internal/pkg/variable/parser/config/config.go @@ -5,6 +5,7 @@ package config import ( "github.com/hashicorp/nomad-pack/sdk/pack" + vault "github.com/hashicorp/vault/api" ) type ParserVersion int @@ -15,6 +16,15 @@ const ( V2 ) +type VariableSourceConfig struct { + Type string + Vault *VaultVariableSourceConfig +} + +type VaultVariableSourceConfig struct { + Path string +} + // ParserConfig contains details of the numerous sources of variables which // should be parsed and merged according to the expected strategy. type ParserConfig struct { @@ -50,4 +60,10 @@ type ParserConfig struct { // IgnoreMissingVars determines whether we error or not on variable overrides // that don't have corresponding vars in the pack. IgnoreMissingVars bool + + // VaultClient is used for Vault-backed template and variable-source access. + VaultClient *vault.Client + + // VariableSource configures an optional external variable source. + VariableSource *VariableSourceConfig } diff --git a/internal/pkg/variable/parser/parser_v2.go b/internal/pkg/variable/parser/parser_v2.go index 2e9f4192..e42054ae 100644 --- a/internal/pkg/variable/parser/parser_v2.go +++ b/internal/pkg/variable/parser/parser_v2.go @@ -4,9 +4,11 @@ package parser import ( + "encoding/json" "errors" "fmt" "os" + "strconv" "strings" "github.com/hashicorp/hcl/v2" @@ -22,6 +24,7 @@ import ( "github.com/hashicorp/nomad-pack/sdk/pack/variables" "github.com/spf13/afero" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" ) type ParserV2 struct { @@ -37,9 +40,10 @@ type ParserV2 struct { // envOverrideVars, fileOverrideVars, cliOverrideVars are the override // variables. The maps are keyed by the pack name they are associated to. - envOverrideVars variables.PackIDKeyedVarMap - fileOverrideVars variables.PackIDKeyedVarMap - flagOverrideVars variables.PackIDKeyedVarMap + sourceOverrideVars variables.PackIDKeyedVarMap + envOverrideVars variables.PackIDKeyedVarMap + fileOverrideVars variables.PackIDKeyedVarMap + flagOverrideVars variables.PackIDKeyedVarMap } func NewParserV2(cfg *config.ParserConfig) (*ParserV2, error) { @@ -63,12 +67,13 @@ func NewParserV2(cfg *config.ParserConfig) (*ParserV2, error) { fs: afero.Afero{ Fs: afero.OsFs{}, }, - cfg: cfg, - rootVars: make(map[pack.ID]map[variables.ID]*variables.Variable), - nomadVars: make(map[pack.ID][]*variables.NomadVariable), - envOverrideVars: make(variables.PackIDKeyedVarMap), - fileOverrideVars: make(variables.PackIDKeyedVarMap), - flagOverrideVars: make(variables.PackIDKeyedVarMap), + cfg: cfg, + rootVars: make(map[pack.ID]map[variables.ID]*variables.Variable), + sourceOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + fileOverrideVars: make(variables.PackIDKeyedVarMap), + flagOverrideVars: make(variables.PackIDKeyedVarMap), + nomadVars: make(map[pack.ID][]*variables.NomadVariable), }, nil } @@ -81,6 +86,9 @@ func (p *ParserV2) Parse() (*ParsedVariables, hcl.Diagnostics) { return nil, diags } + sourceDiags := p.parseVariableSourceOverrides() + diags = packdiags.SafeDiagnosticsExtend(diags, sourceDiags) + // Parse env, file, and CLI overrides. for k, v := range p.cfg.EnvOverrides { envOverrideDiags := p.parseEnvVariable(k, v) @@ -103,7 +111,12 @@ func (p *ParserV2) Parse() (*ParsedVariables, hcl.Diagnostics) { // Iterate all our override variables and merge these into our root // variables with the CLI taking highest priority. - for _, override := range []variables.PackIDKeyedVarMap{p.envOverrideVars, p.fileOverrideVars, p.flagOverrideVars} { + for _, override := range []variables.PackIDKeyedVarMap{ + p.sourceOverrideVars, + p.envOverrideVars, + p.fileOverrideVars, + p.flagOverrideVars, + } { for packName, variables := range override { for _, v := range variables { existing, exists := p.rootVars[packName][v.Name] @@ -351,7 +364,7 @@ func (p *ParserV2) parseVariableImpl(name, rawVal string, tgt variables.PackIDKe return diags } - // If our stored type isn't cty.NilType then attempt to covert the override + // If our stored type isn't cty.NilType then attempt to convert the override // variable, so we know they are compatible. if existing.Type != cty.NilType { var err *hcl.Diagnostic @@ -372,3 +385,175 @@ func (p *ParserV2) parseVariableImpl(name, rawVal string, tgt variables.PackIDKe return nil } + +func (p *ParserV2) parseVariableSourceOverrides() hcl.Diagnostics { + var diags hcl.Diagnostics + + if p.cfg == nil || p.cfg.VariableSource == nil { + return diags + } + + switch p.cfg.VariableSource.Type { + case "": + return diags + case "vault": + if p.cfg.VaultClient == nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "missing Vault client", + Detail: "a Vault-backed variable source was requested, but no Vault client is configured", + }) + } + if p.cfg.VariableSource.Vault == nil || p.cfg.VariableSource.Vault.Path == "" { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "missing Vault variable source path", + Detail: "vault variable source requires a non-empty path", + }) + } + return p.parseVaultVariableSource(p.cfg.VariableSource.Vault.Path) + default: + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "unsupported variable source", + Detail: fmt.Sprintf("unsupported variable source type %q", p.cfg.VariableSource.Type), + }) + } +} + +func (p *ParserV2) parseVaultVariableSource(path string) hcl.Diagnostics { + var diags hcl.Diagnostics + + secret, err := p.cfg.VaultClient.Logical().Read(path) + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to read Vault variable source", + Detail: fmt.Sprintf("failed reading Vault path %q: %v", path, err), + }) + } + if secret == nil || secret.Data == nil { + return diags + } + + data := secret.Data + if nested, ok := data["data"].(map[string]interface{}); ok { + data = nested + } + + for packName, rootVars := range p.rootVars { + for _, rootVar := range rootVars { + raw, ok := data[rootVar.Name.String()] + if !ok { + continue + } + + val, convDiags := p.convertSourceValue(rootVar, raw) + diags = packdiags.SafeDiagnosticsExtend(diags, convDiags) + if convDiags.HasErrors() { + continue + } + + v := &variables.Variable{ + Name: rootVar.Name, + Type: rootVar.Type, + ConstraintType: rootVar.ConstraintType, + TypeDefaults: rootVar.TypeDefaults, + Value: val, + DeclRange: rootVar.DeclRange, + } + p.sourceOverrideVars[packName] = append(p.sourceOverrideVars[packName], v) + } + } + + return diags +} + +func (p *ParserV2) convertSourceValue(rootVar *variables.Variable, raw interface{}) (cty.Value, hcl.Diagnostics) { + var diags hcl.Diagnostics + + targetType := rootVar.ConstraintType + if targetType == cty.NilType { + targetType = rootVar.Type + } + + if targetType == cty.Number { + switch n := raw.(type) { + case json.Number: + f, err := n.Float64() + if err != nil { + return cty.NilVal, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to convert sourced value", + Detail: fmt.Sprintf("failed to convert sourced value for variable %q: %v", rootVar.Name, err), + Subject: rootVar.DeclRange.Ptr(), + }) + } + return cty.NumberFloatVal(f), diags + case float64: + return cty.NumberFloatVal(n), diags + case float32: + return cty.NumberFloatVal(float64(n)), diags + case int: + return cty.NumberIntVal(int64(n)), diags + case int64: + return cty.NumberIntVal(n), diags + case int32: + return cty.NumberIntVal(int64(n)), diags + case int16: + return cty.NumberIntVal(int64(n)), diags + case int8: + return cty.NumberIntVal(int64(n)), diags + case uint: + return cty.NumberUIntVal(uint64(n)), diags + case uint64: + return cty.NumberUIntVal(n), diags + case uint32: + return cty.NumberUIntVal(uint64(n)), diags + case uint16: + return cty.NumberUIntVal(uint64(n)), diags + case uint8: + return cty.NumberUIntVal(uint64(n)), diags + } + } + + if s, ok := raw.(string); ok { + switch targetType { + case cty.Number: + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return cty.NilVal, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to convert sourced value", + Detail: fmt.Sprintf("failed to convert sourced value for variable %q: %v", rootVar.Name, err), + Subject: rootVar.DeclRange.Ptr(), + }) + } + return cty.NumberFloatVal(f), diags + + case cty.Bool: + b, err := strconv.ParseBool(s) + if err != nil { + return cty.NilVal, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to convert sourced value", + Detail: fmt.Sprintf("failed to convert sourced value for variable %q: %v", rootVar.Name, err), + Subject: rootVar.DeclRange.Ptr(), + }) + } + return cty.BoolVal(b), diags + } + } + + val, err := gocty.ToCtyValue(raw, targetType) + if err != nil { + return cty.NilVal, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to convert sourced value", + Detail: fmt.Sprintf("failed to convert sourced value for variable %q: %v", rootVar.Name, err), + Subject: rootVar.DeclRange.Ptr(), + }) + } + + return val, diags +} diff --git a/internal/pkg/variable/parser/parser_v2_test.go b/internal/pkg/variable/parser/parser_v2_test.go index b4ec0ca3..d385c67d 100644 --- a/internal/pkg/variable/parser/parser_v2_test.go +++ b/internal/pkg/variable/parser/parser_v2_test.go @@ -4,7 +4,10 @@ package parser import ( + "encoding/json" "fmt" + "net/http" + "net/http/httptest" "os" "path" "path/filepath" @@ -19,6 +22,7 @@ import ( "github.com/hashicorp/nomad-pack/sdk/pack" "github.com/hashicorp/nomad-pack/sdk/pack/variables" "github.com/hashicorp/nomad/ci" + vault "github.com/hashicorp/vault/api" "github.com/shoenig/test/must" "github.com/spf13/afero" "github.com/zclconf/go-cty/cty" @@ -39,6 +43,21 @@ func testpack(p ...string) *pack.Pack { } } +func testVaultClient(t *testing.T, handler http.HandlerFunc) *vault.Client { + t.Helper() + + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + cfg := vault.DefaultConfig() + cfg.Address = server.URL + + client, err := vault.NewClient(cfg) + must.NoError(t, err) + + return client +} + func TestParserV2_NewParserV2(t *testing.T) { t.Run("fails/with nil config set", func(t *testing.T) { p, err := NewParserV2(nil) @@ -70,6 +89,415 @@ func TestParserV2_NewParserV2(t *testing.T) { }) } +func TestParserV2_Parse_VaultVariableSource(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "us-east-1", + }, + }, + })) + }) + + rootVar := &variables.Variable{ + Name: "region", + Type: cty.String, + ConstraintType: cty.String, + Value: cty.StringVal(""), + DeclRange: hcl.Range{Filename: "variables.hcl"}, + } + + p := &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ + ParentPack: testpack("example"), + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": rootVar, + }, + }, + sourceOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + fileOverrideVars: make(variables.PackIDKeyedVarMap), + flagOverrideVars: make(variables.PackIDKeyedVarMap), + } + + diags := p.parseVariableSourceOverrides() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + + sourceVars := p.sourceOverrideVars["example"] + must.Len(t, 1, sourceVars) + + must.Eq(t, variables.ID("region"), sourceVars[0].Name) + must.Eq(t, cty.StringVal("us-east-1"), sourceVars[0].Value) +} + +func TestParserV2_Parse_EnvOverridesVaultVariableSource(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "vault-region", + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "region" { + type = string + default = "" +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + EnvOverrides: map[string]string{ + "NOMAD_PACK_VAR_region": "env-region", + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["region"] + must.NotNil(t, got) + must.Eq(t, cty.StringVal("env-region"), got.Value) +} + +func TestParserV2_Parse_FileOverridesVaultVariableSource(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "vault-region", + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "region" { + type = string + default = "" +} +`), + } + + tmpDir := t.TempDir() + overrideFilePath := path.Join(tmpDir, "region.override.hcl") + must.NoError(t, os.WriteFile(overrideFilePath, []byte(` +region = "file-region" +`), 0o644)) + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + FileOverrides: []string{overrideFilePath}, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["region"] + must.NotNil(t, got) + must.Eq(t, cty.StringVal("file-region"), got.Value) +} + +func TestParserV2_Parse_FlagOverridesVaultVariableSource(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "vault-region", + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "region" { + type = string + default = "" +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + FlagOverrides: map[string]string{ + "region": "flag-region", + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["region"] + must.NotNil(t, got) + must.Eq(t, cty.StringVal("flag-region"), got.Value) +} + +func TestParserV2_Parse_VaultVariableSource_IgnoresUndeclaredFields(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "region": "us-east-1", + "ignored": "value", + "ignored2": 123, + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "region" { + type = string + default = "" +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["region"] + must.NotNil(t, got) + must.Eq(t, cty.StringVal("us-east-1"), got.Value) +} + +func TestParserV2_Parse_VaultVariableSource_TypeMismatch(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "count": "not-a-number", + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "count" { + type = number + default = 1 +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.True(t, diags.HasErrors()) + must.Nil(t, parsed) +} + +func TestParserV2_Parse_VaultVariableSource_StringNumberToNumber(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "count": "3", + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "count" { + type = number + default = 1 +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["count"] + must.NotNil(t, got) + must.True(t, got.Value.RawEquals(cty.NumberIntVal(3))) +} + +func TestParserV2_Parse_VaultVariableSource_FloatNumberToNumber(t *testing.T) { + vaultClient := testVaultClient(t, func(w http.ResponseWriter, r *http.Request) { + must.Eq(t, "/v1/secret/data/myapp", r.URL.Path) + + w.Header().Set("Content-Type", "application/json") + must.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{ + "count": float64(3), + }, + }, + })) + }) + + rootVarFile := &pack.File{ + Name: "variables.hcl", + Path: "variables.hcl", + Content: []byte(` +variable "count" { + type = number + default = 1 +} +`), + } + + p, err := NewParserV2(&config.ParserConfig{ + ParentPack: testpack("example"), + RootVariableFiles: map[pack.ID]*pack.File{ + "example": rootVarFile, + }, + VaultClient: vaultClient, + VariableSource: &config.VariableSourceConfig{ + Type: "vault", + Vault: &config.VaultVariableSourceConfig{ + Path: "secret/data/myapp", + }, + }, + }) + must.NoError(t, err) + + parsed, diags := p.Parse() + must.False(t, diags.HasErrors(), must.Sprintf("unexpected diagnostics: %v", diags)) + must.NotNil(t, parsed) + + got := parsed.GetVars()["example"]["count"] + must.NotNil(t, got) + must.True(t, got.Value.RawEquals(cty.NumberIntVal(3))) +} + func TestParserV2_parseFlagVariable(t *testing.T) { testCases := []struct { inputParser *ParserV2