feat: Add vaultRead() and vaultKV() template functions to read secrets from HashiCorp Vault (GH-473)

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## UNRELEASED
 
 IMPROVEMENTS:
+* template: Add `vaultRead()` and `vaultKV()` functions to read secrets from HashiCorp Vault with automatic KV v1/v2 detection [[GH-473](https://github.com/hashicorp/nomad-pack/pull/848)]
 * cli: Improved error message when pack lacks `.nomad.tpl` files to clearly explain naming requirements, show template naming convention, and list found template files [[GH-831](https://github.com/hashicorp/nomad-pack/pull/831)]
 * cli: Add registry now honors default main/master branch [[GH-843](https://github.com/hashicorp/nomad-pack/pull/843)]
 * cli: Fixed stale-job reconciliation so nomad-pack run no longer stops active parameterized/periodic child jobs [[GH-855](https://github.com/hashicorp/nomad-pack/pull/855)] 

docs/functions.md

@@ -130,6 +130,89 @@ The `nomadVariable` function retrieves a specific Nomad Variable by path and nam
 password = "[[ .Items.password ]]"
 [[ end ]]
 
+### Vault functions
+
+#### `vaultRead` <a id="vaultRead"></a>
+
+The `vaultRead` function retrieves secrets from HashiCorp Vault with automatic detection of KV v1 and KV v2 secret engines. This function intelligently handles both versions and returns the secret data directly.
+
+**NOTE:** This function requires Vault to be configured via the `VAULT_ADDR` and `VAULT_TOKEN` environment variables. If Vault is not configured, the function will not be available in templates.
+
+##### Parameters
+
+- 1: `string` - The path to the secret in Vault
+
+##### Returns
+
+- `error` or `map[string]interface{}` - The secret data
+
+##### Example
+
+Read a secret from Vault KV v2:
+
+[[ with vaultRead "secret/data/database" ]]
+DB_PASSWORD = "[[ .password ]]"
+DB_USERNAME = "[[ .username ]]"
+[[ end ]]
+
+Use with fallback values:
+
+DB_PASSWORD = "[[ try (vaultRead "secret/data/db" | get "password") "default-password" ]]"
+
+#### `vaultKV` <a id="vaultKV"></a>
+
+The `vaultKV` function retrieves raw secret data from HashiCorp Vault without any automatic version detection or data extraction. This is useful for advanced use cases where you need access to the complete Vault response including metadata.
+
+**NOTE:** This function requires Vault to be configured via the `VAULT_ADDR` and `VAULT_TOKEN` environment variables. If Vault is not configured, the function will not be available in templates.
+
+##### Parameters
+
+- 1: `string` - The path to the secret in Vault
+
+##### Returns
+
+- `error` or `map[string]interface{}` - The raw Vault response
+
+##### Example
+
+Read raw secret data:
+
+[[ $secret := vaultKV "secret/data/app" ]]
+[[ range $key, $value := $secret ]]
+[[ $key ]]: [[ $value ]]
+[[ end ]]
+
+### Vault-backed variable source
+
+Nomad Pack can populate declared pack variables from Vault KV during command execution.
+
+Use the following flags:
+
+- `--var-source=vault`
+- `--vault-var-path=<path>`
+
+Example:
+
+```bash
+nomad-pack render \
+  --no-format=true \
+  --var-source=vault \
+  --vault-var-path=secret/data/myapp \
+  ./fixtures/v2/simple_raw_exec_v2
+```
+
+When using --var-source=vault, --vault-var-path is required.
+
+Currently only vault is supported as a variable source.
+
+Variable precedence is:
+
+1. external variable source
+2. environment variables
+3. variable files
+4. CLI --var
+This means CLI --var values override Vault-sourced values.
+
 ### Region functions
 
 #### `nomadRegions` <a id="nomadRegions"></a>
@@ -629,6 +712,8 @@ These are the additional functions supplied by Nomad Pack itself.
 - [`spewDump`][] - Returns a string representation of a value using `spew.Sdump`.
 - [`spewPrintf`][] - Returns a formatted string representation of a value using `spew.Sprintf`.
 - [`toStringList`][] - Converts a value to a string list.
+- [`vaultKV`][] - Retrieves raw secret data from HashiCorp Vault.
+- [`vaultRead`][] - Retrieves secrets from HashiCorp Vault with automatic KV version detection.
 - [`tpl`][] - Renders a template string using the current template context.
 - [`withContinueOnMethod`][] - Sets the `ContinueOnMethod` flag for a `customSpew`.
 - [`withDisableCapacities`][] - Sets the `DisableCapacities` flag for a `customSpew`.
@@ -670,3 +755,6 @@ These are the additional functions supplied by Nomad Pack itself.
 [`withContinueOnMethod`]: #withContinueOnMethod
 [`withSortKeys`]: #withSortKeys
 [`withSpewKeys`]: #withSpewKeys
+[`vaultKV`]: #vaultKV
+[`vaultRead`]: #vaultRead
+

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.23.0
 	github.com/hashicorp/nomad v1.11.3
 	github.com/hashicorp/nomad/api v0.0.0-20260304165455-489f8b9d1054
+	github.com/hashicorp/vault/api v1.22.0
 	github.com/kr/text v0.2.0
 	github.com/lab47/vterm v0.0.0-20211107042118-80c3d2849f9c
 	github.com/mattn/go-isatty v0.0.20
@@ -241,7 +242,6 @@ require (
 	github.com/hashicorp/raft-autopilot v0.3.0 // indirect
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1 // indirect
 	github.com/hashicorp/serf v0.10.2 // indirect
-	github.com/hashicorp/vault/api v1.22.0 // indirect
 	github.com/hashicorp/vault/api/auth/kubernetes v0.10.0 // indirect
 	github.com/hashicorp/vic v1.5.1-0.20241121050025-d1d58fa204f5 // indirect
 	github.com/hashicorp/yamux v0.1.2 // indirect

internal/cli/cli_test.go

@@ -8,6 +8,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"path/filepath"
@@ -18,6 +20,7 @@ import (
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/command/agent"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/mitchellh/cli"
 	"github.com/shoenig/test/must"
 	"github.com/shoenig/test/wait"
@@ -52,6 +55,36 @@ const (
 	testPlanCmdString = "plan --exit-code-no-changes=90 --exit-code-makes-changes=91 --exit-code-error=92"
 )
 
+func testVaultClientForCLI(t *testing.T, handler http.HandlerFunc) *vault.Client {
+	t.Helper()
+
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+
+	cfg := vault.DefaultConfig()
+	cfg.Address = server.URL
+
+	client, err := vault.NewClient(cfg)
+	must.NoError(t, err)
+
+	return client
+}
+
+func withEnv(t *testing.T, key, value string) {
+	t.Helper()
+
+	prev, ok := os.LookupEnv(key)
+	must.NoError(t, os.Setenv(key, value))
+
+	t.Cleanup(func() {
+		if ok {
+			_ = os.Setenv(key, prev)
+			return
+		}
+		_ = os.Unsetenv(key)
+	})
+}
+
 func TestCLI_CreateTestRegistry(t *testing.T) {
 	// This test is here to help setup the pack registry cache. It needs to be
 	// the first one in the file and can not be `Parallel()`
@@ -88,6 +121,71 @@ func TestCLI_Version(t *testing.T) {
 	must.Zero(t, exitCode)
 }
 
+func TestCLI_Render_VaultVariableSource(t *testing.T) {
+	t.Parallel()
+
+	vaultClient := testVaultClientForCLI(t, func(w http.ResponseWriter, r *http.Request) {
+		must.Eq(t, "/v1/secret/data/myapp", r.URL.Path)
+
+		w.Header().Set("Content-Type", "application/json")
+		must.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"data": map[string]any{
+				"data": map[string]any{
+					"region": "us-east-1",
+					"count":  float64(3),
+				},
+			},
+		}))
+	})
+
+	withEnv(t, "VAULT_ADDR", vaultClient.Address())
+
+	result := runPackCmd(t, []string{
+		"render",
+		"--no-format=true",
+		"--var-source=vault",
+		"--vault-var-path=secret/data/myapp",
+		testfixture.AbsPath(t, "v2/simple_raw_exec_v2"),
+	})
+
+	must.Zero(t, result.exitCode, must.Sprintf("stdout:\n%s\n\nstderr:\n%s\n", result.cmdOut.String(), result.cmdErr.String()))
+	must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String()))
+	must.StrContains(t, result.cmdOut.String(), `region = "us-east-1"`)
+	must.StrContains(t, result.cmdOut.String(), `count = 3`)
+}
+
+func TestCLI_Render_VaultVariableSource_MissingPath(t *testing.T) {
+	t.Parallel()
+
+	result := runPackCmd(t, []string{
+		"render",
+		"--no-format=true",
+		"--var-source=vault",
+		testfixture.AbsPath(t, "v2/simple_raw_exec_v2"),
+	})
+
+	must.Eq(t, 1, result.exitCode)
+	out := result.cmdOut.String()
+	if !strings.Contains(out, "vault variable source requires a non-empty path") && !strings.Contains(out, "a Vault-backed variable source was requested, but no Vault client is configured") {
+		t.Fatalf("expected missing-path or missing-client error, got:\n%s", out)
+	}
+}
+
+func TestCLI_Render_VariableSource_Unsupported(t *testing.T) {
+	t.Parallel()
+
+	result := runPackCmd(t, []string{
+		"render",
+		"--no-format=true",
+		"--var-source=consul",
+		testfixture.AbsPath(t, "v2/simple_raw_exec_v2"),
+	})
+
+	must.Eq(t, 1, result.exitCode)
+	must.StrContains(t, result.cmdOut.String(), `unsupported variable source type "consul"`)
+
+}
+
 func TestCLI_JobRun(t *testing.T) {
 	ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) {
 		expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)}))

internal/cli/commands.go

@@ -71,6 +71,12 @@ type baseCommand struct {
 	// for defined input variables
 	varFiles []string
 
+	// varSourceType configures an optional external variable source.
+	varSourceType string
+
+	// vaultVarPath is the Vault path used for external variable sourcing.
+	vaultVarPath string
+
 	// allowUnsetVars suppresses errors from variables with nil values,
 	// i.e. those that are not set and have no default
 	allowUnsetVars bool
@@ -292,6 +298,20 @@ func (c *baseCommand) flagSet(bit flagSetBit, f func(*flag.Sets)) *flag.Sets {
 					syntax and can be specified multiple times per command.`,
 		})
 
+		f.StringVar(&flag.StringVar{
+			Name:    "var-source",
+			Target:  &c.varSourceType,
+			Default: "",
+			Usage:   `Configures an external variable source. Currently only "vault" is supported.`,
+		})
+
+		f.StringVar(&flag.StringVar{
+			Name:    "vault-var-path",
+			Target:  &c.vaultVarPath,
+			Default: "",
+			Usage:   `Vault path used to populate declared pack variables from an external variable source.`,
+		})
+
 		f.StringVar(&flag.StringVar{
 			Name:    "name",
 			Target:  &c.deploymentName,

internal/cli/helpers.go

@@ -11,13 +11,15 @@ import (
 	"strings"
 
 	"github.com/hashicorp/nomad/api"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/posener/complete"
 
 	"github.com/hashicorp/nomad-pack/internal/pkg/caching"
 	"github.com/hashicorp/nomad-pack/internal/pkg/errors"
 	"github.com/hashicorp/nomad-pack/internal/pkg/manager"
 	"github.com/hashicorp/nomad-pack/internal/pkg/renderer"
 	"github.com/hashicorp/nomad-pack/internal/pkg/variable/parser"
+	parserconfig "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config"
 	"github.com/hashicorp/nomad-pack/internal/runner"
 	"github.com/hashicorp/nomad-pack/internal/runner/job"
 	"github.com/hashicorp/nomad-pack/sdk/pack/variables"
@@ -42,15 +44,43 @@ func initPackCommand(cfg *caching.PackConfig) (errorContext *errors.UIErrorConte
 // generatePackManager is used to generate the pack manager for this Nomad Pack run.
 func generatePackManager(c *baseCommand, client *api.Client, packCfg *caching.PackConfig) *manager.PackManager {
 	// TODO: Refactor to have manager use cache.
+
+	var variableSource *parserconfig.VariableSourceConfig
+
+	switch c.varSourceType {
+	case "":
+		// no external variable source configured
+	case "vault":
+		variableSource = &parserconfig.VariableSourceConfig{
+			Type: "vault",
+			Vault: &parserconfig.VaultVariableSourceConfig{
+				Path: c.vaultVarPath,
+			},
+		}
+	default:
+		variableSource = &parserconfig.VariableSourceConfig{
+			Type: c.varSourceType,
+		}
+	}
+
 	cfg := manager.Config{
 		Path:            packCfg.Path,
 		VariableFiles:   c.varFiles,
 		VariableCLIArgs: c.vars,
 		VariableEnvVars: c.envVars,
 		AllowUnsetVars:  c.allowUnsetVars,
 		UseParserV1:     c.useParserV1,
+		VariableSource:  variableSource,
 	}
-	return manager.NewPackManager(&cfg, client)
+
+	// Create Vault client if configured
+	vaultClient, err := getVaultClient()
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to create Vault client")
+		// Continue without Vault; it's optional
+		vaultClient = nil
+	}
+	return manager.NewPackManager(&cfg, client, vaultClient)
 }
 
 // predictPackName is a complete.Predictor that suggests cached pack names.
@@ -686,6 +716,37 @@ func limit(s string, length int) string {
 	return s[:length]
 }
 
+// getVaultClient creates a Vault client from environment variables.
+// Returns nil client if Vault is not configured (VAULT_ADDR not set).
+// This allows Vault integration to be optional - if VAULT_ADDR is not set,
+// the pack will work normally but Vault template functions won't be available.
+func getVaultClient() (*vault.Client, error) {
+	// check if Vault is configured via VAULT_ADDR environment variable
+	vaultAddr := os.Getenv("VAULT_ADDR")
+	if vaultAddr == "" {
+		// Vault not configured - this is OK, return nil without error
+		return nil, nil
+	}
+
+	//Create default Vault configuration with sensible defaults
+	config := vault.DefaultConfig()
+	config.Address = vaultAddr
+
+	//create Vault client
+	client, err := vault.NewClient(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Vault client: %w", err)
+	}
+
+	//set authentication token if provided via VAULT_TOKEN environment variable
+	vaultToken := os.Getenv("VAULT_TOKEN")
+	if vaultToken != "" {
+		client.SetToken(vaultToken)
+	}
+
+	return client, nil
+}
+
 // addNoParentTemplatesContext adds error details for missing parent templates
 // to an existing error context. It lists any .tpl files discovered and provides
 // naming guidance.