[Issue #473] Add Consul KV variable source

fixtures/v2/simple_raw_exec_v2/variables.hcl

@@ -46,15 +46,15 @@ nomad_variable "app_config" {
   path = "nomad/jobs/simple_raw_exec/config"
   items = {
     database_url = "postgres://localhost:5432/mydb"
-    api_key = "secret-api-key-123"
-    environment = "production"
+    api_key      = "secret-api-key-123"
+    environment  = "production"
   }
 }
 
 nomad_variable "secrets" {
   path = "nomad/jobs/simple_raw_exec/secrets"
   items = {
     admin_password = "super-secret-password"
-    jwt_secret = "jwt-signing-key-xyz"
+    jwt_secret     = "jwt-signing-key-xyz"
   }
 }

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/fatih/color v1.19.0
 	github.com/go-git/go-git/v5 v5.19.1
+	github.com/hashicorp/consul/api v1.33.4
 	github.com/hashicorp/go-getter v1.8.6
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-multierror v1.1.1
@@ -192,7 +193,6 @@ require (
 	github.com/hashicorp/cap v0.12.0 // indirect
 	github.com/hashicorp/cli v1.1.7 // indirect
 	github.com/hashicorp/consul-template v0.41.4 // indirect
-	github.com/hashicorp/consul/api v1.33.4 // indirect
 	github.com/hashicorp/consul/sdk v0.17.2 // indirect
 	github.com/hashicorp/cronexpr v1.1.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect

internal/cli/commands.go

@@ -71,6 +71,10 @@ type baseCommand struct {
 	// for defined input variables
 	varFiles []string
 
+	// varSources is a list of external variable source URLs
+	// (e.g., consul:///path)
+	varSources []string
+
 	// allowUnsetVars suppresses errors from variables with nil values,
 	// i.e. those that are not set and have no default
 	allowUnsetVars bool
@@ -267,6 +271,31 @@ func (c *baseCommand) flagSet(bit flagSetBit, f func(*flag.Sets)) *flag.Sets {
 			Shorthand: "f",
 		})
 
+		// --var-source performs remote reads (e.g. Consul KV) at render time, so
+		// it is only offered by commands that compute a fresh deployment
+		// (run/plan/render). Commands like stop operate on an already-deployed
+		// job and must not depend on a remote source still being reachable or
+		// its keys still existing.
+		if bit&flagSetExternalVarSources != 0 {
+			f.StringSliceVar(&flag.StringSliceVar{
+				Name:    "var-source",
+				Target:  &c.varSources,
+				Default: make([]string, 0),
+				Usage: `Specifies an external variable source as a URL, and may be
+						given more than once to read from several sources. Consul KV
+						is the only source currently supported, using the form
+						consul://<host>:<port>/<path>; for example,
+						consul://localhost:8500/nomad-pack. The host is optional: omit
+						it (consul:///<path>) to use the standard Consul environment
+						configuration, such as CONSUL_HTTP_ADDR and CONSUL_HTTP_TOKEN.
+						Each variable is read from <path>/<variable-name>, so include
+						any per-pack grouping in the path yourself. Variable sources
+						are applied in order of precedence, highest first: --var, then
+						--var-source, then --var-file, then environment variables. A
+						higher-precedence source overrides any lower one.`,
+			})
+		}
+
 		f.BoolVar(&flag.BoolVar{
 			Name:    "allow-unset-vars",
 			Target:  &c.allowUnsetVars,
@@ -435,10 +464,11 @@ func (c *baseCommand) helpUsageMessage() string {
 type flagSetBit uint
 
 const (
-	flagSetNone          flagSetBit = 1 << iota // nolint:deadcode,varcheck,unused
-	flagSetOperation                            // shared flags for operations (run, plan, etc)
-	flagSetNeedsApproval                        // adds the -y flag for commands that require approval to run
-	flagSetNomadClient                          // adds client config flags
+	flagSetNone               flagSetBit = 1 << iota // nolint:deadcode,varcheck,unused
+	flagSetOperation                                 // shared flags for operations (run, plan, etc)
+	flagSetNeedsApproval                             // adds the -y flag for commands that require approval to run
+	flagSetNomadClient                               // adds client config flags
+	flagSetExternalVarSources                        // adds --var-source; only for commands that compute a fresh deployment (run, plan, render)
 )
 
 var (

internal/cli/generate_varfile.go

@@ -142,7 +142,11 @@ func (c *generateVarFileCommand) Run(args []string) int {
 	// until something sets them, like the var file we're trying to create!)
 	c.allowUnsetVars = true
 
-	packManager := generatePackManager(c.baseCommand, nil, c.packConfig)
+	packManager, err := generatePackManager(c.baseCommand, nil, c.packConfig)
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to generate pack manager", errorContext.GetAll()...)
+		return 1
+	}
 	renderOutput, err := renderVariableOverrideFile(packManager, c.ui, errorContext)
 	if err != nil {
 		return 1

internal/cli/helpers.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/nomad-pack/internal/pkg/manager"
 	"github.com/hashicorp/nomad-pack/internal/pkg/renderer"
 	"github.com/hashicorp/nomad-pack/internal/pkg/variable/parser"
+	"github.com/hashicorp/nomad-pack/internal/pkg/variable/source"
 	"github.com/hashicorp/nomad-pack/internal/runner"
 	"github.com/hashicorp/nomad-pack/internal/runner/job"
 	"github.com/hashicorp/nomad-pack/sdk/pack/variables"
@@ -40,17 +41,28 @@ func initPackCommand(cfg *caching.PackConfig) (errorContext *errors.UIErrorConte
 }
 
 // generatePackManager is used to generate the pack manager for this Nomad Pack run.
-func generatePackManager(c *baseCommand, client *api.Client, packCfg *caching.PackConfig) *manager.PackManager {
+func generatePackManager(c *baseCommand, client *api.Client, packCfg *caching.PackConfig) (*manager.PackManager, error) {
+	// Parse external variable source configurations if provided.
+	var externalSourceConfigs []source.SourceConfig
+	if len(c.varSources) > 0 {
+		configs, err := parseVarSourceConfigs(c.varSources)
+		if err != nil {
+			return nil, err
+		}
+		externalSourceConfigs = configs
+	}
+
 	// TODO: Refactor to have manager use cache.
 	cfg := manager.Config{
-		Path:            packCfg.Path,
-		VariableFiles:   c.varFiles,
-		VariableCLIArgs: c.vars,
-		VariableEnvVars: c.envVars,
-		AllowUnsetVars:  c.allowUnsetVars,
-		UseParserV1:     c.useParserV1,
-	}
-	return manager.NewPackManager(&cfg, client)
+		Path:                  packCfg.Path,
+		VariableFiles:         c.varFiles,
+		VariableCLIArgs:       c.vars,
+		VariableEnvVars:       c.envVars,
+		AllowUnsetVars:        c.allowUnsetVars,
+		UseParserV1:           c.useParserV1,
+		ExternalSourceConfigs: externalSourceConfigs,
+	}
+	return manager.NewPackManager(&cfg, client), nil
 }
 
 // predictPackName is a complete.Predictor that suggests cached pack names.

internal/cli/plan.go

@@ -54,7 +54,11 @@ func (c *PlanCommand) Run(args []string) int {
 		return c.exitCodeError
 	}
 
-	packManager := generatePackManager(c.baseCommand, client, c.packConfig)
+	packManager, err := generatePackManager(c.baseCommand, client, c.packConfig)
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to generate pack manager", errorContext.GetAll()...)
+		return c.exitCodeError
+	}
 
 	// load pack
 	r, err := renderPack(
@@ -142,7 +146,7 @@ func (c *PlanCommand) Run(args []string) int {
 func (c *PlanCommand) Flags() *flag.Sets {
 	c.packConfig = &caching.PackConfig{}
 
-	return c.flagSet(flagSetOperation|flagSetNomadClient, func(set *flag.Sets) {
+	return c.flagSet(flagSetOperation|flagSetNomadClient|flagSetExternalVarSources, func(set *flag.Sets) {
 		f := set.NewSet("Plan Options")
 
 		c.jobConfig = &job.CLIConfig{

internal/cli/render.go

@@ -280,7 +280,11 @@ func (c *RenderCommand) Run(args []string) int {
 		c.ui.Error(err.Error())
 		return 1
 	}
-	packManager := generatePackManager(c.baseCommand, client, c.packConfig)
+	packManager, err := generatePackManager(c.baseCommand, client, c.packConfig)
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to generate pack manager", errorContext.GetAll()...)
+		return 1
+	}
 
 	renderOutput, err := renderPack(
 		packManager,
@@ -344,7 +348,7 @@ func (c *RenderCommand) Run(args []string) int {
 }
 
 func (c *RenderCommand) Flags() *flag.Sets {
-	return c.flagSet(flagSetOperation|flagSetNeedsApproval, func(set *flag.Sets) {
+	return c.flagSet(flagSetOperation|flagSetNeedsApproval|flagSetExternalVarSources, func(set *flag.Sets) {
 		c.packConfig = &caching.PackConfig{}
 
 		f := set.NewSet("Render Options")

internal/cli/run.go

@@ -64,7 +64,11 @@ func (c *RunCommand) run() int {
 		return 1
 	}
 
-	packManager := generatePackManager(c.baseCommand, client, c.packConfig)
+	packManager, err := generatePackManager(c.baseCommand, client, c.packConfig)
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to generate pack manager", errorContext.GetAll()...)
+		return 1
+	}
 
 	// Render the pack now, before creating the deployer. If we get an error
 	// we won't make it to the deployer.
@@ -191,7 +195,7 @@ func (c *RunCommand) run() int {
 
 // Flags defines the flag.Sets for the operation.
 func (c *RunCommand) Flags() *flag.Sets {
-	return c.flagSet(flagSetOperation|flagSetNomadClient, func(set *flag.Sets) {
+	return c.flagSet(flagSetOperation|flagSetNomadClient|flagSetExternalVarSources, func(set *flag.Sets) {
 		f := set.NewSet("Run Options")
 
 		c.packConfig = &caching.PackConfig{}

internal/cli/stop.go

@@ -81,7 +81,11 @@ func (c *StopCommand) Run(args []string) int {
 
 	var jobs []*api.Job
 
-	packManager := generatePackManager(c.baseCommand, client, c.packConfig)
+	packManager, err := generatePackManager(c.baseCommand, client, c.packConfig)
+	if err != nil {
+		c.ui.ErrorWithContext(err, "failed to generate pack manager", errorContext.GetAll()...)
+		return 1
+	}
 
 	var r *renderer.Rendered
 

internal/cli/varsource_config.go

@@ -0,0 +1,76 @@
+// Copyright IBM Corp. 2023, 2026
+// SPDX-License-Identifier: MPL-2.0
+
+package cli
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/hashicorp/nomad-pack/internal/pkg/variable/source"
+)
+
+// parseVarSourceConfigs parses variable source URLs into typed source configs.
+// Only the configuration is parsed here; no remote connections are made. The
+// returned configs are built into live sources lazily, at render time, by the
+// variable parser.
+//
+// Supported URL formats:
+//   - consul:///path              (uses the Consul environment address)
+//   - consul://host:port/path     (uses the specified Consul address)
+func parseVarSourceConfigs(urls []string) ([]source.SourceConfig, error) {
+	if len(urls) == 0 {
+		return nil, nil
+	}
+
+	configs := make([]source.SourceConfig, 0, len(urls))
+
+	for _, urlStr := range urls {
+		cfg, err := parseVarSourceConfig(urlStr)
+		if err != nil {
+			return nil, fmt.Errorf("var-source %q: %w", urlStr, err)
+		}
+		configs = append(configs, cfg)
+	}
+
+	return configs, nil
+}
+
+// parseVarSourceConfig parses a single variable source URL into a typed config.
+func parseVarSourceConfig(urlStr string) (source.SourceConfig, error) {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, fmt.Errorf("invalid URL: %w", err)
+	}
+
+	host := u.Host
+	path := strings.Trim(u.Path, "/")
+
+	switch u.Scheme {
+	case "consul":
+		return parseConsulSourceConfig(host, path)
+	default:
+		return nil, fmt.Errorf("unsupported scheme %q (supported: consul)", u.Scheme)
+	}
+}
+
+// parseConsulSourceConfig builds a Consul KV source config from the host and
+// path of a consul:// URL. Each variable is read from <path>/<variable-name>.
+// A non-empty host overrides the Consul environment address; the rest of the
+// Consul configuration, including the ACL token, comes from the standard Consul
+// environment configuration (CONSUL_HTTP_ADDR, CONSUL_HTTP_TOKEN, and so on)
+// when the source is built.
+//   - consul:///path/to/vars        -> host="",              path="path/to/vars"
+//   - consul://localhost:8500/path  -> host="localhost:8500", path="path"
+func parseConsulSourceConfig(host, path string) (source.SourceConfig, error) {
+	if path == "" {
+		return nil, fmt.Errorf("consul URL must include a path (e.g., consul:///nomad-pack)")
+	}
+
+	return source.ConsulSourceConfig{
+		Priority: source.PriorityConsul,
+		Address:  host,
+		Path:     path,
+	}, nil
+}

internal/pkg/manager/manager.go

@@ -14,19 +14,21 @@ import (
 	"github.com/hashicorp/nomad-pack/internal/pkg/renderer"
 	"github.com/hashicorp/nomad-pack/internal/pkg/variable/parser"
 	"github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config"
+	"github.com/hashicorp/nomad-pack/internal/pkg/variable/source"
 	"github.com/hashicorp/nomad-pack/sdk/pack"
 	"github.com/hashicorp/nomad/api"
 )
 
 // Config contains all the user specified parameters needed to correctly run
 // the pack manager.
 type Config struct {
-	Path            string
-	VariableFiles   []string
-	VariableCLIArgs map[string]string
-	VariableEnvVars map[string]string
-	UseParserV1     bool
-	AllowUnsetVars  bool
+	Path                  string
+	VariableFiles         []string
+	VariableCLIArgs       map[string]string
+	VariableEnvVars       map[string]string
+	UseParserV1           bool
+	AllowUnsetVars        bool
+	ExternalSourceConfigs []source.SourceConfig // Lazily-built configs for external sources (Consul, Vault, Nomad)
 }
 
 // PackManager is responsible for loading, parsing, and rendering a Pack and
@@ -67,12 +69,13 @@ func (pm *PackManager) ProcessVariableFiles() (*parser.ParsedVariables, []*error
 	parentName, _, _ := strings.Cut(path.Base(pm.cfg.Path), "@")
 
 	pCfg := &config.ParserConfig{
-		Version:           config.V2,
-		ParentPack:        pm.loadedPack,
-		RootVariableFiles: loadedPack.RootVariableFiles(),
-		EnvOverrides:      pm.cfg.VariableEnvVars,
-		FileOverrides:     pm.cfg.VariableFiles,
-		FlagOverrides:     pm.cfg.VariableCLIArgs,
+		Version:               config.V2,
+		ParentPack:            pm.loadedPack,
+		RootVariableFiles:     loadedPack.RootVariableFiles(),
+		EnvOverrides:          pm.cfg.VariableEnvVars,
+		FileOverrides:         pm.cfg.VariableFiles,
+		FlagOverrides:         pm.cfg.VariableCLIArgs,
+		ExternalSourceConfigs: pm.cfg.ExternalSourceConfigs,
 	}
 
 	if pm.cfg.UseParserV1 {

internal/pkg/variable/parser/config/config.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"github.com/hashicorp/nomad-pack/internal/pkg/variable/source"
 	"github.com/hashicorp/nomad-pack/sdk/pack"
 )
 
@@ -47,6 +48,10 @@ type ParserConfig struct {
 	// all sources. If the same key is supplied twice, the last wins.
 	FlagOverrides map[string]string
 
+	// ExternalSourceConfigs are parsed, lazily-built configurations for
+	// external variable sources (Consul, Vault, Nomad).
+	ExternalSourceConfigs []source.SourceConfig
+
 	// IgnoreMissingVars determines whether we error or not on variable overrides
 	// that don't have corresponding vars in the pack.
 	IgnoreMissingVars bool