config: parsing support for multiple Vault clusters in agent config (#18224)

Add the plumbing we need to accept multiple Vault clusters in Nomad agent
configuration, to support upcoming Nomad Enterprise features. The `vault` blocks
are differentiated by a new `name` field, and if the `name` is omitted it
becomes the "default" Vault configuration. All blocks with the same name are
merged together, as with the existing behavior.

Unfortunately we're still using HCL1 for parsing configuration and the `Decode`
method doesn't parse multiple blocks differentiated only by a field name without
a label. So we've had to add an extra parsing pass, similar to what we've done
for HCL1 jobspecs.

For now, all existing consumers will use the "default" Vault configuration, so
there's no user-facing behavior change in this changeset other than the contents
of the agent self API.

Ref: https://github.com/hashicorp/team-nomad/issues/404

command/agent/agent.go

@@ -507,7 +507,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 
 	// Add the Consul and Vault configs
 	conf.ConsulConfig = agentConfig.Consul
+
 	conf.VaultConfig = agentConfig.Vault
+	for _, vaultConfig := range agentConfig.Vaults {
+		conf.VaultConfigs[vaultConfig.Name] = vaultConfig
+	}
 
 	// Set the TLS config
 	conf.TLSConfig = agentConfig.TLSConfig
@@ -799,7 +803,11 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	}
 
 	conf.ConsulConfig = agentConfig.Consul
+
 	conf.VaultConfig = agentConfig.Vault
+	for _, vaultConfig := range agentConfig.Vaults {
+		conf.VaultConfigs[vaultConfig.Name] = vaultConfig
+	}
 
 	// Set up Telemetry configuration
 	conf.StatsCollectionInterval = agentConfig.Telemetry.collectionInterval

command/agent/agent_endpoint.go

@@ -89,6 +89,11 @@ func (s *HTTPServer) AgentSelfRequest(resp http.ResponseWriter, req *http.Reques
 	if self.Config != nil && self.Config.Vault != nil && self.Config.Vault.Token != "" {
 		self.Config.Vault.Token = "<redacted>"
 	}
+	for _, vaultConfig := range self.Config.Vaults {
+		if vaultConfig.Token != "" {
+			vaultConfig.Token = "<redacted>"
+		}
+	}
 
 	if self.Config != nil && self.Config.ACL != nil && self.Config.ACL.ReplicationToken != "" {
 		self.Config.ACL.ReplicationToken = "<redacted>"

command/agent/command.go

@@ -77,6 +77,7 @@ func (c *Command) readConfig() *Config {
 		ACL:   &ACLConfig{},
 		Audit: &config.AuditConfig{},
 	}
+	cmdConfig.Vaults = map[string]*config.VaultConfig{"default": cmdConfig.Vault}
 
 	flags := flag.NewFlagSet("agent", flag.ContinueOnError)
 	flags.Usage = func() { c.Ui.Error(c.Help()) }

command/agent/config.go

@@ -136,9 +136,17 @@ type Config struct {
 	// discover the current Nomad servers.
 	Consul *config.ConsulConfig `hcl:"consul"`
 
-	// Vault contains the configuration for the Vault Agent and
+	// Vault contains the configuration for the default Vault Agent and
 	// parameters necessary to derive tokens.
-	Vault *config.VaultConfig `hcl:"vault"`
+	//
+	// TODO(tgross): we'll probably want to remove this field once we've added a
+	// selector so that we don't have to maintain it
+	Vault *config.VaultConfig
+
+	// Vaults is a map derived from multiple `vault` blocks, here to support
+	// features in Nomad Enterprise. The default Vault config pointer above will
+	// be found in this map under the name "default"
+	Vaults map[string]*config.VaultConfig
 
 	// UI is used to configure the web UI
 	UI *config.UIConfig `hcl:"ui"`
@@ -1264,7 +1272,7 @@ func DevConfig(mode *devModeConfig) *Config {
 
 // DefaultConfig is the baseline configuration for Nomad.
 func DefaultConfig() *Config {
-	return &Config{
+	cfg := &Config{
 		LogLevel:   "INFO",
 		Region:     "global",
 		Datacenter: "dc1",
@@ -1355,6 +1363,9 @@ func DefaultConfig() *Config {
 		DisableUpdateCheck: pointer.Of(false),
 		Limits:             config.DefaultLimits(),
 	}
+
+	cfg.Vaults = map[string]*config.VaultConfig{"default": cfg.Vault}
+	return cfg
 }
 
 // Listener can be used to get a new listener using a custom bind address.
@@ -1522,13 +1533,9 @@ func (c *Config) Merge(b *Config) *Config {
 		result.Consul = result.Consul.Merge(b.Consul)
 	}
 
-	// Apply the Vault Configuration
-	if result.Vault == nil && b.Vault != nil {
-		vaultConfig := *b.Vault
-		result.Vault = &vaultConfig
-	} else if b.Vault != nil {
-		result.Vault = result.Vault.Merge(b.Vault)
-	}
+	// Apply the Vault Configurations and overwrite the default Vault config
+	result.Vaults = mergeVaultConfigs(result.Vaults, b.Vaults)
+	result.Vault = result.Vaults["default"]
 
 	// Apply the UI Configuration
 	if result.UI == nil && b.UI != nil {
@@ -1579,6 +1586,21 @@ func (c *Config) Merge(b *Config) *Config {
 	return &result
 }
 
+func mergeVaultConfigs(left, right map[string]*config.VaultConfig) map[string]*config.VaultConfig {
+	merged := helper.DeepCopyMap(left)
+	if left == nil {
+		merged = map[string]*config.VaultConfig{}
+	}
+	for name, rConfig := range right {
+		if lConfig, ok := left[name]; ok {
+			merged[name] = lConfig.Merge(rConfig)
+		} else {
+			merged[name] = rConfig.Copy()
+		}
+	}
+	return merged
+}
+
 // Copy returns a deep copy safe for mutation.
 func (c *Config) Copy() *Config {
 	if c == nil {
@@ -1597,6 +1619,7 @@ func (c *Config) Copy() *Config {
 	nc.DisableUpdateCheck = pointer.Copy(c.DisableUpdateCheck)
 	nc.Consul = c.Consul.Copy()
 	nc.Vault = c.Vault.Copy()
+	nc.Vaults = helper.DeepCopyMap(c.Vaults)
 	nc.UI = c.UI.Copy()
 
 	nc.NomadConfig = c.NomadConfig.Copy()

command/agent/config_parse.go

@@ -12,9 +12,11 @@ import (
 	"time"
 
 	"github.com/hashicorp/hcl"
+	"github.com/hashicorp/hcl/hcl/ast"
 	client "github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/mitchellh/mapstructure"
 )
 
 // ParseConfigFile returns an agent.Config from parsed from a file.
@@ -57,6 +59,7 @@ func ParseConfigFile(path string) (*Config, error) {
 		Autopilot: &config.AutopilotConfig{},
 		Telemetry: &Telemetry{},
 		Vault:     &config.VaultConfig{},
+		Vaults:    map[string]*config.VaultConfig{},
 	}
 
 	err = hcl.Decode(c, buf.String())
@@ -156,6 +159,23 @@ func ParseConfigFile(path string) (*Config, error) {
 		return nil, err
 	}
 
+	// Re-parse the file to extract the multiple Vault configurations, which we
+	// need to parse by hand because we don't have a label on the block
+	root, err := hcl.Parse(buf.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse HCL file %s: %w", path, err)
+	}
+	list, ok := root.Node.(*ast.ObjectList)
+	if !ok {
+		return nil, fmt.Errorf("error parsing: root should be an object")
+	}
+	matches := list.Filter("vault")
+	if len(matches.Items) > 0 {
+		if err := parseVaults(c, matches); err != nil {
+			return nil, fmt.Errorf("error parsing 'vault': %w", err)
+		}
+	}
+
 	// report unexpected keys
 	err = extraKeys(c)
 	if err != nil {
@@ -261,6 +281,10 @@ func extraKeys(c *Config) error {
 		helper.RemoveEqualFold(&c.ExtraKeysHCL, "telemetry")
 	}
 
+	// The `vault` blocks are parsed separately from the Decode method, so it
+	// will incorrectly report them as extra keys
+	helper.RemoveEqualFold(&c.ExtraKeysHCL, "vault")
+
 	return helper.UnusedKeys(c)
 }
 
@@ -293,3 +317,35 @@ func finalizeClientTemplateConfig(config *Config) {
 		config.Client.TemplateConfig = nil
 	}
 }
+
+// parseVaults decodes the `vault` blocks. The hcl.Decode method can't parse
+// these correctly as HCL1 because they don't have labels, which would result in
+// all the blocks getting merged regardless of name.
+func parseVaults(c *Config, list *ast.ObjectList) error {
+	if len(list.Items) == 0 {
+		return nil
+	}
+
+	for _, obj := range list.Items {
+		var m map[string]interface{}
+		if err := hcl.DecodeObject(&m, obj.Val); err != nil {
+			return err
+		}
+		v := &config.VaultConfig{}
+		err := mapstructure.WeakDecode(m, v)
+		if err != nil {
+			return err
+		}
+		if v.Name == "" {
+			v.Name = "default"
+		}
+		if exist, ok := c.Vaults[v.Name]; ok {
+			c.Vaults[v.Name] = exist.Merge(v)
+		} else {
+			c.Vaults[v.Name] = v
+		}
+	}
+
+	c.Vault = c.Vaults["default"]
+	return nil
+}

command/agent/config_parse_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 )
 
@@ -235,6 +236,7 @@ var basicConfig = &Config{
 		TimeoutHCL:           "5s",
 	},
 	Vault: &config.VaultConfig{
+		Name:                 "default",
 		Addr:                 "127.0.0.1:9500",
 		AllowUnauthenticated: &trueValue,
 		ConnectionRetryIntv:  config.DefaultVaultConnectRetryIntv,
@@ -249,6 +251,24 @@ var basicConfig = &Config{
 		TaskTokenTTL:         "1s",
 		Token:                "12345",
 	},
+	Vaults: map[string]*config.VaultConfig{
+		"default": {
+			Name:                 "default",
+			Addr:                 "127.0.0.1:9500",
+			AllowUnauthenticated: &trueValue,
+			ConnectionRetryIntv:  config.DefaultVaultConnectRetryIntv,
+			Enabled:              &falseValue,
+			Role:                 "test_role",
+			TLSCaFile:            "/path/to/ca/file",
+			TLSCaPath:            "/path/to/ca",
+			TLSCertFile:          "/path/to/cert/file",
+			TLSKeyFile:           "/path/to/key/file",
+			TLSServerName:        "foobar",
+			TLSSkipVerify:        &trueValue,
+			TaskTokenTTL:         "1s",
+			Token:                "12345",
+		},
+	},
 	TLSConfig: &config.TLSConfig{
 		EnableHTTP:                  true,
 		EnableRPC:                   true,
@@ -360,6 +380,7 @@ var pluginConfig = &Config{
 	DisableAnonymousSignature: false,
 	Consul:                    nil,
 	Vault:                     nil,
+	Vaults:                    map[string]*config.VaultConfig{},
 	TLSConfig:                 nil,
 	HTTPAPIResponseHeaders:    map[string]string{},
 	Sentinel:                  nil,
@@ -512,6 +533,7 @@ func TestConfig_Parse(t *testing.T) {
 			oldDefault := &Config{
 				Consul:    config.DefaultConsulConfig(),
 				Vault:     config.DefaultVaultConfig(),
+				Vaults:    map[string]*config.VaultConfig{"default": config.DefaultVaultConfig()},
 				Autopilot: config.DefaultAutopilotConfig(),
 			}
 			actual = oldDefault.Merge(actual)
@@ -552,6 +574,7 @@ func (c *Config) addDefaults() {
 	}
 	if c.Vault == nil {
 		c.Vault = config.DefaultVaultConfig()
+		c.Vaults = map[string]*config.VaultConfig{"default": c.Vault}
 	}
 	if c.Telemetry == nil {
 		c.Telemetry = &Telemetry{}
@@ -694,10 +717,20 @@ var sample0 = &Config{
 		ClientAutoJoin: pointer.Of(false),
 	},
 	Vault: &config.VaultConfig{
+		Name:    "default",
 		Enabled: pointer.Of(true),
 		Role:    "nomad-cluster",
 		Addr:    "http://host.example.com:8200",
 	},
+	Vaults: map[string]*config.VaultConfig{
+		"default": {
+			Name:    "default",
+			Enabled: pointer.Of(true),
+			Role:    "nomad-cluster",
+			Addr:    "http://host.example.com:8200",
+		},
+	},
+
 	TLSConfig: &config.TLSConfig{
 		EnableHTTP:           true,
 		EnableRPC:            true,
@@ -790,10 +823,19 @@ var sample1 = &Config{
 		ClientAutoJoin: pointer.Of(false),
 	},
 	Vault: &config.VaultConfig{
+		Name:    "default",
 		Enabled: pointer.Of(true),
 		Role:    "nomad-cluster",
 		Addr:    "http://host.example.com:8200",
 	},
+	Vaults: map[string]*config.VaultConfig{
+		"default": {
+			Name:    "default",
+			Enabled: pointer.Of(true),
+			Role:    "nomad-cluster",
+			Addr:    "http://host.example.com:8200",
+		},
+	},
 	TLSConfig: &config.TLSConfig{
 		EnableHTTP:           true,
 		EnableRPC:            true,
@@ -904,3 +946,51 @@ func permutations(arr []string) [][]string {
 	helper(arr, len(arr))
 	return res
 }
+
+func TestConfig_MultipleVault(t *testing.T) {
+
+	// verify the default Vault config is set from the list
+	cfg := DefaultConfig()
+	must.Eq(t, "default", cfg.Vault.Name)
+	must.Equal(t, config.DefaultVaultConfig(), cfg.Vault)
+	must.Nil(t, cfg.Vault.Enabled) // unset
+	must.Eq(t, "https://vault.service.consul:8200", cfg.Vault.Addr)
+	must.Eq(t, "", cfg.Vault.Token)
+
+	must.MapLen(t, 1, cfg.Vaults)
+	must.Equal(t, cfg.Vault, cfg.Vaults["default"])
+	must.True(t, cfg.Vault == cfg.Vaults["default"]) // must be same pointer
+
+	// merge in the user's configuration
+	fc, err := LoadConfig("testdata/basic.hcl")
+	must.NoError(t, err)
+	cfg = cfg.Merge(fc)
+
+	must.Eq(t, "default", cfg.Vault.Name)
+	must.NotNil(t, cfg.Vault.Enabled, must.Sprint("override should set to non-nil"))
+	must.False(t, *cfg.Vault.Enabled)
+	must.Eq(t, "127.0.0.1:9500", cfg.Vault.Addr)
+	must.Eq(t, "12345", cfg.Vault.Token)
+
+	must.MapLen(t, 1, cfg.Vaults)
+	must.Equal(t, cfg.Vault, cfg.Vaults["default"])
+
+	// add an extra Vault config and override fields in the default
+	fc, err = LoadConfig("testdata/extra-vault.hcl")
+	must.NoError(t, err)
+
+	cfg = cfg.Merge(fc)
+
+	must.Eq(t, "default", cfg.Vault.Name)
+	must.True(t, *cfg.Vault.Enabled)
+	must.Eq(t, "127.0.0.1:9500", cfg.Vault.Addr)
+	must.Eq(t, "abracadabra", cfg.Vault.Token)
+
+	must.MapLen(t, 2, cfg.Vaults)
+	must.Equal(t, cfg.Vault, cfg.Vaults["default"])
+
+	must.Eq(t, "alternate", cfg.Vaults["alternate"].Name)
+	must.True(t, *cfg.Vaults["alternate"].Enabled)
+	must.Eq(t, "127.0.0.1:9501", cfg.Vaults["alternate"].Addr)
+	must.Eq(t, "xyzzy", cfg.Vaults["alternate"].Token)
+}

command/agent/config_test.go

@@ -184,6 +184,7 @@ func TestConfig_Merge(t *testing.T) {
 			"Access-Control-Allow-Origin": "*",
 		},
 		Vault: &config.VaultConfig{
+			Name:                 "default",
 			Token:                "1",
 			AllowUnauthenticated: &falseValue,
 			TaskTokenTTL:         "1",
@@ -195,6 +196,21 @@ func TestConfig_Merge(t *testing.T) {
 			TLSSkipVerify:        &falseValue,
 			TLSServerName:        "1",
 		},
+		Vaults: map[string]*config.VaultConfig{
+			"default": {
+				Name:                 "default",
+				Token:                "1",
+				AllowUnauthenticated: &falseValue,
+				TaskTokenTTL:         "1",
+				Addr:                 "1",
+				TLSCaFile:            "1",
+				TLSCaPath:            "1",
+				TLSCertFile:          "1",
+				TLSKeyFile:           "1",
+				TLSSkipVerify:        &falseValue,
+				TLSServerName:        "1",
+			},
+		},
 		Consul: &config.ConsulConfig{
 			ServerServiceName:    "1",
 			ClientServiceName:    "1",
@@ -393,6 +409,7 @@ func TestConfig_Merge(t *testing.T) {
 			"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
 		},
 		Vault: &config.VaultConfig{
+			Name:                 "default",
 			Token:                "2",
 			AllowUnauthenticated: &trueValue,
 			TaskTokenTTL:         "2",
@@ -404,6 +421,21 @@ func TestConfig_Merge(t *testing.T) {
 			TLSSkipVerify:        &trueValue,
 			TLSServerName:        "2",
 		},
+		Vaults: map[string]*config.VaultConfig{
+			"default": {
+				Name:                 "default",
+				Token:                "2",
+				AllowUnauthenticated: &trueValue,
+				TaskTokenTTL:         "2",
+				Addr:                 "2",
+				TLSCaFile:            "2",
+				TLSCaPath:            "2",
+				TLSCertFile:          "2",
+				TLSKeyFile:           "2",
+				TLSSkipVerify:        &trueValue,
+				TLSServerName:        "2",
+			},
+		},
 		Consul: &config.ConsulConfig{
 			ServerServiceName:    "2",
 			ClientServiceName:    "2",

command/agent/testdata/extra-vault.hcl

@@ -0,0 +1,25 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# this unnamed (default) config should merge cleanly onto the basic config
+vault {
+  enabled = true
+  token   = "abracadabra"
+}
+
+# this alternate config should be added as an extra vault config
+vault {
+  name                  = "alternate"
+  address               = "127.0.0.1:9501"
+  allow_unauthenticated = true
+  task_token_ttl        = "5s"
+  enabled               = true
+  token                 = "xyzzy"
+  ca_file               = "/path/to/ca/file"
+  ca_path               = "/path/to/ca"
+  cert_file             = "/path/to/cert/file"
+  key_file              = "/path/to/key/file"
+  tls_server_name       = "barbaz"
+  tls_skip_verify       = true
+  create_from_role      = "test_role2"
+}

command/agent/testdata/sample0.json

@@ -82,6 +82,7 @@
     "verify_server_hostname": true
   },
   "vault": {
+    "name": "default",
     "address": "http://host.example.com:8200",
     "create_from_role": "nomad-cluster",
     "enabled": true