ACLs: allow/deny/default config for Consul/Vault clusters by namespace (#18425)

In Nomad Enterprise when multiple Vault/Consul clusters are configured, cluster admins can control access to clusters for jobs via namespace ACLs, similar to how we've done so for node pools. This changeset updates the ACL configuration structs, but doesn't wire them up.

api/namespace.go

@@ -75,6 +75,8 @@ type Namespace struct {
 	Quota                 string
 	Capabilities          *NamespaceCapabilities          `hcl:"capabilities,block"`
 	NodePoolConfiguration *NamespaceNodePoolConfiguration `hcl:"node_pool_config,block"`
+	VaultConfiguration    *NamespaceVaultConfiguration    `hcl:"vault,block"`
+	ConsulConfiguration   *NamespaceConsulConfiguration   `hcl:"consul,block"`
 	Meta                  map[string]string
 	CreateIndex           uint64
 	ModifyIndex           uint64
@@ -95,6 +97,50 @@ type NamespaceNodePoolConfiguration struct {
 	Denied  []string
 }
 
+// NamespaceVaultConfiguration stores configuration about permissions to Vault
+// clusters for a namespace, for use with Nomad Enterprise.
+type NamespaceVaultConfiguration struct {
+	// Default is the Vault cluster used by jobs in this namespace that don't
+	// specify a cluster of their own.
+	Default string
+
+	// Allowed specifies the Vault clusters that are allowed to be used by jobs
+	// in this namespace. By default, all clusters are allowed. If an empty list
+	// is provided only the namespace's default cluster is allowed. This field
+	// supports wildcard globbing through the use of `*` for multi-character
+	// matching. This field cannot be used with Denied.
+	Allowed []string
+
+	// Denied specifies the Vault clusters that are not allowed to be used by
+	// jobs in this namespace. This field supports wildcard globbing through the
+	// use of `*` for multi-character matching. If specified, any cluster is
+	// allowed to be used, except for those that match any of these patterns.
+	// This field cannot be used with Allowed.
+	Denied []string
+}
+
+// NamespaceConsulConfiguration stores configuration about permissions to Consul
+// clusters for a namespace, for use with Nomad Enterprise.
+type NamespaceConsulConfiguration struct {
+	// Default is the Consul cluster used by jobs in this namespace that don't
+	// specify a cluster of their own.
+	Default string
+
+	// Allowed specifies the Consul clusters that are allowed to be used by jobs
+	// in this namespace. By default, all clusters are allowed. If an empty list
+	// is provided only the namespace's default cluster is allowed. This field
+	// supports wildcard globbing through the use of `*` for multi-character
+	// matching. This field cannot be used with Denied.
+	Allowed []string
+
+	// Denied specifies the Consul clusters that are not allowed to be used by
+	// jobs in this namespace. This field supports wildcard globbing through the
+	// use of `*` for multi-character matching. If specified, any cluster is
+	// allowed to be used, except for those that match any of these patterns.
+	// This field cannot be used with Allowed.
+	Denied []string
+}
+
 // NamespaceIndexSort is a wrapper to sort Namespaces by CreateIndex. We
 // reverse the test so that we get the highest index first.
 type NamespaceIndexSort []*Namespace

command/agent/command.go

@@ -395,6 +395,17 @@ func (c *Command) IsValidConfig(config, cmdConfig *Config) bool {
 		}
 	}
 
+	for _, consul := range config.Consuls {
+		if err := structs.ValidateConsulClusterName(consul.Name); err != nil {
+			c.Ui.Error(fmt.Sprintf("Invalid Consul configuration: %v", err))
+		}
+	}
+	for _, vault := range config.Vaults {
+		if err := structs.ValidateVaultClusterName(vault.Name); err != nil {
+			c.Ui.Error(fmt.Sprintf("Invalid Vault configuration: %v", err))
+		}
+	}
+
 	for _, volumeConfig := range config.Client.HostVolumes {
 		if volumeConfig.Path == "" {
 			c.Ui.Error("Missing path in host_volume config")

command/namespace_apply.go

@@ -227,6 +227,8 @@ func parseNamespaceSpecImpl(result *api.Namespace, list *ast.ObjectList) error {
 	delete(m, "capabilities")
 	delete(m, "meta")
 	delete(m, "node_pool_config")
+	delete(m, "vault")
+	delete(m, "consul")
 
 	// Decode the rest
 	if err := mapstructure.WeakDecode(m, result); err != nil {
@@ -265,6 +267,38 @@ func parseNamespaceSpecImpl(result *api.Namespace, list *ast.ObjectList) error {
 		}
 	}
 
+	vObj := list.Filter("vault")
+	if len(vObj.Items) > 0 {
+		for _, o := range vObj.Elem().Items {
+			ot, ok := o.Val.(*ast.ObjectType)
+			if !ok {
+				break
+			}
+			var vConfig *api.NamespaceVaultConfiguration
+			if err := hcl.DecodeObject(&vConfig, ot.List); err != nil {
+				return err
+			}
+			result.VaultConfiguration = vConfig
+			break
+		}
+	}
+
+	conObj := list.Filter("consul")
+	if len(conObj.Items) > 0 {
+		for _, o := range conObj.Elem().Items {
+			ot, ok := o.Val.(*ast.ObjectType)
+			if !ok {
+				break
+			}
+			var cConfig *api.NamespaceConsulConfiguration
+			if err := hcl.DecodeObject(&cConfig, ot.List); err != nil {
+				return err
+			}
+			result.ConsulConfiguration = cConfig
+			break
+		}
+	}
+
 	if metaO := list.Filter("meta"); len(metaO.Items) > 0 {
 		for _, o := range metaO.Elem().Items {
 			var m map[string]interface{}

command/namespace_apply_test.go

@@ -88,6 +88,16 @@ node_pool_config {
   allowed = ["prod*"]
 }
 
+vault {
+  default = "infra"
+  allowed = ["apps", "infra"]
+}
+
+consul {
+  default = "prod"
+  allowed = ["prod", "apps*"]
+}
+
 meta {
   dept = "eng"
 }`,
@@ -103,6 +113,14 @@ meta {
 					Default: "dev",
 					Allowed: []string{"prod*"},
 				},
+				VaultConfiguration: &api.NamespaceVaultConfiguration{
+					Default: "infra",
+					Allowed: []string{"apps", "infra"},
+				},
+				ConsulConfiguration: &api.NamespaceConsulConfiguration{
+					Default: "prod",
+					Allowed: []string{"prod", "apps*"},
+				},
 				Meta: map[string]string{
 					"dept": "eng",
 				},

command/namespace_status.go

@@ -164,6 +164,36 @@ func (c *NamespaceStatusCommand) Run(args []string) int {
 		c.Ui.Output(formatKV(npConfigOut))
 	}
 
+	if ns.VaultConfiguration != nil {
+		c.Ui.Output(c.Colorize().Color("\n[bold]Vault Configuration[reset]"))
+		vConfig := ns.VaultConfiguration
+		vConfigOut := []string{
+			fmt.Sprintf("Default|%s", vConfig.Default),
+		}
+		if len(vConfig.Allowed) > 0 {
+			vConfigOut = append(vConfigOut, fmt.Sprintf("Allowed|%s", strings.Join(vConfig.Allowed, ", ")))
+		}
+		if len(vConfig.Denied) > 0 {
+			vConfigOut = append(vConfigOut, fmt.Sprintf("Denied|%s", strings.Join(vConfig.Denied, ", ")))
+		}
+		c.Ui.Output(formatKV(vConfigOut))
+	}
+
+	if ns.ConsulConfiguration != nil {
+		c.Ui.Output(c.Colorize().Color("\n[bold]Consul Configuration[reset]"))
+		cConfig := ns.ConsulConfiguration
+		cConfigOut := []string{
+			fmt.Sprintf("Default|%s", cConfig.Default),
+		}
+		if len(cConfig.Allowed) > 0 {
+			cConfigOut = append(cConfigOut, fmt.Sprintf("Allowed|%s", strings.Join(cConfig.Allowed, ", ")))
+		}
+		if len(cConfig.Denied) > 0 {
+			cConfigOut = append(cConfigOut, fmt.Sprintf("Denied|%s", strings.Join(cConfig.Denied, ", ")))
+		}
+		c.Ui.Output(formatKV(cConfigOut))
+	}
+
 	return 0
 }
 

nomad/structs/consul.go

@@ -3,6 +3,11 @@
 
 package structs
 
+import (
+	"fmt"
+	"regexp"
+)
+
 // Consul represents optional per-group consul configuration.
 type Consul struct {
 	// Namespace in which to operate in Consul.
@@ -44,6 +49,20 @@ func (c *Consul) Validate() error {
 	return nil
 }
 
+var (
+	// validConsulVaultClusterName is the rule used to validate a Consul or
+	// Vault cluster name.
+	validConsulVaultClusterName = regexp.MustCompile("^[a-zA-Z0-9-_]{1,128}$")
+)
+
+func ValidateConsulClusterName(cluster string) error {
+	if !validConsulVaultClusterName.MatchString(cluster) {
+		return fmt.Errorf("invalid name %q, must match regex %s", cluster, validConsulVaultClusterName)
+	}
+
+	return nil
+}
+
 // ConsulUsage is provides meta information about how Consul is used by a job,
 // noting which connect services and normal services will be registered, and
 // whether the keystore will be read via template.

nomad/structs/namespace.go

@@ -0,0 +1,48 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package structs
+
+// NamespaceVaultConfiguration stores configuration about permissions to Vault
+// clusters for a namespace, for use with Nomad Enterprise.
+type NamespaceVaultConfiguration struct {
+	// Default is the Vault cluster used by jobs in this namespace that don't
+	// specify a cluster of their own.
+	Default string
+
+	// Allowed specifies the Vault clusters that are allowed to be used by jobs
+	// in this namespace. By default, all clusters are allowed. If an empty list
+	// is provided only the namespace's default cluster is allowed. This field
+	// supports wildcard globbing through the use of `*` for multi-character
+	// matching. This field cannot be used with Denied.
+	Allowed []string
+
+	// Denied specifies the Vault clusters that are not allowed to be used by
+	// jobs in this namespace. This field supports wildcard globbing through the
+	// use of `*` for multi-character matching. If specified, any cluster is
+	// allowed to be used, except for those that match any of these patterns.
+	// This field cannot be used with Allowed.
+	Denied []string
+}
+
+// NamespaceConsulConfiguration stores configuration about permissions to Consul
+// clusters for a namespace, for use with Nomad Enterprise.
+type NamespaceConsulConfiguration struct {
+	// Default is the Consul cluster used by jobs in this namespace that don't
+	// specify a cluster of their own.
+	Default string
+
+	// Allowed specifies the Consul clusters that are allowed to be used by jobs
+	// in this namespace. By default, all clusters are allowed. If an empty list
+	// is provided only the namespace's default cluster is allowed. This field
+	// supports wildcard globbing through the use of `*` for multi-character
+	// matching. This field cannot be used with Denied.
+	Allowed []string
+
+	// Denied specifies the Consul clusters that are not allowed to be used by
+	// jobs in this namespace. This field supports wildcard globbing through the
+	// use of `*` for multi-character matching. If specified, any cluster is
+	// allowed to be used, except for those that match any of these patterns.
+	// This field cannot be used with Allowed.
+	Denied []string
+}

nomad/structs/structs.go

@@ -5380,6 +5380,9 @@ type Namespace struct {
 	// pools.
 	NodePoolConfiguration *NamespaceNodePoolConfiguration
 
+	VaultConfiguration  *NamespaceVaultConfiguration
+	ConsulConfiguration *NamespaceConsulConfiguration
+
 	// Meta is the set of metadata key/value pairs that attached to the namespace
 	Meta map[string]string
 
@@ -5444,6 +5447,26 @@ func (n *Namespace) Validate() error {
 		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid node pool configuration: %v", e))
 	}
 
+	err = n.VaultConfiguration.Validate()
+	switch e := err.(type) {
+	case *multierror.Error:
+		for _, vErr := range e.Errors {
+			mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid vault configuration: %v", vErr))
+		}
+	case error:
+		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid vault configuration: %v", e))
+	}
+
+	err = n.ConsulConfiguration.Validate()
+	switch e := err.(type) {
+	case *multierror.Error:
+		for _, cErr := range e.Errors {
+			mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid consul configuration: %v", cErr))
+		}
+	case error:
+		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid consul configuration: %v", e))
+	}
+
 	return mErr.ErrorOrNil()
 }
 
@@ -5477,6 +5500,26 @@ func (n *Namespace) SetHash() []byte {
 		}
 	}
 
+	if n.VaultConfiguration != nil {
+		_, _ = hash.Write([]byte(n.VaultConfiguration.Default))
+		for _, cluster := range n.VaultConfiguration.Allowed {
+			_, _ = hash.Write([]byte(cluster))
+		}
+		for _, cluster := range n.VaultConfiguration.Denied {
+			_, _ = hash.Write([]byte(cluster))
+		}
+	}
+
+	if n.ConsulConfiguration != nil {
+		_, _ = hash.Write([]byte(n.ConsulConfiguration.Default))
+		for _, cluster := range n.ConsulConfiguration.Allowed {
+			_, _ = hash.Write([]byte(cluster))
+		}
+		for _, cluster := range n.ConsulConfiguration.Denied {
+			_, _ = hash.Write([]byte(cluster))
+		}
+	}
+
 	// sort keys to ensure hash stability when meta is stored later
 	var keys []string
 	for k := range n.Meta {
@@ -5514,6 +5557,19 @@ func (n *Namespace) Copy() *Namespace {
 		np.Allowed = slices.Clone(n.NodePoolConfiguration.Allowed)
 		np.Denied = slices.Clone(n.NodePoolConfiguration.Denied)
 	}
+	if n.VaultConfiguration != nil {
+		nv := new(NamespaceVaultConfiguration)
+		*nv = *n.VaultConfiguration
+		nv.Allowed = slices.Clone(n.VaultConfiguration.Allowed)
+		nv.Denied = slices.Clone(n.VaultConfiguration.Denied)
+	}
+	if n.ConsulConfiguration != nil {
+		nc := new(NamespaceConsulConfiguration)
+		*nc = *n.ConsulConfiguration
+		nc.Allowed = slices.Clone(n.ConsulConfiguration.Allowed)
+		nc.Denied = slices.Clone(n.ConsulConfiguration.Denied)
+	}
+
 	if n.Meta != nil {
 		nc.Meta = make(map[string]string, len(n.Meta))
 		for k, v := range n.Meta {

nomad/structs/structs_ce.go

@@ -24,6 +24,24 @@ func (n *NamespaceNodePoolConfiguration) Validate() error {
 	return nil
 }
 
+func (n *NamespaceVaultConfiguration) Canonicalize() {}
+
+func (n *NamespaceVaultConfiguration) Validate() error {
+	if n != nil {
+		return errors.New("Multi-Cluster Vault is unlicensed.")
+	}
+	return nil
+}
+
+func (n *NamespaceConsulConfiguration) Canonicalize() {}
+
+func (n *NamespaceConsulConfiguration) Validate() error {
+	if n != nil {
+		return errors.New("Multi-Cluster Consul is unlicensed.")
+	}
+	return nil
+}
+
 func (m *Multiregion) Validate(jobType string, jobDatacenters []string) error {
 	if m != nil {
 		return errors.New("Multiregion jobs are unlicensed.")

nomad/structs/structs_ce_test.go

@@ -31,6 +31,26 @@ func TestNamespace_Validate_Oss(t *testing.T) {
 			},
 			expectedErr: "unlicensed",
 		},
+		{
+			name: "vault config not allowed",
+			namespace: &Namespace{
+				Name: "test",
+				VaultConfiguration: &NamespaceVaultConfiguration{
+					Default: "dev",
+				},
+			},
+			expectedErr: "unlicensed",
+		},
+		{
+			name: "consul config not allowed",
+			namespace: &Namespace{
+				Name: "test",
+				ConsulConfiguration: &NamespaceConsulConfiguration{
+					Default: "dev",
+				},
+			},
+			expectedErr: "unlicensed",
+		},
 	}
 
 	for _, tc := range cases {

nomad/structs/structs_test.go

@@ -101,6 +101,14 @@ func TestNamespace_SetHash(t *testing.T) {
 			Default: "dev",
 			Allowed: []string{"default"},
 		},
+		VaultConfiguration: &NamespaceVaultConfiguration{
+			Default: "default",
+			Allowed: []string{"default"},
+		},
+		ConsulConfiguration: &NamespaceConsulConfiguration{
+			Default: "default",
+			Allowed: []string{"default"},
+		},
 		Meta: map[string]string{
 			"a": "b",
 			"c": "d",
@@ -150,6 +158,24 @@ func TestNamespace_SetHash(t *testing.T) {
 	must.NotNil(t, ns.Hash)
 	must.Eq(t, out6, ns.Hash)
 	must.NotEq(t, out5, out6)
+
+	ns.VaultConfiguration.Default = "infra"
+	ns.VaultConfiguration.Allowed = []string{}
+	ns.VaultConfiguration.Denied = []string{"all"}
+	out7 := ns.SetHash()
+	must.NotNil(t, out7)
+	must.NotNil(t, ns.Hash)
+	must.Eq(t, out7, ns.Hash)
+	must.NotEq(t, out6, out7)
+
+	ns.ConsulConfiguration.Default = "infra"
+	ns.ConsulConfiguration.Allowed = []string{}
+	ns.ConsulConfiguration.Denied = []string{"all"}
+	out8 := ns.SetHash()
+	must.NotNil(t, out8)
+	must.NotNil(t, ns.Hash)
+	must.Eq(t, out8, ns.Hash)
+	must.NotEq(t, out7, out8)
 }
 
 func TestNamespace_Copy(t *testing.T) {
@@ -167,6 +193,14 @@ func TestNamespace_Copy(t *testing.T) {
 			Default: "dev",
 			Allowed: []string{"default"},
 		},
+		VaultConfiguration: &NamespaceVaultConfiguration{
+			Default: "default",
+			Allowed: []string{"default"},
+		},
+		ConsulConfiguration: &NamespaceConsulConfiguration{
+			Default: "default",
+			Allowed: []string{"default"},
+		},
 		Meta: map[string]string{
 			"a": "b",
 			"c": "d",
@@ -183,6 +217,12 @@ func TestNamespace_Copy(t *testing.T) {
 	nsCopy.NodePoolConfiguration.Default = "default"
 	nsCopy.NodePoolConfiguration.Allowed = []string{}
 	nsCopy.NodePoolConfiguration.Denied = []string{"dev"}
+	nsCopy.VaultConfiguration.Default = "infra"
+	nsCopy.VaultConfiguration.Allowed = []string{}
+	nsCopy.VaultConfiguration.Denied = []string{"dev"}
+	nsCopy.ConsulConfiguration.Default = "infra"
+	nsCopy.ConsulConfiguration.Allowed = []string{}
+	nsCopy.ConsulConfiguration.Denied = []string{"dev"}
 	nsCopy.Meta["a"] = "z"
 	must.NotEq(t, ns, nsCopy)
 

nomad/structs/vault.go

@@ -65,3 +65,11 @@ func DecodeVaultSecretData(s *vapi.Secret, out interface{}) error {
 
 	return nil
 }
+
+func ValidateVaultClusterName(cluster string) error {
+	if !validConsulVaultClusterName.MatchString(cluster) {
+		return fmt.Errorf("invalid name %q, must match regex %s", cluster, validConsulVaultClusterName)
+	}
+
+	return nil
+}