workload identity: add support for extra claims config for Vault (#23675)

Although we encourage users to use Vault roles, sometimes they're going to want
to assign policies based on entity and pre-create entities and aliases based on
claims. This allows them to use single default role (or at least small number of
them) that has a templated policy, but have an escape hatch from that.

When defining Vault entities the `user_claim` must be unique. When writing Vault
binding rules for use with Nomad workload identities the binding rule won't be
able to create a 1:1 mapping because the selector language allows accessing only
a single field. The `nomad_job_id` claim isn't sufficient to uniquely identify a
job because of namespaces. It's possible to create a JWT auth role with
`bound_claims` to avoid this becoming a security problem, but this doesn't allow
for correct accounting of user claims.

Add support for an `extra_claims` block on the server's `default_identity`
blocks for Vault. This allows a cluster administrator to add a custom claim on
all allocations. The values for these claims are interpolatable with a limited
subset of fields, similar to how we interpolate the task environment.

Fixes: https://github.com/hashicorp/nomad/issues/23510
Ref: https://hashicorp.atlassian.net/browse/NET-10372
Ref: https://hashicorp.atlassian.net/browse/NET-10387

e2e/vaultcompat/cluster_setup_test.go

@@ -36,7 +36,7 @@ func roleWID(policies []string) map[string]any {
 	return map[string]any{
 		"role_type":               "jwt",
 		"bound_audiences":         "vault.io",
-		"user_claim":              "/nomad_job_id",
+		"user_claim":              "/extra_claims/nomad_workload_id",
 		"user_claim_json_pointer": true,
 		"claim_mappings": map[string]any{
 			"nomad_namespace": "nomad_namespace",

e2e/vaultcompat/input/restricted_jwt.hcl

@@ -0,0 +1,38 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "restricted_jwt" {
+  type = "batch"
+
+  // Tasks in this group are expected to succeed and run to completion.
+  group "success" {
+    vault {}
+
+    count = 2
+
+    // Task default_identity uses the default workload identity injected by the
+    // server and the inherits the Vault configuration from the group.
+    task "authorized" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      // Vault has an alias that maps this job's nomad_workload_id to an entity
+      // with a policy that allows access to these secrets
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      restart {
+        attempts = 0
+        mode     = "fail"
+      }
+    }
+  }
+}

e2e/vaultcompat/run_ce_test.go

@@ -63,4 +63,5 @@ func testVaultJWT(t *testing.T, b build) {
 
 	// Run test job.
 	runJob(t, nc, "input/cat_jwt.hcl", "default", validateJWTAllocs)
+	runJob(t, nc, "input/restricted_jwt.hcl", "default", validateJWTAllocs)
 }

e2e/vaultcompat/vaultcompat_test.go

@@ -239,6 +239,20 @@ func setupVaultJWT(t *testing.T, vc *vaultapi.Client, jwksURL string) {
 	rolePath = fmt.Sprintf("auth/%s/role/nomad-restricted", jwtPath)
 	_, err = logical.Write(rolePath, roleWID([]string{"nomad-restricted"}))
 	must.NoError(t, err)
+
+	entityOut, err := logical.Write("identity/entity", map[string]any{
+		"name":     "default:restricted_jwt",
+		"policies": []string{"nomad-restricted"},
+	})
+	must.NoError(t, err)
+	entityID := entityOut.Data["id"]
+
+	_, err = logical.Write("identity/entity-alias", map[string]any{
+		"name":           "default:restricted_jwt",
+		"canonical_id":   entityID,
+		"mount_accessor": jwtAuthAccessor,
+	})
+	must.NoError(t, err)
 }
 
 func startNomad(t *testing.T, cb func(*testutil.TestServerConfig)) (func(), *nomadapi.Client) {
@@ -285,6 +299,9 @@ func configureNomadVaultJWT(vc *vaultapi.Client) func(*testutil.TestServerConfig
 			DefaultIdentity: &testutil.WorkloadIdentityConfig{
 				Audience: []string{"vault.io"},
 				TTL:      "10m",
+				ExtraClaims: map[string]string{
+					"nomad_workload_id": "${job.namespace}:${job.id}",
+				},
 			},
 
 			// Client configs.

nomad/alloc_endpoint.go

@@ -619,11 +619,23 @@ func (a *Alloc) signTasks(
 		}
 
 		widFound = true
-		claims := structs.NewIdentityClaimsBuilder(alloc.Job, alloc, &idReq.WIHandle, wid).
+		builder := structs.NewIdentityClaimsBuilder(alloc.Job, alloc, &idReq.WIHandle, wid).
 			WithTask(task).
-			WithConsul().
-			WithVault().
-			Build(now)
+			WithConsul()
+
+		var node *structs.Node
+		node, err = a.srv.State().NodeByID(nil, alloc.NodeID)
+		if err != nil {
+			return
+		}
+		builder.WithNode(node)
+
+		vaultCfg := a.srv.GetConfig().GetVaultForIdentity(wid)
+		if vaultCfg != nil && vaultCfg.DefaultIdentity != nil {
+			builder.WithVault(vaultCfg.DefaultIdentity.ExtraClaims)
+		}
+
+		claims := builder.Build(now)
 		err = a.signClaims(claims, idReq, reply)
 		break
 	}

nomad/config.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"runtime"
 	"slices"
+	"strings"
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
@@ -503,6 +504,21 @@ func (c *Config) VaultIdentityConfig(cluster string) *structs.WorkloadIdentity {
 	return workloadIdentityFromConfig(conf.DefaultIdentity)
 }
 
+// GetVaultForIdentity reverses VaultIdentityConfig and finds the Vault
+// configuration that goes with a particular workload identity intended for
+// Vault
+func (c *Config) GetVaultForIdentity(wi *structs.WorkloadIdentity) *config.VaultConfig {
+	if !wi.IsVault() {
+		return nil
+	}
+	cluster := strings.TrimPrefix(wi.Name, structs.WorkloadIdentityVaultPrefix)
+	if cluster == "" {
+		return nil
+	}
+	conf := c.VaultConfigs[cluster]
+	return conf
+}
+
 func (c *Config) GetDefaultConsul() *config.ConsulConfig {
 	return c.ConsulConfigs[structs.ConsulDefaultCluster]
 }

nomad/structs/config/workload_id.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"maps"
 	"slices"
 	"time"
 
@@ -37,6 +38,10 @@ type WorkloadIdentityConfig struct {
 	// this identity (eg the JWT "exp" claim).
 	TTL    *time.Duration `mapstructure:"-"`
 	TTLHCL string         `mapstructure:"ttl" json:"-"`
+
+	// ExtraClaims allows a WI configuration to carry extra claims configured by
+	// the cluster administrator. Note this field is not available on jobspecs.
+	ExtraClaims map[string]string `mapstructure:"extra_claims"`
 }
 
 func (wi *WorkloadIdentityConfig) Copy() *WorkloadIdentityConfig {
@@ -56,6 +61,7 @@ func (wi *WorkloadIdentityConfig) Copy() *WorkloadIdentityConfig {
 	if wi.TTL != nil {
 		nwi.TTL = pointer.Of(*wi.TTL)
 	}
+	nwi.ExtraClaims = maps.Clone(wi.ExtraClaims)
 
 	return nwi
 }
@@ -83,6 +89,9 @@ func (wi *WorkloadIdentityConfig) Equal(other *WorkloadIdentityConfig) bool {
 	if wi.TTLHCL != other.TTLHCL {
 		return false
 	}
+	if !maps.Equal(wi.ExtraClaims, other.ExtraClaims) {
+		return false
+	}
 
 	return true
 }
@@ -114,5 +123,12 @@ func (wi *WorkloadIdentityConfig) Merge(other *WorkloadIdentityConfig) *Workload
 		result.TTLHCL = other.TTLHCL
 	}
 
+	if wi.ExtraClaims == nil {
+		result.ExtraClaims = map[string]string{}
+	}
+	for k, v := range other.ExtraClaims {
+		result.ExtraClaims[k] = v
+	}
+
 	return result
 }

nomad/structs/workload_id.go

@@ -76,6 +76,10 @@ type IdentityClaims struct {
 	VaultNamespace  string `json:"vault_namespace,omitempty"`
 	VaultRole       string `json:"vault_role,omitempty"`
 
+	// ExtraClaims are added based on this identity's
+	// WorkloadIdentityConfiguration, controlled by server configuration
+	ExtraClaims map[string]string `json:"extra_claims,omitempty"`
+
 	jwt.Claims
 }
 
@@ -93,6 +97,8 @@ type IdentityClaimsBuilder struct {
 	serviceName string
 	consul      *Consul
 	vault       *Vault
+	node        *Node
+	extras      map[string]string
 }
 
 // NewIdentityClaimsBuilder returns an initialized IdentityClaimsBuilder for the
@@ -111,6 +117,7 @@ func NewIdentityClaimsBuilder(job *Job, alloc *Allocation, wihandle *WIHandle, w
 		wihandle: wihandle,
 		wid:      wid,
 		tg:       tg,
+		extras:   map[string]string{},
 	}
 }
 
@@ -125,11 +132,14 @@ func (b *IdentityClaimsBuilder) WithTask(task *Task) *IdentityClaimsBuilder {
 
 // WithVault adds the task's vault block to the builder context. This should
 // only be called after WithTask.
-func (b *IdentityClaimsBuilder) WithVault() *IdentityClaimsBuilder {
+func (b *IdentityClaimsBuilder) WithVault(extraClaims map[string]string) *IdentityClaimsBuilder {
 	if !b.wid.IsVault() || b.task == nil {
 		return b
 	}
 	b.vault = b.task.Vault
+	for k, v := range extraClaims {
+		b.extras[k] = v
+	}
 	return b
 }
 
@@ -162,11 +172,18 @@ func (b *IdentityClaimsBuilder) WithService(service *Service) *IdentityClaimsBui
 	return b
 }
 
+// WithNode add the allocation's node to the builder context.
+func (b *IdentityClaimsBuilder) WithNode(node *Node) *IdentityClaimsBuilder {
+	b.node = node
+	return b
+}
+
 // Build is the terminal method for the builder and sets all the derived values
 // on the claim. The claim ID is random (nondeterministic) so multiple calls
 // with the same values will not return equal claims by design. JWT IDs should
 // never collide.
 func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
+	b.interpolate()
 
 	jwtnow := jwt.NewNumericDate(now.UTC())
 	claims := &IdentityClaims{
@@ -178,6 +195,7 @@ func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
 			NotBefore: jwtnow,
 			IssuedAt:  jwtnow,
 		},
+		ExtraClaims: b.extras,
 	}
 	// If this is a child job, use the parent's ID
 	if b.job.ParentID != "" {
@@ -203,6 +221,40 @@ func (b *IdentityClaimsBuilder) Build(now time.Time) *IdentityClaims {
 	return claims
 }
 
+func strAttrGet[T any](x *T, fn func(x *T) string) string {
+	if x != nil {
+		return fn(x)
+	}
+	return ""
+}
+
+func (b *IdentityClaimsBuilder) interpolate() {
+	if len(b.extras) == 0 {
+		return
+	}
+	r := strings.NewReplacer(
+		// attributes that always exist
+		"${job.region}", b.job.Region,
+		"${job.namespace}", b.job.Namespace,
+		"${job.id}", b.job.ID,
+		"${job.node_pool}", b.job.NodePool,
+		"${group.name}", b.tg.Name,
+
+		// attributes that conditionally exist
+		"${node.id}", strAttrGet(b.node, func(n *Node) string { return n.ID }),
+		"${node.datacenter}", strAttrGet(b.node, func(n *Node) string { return n.Datacenter }),
+		"${node.pool}", strAttrGet(b.node, func(n *Node) string { return n.NodePool }),
+		"${node.class}", strAttrGet(b.node, func(n *Node) string { return n.NodeClass }),
+		"${task.name}", strAttrGet(b.task, func(t *Task) string { return t.Name }),
+		"${vault.cluster}", strAttrGet(b.vault, func(v *Vault) string { return v.Cluster }),
+		"${vault.namespace}", strAttrGet(b.vault, func(v *Vault) string { return v.Namespace }),
+		"${vault.role}", strAttrGet(b.vault, func(v *Vault) string { return v.Role }),
+	)
+	for k, v := range b.extras {
+		b.extras[k] = r.Replace(v)
+	}
+}
+
 // setSubject creates the standard subject claim for workload identities.
 func (claims *IdentityClaims) setSubject(job *Job, group, widentifier, id string) {
 	claims.Subject = strings.Join([]string{
@@ -260,6 +312,10 @@ type WorkloadIdentity struct {
 	// TTL is used to determine the expiration of the credentials created for
 	// this identity (eg the JWT "exp" claim).
 	TTL time.Duration
+
+	// Note: ExtraClaims is available on config/WorkloadIdentity but not
+	// available here on jobspecs because that might allow a job author to
+	// escalate their privileges if they know what claim mappings to expect.
 }
 
 // IsConsul returns true if the identity name starts with the standard prefix

nomad/structs/workload_id_test.go

@@ -184,6 +184,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:group-service:consul-service_group-service-http",
 				Audience: jwt.Audience{"group-service.consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// group: no consul.
 		// task:  no consul, no vault.
@@ -195,6 +196,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:task:default-identity",
 				Audience: jwt.Audience{"example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		"job/group/task/alt-identity": {
 			Namespace: "default",
@@ -204,6 +206,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:task:alt-identity",
 				Audience: jwt.Audience{"alt.example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// No ConsulNamespace because there is no consul block at either task
 		// or group level.
@@ -216,6 +219,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:task:consul_default",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// No VaultNamespace because there is no vault block at either task
 		// or group level.
@@ -229,6 +233,9 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:task:vault_default",
 				Audience: jwt.Audience{"vault.io"},
 			},
+			ExtraClaims: map[string]string{
+				"nomad_workload_id": "global:default:job",
+			},
 		},
 		"job/group/task/services/task-service": {
 			Namespace:   "default",
@@ -238,6 +245,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:task-service:consul-service_task-task-service-http",
 				Audience: jwt.Audience{"task-service.consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// group: no consul.
 		// task:  with consul, with vault.
@@ -249,6 +257,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:consul-vault-task:default-identity",
 				Audience: jwt.Audience{"example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// Use task-level Consul namespace.
 		"job/group/consul-vault-task/consul_default": {
@@ -260,6 +269,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:consul-vault-task:consul_default",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// Use task-level Vault namespace.
 		"job/group/consul-vault-task/vault_default": {
@@ -272,6 +282,9 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:consul-vault-task:vault_default",
 				Audience: jwt.Audience{"vault.io"},
 			},
+			ExtraClaims: map[string]string{
+				"nomad_workload_id": "global:default:job",
+			},
 		},
 		// Use task-level Consul namespace for task services.
 		"job/group/consul-vault-task/services/consul-vault-task-service": {
@@ -283,6 +296,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:group:consul-vault-task-service:consul-service_consul-vault-task-service-http",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// group: with consul.
 		// Use group-level Consul namespace for group services.
@@ -295,6 +309,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:group-service:consul-service_group-service-http",
 				Audience: jwt.Audience{"group-service.consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// group: with consul.
 		// task:  no consul, no vault.
@@ -306,6 +321,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:task:default-identity",
 				Audience: jwt.Audience{"example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		"job/consul-group/task/alt-identity": {
 			Namespace: "default",
@@ -315,6 +331,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:task:alt-identity",
 				Audience: jwt.Audience{"alt.example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// Use group-level Consul namespace because task doesn't have a consul
 		// block.
@@ -327,6 +344,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:task:consul_default",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		"job/consul-group/task/vault_default": {
 			Namespace: "default",
@@ -337,6 +355,9 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:task:vault_default",
 				Audience: jwt.Audience{"vault.io"},
 			},
+			ExtraClaims: map[string]string{
+				"nomad_workload_id": "global:default:job",
+			},
 		},
 		// Use group-level Consul namespace for task service because task
 		// doesn't have a consul block.
@@ -349,6 +370,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:task-service:consul-service_task-task-service-http",
 				Audience: jwt.Audience{"task-service.consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// group: no consul.
 		// task:  with consul, with vault.
@@ -360,6 +382,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:consul-vault-task:default-identity",
 				Audience: jwt.Audience{"example.com"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		// Use task-level Consul namespace.
 		"job/consul-group/consul-vault-task/consul_default": {
@@ -371,6 +394,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:consul-vault-task:consul_default",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 		"job/consul-group/consul-vault-task/vault_default": {
 			VaultNamespace: "vault-namespace",
@@ -382,6 +406,9 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:consul-vault-task:vault_default",
 				Audience: jwt.Audience{"vault.io"},
 			},
+			ExtraClaims: map[string]string{
+				"nomad_workload_id": "global:default:job",
+			},
 		},
 		// Use task-level Consul namespace for task services.
 		"job/consul-group/consul-vault-task/services/consul-task-service": {
@@ -393,6 +420,7 @@ func TestNewIdentityClaims(t *testing.T) {
 				Subject:  "global:default:job:consul-group:consul-task-service:consul-service_consul-vault-task-consul-task-service-http",
 				Audience: jwt.Audience{"consul.io"},
 			},
+			ExtraClaims: map[string]string{},
 		},
 	}
 
@@ -492,7 +520,9 @@ func TestNewIdentityClaims(t *testing.T) {
 				WithTask(tc.task).
 				WithService(tc.svc).
 				WithConsul().
-				WithVault().
+				WithVault(map[string]string{
+					"nomad_workload_id": "${job.region}:${job.namespace}:${job.id}",
+				}).
 				Build(now)
 
 			must.Eq(t, tc.expectedClaims, got, must.Cmp(cmpopts.IgnoreFields(

testutil/server.go

@@ -64,10 +64,11 @@ type Consul struct {
 
 // WorkloadIdentityConfig is the configuration for default workload identities.
 type WorkloadIdentityConfig struct {
-	Audience []string `json:"aud"`
-	Env      bool     `json:"env"`
-	File     bool     `json:"file"`
-	TTL      string   `json:"ttl"`
+	Audience    []string          `json:"aud"`
+	Env         bool              `json:"env"`
+	File        bool              `json:"file"`
+	TTL         string            `json:"ttl"`
+	ExtraClaims map[string]string `json:"extra_claims,omitempty"`
 }
 
 // Advertise is used to configure the addresses to advertise

website/content/docs/configuration/vault.mdx

@@ -32,6 +32,10 @@ vault {
   default_identity {
     aud = ["vault.io"]
     ttl = "1h"
+
+    extra_claims {
+      unique_id = "${job.region}:${job.namespace}:${job.id}"
+    }
   }
 }
 ```
@@ -186,6 +190,28 @@ will be removed in a future release.
 - `ttl` `(string: "")` - Specifies for how long the workload identity should be
   considered as valid before expiring.
 
+- `extra_claims` `(map[string]string: optional)` - A set of key-value pairs that
+  will be provided as extra identity claims for workloads. You can use the keys
+  as [user claims in Vault role configurations][vault-jwt-user-claim]. The
+  values are interpolated. For example, if you include the extra claim
+  `unique_id = "${job.region}:${job.namespace}:${job.id}"`, you could set the
+  user claim field to `/extra_claims/unique_id` to map that identifier to an
+  entity alias. The available attributes for interpolation are:
+
+  - `${job.region}` - The region where the job is running.
+  - `${job.namespace}` - The job's namespace.
+  - `${job.id}` - The job's ID.
+  - `${job.node_pool}` - The node pool where the allocation is running.
+  - `${group.name}` - The task group name of the task using Vault.
+  - `${task.name}` - The name of the task using Vault.
+  - `${node.id}` - The ID of the node where the allocation is running.
+  - `${node.datacenter}` - The datacenter of the node where the allocation is running.
+  - `${node.pool}` - The node pool of the node where the allocation is running.
+  - `${node.class` - The class of the node where the allocation is running.
+  - `${vault.cluster}` - The Vault cluster name.
+  - `${vault.namespace}` - The Vault namespace.
+  - `${vault.role}` - The Vault role.
+
 ### Token-based Authentication
 
 ~> **Warning:** The token-based authentication flow is deprecated and will be
@@ -316,3 +342,4 @@ can be accomplished by sending the process a `SIGHUP` signal.
 [vault_bound_aud]: /vault/api-docs/auth/jwt#bound_audiences
 [vault_auth_enable_path]: /vault/docs/commands/auth/enable#path
 [workload_id]: /nomad/docs/concepts/workload-identity
+[vault-jwt-user-claim]: /vault/api-docs/auth/jwt#user_claim