E2E: make vault.create_from_role unique per cluster (#20267)

If a E2E cluster is destroyed after a different one has been created, the role
and policy we create in Vault for the cluster will be deleted and Vault-related
tests will fail. Note that before 1.9, we should figure out a way to give HCP
Vault access to the JWKS endpoint and have a different set of policies, but
we'll need to have a role-per-cluster in that case as well.

Fixes: https://github.com/hashicorp/nomad-e2e/issues/138 (internal)

e2e/terraform/etc/acls/vault/nomad-policy.hcl

@@ -3,13 +3,13 @@
 
 # Allow creating tokens under "nomad-tasks" role. The role name should be
 # updated if "nomad-tasks" is not used.
-path "auth/token/create/nomad-tasks" {
+path "auth/token/create/${role}" {
   capabilities = ["update"]
 }
 
-# Allow looking up "nomad-tasks" role. The role name should be updated if
-# "nomad-tasks" is not used.
-path "auth/token/roles/nomad-tasks" {
+# Allow looking up "${role}" role. The role name should be updated if
+# "${role}" is not used.
+path "auth/token/roles/${role}" {
   capabilities = ["read"]
 }
 

e2e/terraform/etc/nomad.d/vault.hcl

@@ -5,7 +5,7 @@ vault {
   enabled          = true
   address          = "${url}"
   task_token_ttl   = "1h"
-  create_from_role = "nomad-tasks"
+  create_from_role = "${role}"
   namespace        = "${namespace}"
   token            = "${token}"
 }

e2e/terraform/hcp_vault.tf

@@ -11,15 +11,14 @@ data "hcp_vault_cluster" "e2e_shared_vault" {
   cluster_id = var.hcp_vault_cluster_id
 }
 
-# Nomad servers configuration for Vault
-
+# Vault policy for the Nomad cluster, which allows it to mint derived tokens for
+# tasks. It's interpolated with the random cluster name to avoid collisions
+# between concurrent E2E clusters
 resource "vault_policy" "nomad" {
-  name   = "${local.random_name}-nomad-server"
-  policy = data.local_file.vault_policy_for_nomad.content
-}
-
-data "local_file" "vault_policy_for_nomad" {
-  filename = "${path.root}/etc/acls/vault/nomad-policy.hcl"
+  name = "${local.random_name}-nomad-server"
+  policy = templatefile("${path.root}/etc/acls/vault/nomad-policy.hcl", {
+    role = "nomad-tasks-${local.random_name}"
+  })
 }
 
 resource "vault_token" "nomad" {
@@ -29,11 +28,11 @@ resource "vault_token" "nomad" {
   ttl       = "72h"
 }
 
-# this is the role that Nomad will use for derived tokens. It's not
-# allowed access to nomad-policy so that only mint tokens for tasks,
-# not for new clusters
+# The default role that Nomad will use for derived tokens. It's not allowed
+# access to nomad-policy so that it can only mint tokens for tasks, not for new
+# clusters
 resource "vault_token_auth_backend_role" "nomad_cluster" {
-  role_name           = "nomad-tasks"
+  role_name           = "nomad-tasks-${local.random_name}"
   disallowed_policies = [vault_policy.nomad.name]
   orphan              = true
   token_period        = "259200"
@@ -41,11 +40,13 @@ resource "vault_token_auth_backend_role" "nomad_cluster" {
   token_max_ttl       = "0"
 }
 
+# Nomad agent configuration for Vault
 resource "local_sensitive_file" "nomad_config_for_vault" {
   content = templatefile("etc/nomad.d/vault.hcl", {
     token     = vault_token.nomad.client_token
     url       = data.hcp_vault_cluster.e2e_shared_vault.vault_private_endpoint_url
     namespace = var.hcp_vault_namespace
+    role      = "nomad-tasks-${local.random_name}"
   })
   filename        = "uploads/shared/nomad.d/vault.hcl"
   file_permission = "0600"