func: add workload to test vault workload identity

enos/enos-modules.hcl

@@ -24,3 +24,7 @@ module "upgrade_servers" {
 module "upgrade_client" {
   source = "./modules/upgrade_client"
 }
+
+module "get_vault_env" {
+  source = "../e2e/terraform/hcp-vault-auth"
+}

enos/enos-scenario-upgrade.hcl

@@ -118,6 +118,15 @@ scenario "upgrade" {
     ]
   }
 
+  step "get_vault_env" {
+    
+    description = <<-EOF
+    Get the HCP vault address and token
+    EOF
+
+    module = module.get_vault_env
+  }
+
   step "run_initial_workloads" {
     depends_on = [step.initial_test_cluster_health]
 
@@ -135,6 +144,10 @@ scenario "upgrade" {
       availability_zone = var.availability_zone
       consul_addr       = step.provision_cluster.consul_addr
       consul_token      = step.provision_cluster.consul_token
+      vault_token       = step.get_vault_env.vault_token
+      vault_addr        = step.get_vault_env.vault_addr
+      // The provision_cluster module enables a kv v2 secrets engine using the cluster name as path.
+      vault_mount_path  = step.provision_cluster.cluster_unique_identifier 
 
       workloads = {
         service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3, type = "service" }
@@ -183,6 +196,12 @@ scenario "upgrade" {
           pre_script  = "scripts/configure-variables-acls.sh"
         }
 
+        gets_secret = {
+          job_spec = "jobs/vault-secrets.nomad.hcl",
+          alloc_count = 3,
+          type = "service",
+          pre_script = "scripts/populate_secret.sh"
+        }
       }
     }
 
@@ -228,7 +247,7 @@ scenario "upgrade" {
     ]
   }
 
-  step "fetch_upgrade_binary" {
+  /*   step "fetch_upgrade_binary" {
     depends_on = [step.provision_cluster, step.workloads_test_cluster_health]
 
     description = <<-EOF
@@ -529,7 +548,7 @@ scenario "upgrade" {
       quality.nomad_allocs_status,
       quality.nomad_reschedule_alloc,
     ]
-  }
+  } */
 
   output "servers" {
     value = step.provision_cluster.servers

enos/modules/run_workloads/jobs/vault-secrets.nomad.hcl

@@ -0,0 +1,73 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "alloc_count" {
+  type = number
+}
+
+job "get-secret" {
+  group "group" {
+    count = var.alloc_count
+
+    network {
+      port "web" {
+        to = 8001
+      }
+    }
+
+    service {
+      provider = "consul"
+      name     = "writes-vars-checker"
+      port     = "web"
+      task     = "task"
+
+      /* check {
+        type     = "script"
+        interval = "10s"
+        timeout  = "1s"
+        command  = "/bin/sh"
+        args     = ["/local/read-script.sh"]
+
+        # this check will read from the Task API, so we need to ensure that we
+        # can tolerate the listener going away during client upgrades
+        check_restart {
+          limit = 10
+        }
+      } */
+    }
+
+    task "read-secrets" {
+      driver = "raw_exec"
+
+      config {
+        command = "/bin/bash"
+        args    = ["-c", "cat local/config.json && sleep 30"]
+      }
+
+      vault {}
+
+      template {
+        destination = "local/config.json"
+        change_mode = "restart"
+
+        data = <<EOT
+{{ with secret "{{}}/data/default/get-secret" }}
+{
+  "username": "{{ .Data.data.username }}",
+  "password": "{{ .Data.data.password }}"
+}
+{{ end }}
+EOT
+      }
+
+      identity {
+        env = true
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

enos/modules/run_workloads/main.tf

@@ -16,11 +16,20 @@ locals {
     NOMAD_CLIENT_CERT = var.cert_file
     NOMAD_CLIENT_KEY  = var.key_file
     NOMAD_TOKEN       = var.nomad_token
+  }
+
+  consul_env = {
     CONSUL_HTTP_TOKEN = var.consul_token
     CONSUL_CACERT     = var.ca_file
     CONSUL_HTTP_ADDR  = var.consul_addr
   }
 
+  vault_env = {
+    VAULT_TOKEN = var.vault_token
+    VAULT_PATH  = var.vault_mount_path
+    VAULT_ADDR  = var.vault_addr
+  }
+
   system_job_count     = length({ for k, v in var.workloads : k => v if v.type == "system" })
   service_batch_allocs = sum([for wl in var.workloads : wl.alloc_count])
 }
@@ -59,7 +68,11 @@ resource "enos_local_exec" "workloads" {
   ]
   for_each = var.workloads
 
-  environment = local.nomad_env
+  environment = merge(
+    local.nomad_env,
+    local.vault_env,
+    local.consul_env,
+  )
 
   inline = [
     each.value.pre_script != null ? abspath("${path.module}/${each.value.pre_script}") : "echo ok",

enos/modules/run_workloads/scripts/populates_secret.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -euo pipefail
+
+vault kv put "$VAULT_PATH/default/get-secret" username="admin" password="supersecret"

enos/modules/run_workloads/variables.tf

@@ -45,6 +45,24 @@ variable "availability_zone" {
   type        = string
 }
 
+variable "vault_addr" {
+  description = "The Vault API HTTP address."
+  type        = string
+  default     = "http://localhost:8200"
+}
+
+variable "vault_token" {
+  description = "The Secret ID of an ACL token to make requests to Vault with"
+  type        = string
+  sensitive   = true
+}
+
+variable "vault_mount_path" {
+  description = "The path where the provision_cluster modules enables a secrets engine "
+  type        = string
+  default     = "admin"
+}
+
 variable "workloads" {
   description = "A map of workloads to provision"
 
@@ -55,4 +73,11 @@ variable "workloads" {
     pre_script  = optional(string)
     post_script = optional(string)
   }))
+
+  validation {
+    condition = alltrue([
+      for w in values(var.workloads) : contains(["service", "batch", "system"], w.type)
+    ])
+    error_message = "Each workload must have a 'type' value of either service, batch or system"
+  }
 }