func: add check script for vault workload

enos/enos-scenario-upgrade.hcl

@@ -119,7 +119,7 @@ scenario "upgrade" {
   }
 
   step "get_vault_env" {
-    
+
     description = <<-EOF
     Get the HCP vault address and token
     EOF
@@ -147,7 +147,7 @@ scenario "upgrade" {
       vault_token       = step.get_vault_env.vault_token
       vault_addr        = step.get_vault_env.vault_addr
       // The provision_cluster module enables a kv v2 secrets engine using the cluster name as path.
-      vault_mount_path  = step.provision_cluster.cluster_unique_identifier 
+      vault_mount_path = step.provision_cluster.cluster_unique_identifier
 
       workloads = {
         service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3, type = "service" }
@@ -197,10 +197,10 @@ scenario "upgrade" {
         }
 
         gets_secret = {
-          job_spec = "jobs/vault-secrets.nomad.hcl",
+          job_spec    = "jobs/vault-secrets.nomad.hcl",
           alloc_count = 3,
-          type = "service",
-          pre_script = "scripts/populate_secret.sh"
+          type        = "service",
+          pre_script  = "scripts/populates_secret.sh"
         }
       }
     }
@@ -247,7 +247,7 @@ scenario "upgrade" {
     ]
   }
 
-  /*   step "fetch_upgrade_binary" {
+  step "fetch_upgrade_binary" {
     depends_on = [step.provision_cluster, step.workloads_test_cluster_health]
 
     description = <<-EOF
@@ -548,7 +548,7 @@ scenario "upgrade" {
       quality.nomad_allocs_status,
       quality.nomad_reschedule_alloc,
     ]
-  } */
+  }
 
   output "servers" {
     value = step.provision_cluster.servers

enos/modules/run_workloads/main.tf

@@ -61,10 +61,18 @@ resource "enos_local_exec" "get_allocs" {
   inline = ["nomad alloc status -json | jq '[.[] | select(.ClientStatus == \"running\")] | length'"]
 }
 
+resource "local_file" "vault_workload" {
+  filename = "${path.module}/jobs/vault-secrets.nomad.hcl"
+  content = templatefile("${path.module}/templates/vault-secrets.nomad.hcl.tpl", {
+    secret_path = "${var.vault_mount_path}/default/get-secret"
+  })
+}
+
 resource "enos_local_exec" "workloads" {
   depends_on = [
     enos_local_exec.get_jobs,
     enos_local_exec.get_allocs,
+    local_file.vault_workload
   ]
   for_each = var.workloads
 

enos/modules/run_workloads/scripts/populates_secret.sh

@@ -4,4 +4,8 @@
 
 set -euo pipefail
 
-vault kv put "$VAULT_PATH/default/get-secret" username="admin" password="supersecret"
+# Path enabled by the provision_cluster module: 
+# https://github.com/hashicorp/nomad/e2e/terraform/provision-infra/hcp_vault.tf
+secret_path="$VAULT_PATH/default/get-secret"
+
+vault kv put "$secret_path" username="admin" password="supersecret"

enos/modules/run_workloads/templates/vault-secrets.nomad.hcl.tpl

@@ -9,6 +9,13 @@ job "get-secret" {
   group "group" {
     count = var.alloc_count
 
+  restart {
+    interval         = "5s"
+    delay            = "1s"
+    mode             = "delay"
+    render_templates = true
+  }
+
     network {
       port "web" {
         to = 8001
@@ -17,44 +24,42 @@ job "get-secret" {
 
     service {
       provider = "consul"
-      name     = "writes-vars-checker"
+      name     = "get-secret"
       port     = "web"
-      task     = "task"
+      task     = "read-secrets"
 
-      /* check {
-        type     = "script"
+    check {
         interval = "10s"
         timeout  = "1s"
-        command  = "/bin/sh"
-        args     = ["/local/read-script.sh"]
 
-        # this check will read from the Task API, so we need to ensure that we
-        # can tolerate the listener going away during client upgrades
-        check_restart {
-          limit = 10
-        }
-      } */
+    type    = "script"
+    command = "/bin/bash"
+    args    = ["-c", "test -f local/config.json"]
+
+      }
     }
 
     task "read-secrets" {
       driver = "raw_exec"
 
-      config {
+     config {
         command = "/bin/bash"
-        args    = ["-c", "cat local/config.json && sleep 30"]
+        args    = ["-c", "while true; do cat local/config.json; sleep 1; done"]
       }
 
       vault {}
 
       template {
         destination = "local/config.json"
-        change_mode = "restart"
+        change_mode   = "signal"
+        change_signal = "SIGHUP"
 
         data = <<EOT
-{{ with secret "{{}}/data/default/get-secret" }}
+{{ with secret "${secret_path}" }}
 {
   "username": "{{ .Data.data.username }}",
   "password": "{{ .Data.data.password }}"
+  {{ timestamp "unix" }}
 }
 {{ end }}
 EOT