Add new workloads (#25106)

* func: Add more workloads

* Update jobs.sh

* Update versions.sh

* style: format

* Update enos/modules/test_cluster_health/scripts/allocs.sh

Co-authored-by: Tim Gross <tgross@hashicorp.com>

* docs: improve outputs descriptions

* func: change docker workloads to be redis boxes and add healthchecks

* func: register the services on consul

* style: format

---------

Co-authored-by: Tim Gross <tgross@hashicorp.com>

enos/enos-scenario-upgrade.hcl

@@ -94,6 +94,14 @@ scenario "upgrade" {
       cert_file   = step.provision_cluster.cert_file
       key_file    = step.provision_cluster.key_file
       nomad_token = step.provision_cluster.nomad_token
+      workloads = {
+        service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3, type = "service" }
+        service_docker   = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3, type = "service" }
+        system_docker    = { job_spec = "jobs/docker-system.nomad.hcl", alloc_count = 0, type = "system" }
+        batch_docker     = { job_spec = "jobs/docker-batch.nomad.hcl", alloc_count = 3, type = "batch" }
+        batch_raw_exec   = { job_spec = "jobs/raw-exec-batch.nomad.hcl", alloc_count = 3, type = "batch" }
+        system_raw_exec  = { job_spec = "jobs/raw-exec-system.nomad.hcl", alloc_count = 0, type = "system" }
+      }
     }
 
     verifies = [
@@ -365,4 +373,8 @@ EOF
   output "binary_path" {
     value = step.copy_initial_binary.binary_path
   }
+
+  output "allocs" {
+    value = step.run_initial_workloads.allocs_count
+  }
 }

enos/modules/run_workloads/jobs/docker-batch.nomad.hcl

@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+job "batch-docker" {
+  type = "batch"
+
+  group "batch-docker" {
+    count = var.alloc_count
+
+    network {
+      port "db" {
+        to = 6377
+      }
+    }
+
+    service {
+      provider = "consul"
+      name     = "batch-docker"
+      port     = "db"
+
+      check {
+        name     = "service-docker_probe"
+        type     = "tcp"
+        interval = "10s"
+        timeout  = "1s"
+      }
+    }
+
+    task "batch-docker" {
+      driver = "docker"
+
+      config {
+        image = "redis:latest"
+        ports = ["db"]
+        args  = ["--port", "6377"]
+        labels {
+          workload = "docker-batch"
+        }
+      }
+
+      resources {
+        cpu    = 50
+        memory = 64
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/docker-service.nomad.hcl

@@ -6,22 +6,42 @@ variable "alloc_count" {
 }
 
 job "service-docker" {
-
   group "service-docker" {
     count = var.alloc_count
-    task "alpine" {
+
+    network {
+      port "db" {
+        to = 6379
+      }
+    }
+
+    service {
+      provider = "consul"
+      name     = "service-docker"
+      port     = "db"
+
+      check {
+        name     = "service-docker_probe"
+        type     = "tcp"
+        interval = "10s"
+        timeout  = "1s"
+      }
+    }
+
+    task "service-docker" {
       driver = "docker"
 
       config {
-        image   = "alpine:latest"
-        command = "sh"
-        args    = ["-c", "while true; do sleep 30000; done"]
-
+        image = "redis:7.2"
+        ports = ["db"]
+        labels {
+          workload = "docker-service"
+        }
       }
 
       resources {
-        cpu    = 100
-        memory = 128
+        cpu    = 50
+        memory = 64
       }
     }
   }

enos/modules/run_workloads/jobs/docker-system.nomad.hcl

@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+job "system-docker" {
+  type = "system"
+
+  group "system-docker" {
+
+    network {
+      port "db" {
+        to = 6378
+      }
+    }
+
+    service {
+      provider = "consul"
+      name     = "system-docker"
+      port     = "db"
+
+      check {
+        name     = "system-docker_probe"
+        type     = "tcp"
+        interval = "10s"
+        timeout  = "1s"
+      }
+    }
+
+
+    task "system-docker" {
+      driver = "docker"
+
+      config {
+        image = "redis:7.2"
+        ports = ["db"]
+        args  = ["--port", "6378"]
+        labels {
+          workload = "docker-system"
+        }
+      }
+
+      resources {
+        cpu    = 50
+        memory = 64
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/raw-exec-batch.nomad.hcl

@@ -0,0 +1,41 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+job "batch-raw-exec" {
+  type = "batch"
+
+  group "batch-raw-exec" {
+    count = var.alloc_count
+
+    task "batch-raw-exec" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "./local/runme.sh"]
+      }
+
+      template {
+        data        = <<EOH
+#!/bin/bash
+
+while true; do
+    sleep 30000  
+done
+EOH
+        destination = "local/runme.sh"
+        perms       = "755"
+      }
+
+      resources {
+        cpu    = 50
+        memory = 64
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/raw-exec-service.nomad.hcl

@@ -35,6 +35,11 @@ EOH
         destination = "local/runme.sh"
         perms       = "755"
       }
+
+      resources {
+        cpu    = 50
+        memory = 64
+      }
     }
   }
 }

enos/modules/run_workloads/jobs/raw-exec-system.nomad.hcl

@@ -0,0 +1,40 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+job "system-raw-exec" {
+  type = "system"
+
+  group "system-raw-exec" {
+
+    task "system-raw-exec" {
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["-c", "./local/runme.sh"]
+      }
+
+      template {
+        data        = <<EOH
+#!/bin/bash
+
+while true; do
+    sleep 30000  
+done
+EOH
+        destination = "local/runme.sh"
+        perms       = "755"
+      }
+
+      resources {
+        cpu    = 50
+        memory = 64
+      }
+    }
+  }
+}

enos/modules/run_workloads/main.tf

@@ -10,11 +10,16 @@ terraform {
 }
 
 locals {
-  nomad_env = { NOMAD_ADDR = var.nomad_addr
+  nomad_env = {
+    NOMAD_ADDR        = var.nomad_addr
     NOMAD_CACERT      = var.ca_file
     NOMAD_CLIENT_CERT = var.cert_file
     NOMAD_CLIENT_KEY  = var.key_file
-  NOMAD_TOKEN = var.nomad_token }
+    NOMAD_TOKEN       = var.nomad_token
+  }
+
+  system_job_count     = length({ for k, v in var.workloads : k => v if v.type == "system" })
+  service_batch_allocs = sum([for wl in var.workloads : wl.alloc_count])
 }
 
 resource "enos_local_exec" "wait_for_nomad_api" {
@@ -23,8 +28,27 @@ resource "enos_local_exec" "wait_for_nomad_api" {
   scripts = [abspath("${path.module}/scripts/wait_for_nomad_api.sh")]
 }
 
+resource "enos_local_exec" "get_nodes" {
+  environment = local.nomad_env
+
+  inline = ["nomad node status -json | jq '[.[] | select(.Status == \"ready\")] | length'"]
+}
+
+resource "enos_local_exec" "get_jobs" {
+  environment = local.nomad_env
+
+  inline = ["nomad job status| awk '$4 == \"running\" {count++} END {print count+0}'"]
+}
+
+resource "enos_local_exec" "get_allocs" {
+  environment = local.nomad_env
+
+  inline = ["nomad alloc status -json | jq '[.[] | select(.ClientStatus == \"running\")] | length'"]
+}
+
 resource "enos_local_exec" "workloads" {
-  for_each = var.workloads
+  depends_on = [enos_local_exec.get_jobs, enos_local_exec.get_allocs]
+  for_each   = var.workloads
 
   environment = local.nomad_env
 

enos/modules/run_workloads/outputs.tf

@@ -1,16 +1,27 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
-/* output "jobs_count" {
-  value = length(local.job_names)
-} */
-
 output "jobs_count" {
   description = "The number of jobs thar should be running in the cluster"
+  value       = length(var.workloads) + tonumber(coalesce(chomp(enos_local_exec.get_jobs.stdout)))
+}
+
+output "new_jobs_count" {
+  description = "The number of jobs that were triggered by the module"
   value       = length(var.workloads)
 }
 
 output "allocs_count" {
   description = "The number of allocs that should be running in the cluster"
-  value       = sum([for wl in var.workloads : wl.alloc_count])
+  value       = local.system_job_count * tonumber(coalesce(chomp(enos_local_exec.get_nodes.stdout))) + local.service_batch_allocs + tonumber(coalesce(chomp(enos_local_exec.get_allocs.stdout)))
+}
+
+output "nodes" {
+  description = "Number of current clients in the cluster"
+  value       = chomp(enos_local_exec.get_nodes.stdout)
+}
+
+output "new_allocs_count" {
+  description = "The number of allocs that will be added to the cluster after all the workloads are run"
+  value       = local.system_job_count * tonumber(coalesce(chomp(enos_local_exec.get_nodes.stdout), "0")) + local.service_batch_allocs
 }

enos/modules/run_workloads/variables.tf

@@ -34,10 +34,6 @@ variable "workloads" {
   type = map(object({
     job_spec    = string
     alloc_count = number
+    type        = string
   }))
-
-  default = {
-    service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3 }
-    service_docker   = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3 }
-  }
 }

enos/modules/test_cluster_health/scripts/allocs.sh

@@ -48,6 +48,10 @@ done
 
 echo "All ALLOCS are running."
 
+if [ "$allocs_length" -eq 0 ]; then
+    exit 0
+fi
+
 # Quality: nomad_reschedule_alloc: A POST / PUT call to /v1/allocation/:alloc_id/stop results in the stopped allocation being rescheduled
 
 random_index=$((RANDOM % allocs_length))

enos/modules/test_cluster_health/scripts/jobs.sh

@@ -18,7 +18,7 @@ if [ -z "$jobs_length" ];  then
 fi
 
 if [ "$jobs_length" -ne "$JOB_COUNT" ]; then
-    error_exit "The number  of running jobs ($jobs_length) does not match the expected count ($JOB_COUNT)\n$(nomad job status | awk 'NR > 1 && $4 != "running" {print $4}')"
+    error_exit "The number  of running jobs ($jobs_length) does not match the expected count ($JOB_COUNT) $(nomad job status | awk 'NR > 1 && $4 != "running" {print $4}') "
 fi
 
 echo "All JOBS are running."