diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index b703e12fe..1cc91aa77 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -94,6 +94,14 @@ scenario "upgrade" { cert_file = step.provision_cluster.cert_file key_file = step.provision_cluster.key_file nomad_token = step.provision_cluster.nomad_token + workloads = { + service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3, type = "service" } + service_docker = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3, type = "service" } + system_docker = { job_spec = "jobs/docker-system.nomad.hcl", alloc_count = 0, type = "system" } + batch_docker = { job_spec = "jobs/docker-batch.nomad.hcl", alloc_count = 3, type = "batch" } + batch_raw_exec = { job_spec = "jobs/raw-exec-batch.nomad.hcl", alloc_count = 3, type = "batch" } + system_raw_exec = { job_spec = "jobs/raw-exec-system.nomad.hcl", alloc_count = 0, type = "system" } + } } verifies = [ @@ -365,4 +373,8 @@ EOF output "binary_path" { value = step.copy_initial_binary.binary_path } + + output "allocs" { + value = step.run_initial_workloads.allocs_count + } } diff --git a/enos/modules/run_workloads/jobs/docker-batch.nomad.hcl b/enos/modules/run_workloads/jobs/docker-batch.nomad.hcl new file mode 100644 index 000000000..ebaa3195c --- /dev/null +++ b/enos/modules/run_workloads/jobs/docker-batch.nomad.hcl @@ -0,0 +1,51 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 +variable "alloc_count" { + type = number + default = 1 +} + +job "batch-docker" { + type = "batch" + + group "batch-docker" { + count = var.alloc_count + + network { + port "db" { + to = 6377 + } + } + + service { + provider = "consul" + name = "batch-docker" + port = "db" + + check { + name = "service-docker_probe" + type = "tcp" + interval = "10s" + timeout = "1s" + } + } + + task "batch-docker" { + driver = "docker" + + config { + image = "redis:latest" + ports = ["db"] + args = ["--port", "6377"] + labels { + workload = "docker-batch" + } + } + + resources { + cpu = 50 + memory = 64 + } + } + } +} diff --git a/enos/modules/run_workloads/jobs/docker-service.nomad.hcl b/enos/modules/run_workloads/jobs/docker-service.nomad.hcl index 2cec24fbd..6cf2613cb 100644 --- a/enos/modules/run_workloads/jobs/docker-service.nomad.hcl +++ b/enos/modules/run_workloads/jobs/docker-service.nomad.hcl @@ -6,22 +6,42 @@ variable "alloc_count" { } job "service-docker" { - group "service-docker" { count = var.alloc_count - task "alpine" { + + network { + port "db" { + to = 6379 + } + } + + service { + provider = "consul" + name = "service-docker" + port = "db" + + check { + name = "service-docker_probe" + type = "tcp" + interval = "10s" + timeout = "1s" + } + } + + task "service-docker" { driver = "docker" config { - image = "alpine:latest" - command = "sh" - args = ["-c", "while true; do sleep 30000; done"] - + image = "redis:7.2" + ports = ["db"] + labels { + workload = "docker-service" + } } resources { - cpu = 100 - memory = 128 + cpu = 50 + memory = 64 } } } diff --git a/enos/modules/run_workloads/jobs/docker-system.nomad.hcl b/enos/modules/run_workloads/jobs/docker-system.nomad.hcl new file mode 100644 index 000000000..847b75cc2 --- /dev/null +++ b/enos/modules/run_workloads/jobs/docker-system.nomad.hcl @@ -0,0 +1,51 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 +variable "alloc_count" { + type = number + default = 1 +} + +job "system-docker" { + type = "system" + + group "system-docker" { + + network { + port "db" { + to = 6378 + } + } + + service { + provider = "consul" + name = "system-docker" + port = "db" + + check { + name = "system-docker_probe" + type = "tcp" + interval = "10s" + timeout = "1s" + } + } + + + task "system-docker" { + driver = "docker" + + config { + image = "redis:7.2" + ports = ["db"] + args = ["--port", "6378"] + labels { + workload = "docker-system" + } + } + + resources { + cpu = 50 + memory = 64 + } + } + } +} diff --git a/enos/modules/run_workloads/jobs/raw-exec-batch.nomad.hcl b/enos/modules/run_workloads/jobs/raw-exec-batch.nomad.hcl new file mode 100644 index 000000000..1e931c8a4 --- /dev/null +++ b/enos/modules/run_workloads/jobs/raw-exec-batch.nomad.hcl @@ -0,0 +1,41 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +variable "alloc_count" { + type = number + default = 1 +} + +job "batch-raw-exec" { + type = "batch" + + group "batch-raw-exec" { + count = var.alloc_count + + task "batch-raw-exec" { + driver = "raw_exec" + + config { + command = "bash" + args = ["-c", "./local/runme.sh"] + } + + template { + data = < v if v.type == "system" }) + service_batch_allocs = sum([for wl in var.workloads : wl.alloc_count]) } resource "enos_local_exec" "wait_for_nomad_api" { @@ -23,8 +28,27 @@ resource "enos_local_exec" "wait_for_nomad_api" { scripts = [abspath("${path.module}/scripts/wait_for_nomad_api.sh")] } +resource "enos_local_exec" "get_nodes" { + environment = local.nomad_env + + inline = ["nomad node status -json | jq '[.[] | select(.Status == \"ready\")] | length'"] +} + +resource "enos_local_exec" "get_jobs" { + environment = local.nomad_env + + inline = ["nomad job status| awk '$4 == \"running\" {count++} END {print count+0}'"] +} + +resource "enos_local_exec" "get_allocs" { + environment = local.nomad_env + + inline = ["nomad alloc status -json | jq '[.[] | select(.ClientStatus == \"running\")] | length'"] +} + resource "enos_local_exec" "workloads" { - for_each = var.workloads + depends_on = [enos_local_exec.get_jobs, enos_local_exec.get_allocs] + for_each = var.workloads environment = local.nomad_env diff --git a/enos/modules/run_workloads/outputs.tf b/enos/modules/run_workloads/outputs.tf index b34a38e1d..76c7814d0 100644 --- a/enos/modules/run_workloads/outputs.tf +++ b/enos/modules/run_workloads/outputs.tf @@ -1,16 +1,27 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -/* output "jobs_count" { - value = length(local.job_names) -} */ - output "jobs_count" { description = "The number of jobs thar should be running in the cluster" + value = length(var.workloads) + tonumber(coalesce(chomp(enos_local_exec.get_jobs.stdout))) +} + +output "new_jobs_count" { + description = "The number of jobs that were triggered by the module" value = length(var.workloads) } output "allocs_count" { description = "The number of allocs that should be running in the cluster" - value = sum([for wl in var.workloads : wl.alloc_count]) + value = local.system_job_count * tonumber(coalesce(chomp(enos_local_exec.get_nodes.stdout))) + local.service_batch_allocs + tonumber(coalesce(chomp(enos_local_exec.get_allocs.stdout))) +} + +output "nodes" { + description = "Number of current clients in the cluster" + value = chomp(enos_local_exec.get_nodes.stdout) +} + +output "new_allocs_count" { + description = "The number of allocs that will be added to the cluster after all the workloads are run" + value = local.system_job_count * tonumber(coalesce(chomp(enos_local_exec.get_nodes.stdout), "0")) + local.service_batch_allocs } diff --git a/enos/modules/run_workloads/variables.tf b/enos/modules/run_workloads/variables.tf index 6281d988c..6bd96875e 100644 --- a/enos/modules/run_workloads/variables.tf +++ b/enos/modules/run_workloads/variables.tf @@ -34,10 +34,6 @@ variable "workloads" { type = map(object({ job_spec = string alloc_count = number + type = string })) - - default = { - service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3 } - service_docker = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3 } - } } diff --git a/enos/modules/test_cluster_health/scripts/allocs.sh b/enos/modules/test_cluster_health/scripts/allocs.sh index f8cc5abe5..fad3572cc 100755 --- a/enos/modules/test_cluster_health/scripts/allocs.sh +++ b/enos/modules/test_cluster_health/scripts/allocs.sh @@ -48,6 +48,10 @@ done echo "All ALLOCS are running." +if [ "$allocs_length" -eq 0 ]; then + exit 0 +fi + # Quality: nomad_reschedule_alloc: A POST / PUT call to /v1/allocation/:alloc_id/stop results in the stopped allocation being rescheduled random_index=$((RANDOM % allocs_length)) diff --git a/enos/modules/test_cluster_health/scripts/jobs.sh b/enos/modules/test_cluster_health/scripts/jobs.sh index 167a6650f..dbb46c578 100755 --- a/enos/modules/test_cluster_health/scripts/jobs.sh +++ b/enos/modules/test_cluster_health/scripts/jobs.sh @@ -18,7 +18,7 @@ if [ -z "$jobs_length" ]; then fi if [ "$jobs_length" -ne "$JOB_COUNT" ]; then - error_exit "The number of running jobs ($jobs_length) does not match the expected count ($JOB_COUNT)\n$(nomad job status | awk 'NR > 1 && $4 != "running" {print $4}')" + error_exit "The number of running jobs ($jobs_length) does not match the expected count ($JOB_COUNT) $(nomad job status | awk 'NR > 1 && $4 != "running" {print $4}') " fi echo "All JOBS are running."