From 997358d855d25d0ba61eaf08492b512ad14e6fa2 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 9 Jan 2025 08:41:22 -0500 Subject: [PATCH] E2E: dynamic host volumes workflows (#24816) Initial end-to-end tests for dynamic host volumes. This includes tests for two workflows: * One where a dynamic host volume is created by a plugin and then mounted by a job. * Another where a dynamic host volume is created out-of-band and registered by a job, then mounted by another job. This changeset also moves the existing `volumes` E2E test package to the better-named `volume_mounts`. Ref: https://hashicorp.atlassian.net/browse/NET-11551 --- e2e/dynamic_host_volumes/doc.go | 7 + .../dynamic_host_volumes_test.go | 145 ++++++++++++++++++ .../input/mount-created.nomad.hcl | 42 +++++ .../input/mount-registered.nomad.hcl | 42 +++++ .../input/register-volumes.nomad.hcl | 76 +++++++++ .../input/register-volumes.policy.hcl | 25 +++ .../input/volume-create.nomad.hcl | 11 ++ e2e/e2e_test.go | 2 +- e2e/{volumes => volume_mounts}/doc.go | 2 +- .../input/volumes.nomad | 0 .../volumes_test.go | 2 +- 11 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 e2e/dynamic_host_volumes/doc.go create mode 100644 e2e/dynamic_host_volumes/dynamic_host_volumes_test.go create mode 100644 e2e/dynamic_host_volumes/input/mount-created.nomad.hcl create mode 100644 e2e/dynamic_host_volumes/input/mount-registered.nomad.hcl create mode 100644 e2e/dynamic_host_volumes/input/register-volumes.nomad.hcl create mode 100644 e2e/dynamic_host_volumes/input/register-volumes.policy.hcl create mode 100644 e2e/dynamic_host_volumes/input/volume-create.nomad.hcl rename e2e/{volumes => volume_mounts}/doc.go (90%) rename e2e/{volumes => volume_mounts}/input/volumes.nomad (100%) rename e2e/{volumes => volume_mounts}/volumes_test.go (99%) diff --git a/e2e/dynamic_host_volumes/doc.go b/e2e/dynamic_host_volumes/doc.go new file mode 100644 index 000000000..6df947495 --- /dev/null +++ b/e2e/dynamic_host_volumes/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package dynamic_host_volumes + +// This package contains only tests, so this is a placeholder file to +// make sure builds don't fail with "no non-test Go files in" errors diff --git a/e2e/dynamic_host_volumes/dynamic_host_volumes_test.go b/e2e/dynamic_host_volumes/dynamic_host_volumes_test.go new file mode 100644 index 000000000..886623cfc --- /dev/null +++ b/e2e/dynamic_host_volumes/dynamic_host_volumes_test.go @@ -0,0 +1,145 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package dynamic_host_volumes + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/shoenig/test/must" + "github.com/shoenig/test/wait" +) + +// TestDynamicHostVolumes_CreateWorkflow tests the workflow where a dynamic host +// volume is created by a plugin and then mounted by a job. +func TestDynamicHostVolumes_CreateWorkflow(t *testing.T) { + + nomad := e2eutil.NomadClient(t) + e2eutil.WaitForLeader(t, nomad) + e2eutil.WaitForNodesReady(t, nomad, 1) + + out, err := e2eutil.Command("nomad", "volume", "create", + "-detach", "input/volume-create.nomad.hcl") + must.NoError(t, err) + + split := strings.Split(out, " ") + volID := strings.TrimSpace(split[len(split)-1]) + t.Logf("created volume: %q\n", volID) + + t.Cleanup(func() { + _, err := e2eutil.Command("nomad", "volume", "delete", "-type", "host", volID) + must.NoError(t, err) + }) + + out, err = e2eutil.Command("nomad", "volume", "status", "-type", "host", volID) + must.NoError(t, err) + + nodeID, err := e2eutil.GetField(out, "Node ID") + must.NoError(t, err) + + must.Wait(t, wait.InitialSuccess( + wait.ErrorFunc(func() error { + node, _, err := nomad.Nodes().Info(nodeID, nil) + if err != nil { + return err + } + _, ok := node.HostVolumes["created-volume"] + if !ok { + return fmt.Errorf("node %q did not fingerprint volume %q", nodeID, volID) + } + vol, _, err := nomad.HostVolumes().Get(volID, nil) + if err != nil { + return err + } + if vol.State != "ready" { + return fmt.Errorf("node fingerprinted volume but status was not updated") + } + return nil + }), + wait.Timeout(5*time.Second), + wait.Gap(50*time.Millisecond), + )) + + _, cleanup := jobs3.Submit(t, "./input/mount-created.nomad.hcl") + t.Cleanup(cleanup) +} + +// TestDynamicHostVolumes_RegisterWorkflow tests the workflow where a dynamic +// host volume is created out-of-band and registered by a job, then mounted by +// another job. +func TestDynamicHostVolumes_RegisterWorkflow(t *testing.T) { + + nomad := e2eutil.NomadClient(t) + e2eutil.WaitForLeader(t, nomad) + e2eutil.WaitForNodesReady(t, nomad, 1) + + submitted, cleanup := jobs3.Submit(t, "./input/register-volumes.nomad.hcl", + jobs3.Dispatcher(), + ) + t.Cleanup(cleanup) + + _, err := e2eutil.Command( + "nomad", "acl", "policy", "apply", + "-namespace", "default", "-job", submitted.JobID(), + "register-volumes-policy", "./input/register-volumes.policy.hcl") + must.NoError(t, err) + + must.NoError(t, e2eutil.Dispatch(submitted.JobID(), + map[string]string{ + "vol_name": "registered-volume", + "vol_size": "1G", + "vol_path": "/tmp/registered-volume", + }, "")) + + out, err := e2eutil.Command("nomad", "volume", "status", "-verbose", "-type", "host") + must.NoError(t, err) + vols, err := e2eutil.ParseColumns(out) + must.NoError(t, err) + + var volID string + var nodeID string + for _, vol := range vols { + if vol["Name"] == "registered-volume" { + volID = vol["ID"] + nodeID = vol["Node ID"] + } + } + must.NotEq(t, "", volID) + + t.Cleanup(func() { + _, err := e2eutil.Command("nomad", "volume", "delete", "-type", "host", volID) + must.NoError(t, err) + }) + + must.Wait(t, wait.InitialSuccess( + wait.ErrorFunc(func() error { + node, _, err := nomad.Nodes().Info(nodeID, nil) + if err != nil { + return err + } + _, ok := node.HostVolumes["registered-volume"] + if !ok { + return fmt.Errorf("node %q did not fingerprint volume %q", nodeID, volID) + } + vol, _, err := nomad.HostVolumes().Get(volID, nil) + if err != nil { + return err + } + if vol.State != "ready" { + return fmt.Errorf("node fingerprinted volume but status was not updated") + } + return nil + }), + wait.Timeout(5*time.Second), + wait.Gap(50*time.Millisecond), + )) + + _, cleanup2 := jobs3.Submit(t, "./input/mount-registered.nomad.hcl") + t.Cleanup(cleanup2) + +} diff --git a/e2e/dynamic_host_volumes/input/mount-created.nomad.hcl b/e2e/dynamic_host_volumes/input/mount-created.nomad.hcl new file mode 100644 index 000000000..94456f698 --- /dev/null +++ b/e2e/dynamic_host_volumes/input/mount-created.nomad.hcl @@ -0,0 +1,42 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "example" { + + group "web" { + + network { + mode = "bridge" + port "www" { + to = 8001 + } + } + + volume "data" { + type = "host" + source = "created-volume" + } + + task "http" { + + driver = "docker" + config { + image = "busybox:1" + command = "httpd" + args = ["-v", "-f", "-p", "8001", "-h", "/var/www"] + ports = ["www"] + } + + volume_mount { + volume = "data" + destination = "/var/www" + } + + resources { + cpu = 128 + memory = 128 + } + + } + } +} diff --git a/e2e/dynamic_host_volumes/input/mount-registered.nomad.hcl b/e2e/dynamic_host_volumes/input/mount-registered.nomad.hcl new file mode 100644 index 000000000..2d635aeef --- /dev/null +++ b/e2e/dynamic_host_volumes/input/mount-registered.nomad.hcl @@ -0,0 +1,42 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "example" { + + group "web" { + + network { + mode = "bridge" + port "www" { + to = 8001 + } + } + + volume "data" { + type = "host" + source = "registered-volume" + } + + task "http" { + + driver = "docker" + config { + image = "busybox:1" + command = "httpd" + args = ["-v", "-f", "-p", "8001", "-h", "/var/www"] + ports = ["www"] + } + + volume_mount { + volume = "data" + destination = "/var/www" + } + + resources { + cpu = 128 + memory = 128 + } + + } + } +} diff --git a/e2e/dynamic_host_volumes/input/register-volumes.nomad.hcl b/e2e/dynamic_host_volumes/input/register-volumes.nomad.hcl new file mode 100644 index 000000000..b6c808142 --- /dev/null +++ b/e2e/dynamic_host_volumes/input/register-volumes.nomad.hcl @@ -0,0 +1,76 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +job "register-volumes" { + + type = "batch" + + parameterized { + meta_required = ["vol_name", "vol_size", "vol_path"] + } + + group "group" { + + restart { + attempts = 0 + mode = "fail" + } + + task "task" { + + driver = "raw_exec" + + config { + command = "bash" + args = ["${NOMAD_TASK_DIR}/register.sh", "${node.unique.id}"] + } + + template { + destination = "local/register.sh" + data = <