E2E: dynamic host volumes workflows (#24816)

Initial end-to-end tests for dynamic host volumes. This includes tests for two
workflows:

* One where a dynamic host volume is created by a plugin and then mounted by a job.
* Another where a dynamic host volume is created out-of-band and registered by a
  job, then mounted by another job.

This changeset also moves the existing `volumes` E2E test package to the
better-named `volume_mounts`.

Ref: https://hashicorp.atlassian.net/browse/NET-11551

e2e/dynamic_host_volumes/doc.go

@@ -0,0 +1,7 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package dynamic_host_volumes
+
+// This package contains only tests, so this is a placeholder file to
+// make sure builds don't fail with "no non-test Go files in" errors

e2e/dynamic_host_volumes/dynamic_host_volumes_test.go

@@ -0,0 +1,145 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package dynamic_host_volumes
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/shoenig/test/must"
+	"github.com/shoenig/test/wait"
+)
+
+// TestDynamicHostVolumes_CreateWorkflow tests the workflow where a dynamic host
+// volume is created by a plugin and then mounted by a job.
+func TestDynamicHostVolumes_CreateWorkflow(t *testing.T) {
+
+	nomad := e2eutil.NomadClient(t)
+	e2eutil.WaitForLeader(t, nomad)
+	e2eutil.WaitForNodesReady(t, nomad, 1)
+
+	out, err := e2eutil.Command("nomad", "volume", "create",
+		"-detach", "input/volume-create.nomad.hcl")
+	must.NoError(t, err)
+
+	split := strings.Split(out, " ")
+	volID := strings.TrimSpace(split[len(split)-1])
+	t.Logf("created volume: %q\n", volID)
+
+	t.Cleanup(func() {
+		_, err := e2eutil.Command("nomad", "volume", "delete", "-type", "host", volID)
+		must.NoError(t, err)
+	})
+
+	out, err = e2eutil.Command("nomad", "volume", "status", "-type", "host", volID)
+	must.NoError(t, err)
+
+	nodeID, err := e2eutil.GetField(out, "Node ID")
+	must.NoError(t, err)
+
+	must.Wait(t, wait.InitialSuccess(
+		wait.ErrorFunc(func() error {
+			node, _, err := nomad.Nodes().Info(nodeID, nil)
+			if err != nil {
+				return err
+			}
+			_, ok := node.HostVolumes["created-volume"]
+			if !ok {
+				return fmt.Errorf("node %q did not fingerprint volume %q", nodeID, volID)
+			}
+			vol, _, err := nomad.HostVolumes().Get(volID, nil)
+			if err != nil {
+				return err
+			}
+			if vol.State != "ready" {
+				return fmt.Errorf("node fingerprinted volume but status was not updated")
+			}
+			return nil
+		}),
+		wait.Timeout(5*time.Second),
+		wait.Gap(50*time.Millisecond),
+	))
+
+	_, cleanup := jobs3.Submit(t, "./input/mount-created.nomad.hcl")
+	t.Cleanup(cleanup)
+}
+
+// TestDynamicHostVolumes_RegisterWorkflow tests the workflow where a dynamic
+// host volume is created out-of-band and registered by a job, then mounted by
+// another job.
+func TestDynamicHostVolumes_RegisterWorkflow(t *testing.T) {
+
+	nomad := e2eutil.NomadClient(t)
+	e2eutil.WaitForLeader(t, nomad)
+	e2eutil.WaitForNodesReady(t, nomad, 1)
+
+	submitted, cleanup := jobs3.Submit(t, "./input/register-volumes.nomad.hcl",
+		jobs3.Dispatcher(),
+	)
+	t.Cleanup(cleanup)
+
+	_, err := e2eutil.Command(
+		"nomad", "acl", "policy", "apply",
+		"-namespace", "default", "-job", submitted.JobID(),
+		"register-volumes-policy", "./input/register-volumes.policy.hcl")
+	must.NoError(t, err)
+
+	must.NoError(t, e2eutil.Dispatch(submitted.JobID(),
+		map[string]string{
+			"vol_name": "registered-volume",
+			"vol_size": "1G",
+			"vol_path": "/tmp/registered-volume",
+		}, ""))
+
+	out, err := e2eutil.Command("nomad", "volume", "status", "-verbose", "-type", "host")
+	must.NoError(t, err)
+	vols, err := e2eutil.ParseColumns(out)
+	must.NoError(t, err)
+
+	var volID string
+	var nodeID string
+	for _, vol := range vols {
+		if vol["Name"] == "registered-volume" {
+			volID = vol["ID"]
+			nodeID = vol["Node ID"]
+		}
+	}
+	must.NotEq(t, "", volID)
+
+	t.Cleanup(func() {
+		_, err := e2eutil.Command("nomad", "volume", "delete", "-type", "host", volID)
+		must.NoError(t, err)
+	})
+
+	must.Wait(t, wait.InitialSuccess(
+		wait.ErrorFunc(func() error {
+			node, _, err := nomad.Nodes().Info(nodeID, nil)
+			if err != nil {
+				return err
+			}
+			_, ok := node.HostVolumes["registered-volume"]
+			if !ok {
+				return fmt.Errorf("node %q did not fingerprint volume %q", nodeID, volID)
+			}
+			vol, _, err := nomad.HostVolumes().Get(volID, nil)
+			if err != nil {
+				return err
+			}
+			if vol.State != "ready" {
+				return fmt.Errorf("node fingerprinted volume but status was not updated")
+			}
+			return nil
+		}),
+		wait.Timeout(5*time.Second),
+		wait.Gap(50*time.Millisecond),
+	))
+
+	_, cleanup2 := jobs3.Submit(t, "./input/mount-registered.nomad.hcl")
+	t.Cleanup(cleanup2)
+
+}

e2e/dynamic_host_volumes/input/mount-created.nomad.hcl

@@ -0,0 +1,42 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "example" {
+
+  group "web" {
+
+    network {
+      mode = "bridge"
+      port "www" {
+        to = 8001
+      }
+    }
+
+    volume "data" {
+      type   = "host"
+      source = "created-volume"
+    }
+
+    task "http" {
+
+      driver = "docker"
+      config {
+        image   = "busybox:1"
+        command = "httpd"
+        args    = ["-v", "-f", "-p", "8001", "-h", "/var/www"]
+        ports   = ["www"]
+      }
+
+      volume_mount {
+        volume      = "data"
+        destination = "/var/www"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 128
+      }
+
+    }
+  }
+}

e2e/dynamic_host_volumes/input/mount-registered.nomad.hcl

@@ -0,0 +1,42 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "example" {
+
+  group "web" {
+
+    network {
+      mode = "bridge"
+      port "www" {
+        to = 8001
+      }
+    }
+
+    volume "data" {
+      type   = "host"
+      source = "registered-volume"
+    }
+
+    task "http" {
+
+      driver = "docker"
+      config {
+        image   = "busybox:1"
+        command = "httpd"
+        args    = ["-v", "-f", "-p", "8001", "-h", "/var/www"]
+        ports   = ["www"]
+      }
+
+      volume_mount {
+        volume      = "data"
+        destination = "/var/www"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 128
+      }
+
+    }
+  }
+}

e2e/dynamic_host_volumes/input/register-volumes.nomad.hcl

@@ -0,0 +1,76 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+job "register-volumes" {
+
+  type = "batch"
+
+  parameterized {
+    meta_required = ["vol_name", "vol_size", "vol_path"]
+  }
+
+  group "group" {
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    task "task" {
+
+      driver = "raw_exec"
+
+      config {
+        command = "bash"
+        args    = ["${NOMAD_TASK_DIR}/register.sh", "${node.unique.id}"]
+      }
+
+      template {
+        destination = "local/register.sh"
+        data        = <<EOT
+#!/usr/bin/env bash
+set -ex
+
+NODE_ID=$1
+mkdir -p "${NOMAD_META_vol_path}"
+
+sed -e "s~NODE_ID~$NODE_ID~" \
+    -e "s~VOL_NAME~${NOMAD_META_vol_name}~" \
+    -e "s~VOL_SIZE~${NOMAD_META_vol_size}~" \
+    -e "s~VOL_PATH~${NOMAD_META_vol_path}~" \
+    local/volume.hcl | nomad volume register -
+
+        EOT
+
+      }
+
+      template {
+        destination = "local/volume.hcl"
+        data        = <<EOT
+name      = "VOL_NAME"
+node_id   = "NODE_ID"
+type      = "host"
+host_path = "VOL_PATH"
+capacity  = "VOL_SIZE"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+        EOT
+
+      }
+
+
+      identity {
+        env = true
+      }
+
+      resources {
+        cpu    = 100
+        memory = 100
+      }
+
+    }
+  }
+}

e2e/dynamic_host_volumes/input/register-volumes.policy.hcl

@@ -0,0 +1,25 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+namespace "*" {
+  policy = "write"
+  capabilities = [
+    "host-volume-register",
+  ]
+}
+
+agent {
+  policy = "read"
+}
+
+operator {
+  policy = "read"
+}
+
+node {
+  policy = "read"
+}
+
+node_pool "*" {
+  policy = "read"
+}

e2e/dynamic_host_volumes/input/volume-create.nomad.hcl

@@ -0,0 +1,11 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+name      = "created-volume"
+type      = "host"
+plugin_id = "mkdir"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}

e2e/e2e_test.go

@@ -46,7 +46,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/spread"
 	_ "github.com/hashicorp/nomad/e2e/vaultsecrets"
-	_ "github.com/hashicorp/nomad/e2e/volumes"
+	_ "github.com/hashicorp/nomad/e2e/volume_mounts"
 )
 
 func TestE2E(t *testing.T) {

e2e/volume_mounts/doc.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package volumes
+package volume_mounts
 
 // This package contains only tests, so this is a placeholder file to
 // make sure builds don't fail with "no non-test Go files in" errors

e2e/volume_mounts/volumes_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package volumes
+package volume_mounts
 
 import (
 	"fmt"