e2e: add initial tests for secrets block (#26397)

e2e/e2e_test.go

@@ -44,6 +44,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/podman"
 	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/scaling"
+	_ "github.com/hashicorp/nomad/e2e/secret"
 	_ "github.com/hashicorp/nomad/e2e/spread"
 	_ "github.com/hashicorp/nomad/e2e/vaultsecrets"
 	_ "github.com/hashicorp/nomad/e2e/volume_mounts"

e2e/secret/doc.go

@@ -0,0 +1,8 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package secret provides end-to-end tests for Nomad workloads with secret blocks.
+//
+// In order to run this test suite only, from the e2e directory you can trigger
+// go test -v ./secret
+package secret

e2e/secret/input/nomad_secret.hcl

@@ -0,0 +1,41 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "nomad_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      secret "testsecret" {
+        provider = "nomad"
+        path     = "SECRET_PATH"
+        config {
+          namespace = "default"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.key}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/input/vault_secret.hcl

@@ -0,0 +1,43 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "vault_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      vault {}
+
+      secret "testsecret" {
+        provider = "vault"
+        path     = "SECRET_PATH"
+        config {
+          engine = "kv_v2"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.key}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/secret_test.go

@@ -0,0 +1,118 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secret
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/shoenig/test/must"
+)
+
+const ns = "default"
+
+func TestVaultSecret(t *testing.T) {
+	// Lookup the cluster ID which is the KV backend path start.
+	clusterID, found := os.LookupEnv("CLUSTER_UNIQUE_IDENTIFIER")
+	if !found {
+		t.Fatal("CLUSTER_UNIQUE_IDENTIFIER env var not set")
+	}
+
+	// Generate our pathing for Vault and a secret value that we will check as
+	// part of the test.
+	secretCLIPath := filepath.Join(ns, "vault_secret", "testsecret")
+	secretFullPath := filepath.Join(clusterID, "data", secretCLIPath)
+	secretValue := uuid.Generate()
+
+	// Create the secret at the correct mount point for this E2E cluster and use
+	// the metadata delete command to permanently delete this when the test exits
+	e2e.MustCommand(t, "vault kv put -mount=%s %s key=%s", clusterID, secretCLIPath, secretValue)
+	e2e.CleanupCommand(t, "vault kv metadata delete -mount=%s %s", clusterID, secretCLIPath)
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/vault_secret.hcl",
+		jobs3.DisableRandomJobID(),
+		jobs3.Namespace(ns),
+		jobs3.Detach(),
+		jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath),
+	)
+	t.Cleanup(cleanJob)
+
+	// Ensure the placed allocation reaches the running state. If the test fails
+	// here, it's likely due to permissions or pathing of the secret errors.
+	must.NoError(
+		t,
+		e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}),
+		must.Sprint("expected running allocation"),
+	)
+
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env")
+	must.NoError(t, err)
+	must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}
+
+func TestNomadSecret(t *testing.T) {
+	// Generate our pathing for Vault and a secret value that we will check as
+	// part of the test.
+	secretFullPath := filepath.Join("nomad_secret", "testsecret")
+	secretValue := uuid.Generate()
+
+	nomadClient := e2e.NomadClient(t)
+
+	opts := &api.WriteOptions{Namespace: ns}
+	_, _, err := nomadClient.Variables().Create(&api.Variable{
+		Namespace: ns,
+		Path:      secretFullPath,
+		Items:     map[string]string{"key": secretValue},
+	}, opts)
+	must.NoError(t, err)
+
+	// create an ACL policy and attach it to the job ID this test will run
+	myNamespacePolicy := api.ACLPolicy{
+		Name:        "secret-block-policy",
+		Rules:       fmt.Sprintf(`namespace "%s" {variables {path "*" {capabilities = ["read"]}}}`, ns),
+		Description: "This namespace is for secrets block e2e testing",
+		JobACL: &api.JobACL{
+			Namespace: ns,
+			JobID:     "nomad_secret",
+		},
+	}
+	_, err = nomadClient.ACLPolicies().Upsert(&myNamespacePolicy, nil)
+	must.NoError(t, err)
+
+	t.Cleanup(func() {
+		nomadClient.ACLPolicies().Delete("secret-block-policy", nil)
+	})
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/nomad_secret.hcl",
+		jobs3.DisableRandomJobID(),
+		jobs3.Namespace(ns),
+		jobs3.Detach(),
+		jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath),
+	)
+	t.Cleanup(cleanJob)
+
+	// Ensure the placed allocation reaches the running state. If the test fails
+	// here, it's likely due to permissions or pathing of the secret errors.
+	must.NoError(
+		t,
+		e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}),
+		must.Sprint("expected running allocation"),
+	)
+
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env")
+	must.NoError(t, err)
+	must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}