Merge pull request #26681 from hashicorp/NMD-760-nomad-secrets-block

Secrets Block: merge feature branch to main

e2e/e2e_test.go

@@ -44,6 +44,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/podman"
 	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/scaling"
+	_ "github.com/hashicorp/nomad/e2e/secret"
 	_ "github.com/hashicorp/nomad/e2e/spread"
 	_ "github.com/hashicorp/nomad/e2e/vaultsecrets"
 	_ "github.com/hashicorp/nomad/e2e/volume_mounts"

e2e/secret/doc.go

@@ -0,0 +1,8 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package secret provides end-to-end tests for Nomad workloads with secret blocks.
+//
+// In order to run this test suite only, from the e2e directory you can trigger
+// go test -v ./secret
+package secret

e2e/secret/input/custom_secret.hcl

@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "secret_value" {
+  type        = string
+  description = "The value of the randomly generated secret for this test"
+}
+
+job "custom_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  update {
+    min_healthy_time = "1s"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      secret "testsecret" {
+        provider = "test_secret_plugin"
+        path     = "some/path"
+        env {
+          // The custom plugin will output this as part of the result field
+          TEST_ENV = "${var.secret_value}"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.TEST_ENV}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/input/nomad_secret.hcl

@@ -0,0 +1,50 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "secret_path" {
+  type        = string
+  description = "The path of the vault secret"
+}
+
+job "nomad_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  update {
+    min_healthy_time = "1s"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      secret "testsecret" {
+        provider = "nomad"
+        path     = "${var.secret_path}"
+        config {
+          namespace = "default"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.key}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/input/vault_secret.hcl

@@ -0,0 +1,52 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "secret_path" {
+  type        = string
+  description = "The path of the vault secret"
+}
+
+job "vault_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  update {
+    min_healthy_time = "1s"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      vault {}
+
+      secret "testsecret" {
+        provider = "vault"
+        path     = "${var.secret_path}"
+        config {
+          engine = "kv_v2"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.key}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/secret_test.go

@@ -0,0 +1,117 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secret
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/shoenig/test/must"
+)
+
+const ns = "default"
+
+func TestVaultSecret(t *testing.T) {
+	// Lookup the cluster ID which is the KV backend path start.
+	clusterID, found := os.LookupEnv("CLUSTER_UNIQUE_IDENTIFIER")
+	if !found {
+		t.Fatal("CLUSTER_UNIQUE_IDENTIFIER env var not set")
+	}
+
+	// Generate our pathing for Vault and a secret value that we will check as
+	// part of the test.
+	secretCLIPath := filepath.Join(ns, "vault_secret", "testsecret")
+	secretFullPath := filepath.Join(clusterID, "data", secretCLIPath)
+	secretValue := uuid.Generate()
+
+	// Create the secret at the correct mount point for this E2E cluster and use
+	// the metadata delete command to permanently delete this when the test exits
+	e2e.MustCommand(t, "vault kv put -mount=%s %s key=%s", clusterID, secretCLIPath, secretValue)
+	e2e.CleanupCommand(t, "vault kv metadata delete -mount=%s %s", clusterID, secretCLIPath)
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/vault_secret.hcl",
+		jobs3.DisableRandomJobID(), // our path won't match the secret path with a random jobID
+		jobs3.Namespace(ns),
+		jobs3.Var("secret_path", secretFullPath),
+	)
+	t.Cleanup(cleanJob)
+
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}
+
+func TestNomadSecret(t *testing.T) {
+	// Generate our pathing for Vault and a secret value that we will check as
+	// part of the test.
+	secretFullPath := filepath.Join("nomad_secret", "testsecret")
+	secretValue := uuid.Generate()
+
+	nomadClient := e2e.NomadClient(t)
+
+	opts := &api.WriteOptions{Namespace: ns}
+	_, _, err := nomadClient.Variables().Create(&api.Variable{
+		Namespace: ns,
+		Path:      secretFullPath,
+		Items:     map[string]string{"key": secretValue},
+	}, opts)
+	must.NoError(t, err)
+
+	// create an ACL policy and attach it to the job ID this test will run
+	myNamespacePolicy := api.ACLPolicy{
+		Name:        "secret-block-policy",
+		Rules:       fmt.Sprintf(`namespace "%s" {variables {path "*" {capabilities = ["read"]}}}`, ns),
+		Description: "This namespace is for secrets block e2e testing",
+		JobACL: &api.JobACL{
+			Namespace: ns,
+			JobID:     "nomad_secret",
+		},
+	}
+	_, err = nomadClient.ACLPolicies().Upsert(&myNamespacePolicy, nil)
+	must.NoError(t, err)
+
+	t.Cleanup(func() {
+		nomadClient.ACLPolicies().Delete("secret-block-policy", nil)
+	})
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/nomad_secret.hcl",
+		jobs3.DisableRandomJobID(),
+		jobs3.Namespace(ns),
+		jobs3.Var("secret_path", secretFullPath),
+	)
+	t.Cleanup(cleanJob)
+
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}
+
+func TestPluginSecret(t *testing.T) {
+	// Generate a uuid value for the secret plugins env block which it will output
+	// as a part of the result field.
+	secretValue := uuid.Generate()
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/custom_secret.hcl",
+		jobs3.DisableRandomJobID(),
+		jobs3.Namespace(ns),
+		jobs3.Var("secret_value", secretValue),
+	)
+	t.Cleanup(cleanJob)
+
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}

e2e/terraform/packer/ubuntu-jammy-amd64/common-plugins/test_secret_plugin.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+fingerprint() {
+  echo {\"type\": \"secrets\", \"version\": \"0.0.1\"}
+}
+
+fetch() {
+  # return any passed environment variables as output
+  echo '{"result":{'$(printenv | awk -F= '{printf "\"%s\":\"%s\",", $1, $2}' | sed 's/,$//')'}}'
+}
+
+case "$1" in
+  fingerprint)
+    fingerprint
+    ;;
+  fetch)
+    fetch
+    ;;
+  *)
+    exit 1
+esac
+

e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh

@@ -113,6 +113,10 @@ echo "Installing additional CNI network configs"
 # copy of nomad's "bridge" for connect+cni test (e2e/connect/)
 sudo mv /tmp/linux/cni/nomad_bridge_copy.conflist /opt/cni/config/
 
+echo "Installing CPI test plugins"
+mkdir_for_root /opt/nomad/data/common_plugins/secrets
+sudo mv /tmp/linux/common-plugins/test_secret_plugin.sh /opt/nomad/data/common_plugins/secrets/test_secret_plugin
+
 # Podman
 echo "Installing Podman"
 sudo apt-get -y install podman catatonit