diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 17242c74e..1f2ecf079 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -44,6 +44,7 @@ import ( _ "github.com/hashicorp/nomad/e2e/podman" _ "github.com/hashicorp/nomad/e2e/rescheduling" _ "github.com/hashicorp/nomad/e2e/scaling" + _ "github.com/hashicorp/nomad/e2e/secret" _ "github.com/hashicorp/nomad/e2e/spread" _ "github.com/hashicorp/nomad/e2e/vaultsecrets" _ "github.com/hashicorp/nomad/e2e/volume_mounts" diff --git a/e2e/secret/doc.go b/e2e/secret/doc.go new file mode 100644 index 000000000..eaa9b3f70 --- /dev/null +++ b/e2e/secret/doc.go @@ -0,0 +1,8 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +// Package secret provides end-to-end tests for Nomad workloads with secret blocks. +// +// In order to run this test suite only, from the e2e directory you can trigger +// go test -v ./secret +package secret diff --git a/e2e/secret/input/nomad_secret.hcl b/e2e/secret/input/nomad_secret.hcl new file mode 100644 index 000000000..8db05eb2f --- /dev/null +++ b/e2e/secret/input/nomad_secret.hcl @@ -0,0 +1,41 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "nomad_secret" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + + task "task" { + + driver = "docker" + + config { + image = "busybox:1" + command = "/bin/sh" + args = ["-c", "sleep 300"] + } + + secret "testsecret" { + provider = "nomad" + path = "SECRET_PATH" + config { + namespace = "default" + } + } + + env { + TEST_SECRET = "${secret.testsecret.key}" + } + + resources { + cpu = 128 + memory = 64 + } + } + } +} diff --git a/e2e/secret/input/vault_secret.hcl b/e2e/secret/input/vault_secret.hcl new file mode 100644 index 000000000..ea2bcfa30 --- /dev/null +++ b/e2e/secret/input/vault_secret.hcl @@ -0,0 +1,43 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "vault_secret" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + + task "task" { + + driver = "docker" + + config { + image = "busybox:1" + command = "/bin/sh" + args = ["-c", "sleep 300"] + } + + vault {} + + secret "testsecret" { + provider = "vault" + path = "SECRET_PATH" + config { + engine = "kv_v2" + } + } + + env { + TEST_SECRET = "${secret.testsecret.key}" + } + + resources { + cpu = 128 + memory = 64 + } + } + } +} diff --git a/e2e/secret/secret_test.go b/e2e/secret/secret_test.go new file mode 100644 index 000000000..0b2c39796 --- /dev/null +++ b/e2e/secret/secret_test.go @@ -0,0 +1,118 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package secret + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/hashicorp/nomad/api" + e2e "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/shoenig/test/must" +) + +const ns = "default" + +func TestVaultSecret(t *testing.T) { + // Lookup the cluster ID which is the KV backend path start. + clusterID, found := os.LookupEnv("CLUSTER_UNIQUE_IDENTIFIER") + if !found { + t.Fatal("CLUSTER_UNIQUE_IDENTIFIER env var not set") + } + + // Generate our pathing for Vault and a secret value that we will check as + // part of the test. + secretCLIPath := filepath.Join(ns, "vault_secret", "testsecret") + secretFullPath := filepath.Join(clusterID, "data", secretCLIPath) + secretValue := uuid.Generate() + + // Create the secret at the correct mount point for this E2E cluster and use + // the metadata delete command to permanently delete this when the test exits + e2e.MustCommand(t, "vault kv put -mount=%s %s key=%s", clusterID, secretCLIPath, secretValue) + e2e.CleanupCommand(t, "vault kv metadata delete -mount=%s %s", clusterID, secretCLIPath) + + submission, cleanJob := jobs3.Submit(t, + "./input/vault_secret.hcl", + jobs3.DisableRandomJobID(), + jobs3.Namespace(ns), + jobs3.Detach(), + jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath), + ) + t.Cleanup(cleanJob) + + // Ensure the placed allocation reaches the running state. If the test fails + // here, it's likely due to permissions or pathing of the secret errors. + must.NoError( + t, + e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}), + must.Sprint("expected running allocation"), + ) + + // Validate the nomad variable was read and parsed into the expected + // environment variable + out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env") + must.NoError(t, err) + must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue)) +} + +func TestNomadSecret(t *testing.T) { + // Generate our pathing for Vault and a secret value that we will check as + // part of the test. + secretFullPath := filepath.Join("nomad_secret", "testsecret") + secretValue := uuid.Generate() + + nomadClient := e2e.NomadClient(t) + + opts := &api.WriteOptions{Namespace: ns} + _, _, err := nomadClient.Variables().Create(&api.Variable{ + Namespace: ns, + Path: secretFullPath, + Items: map[string]string{"key": secretValue}, + }, opts) + must.NoError(t, err) + + // create an ACL policy and attach it to the job ID this test will run + myNamespacePolicy := api.ACLPolicy{ + Name: "secret-block-policy", + Rules: fmt.Sprintf(`namespace "%s" {variables {path "*" {capabilities = ["read"]}}}`, ns), + Description: "This namespace is for secrets block e2e testing", + JobACL: &api.JobACL{ + Namespace: ns, + JobID: "nomad_secret", + }, + } + _, err = nomadClient.ACLPolicies().Upsert(&myNamespacePolicy, nil) + must.NoError(t, err) + + t.Cleanup(func() { + nomadClient.ACLPolicies().Delete("secret-block-policy", nil) + }) + + submission, cleanJob := jobs3.Submit(t, + "./input/nomad_secret.hcl", + jobs3.DisableRandomJobID(), + jobs3.Namespace(ns), + jobs3.Detach(), + jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath), + ) + t.Cleanup(cleanJob) + + // Ensure the placed allocation reaches the running state. If the test fails + // here, it's likely due to permissions or pathing of the secret errors. + must.NoError( + t, + e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}), + must.Sprint("expected running allocation"), + ) + + // Validate the nomad variable was read and parsed into the expected + // environment variable + out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env") + must.NoError(t, err) + must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue)) +}